I am searching for a layer that performs an element-wise division of the input but of course the parameters of this division must be learned, just as those of a standard conv2D layer.
I found this:
https://www.tensorflow.org/api_docs/python/tf/keras/layers/Multiply
but i don't think its what i want, because i want the division parameters to be learned, not divide 2 layers.
With a dense layer, dot products are computed, which is NOT what I want. I am looking for ELEMENT-WISE multiplication/division.
The sample code for a Custom Layer that performs an element-wise division of the input with the parameters(Weights) of this division learned during Training, is shown below:
%tensorflow_version 2.x
from tensorflow import keras
from tensorflow.keras import Input
from tensorflow.keras.layers import Flatten, Dense
from tensorflow.keras.models import Model, Sequential
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras.layers import Layer
import numpy as np
class MyLayer(Layer):
def __init__(self, output_dims, **kwargs):
self.output_dims = output_dims
super(MyLayer, self).__init__(**kwargs)
def build(self, input_shape):
# Create a trainable weight variable for this layer.
self.kernel = self.add_weight(name='kernel',
shape=self.output_dims,
initializer='ones',
trainable=True)
super(MyLayer, self).build(input_shape) # Be sure to call this somewhere!
def call(self, x):
# Dividing Input with Weights
return tf.divide(x, self.kernel)
def compute_output_shape(self, input_shape):
return (self.output_dims)
mInput = np.array([[1,2,3,4]])
inShape = (4,)
net = Sequential()
outShape = (4,)
l1 = MyLayer(outShape, input_shape= inShape)
net.add(l1)
net.compile(loss='mean_absolute_error', optimizer='adam', metrics=['accuracy'])
p = net.predict(x=mInput, batch_size=1)
print(p)
Hope this helps. Happy Learning!
Related
Here is a demo code (not work). I want the filters (output_channels) equal to the input_channels, where I do not know the input_channels before hand. (i.e., it is dynamic).
import tensorflow.keras as keras
class CustomLayer(keras.layers.Layer):
def __init__(self, strides):
super().__init()
self.conv = keras.layers.Conv2D(
filters=input_channels,
strides=strides
)
def call(self, inputs):
return self.conv(inputs)
I am new to TensorFlow and Keras, I have been making a dilated resnet and wanted to add instance normalization on a layer but I could not as it keeps throwing errors.
I am using tensorflow 1.15 and keras 2.1. I commented out the BatchNormalization part which works and I tried to add instance normalization but it cannot find the module.
Thanks a lot for your suggestions
from keras.layers import Conv2D
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Nadam, Adam
from keras.layers import Input, Dense, Reshape, Activation, Flatten, Embedding, Dropout, Lambda, add, concatenate, Concatenate, ConvLSTM2D, LSTM, average, MaxPooling2D, multiply, MaxPooling3D
from keras.layers import GlobalAveragePooling2D, Permute
from keras.layers.advanced_activations import LeakyReLU, PReLU
from keras.layers.convolutional import UpSampling2D, Conv2D, Conv1D
from keras.models import Sequential, Model
from keras.utils import multi_gpu_model
from keras.utils.generic_utils import Progbar
from keras.constraints import maxnorm
from keras.activations import tanh, softmax
from keras import metrics, initializers, utils, regularizers
import tensorflow as tf
import numpy as np
import math
import os
import sys
import random
import keras.backend as K
epsilon = K.epsilon()
def basic_block_conv2D_norm_elu(filters, kernel_size, kernel_regularizer=regularizers.l2(1e-4),act_func="elu", normalize="Instance", dropout='0.15',
strides=1,use_bias = True,kernel_initializer = "he_normal",_dilation_rate=0):
def f(input):
if kernel_regularizer == None:
if _dilation_rate == 0:
conv = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides,
padding="same", use_bias=use_bias)(input)
else:
conv = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides,
padding="same", use_bias=use_bias,dilation_rate=_dilation_rate)(input)
else:
if _dilation_rate == 0:
conv = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides,
kernel_initializer=kernel_initializer, padding="same", use_bias=use_bias,
kernel_regularizer=kernel_regularizer)(input)
else:
conv = Conv2D(filters=filters, kernel_size=kernel_size, strides=strides,
kernel_initializer=kernel_initializer, padding="same", use_bias=use_bias,
kernel_regularizer=kernel_regularizer, dilation_rate=_dilation_rate)(input)
if dropout != None:
dropout_layer = Dropout(0.15)(conv)
if normalize == None and dropout != None:
norm_layer = conv(dropout_layer)
else:
norm_layer = InstanceNormalization()(dropout_layer)
# norm_layer = BatchNormalization()(dropout_layer)
return Activation(act_func)(norm_layer)
return f
There is no such thing as InstanceNormalization(). In Keras you do not have a separate layer for InstanceNormalisation. (Which doesn't mean that you can't apply InstanceNormalisation )
In Keras we have tf.keras.layers.BatchNormalization layer which can be used to apply any type of normalization.
This layer has following parameters:
axis=-1,
momentum=0.99,
epsilon=0.001,
center=True,
scale=True,
beta_initializer="zeros",
gamma_initializer="ones",
moving_mean_initializer="zeros",
moving_variance_initializer="ones",
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
**kwargs
)
Now you can change your axis parameter to generate the Instance normalisation Layer or any other type of normalisation.
The formula for BatchNormalisation and Instance Normalisation is given as:
Now, Let's Assume you have Channel first implementation i.e. [B,C,H,W] If you want to calculate BatchNormalisation then you need to give your channel axis as the axis in the BatchNormalisation() layer. In this case it will calculate C means and standard deviations
BatchNormalisation layer : tf.keras.layers.BatchNormalization(axis=1)
And If you want to calculate InstanceNormalisation then Just give set your axis as the axis of Batch and Channel. In this case it will calculate B*C means and standard deviations
InstanceNormalisation layer: tf.keras.layers.BatchNormalization(axis=[0,1])
Update 1
While using batch Normalisation you must keep training =1 if you want to use it as InstanceNormalisation
Update 2
You can directly use the inbuilt InstanceNormalisation
given as below
https://www.tensorflow.org/addons/api_docs/python/tfa/layers/InstanceNormalization
I am transitioning from Pytorch to TensorFlow 1.12 and would like to know whether it is possible to define tf.keras.Sequential classes within a tf.keras.Model and run those in eager mode.
I constructed this minimum non-working example and would be grateful if someone could advise where I am going wrong. I have also used tf.contrib.eager.Network classes (with more success) however, since they are scheduled for deprecation I tried avoiding those.
import numpy as np
import tensorflow as tf
import tensorflow.contrib.eager as tfe
from keras.models import Sequential
from keras.layers import Dense, Activation
from tensorflow.train import AdamOptimizer
tf.enable_eager_execution()
class MLP(tf.keras.Model):
def __init__(self, in_dim, out_dim, hidden_dim, num_layers, activation):
super(MLP, self).__init__()
model = Sequential()
in_features = in_dim
for layer in range(num_layers):
model.add(Dense(hidden_dim,))
model.add(Activation(activation))
in_features = hidden_dim
model.add(Dense(out_dim, input_shape=(hidden_dim,)))
self.model = model
def call(self, inputs):
return self.model(inputs)
model = MLP(10, 1, 20, 4, 'relu')
optim = AdamOptimizer(learning_rate=1e-4)
for v in model.variables:
print(v)
z = tf.convert_to_tensor(np.random.randn(100, 10), dtype=tf.float32)
with tfe.GradientTape() as tape:
tape.watch(z)
u = model(z)
loss = tf.reduce_mean(tf.abs(u))
grad = tape.gradient(loss, model.trainable_variables)
optim.apply_gradients(zip(grad, model.trainable_variables))
print(loss.numpy())
Use
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation
instead of:
from keras.models import Sequential
from keras.layers import Dense, Activation
The below code throws me an error "AttributeError: can't set attribute". I think this is because I am trying to put TensorFlow layers into an ordinary list.
Does anyone know how I can get around this and be able to create a list of layers? I don't want to use Sequential because it is less flexible.
In PyTorch they have ModuleLists which you can use instead of a list, is there an equivalent in TensorFlow I can use?
!pip install tensorflow-gpu==2.0.0-alpha0
import tensorflow as tf
from tensorflow.keras.layers import Dense, Flatten, Conv2D
from tensorflow.keras import Model
class MyModel(Model):
def __init__(self):
super(MyModel, self).__init__()
self.layers = self.create_layers()
def create_layers(self):
layers = [Conv2D(32, 3, activation='relu'), Flatten(),
Dense(128, activation='relu'), Dense(10, activation='softmax')]
return layers
def call(self, x):
for layer in self.layers:
x = layer(x)
return x
model = MyModel()
full problem
layers is a reserved name for the layers of the model. Consider using another attribute for the model.
import tensorflow as tf
from tensorflow.keras.layers import Dense, Flatten, Conv2D
from tensorflow.keras import Model
class MyModel(Model):
def __init__(self):
super(MyModel, self).__init__()
self.layers_custom = self.create_layers()
def create_layers(self):
layers = [Conv2D(32, 3, activation='relu'), Flatten(),
Dense(128, activation='relu'), Dense(10, activation='softmax')]
return layers
def call(self, x):
for layer in self.layers_custom:
x = layer(x)
return x
model = MyModel()
print(model.layers)
print(model.layers_custom)
Just see this example in tf2.0's tutorial about how to make a list of layers
https://www.tensorflow.org/tutorials/generative/pix2pix
I am attempting to use the Gamma function from tfp in a custom Keras loss function using the log_prob method, but the function always returns nan when training starts.
I have tested the loss function and seems to work fine:
import tensorflow as tf
import tensorflow_probability as tfp
tf.enable_eager_execution()
def gamma_loss(y_true, alpha, beta):
gamma_distr = tfp.distributions.Gamma(concentration=alpha, rate=beta)
log_lik_gamma = gamma_distr.log_prob(y_true)
return -tf.reduce_mean(log_lik_gamma)
gamma_loss(100, 2, 2).numpy()
# 194.00854
The problem may be related to the parameters (alpha and beta) that I am passing to the function and that are produced by the final (custom) layer of the model I am using.
This is the full snippet:
import tensorflow as tf
from tensorflow.keras import backend as K
from tensorflow.keras.layers import Input, Dense, Layer, Concatenate
from tensorflow.keras.models import Model
from tensorflow.keras.initializers import glorot_normal
import tensorflow_probability as tfp
from sklearn.datasets import make_regression
class GammaLayer(Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(GammaLayer, self).__init__(**kwargs)
def build(self, input_shape):
n_weight_rows = 4
self.kernel_2 = self.add_weight(name='kernel_2',
shape=(n_weight_rows, self.output_dim),
initializer=glorot_normal(),
trainable=True)
self.kernel_3 = self.add_weight(name='kernel_3',
shape=(n_weight_rows, self.output_dim),
initializer=glorot_normal(),
trainable=True)
self.bias_2 = self.add_weight(name='bias_2',
shape=(self.output_dim,),
initializer=glorot_normal(),
trainable=True)
self.bias_3 = self.add_weight(name='bias_3',
shape=(self.output_dim,),
initializer=glorot_normal(),
trainable=True)
super(GammaLayer, self).build(input_shape)
def call(self, x):
# Here i use softplus to make the parameters strictly positive
alpha = tf.math.softplus(K.dot(x, self.kernel_2) + self.bias_2)
beta = tf.math.softplus(K.dot(x, self.kernel_3) + self.bias_3)
return [alpha, beta]
def compute_output_shape(self, input_shape):
"""
The assumption is that the output is always one-dimensional
"""
return [(input_shape[0], self.output_dim), (input_shape[0], self.output_dim)]
def gamma_loss(y_true, y_pred):
alpha, beta = y_pred[0], y_pred[1]
gamma_distr = tfp.distributions.Gamma(concentration=alpha, rate=beta)
return -tf.reduce_mean(gamma_distr.log_prob(y_true))
X, y = make_regression(n_samples=1000, n_features=3, noise=0.1)
inputs = Input(shape=(3,))
x = Dense(6, activation='relu')(inputs)
x = Dense(4, activation='relu')(x)
x = GammaLayer(1, name='main_output')(x)
output_params = Concatenate(1, name="pvec")(x)
model = Model(inputs, output_params)
model.compile(loss=gamma_loss, optimizer='adam')
model.fit(X, y, epochs=30, batch_size=10) ```
Can you try adding an additional 1e-6 or so outside the softplus? For very negative values, softplus becomes quite close to zero.