I am trying to use Tensorflow's 2.0 new MirroredStrategy but I am receiving an error saying:
ValueError: We currently do not support distribution strategy with a `Sequential` model that is created without `input_shape`/`input_dim` set in its first layer or a subclassed model.
Model:
class Model(kr.Model):
def __init__(self, input_shape, conv_sizes, num_outputs):
super().__init__('model_1')
self.num_outputs = num_outputs
rows, cols, depth = input_shape
self.one_hot = kl.Lambda(lambda x: tf.one_hot(tf.cast(x, 'int32'), num_outputs), input_shape=(rows, cols))
self.concat = kl.Concatenate(axis=-1)
vision_layers = []
for i, (filters, kernel, stride) in enumerate(conv_sizes):
if not i:
depth += num_outputs - 1
vision_layers += [kl.Conv2D(filters, kernel, stride, activation='relu',
input_shape=(rows, cols, depth))]
else:
vision_layers += [kl.Conv2D(filters, kernel, stride, activation='relu')]
vision_layers += [kl.MaxPool2D(pool_size=(2, 2))]
flatten = kl.Flatten()
dense = kl.Dense(num_outputs)
self.net = kr.Sequential(vision_layers+[flatten]+[dense])
self.build(input_shape=(None, ) + input_shape)
def call(self, inputs):
one_hot = self.one_hot(inputs[:, :, :, -1])
return self.net(self.concat([inputs[:, :, :, :-1], one_hot]))
Reproduction code:
model_args = {'conv_sizes': [(32, (2, 2), 1), (32, (2, 2), 1), (32, (2, 2), 1)],
'input_shape': (50, 50, 6),
'num_outputs': 5}
def dummy_loss(values, targets):
return tf.reduce_sum(values-targets, axis=-1)
mirrored_strategy = tf.distribute.MirroredStrategy()
with mirrored_strategy.scope():
model = Model(**model_args)
model.compile(optimizer=kr.optimizers.Adam(learning_rate=0.01), loss=dummy_loss)
Output:
Traceback (most recent call last):
File "/home/joao/anaconda3/envs/tf2/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3296, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-4-dc492e7c638b>", line 18, in <module>
model.compile(optimizer=kr.optimizers.Adam(learning_rate=0.01), loss=dummy_loss)
File "/home/joao/anaconda3/envs/tf2/lib/python3.6/site-packages/tensorflow/python/training/tracking/base.py", line 456, in _method_wrapper
result = method(self, *args, **kwargs)
File "/home/joao/anaconda3/envs/tf2/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py", line 263, in compile
'We currently do not support distribution strategy with a '
ValueError: We currently do not support distribution strategy with a `Sequential` model that is created without `input_shape`/`input_dim` set in its first layer or a subclassed model.
Model Summary (model.summary()):
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
lambda (Lambda) multiple 0
_________________________________________________________________
concatenate (Concatenate) multiple 0
_________________________________________________________________
sequential (Sequential) (None, 5) 13573
=================================================================
Total params: 13,573
Trainable params: 13,573
Non-trainable params: 0
I would do away with the Sequential approach and use the Model class directly:
def create_model(input_shape, conv_sizes, fc_sizes, num_outputs):
num_outputs = num_outputs
rows, cols, depth = input_shape
input_layer = kl.Input(shape=(rows, cols, depth))
actions = tf.slice(input_layer, [0, 0, 0, depth - 1], [-1, rows, cols, 1])
non_actions = tf.slice(input_layer, [0, 0, 0, 0], [-1, rows, cols, depth - 1])
one_hot = kl.Lambda(lambda x: tf.one_hot(tf.cast(x, 'int32'), num_outputs),
input_shape=(rows, cols))(actions)
concat = kl.Concatenate(axis=-1)([non_actions, tf.reshape(one_hot, (-1, rows, cols, num_outputs))])
vision_layer = concat
for i, (filters, kernel, stride) in enumerate(conv_sizes):
vision_layer = kl.Conv2D(filters, kernel, stride, activation='relu')(vision_layer)
vision_layer = kl.MaxPool2D(pool_size=(2, 2))(vision_layer)
flatten = kl.Flatten()(vision_layer)
dense = kl.Dense(num_outputs)(flatten)
return kr.Model(inputs=input_layer, outputs=[dense])
Related
I am getting the following error:
Node: 'BGNet/dense/BiasAdd'
Matrix size-incompatible: In[0]: [1120,0], In[1]: [2048,1024]
[[{{node BGNet/dense/BiasAdd}}]] [Op:__inference_train_function_11676]
I found the root in this part of the model:
File "<ipython-input-14-3dcbdf5337b8>", line 69, in call
f = self.dense(f)
This is my custom multi model:
class BGNet(tf.keras.Model):
def __init__(self, img_h, img_w, img_c, batch_size, classes):
super(BGNet, self).__init__(name='BGNet')
self.img_h = img_h
self.img_w = img_w
self.img_c = img_c
self.batch_size = batch_size
self.classes = classes
# (224, 224, 3)
self.bgblock0 = BGBlock(f=[32, 32, 32, 32],
k=[7, 5, 5, 5],
d=[1, 2, 2, 1],
stage=0)
# (112, 112, 32)
self.bgblock1 = BGBlock(f=[64, 64, 64, 64],
k=[5, 5, 5, 3],
d=[2, 1, 1, 2],
stage=1)
# (56, 56, 64)
self.bgblock2 = BGBlock(f=[128, 128, 128, 128],
k=[5, 5, 3, 3],
d=[2, 1, 2, 1],
stage=2)
# (28, 28, 128)
self.bgblock3 = BGBlock(f=[256, 256, 256, 256],
k=[5, 3, 3, 3,],
d=[1, 2, 1, 2],
stage=3)
# (14, 14, 256)
self.bgblock4 = BGBlock(f=[512, 512, 512],
k=[3, 3, 3],
d=[1, 1, 2],
stage=4)
# (7, 7, 512)
self.bgblock5 = BGBlock(f=[1024, 1024, 1024],
k=[3, 3, 1],
d=[2, 1, 1],
stage=5)
# (4, 4, 1024)
self.bgblock6 = BGBlock(f=[2048, 2048],
k=[1, 1],
d=[1, 2],
stage=6)
# (2, 2, 2048)
self.flatten = tf.keras.layers.Flatten(name='flatten')
self.dense = tf.keras.layers.Dense(1024, activation='tanh', name='dense')
self.dropout = tf.keras.layers.Dropout(0.2, name='dropout')
self.prob = tf.keras.layers.Dense(1, activation='sigmoid', name='prob')
self.concat1 = tf.keras.layers.Concatenate(axis=-1, name='concat1')
self.bbox1 = tf.keras.layers.Dense(512, activation='relu', name='bbox1')
self.bbox2 = tf.keras.layers.Dropout(0.1, name='bbox2')
self.bbox3 = tf.keras.layers.Dense(256, activation='sigmoid', name='bbox3')
self.bbox = tf.keras.layers.Dense(4, name='bbox')
self.concat2 = tf.keras.layers.Concatenate(axis=-1, name='concat2')
self.cat = tf.keras.layers.Dense(len(self.classes), activation='softmax', name='cat')
def call(self, input_tensor, training=True):
x = self.bgblock0(input_tensor)
x = self.bgblock1(x)
x = self.bgblock2(x)
x = self.bgblock3(x)
x = self.bgblock4(x)
x = self.bgblock5(x)
x = self.bgblock6(x)
f = self.flatten(x)
f = self.dense(f)
f = self.dropout(f)
p = self.prob(f)
b = self.concat1([f, p])
b = self.bbox1(b)
b = self.bbox2(b)
b = self.bbox3(b)
b = self.bbox(b)
c = self.concat2([f, b])
c = self.cat(c)
return {'prob': p, 'bbox': b, 'class': c}
model1 = BGNet(H, W, C, B, N)
model1.build(input_shape=(B, H, W, C))
model1.call(tf.keras.layers.Input(shape=(H, W, C), batch_size=B))
model1.summary(print_fn=tf.print, expand_nested=True, show_trainable=True)
The custom (BGBlocks) blocks are not that important but if you are curious they are convolution blocks consisting of conv2d, batchnorm, activation and pooling layers
The model produces 3 outputs of different size vector while sharing the first dense layers. The output layers first predict the confidence score(prob in loss) of the an object being in the image. Next they predict the bounding box(bbox in loss) and finally the class(class in loss) of the bounded object.
The main issue is after the flatten layer. The model builds without errors with input images of (224, 224, 3). This is how the summary of the model looks: model.summary() image
I have even created a custom IOU (Intersection Over Union) for bounding boxes to be used as model metric. The losses are simple, inbuilt and as follows:
loss = {'prob': 'binary_crossentropy', 'bbox': 'mse', 'class': 'categorical_crossentropy'}
Hoe can I resolve this error?
I want to make a custom metric as in https://www.tensorflow.org/guide/keras/train_and_evaluate#specifying_a_loss_metrics_and_an_optimizer
My code looks like this
class IOU(tf.keras.metrics.Metric):
def __init__(self, name='iou_part', **kwargs):
super(IOU, self).__init__(name=name, **kwargs)
self.iou = self.add_weight(name='iou_part', initializer='zeros')
self.template_width = 115
self.template_height = 75
self.frame_width = 1280
self.frame_height = 720
self.corners = tf.constant([[-0.5, 0.1], [-0.5, 0.5], [0.5, 0.5], [0.5, 0.1]], dtype=tf.float32)
self.epsilon = 1e-6
def update_state(self, y_true, y_pred, sample_weight=None):
batch_size = y_true.shape[0]
fake_frame = tf.ones((batch_size, 1, self.frame_height, self.frame_width))
fake_template = tf.ones((batch_size, 1, self.template_height, self.template_width))
target = get_perspective_transform(self.corners, tf.reshape(y_true, (-1, 2, 4)))
output = get_perspective_transform(self.corners, tf.reshape(y_pred, (-1, 2, 4)))
## Compute IOU
However this gives the error 'TypeError: Expected int32, got None of type 'NoneType' instead.' This is because y_true is (None, 4, 2) when model.compile(....) is executed. What is the correct way to get the batch size into the metric?
I am trying to implement a patch creation function with using tensorflow's extract_image_patches function but dynamic output shape is not same as my expectation.
Let me tell briefly what it does. Input shape is supposed to be
6000x4000. We first find its greatest common denominator. It turns out it is 3. then we pass '64' argument to our function to create patches with size of 3x64,2x64=192,128. This returns us 31x31 distinct patches. Everything works ok with static output, but when it comes to dynamic output things are not ok. I could not find which part caused a different dynamic output.
# input_shape_inbuild: (None, 6000, 4000, 1)
# ---LAYER---
# Input Size: (None, 6000, 4000, 1)
# Patch Size: (x,y) = 192, 128
# Aspect ratio: (3, 2)
!wget https://www.fujifilm.com/products/digital_cameras/x/fujifilm_x_pro2/sample_images/img/index/ff_x_pro2_001.JPG
img = cv2.imread('ff_x_pro2_001.JPG', 0)
img = tf.reshape(img, [1,img.shape[0],img.shape[1],1])
***tensorflow takes images as (y, x)
so a 6000x4000 im is given as tf.func(4000, 6000)
# Here I define custom layer in tensorflow.
class create_patches(Layer):
def __init__(self, patchMultiplier):
super(create_patches, self).__init__()
self.patchMultiplier = patchMultiplier
def build(self, input_shape):
print('input_shape_inbuild: ', input_shape)
def aspect_ratio(width, height):
#find greatest common divider of input_shape
def gcd(x, y):
while y != 0:
(x, y) = (y, x % y)
return x
r = gcd(width, height)
x = int(width/r)
y = int(height/r)
return x, y
self.aspect_ratio = aspect_ratio(input_shape[1], input_shape[2])
self.patchSize_x = self.aspect_ratio[0] * self.patchMultiplier
self.patchSize_y = self.aspect_ratio[1] * self.patchMultiplier
def call(self, inputs):
print('---LAYER---')
print('Input Size:', inputs._keras_shape)
print('Patch Size: (x,y) = {}, {}'.format(self.patchSize_x, self.patchSize_y))
print('Aspect ratio: {}'.format(self.aspect_ratio))
print('---LAYER---')
#call tf.extract_image_patches to return it.
out = tf.extract_image_patches(images=inputs,
ksizes=[1, self.patchSize_y, self.patchSize_x, 1],
strides=[1, self.patchSize_y, self.patchSize_x, 1],
rates=[1, 1, 1, 1],
padding='VALID')
return out
def compute_output_shape(self, input_shape):
"""
ksize_cols = patchSize_x
ksize_rows = patchSize_y
"""
#output shape=[batch, out_rows, out_cols, ksize_rows * ksize_cols * depth]
"""
shape = (self.patchSize_x, self.patchSize_y,
(input_shape[1]/self.patchSize_x) * (input_shape[2]/self.patchSize_y))
"""
shape =(input_shape[0],
input_shape[1]/self.patchSize_x, # patch row count
input_shape[2]/self.patchSize_y, # patch col count
self.patchSize_x * self.patchSize_y) # patch pixel count
return shape
#here is input with 6000x4000 pixels.
input_shape_1 = Input(shape=(6000, 4000, 1))
#here I fed input to my custom layer.
x1 = create_patches(64)(input_shape_1)
print('Output shape: ', x1.shape)
# here I build a model to see static output
f = K.function([input_shape_1], [x1])
import numpy as np
#result = f([np.random.randint(256, size=(1,4000,6000,1))])
result = f([img])
#print(result)
result = np.array(result)
# [batch, out_rows, out_cols, ksize_rows * ksize_cols * depth]
# Result shape: (1, 1, 125, 125, 1536)
print('Result shape: ', result.shape, '\n\n')
#print(result[:, :, :, 0].shape)
here is output I get.
input_shape_inbuild: (None, 6000, 4000, 1)
---LAYER---
Input Size: (None, 6000, 4000, 1)
Patch Size: (x,y) = 192, 128
Aspect ratio: (3, 2)
---LAYER---
Output shape: (?, 46, 20, 24576)
Result shape: (1, 1, 31, 31, 24576)
#####Result Shape is as I expected but at output shape I could not resolve where 46 and 20 come from. Could you tell me why it is like this?
I'm trying to create a char cnn using Keras. That type of cnn requires you to use Convolutional1D layer. But all the ways I try to add them to my model, it gives me errors at creation stage. Here is my code:
def char_cnn(n_vocab, max_len, n_classes):
conv_layers = [[256, 7, 3],
[256, 7, 3],
[256, 3, None],
[256, 3, None],
[256, 3, None],
[256, 3, 3]]
fully_layers = [1024, 1024]
th = 1e-6
embedding_size = 128
inputs = Input(shape=(max_len,), name='sent_input', dtype='int64')
# Embedding layer
x = Embedding(n_vocab, embedding_size, input_length=max_len)(inputs)
# Convolution layers
for cl in conv_layers:
x = Convolution1D(cl[0], cl[1])(x)
x = ThresholdedReLU(th)(x)
if not cl[2] is None:
x = MaxPooling1D(cl[2])(x)
x = Flatten()(x)
#Fully connected layers
for fl in fully_layers:
x = Dense(fl)(x)
x = ThresholdedReLU(th)(x)
x = Dropout(0.5)(x)
predictions = Dense(n_classes, activation='softmax')(x)
model = Model(input=inputs, output=predictions)
model.compile(optimizer='adam', loss='categorical_crossentropy')
return model
And here is the error I receive when I try to call char_cnn function
InvalidArgumentError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/common_shapes.py in _call_cpp_shape_fn_impl(op, input_tensors_needed, input_tensors_as_shapes_needed, require_shape_fn)
685 graph_def_version, node_def_str, input_shapes, input_tensors,
--> 686 input_tensors_as_shapes, status)
687 except errors.InvalidArgumentError as err:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/errors_impl.py in __exit__(self, type_arg, value_arg, traceback_arg)
515 compat.as_text(c_api.TF_Message(self.status.status)),
--> 516 c_api.TF_GetCode(self.status.status))
517 # Delete the underlying status object from memory otherwise it stays alive
InvalidArgumentError: Negative dimension size caused by subtracting 3 from 1 for 'conv1d_26/convolution/Conv2D' (op: 'Conv2D') with input shapes: [?,1,1,256], [1,3,256,256].
How to fix it?
Your downsampling is too aggressive and the key argument here is max_len: when it's too small, the sequence becomes too short to perform either a convolution or a max-pooling. You set pool_size=3, hence it shrinks the sequence by a factor of 3 after each pooling (see the example below). I suggest you try pool_size=2.
The minimal max_len that this network can handle is max_len=123. In this case x shape is transformed in the following way (according to conv_layers):
(?, 123, 128)
(?, 39, 256)
(?, 11, 256)
(?, 9, 256)
(?, 7, 256)
(?, 5, 256)
Setting a smaller value, like max_len=120 causes x.shape=(?, 4, 256) before the last layer and this can't be performed.
I am trying to build deconvolution network using tensorflow.
here is my code.
def decoder(self, activations):
with tf.variable_scope("Decoder") as scope:
h0 = conv2d(activations, 128, name = "d_h0_conv_1")
h0 = lrelu(h0)
shape = activations.get_shape().as_list()
h0 = deconv2d(h0, [shape[0], 2 * shape[1], 2 * shape[2], 128], name = "d_h0_deconv_1")
h0 = lrelu(h0)
h1 = conv2d(h0, 128, name = "d_h1_conv_1")
h1 = lrelu(h1)
h1 = conv2d(h1, 64, name = "d_h1_conv_2")
h1 = lrelu(h1)
shape = h1.get_shape().as_list()
h1 = deconv2d(h1, [shape[0], 2 * shape[1], 2 * shape[2], 64], name = "d_h1_deconv_1")
h1 = lrelu(h1)
h2 = conv2d(h1, 64, name = "d_h2_conv_1")
h2 = lrelu(h2)
h2 = conv2d(h2, 3, name = "d_h2_conv_2")
output = h2
print shape
return output
the parameter activation is basically activation from VGG19 network.
Here is the deconv2d() function
def deconv2d(input_, output_shape,
k_h=3, k_w=3, d_h=1, d_w=1, stddev=0.02,
name="deconv2d", with_w=False):
with tf.variable_scope(name):
# filter : [height, width, output_channels, in_channels]
w = tf.get_variable('w', [k_h, k_w, output_shape[-1], input_.get_shape()[-1]],
initializer=tf.contrib.layers.variance_scaling_initializer())
deconv = tf.nn.conv2d_transpose(input_, w, output_shape=output_shape,
strides=[1, d_h, d_w, 1], padding='SAME')
biases = tf.get_variable('biases', [output_shape[-1]], initializer=tf.constant_initializer(0.0))
deconv = tf.reshape(tf.nn.bias_add(deconv, biases), deconv.get_shape())
return deconv
and this is loss
with tf.name_scope("total_loss"):
self.loss = tf.nn.l2_loss(self.output - self.images)
It does not produce output shape compatible error.
However, with optimization,
with tf.variable_scope("Optimizer"):
optimizer = tf.train.AdamOptimizer(config.learning_rate)
grad_and_vars = optimizer.compute_gradients(self.loss, var_list = self.d_vars)
self.d_optim = optimizer.apply_gradients(grad_and_vars)
The tensorflow produces the error,
Traceback (most recent call last):
File "main.py", line 74, in <module>
tf.app.run()
File "/usr/local/lib/python2.7/dist- packages/tensorflow/python/platform/app.py", line 44, in run
_sys.exit(main(_sys.argv[:1] + flags_passthrough))
File "main.py", line 59, in main
dcgan.train(FLAGS)
File "/home/junyonglee/workspace/bi_sim/sumGAN/model.py", line 121, in train
grad_and_vars = optimizer.compute_gradients(self.loss, var_list = self.d_vars)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/optimizer.py", line 354, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gradients_impl.py", line 500, in gradients
in_grad.set_shape(t_in.get_shape())
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 425, in set_shape
self._shape = self._shape.merge_with(shape)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/tensor_shape.py", line 585, in merge_with
(self, other))
ValueError: Shapes (30, 256, 256, 64) and (30, 128, 128, 64) are not compatible
The output size of the decoder is (30, 256, 256 3) where 30 is the batch size.
It looks like at layer "d_h1_deconv_1", the global gradient (gradient flow into the op unit) is shape of (30, 256, 256, 64) where the local gradient (gradient wrt the inputs) is shape of (30, 128, 128, 64), which is very obvious fact that it is doing transposed convolution.
Does anyone know how to properly backprop using conv2d_transpose()?
Thank you!
Can you show us your deconv2d function? Without it I can't offer you a lot of advise.
Here are two ways I implemented such a deconvolution function:
def transpose_deconvolution_layer(input_tensor,used_weights,new_shape,stride,scope_name):
with tf.variable_scope(scope_name):
output = tf.nn.conv2d_transpose(input_tensor, used_weights, output_shape=new_shape,strides=[1,stride,stride,1], padding='SAME')
output = tf.nn.relu(output)
return output
def resize_deconvolution_layer(input_tensor,used_weights,new_shape,stride,scope_name):
with tf.variable_scope(scope_name):
output = tf.image.resize_images(input_tensor,(new_shape[1],new_shape[2]))#tf.nn.conv2d_transpose(input_tensor, used_weights, output_shape=new_shape,strides=[1,stride,stride,1], padding='SAME')
output, unused_weights = conv_layer(output,3,new_shape[3]*2,new_shape[3],1,scope_name+"_awesome_deconv")
return output
Please test if this works.
If you want to know more about why I programmed two, check this article: http://www.pinchofintelligence.com/photorealistic-neural-network-gameboy/ and this article: http://distill.pub/2016/deconv-checkerboard/
Let me know if this helped!
Kind regards