How to change a saved model input shape in Tensorflow? - tensorflow

I want to make this repo https://github.com/ildoonet/tf-pose-estimation run with Intel Movidius, so I tried convert the pb model using mvNCCompile.
The problem is mvNCCompile require a fixed input shape but the model I have is a dynamic one.
I tried this
graph_path = 'models/graph/mobilenet_thin/graph_opt.pb'
with tf.gfile.GFile(graph_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.get_default_graph()
tf.import_graph_def(graph_def, name='TfPoseEstimator')
x = graph.get_tensor_by_name('TfPoseEstimator/image:0')
x.set_shape([1, 368, 368, 3])
x = graph.get_tensor_by_name('TfPoseEstimator/MobilenetV1/Conv2d_0/Conv2D:0')
x.set_shape([1, 368, 368, 24])
and got this
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_0/weights:0' shape=(3, 3, 3, 24) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/image:0' shape=(1, 368, 368, 3) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_0/Conv2D:0' shape=(1, 368, 368, 24) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_0/Conv2D_bn_offset:0' shape=(24,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_0/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 24) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_0/Relu:0' shape=(?, ?, ?, 24) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_1_depthwise/depthwise_weights:0' shape=(3, 3, 24, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_1_pointwise/weights:0' shape=(1, 1, 24, 48) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_1_depthwise/depthwise:0' shape=(?, ?, ?, 24) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_1_pointwise/Conv2D:0' shape=(?, ?, ?, 48) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_1_pointwise/Conv2D_bn_offset:0' shape=(48,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_1_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 48) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_1_pointwise/Relu:0' shape=(?, ?, ?, 48) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_2_depthwise/depthwise_weights:0' shape=(3, 3, 48, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_2_pointwise/weights:0' shape=(1, 1, 48, 96) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_2_depthwise/depthwise:0' shape=(?, ?, ?, 48) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_2_pointwise/Conv2D:0' shape=(?, ?, ?, 96) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_2_pointwise/Conv2D_bn_offset:0' shape=(96,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_2_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 96) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_2_pointwise/Relu:0' shape=(?, ?, ?, 96) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_3_depthwise/depthwise_weights:0' shape=(3, 3, 96, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_3_pointwise/weights:0' shape=(1, 1, 96, 96) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_3_depthwise/depthwise:0' shape=(?, ?, ?, 96) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_3_pointwise/Conv2D:0' shape=(?, ?, ?, 96) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_3_pointwise/Conv2D_bn_offset:0' shape=(96,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_3_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 96) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_3_pointwise/Relu:0' shape=(?, ?, ?, 96) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_4_depthwise/depthwise_weights:0' shape=(3, 3, 96, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_4_pointwise/weights:0' shape=(1, 1, 96, 192) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_4_depthwise/depthwise:0' shape=(?, ?, ?, 96) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_4_pointwise/Conv2D:0' shape=(?, ?, ?, 192) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_4_pointwise/Conv2D_bn_offset:0' shape=(192,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_4_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 192) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_4_pointwise/Relu:0' shape=(?, ?, ?, 192) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_5_depthwise/depthwise_weights:0' shape=(3, 3, 192, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_5_pointwise/weights:0' shape=(1, 1, 192, 192) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_5_depthwise/depthwise:0' shape=(?, ?, ?, 192) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_5_pointwise/Conv2D:0' shape=(?, ?, ?, 192) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_5_pointwise/Conv2D_bn_offset:0' shape=(192,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_5_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 192) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_5_pointwise/Relu:0' shape=(?, ?, ?, 192) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_6_depthwise/depthwise_weights:0' shape=(3, 3, 192, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_6_pointwise/weights:0' shape=(1, 1, 192, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_6_depthwise/depthwise:0' shape=(?, ?, ?, 192) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_6_pointwise/Conv2D:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_6_pointwise/Conv2D_bn_offset:0' shape=(384,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_6_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_6_pointwise/Relu:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_7_depthwise/depthwise_weights:0' shape=(3, 3, 384, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_7_pointwise/weights:0' shape=(1, 1, 384, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_7_depthwise/depthwise:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_7_pointwise/Conv2D:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_7_pointwise/Conv2D_bn_offset:0' shape=(384,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_7_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_7_pointwise/Relu:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_8_depthwise/depthwise_weights:0' shape=(3, 3, 384, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_8_pointwise/weights:0' shape=(1, 1, 384, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_8_depthwise/depthwise:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_8_pointwise/Conv2D:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_8_pointwise/Conv2D_bn_offset:0' shape=(384,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_8_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_8_pointwise/Relu:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_9_depthwise/depthwise_weights:0' shape=(3, 3, 384, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_9_pointwise/weights:0' shape=(1, 1, 384, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_9_depthwise/depthwise:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_9_pointwise/Conv2D:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_9_pointwise/Conv2D_bn_offset:0' shape=(384,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_9_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_9_pointwise/Relu:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_10_depthwise/depthwise_weights:0' shape=(3, 3, 384, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_10_pointwise/weights:0' shape=(1, 1, 384, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_10_depthwise/depthwise:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_10_pointwise/Conv2D:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_10_pointwise/Conv2D_bn_offset:0' shape=(384,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_10_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_10_pointwise/Relu:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_11_depthwise/depthwise_weights:0' shape=(3, 3, 384, 1) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_11_pointwise/weights:0' shape=(1, 1, 384, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_11_depthwise/depthwise:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_11_pointwise/Conv2D:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_11_pointwise/Conv2D_bn_offset:0' shape=(384,) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_11_pointwise/BatchNorm/FusedBatchNorm:0' shape=(?, ?, ?, 384) dtype=float32>,)
(<tf.Tensor 'TfPoseEstimator/MobilenetV1/Conv2d_11_pointwise/Relu:0' shape=(?, ?, ?, 384) dtype=float32>,)
Another layers beside TfPoseEstimator/image:0 and TfPoseEstimator/MobilenetV1/Conv2d_0/Conv2D:0 still have ? shape.
I'm very new in Tensorflow so this might be a stupid question, but how to change the input shape of a saved model?


I manage to solve this problem using this.
import tensorflow as tf
if __name__ == '__main__':
graph_path = 't/tf_model.pb'
with tf.gfile.GFile(graph_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.get_default_graph()
tf_new_image = tf.placeholder(shape=(1, 368, 368, 3), dtype='float32', name='new_image')
tf.import_graph_def(graph_def, name='TfPoseEstimator', input_map={"image:0": tf_new_image})
tf.train.write_graph(graph, "t", "mobilenet_thin_model.pb", as_text=False)


With tf2.x I belive you can change it to concrete func:
imported = tf.saved_model.load('/path/to/saved_model')
concrete_func = imported.signatures["serving_default"]
concrete_func.inputs[0].set_shape([1, 368, 368, 3])

Related

Using a model from TF Hub as a convolutional feature extractor

I would like to build a custom model on top of ResNet-50 extractor (its intermediate layer).
Here how I am trying to do that:
def get_model(img_shape, num_classes):
inputs = Input(shape=img_shape)
backbone = hub.load("https://tfhub.dev/google/imagenet/resnet_v2_50/feature_vector/3").signatures["image_feature_vector"]
x = backbone(inputs)['resnet_v2_50/block3/unit_1/bottleneck_v2/shortcut']
# Add a per-pixel classification layer
outputs = Conv2D(num_classes + 1, 3, activation="softmax", padding="same")(x)
model = tf.keras.Model(inputs=inputs, outputs=outputs)
return model
This code works. However, it runs very long (7 min in Colab) and the model summary I get look like this:
input_14 (InputLayer) [(None, 224, 224, 3)] 0
tf_op_layer_StatefulPartiti [(None, 2048), 0
onedCall_16 (TensorFlowOpLa (None, 28, 28, 256),
yer) (None, 56, 56, 256),
(None, 56, 56, 64),
(None, 56, 56, 64),
(None, 56, 56, 256),
(None, 56, 56, 256),
(None, 56, 56, 256),
(None, 56, 56, 64),
(None, 56, 56, 64),
(None, 56, 56, 256),
(None, 28, 28, 256),
(None, 56, 56, 64),
(None, 28, 28, 64),
(None, 28, 28, 256),
(None, 14, 14, 512),
(None, 28, 28, 512),
(None, 28, 28, 128),
(None, 28, 28, 128),
(None, 28, 28, 512),
(None, 28, 28, 512),
(None, 28, 28, 512),
(None, 28, 28, 128),
(None, 28, 28, 128),
(None, 28, 28, 512),
(None, 28, 28, 512),
(None, 28, 28, 128),
(None, 28, 28, 128),
(None, 28, 28, 512),
(None, 14, 14, 512),
(None, 28, 28, 128),
(None, 14, 14, 128),
(None, 14, 14, 512),
(None, 7, 7, 1024),
(None, 14, 14, 1024),
(None, 14, 14, 256),
(None, 14, 14, 256),
(None, 14, 14, 1024),
(None, 14, 14, 1024),
(None, 14, 14, 1024),
(None, 14, 14, 256),
(None, 14, 14, 256),
(None, 14, 14, 1024),
(None, 14, 14, 1024),
(None, 14, 14, 256),
(None, 14, 14, 256),
(None, 14, 14, 1024),
(None, 14, 14, 1024),
(None, 14, 14, 256),
(None, 14, 14, 256),
(None, 14, 14, 1024),
(None, 14, 14, 1024),
(None, 14, 14, 256),
(None, 14, 14, 256),
(None, 14, 14, 1024),
(None, 7, 7, 1024),
(None, 14, 14, 256),
(None, 7, 7, 256),
(None, 7, 7, 1024),
(None, 7, 7, 2048),
(None, 7, 7, 2048),
(None, 7, 7, 512),
(None, 7, 7, 512),
(None, 7, 7, 2048),
(None, 7, 7, 2048),
(None, 7, 7, 2048),
(None, 7, 7, 512),
(None, 7, 7, 512),
(None, 7, 7, 2048),
(None, 7, 7, 2048),
(None, 7, 7, 512),
(None, 7, 7, 512),
(None, 7, 7, 2048),
(None, 112, 112, 64),
(None, 1, 1, 2048)]
conv2d_3 (Conv2D) (None, 14, 14, 2) 18434
To me that looks like a hanging outputs from all intermediate layers of ResNet. Also, the execution time is somewhat too long for instantiation such a simple model.
How to avoid getting list of tensors as TensorFlowOpLayer and just link a single output to the rest of the model?

How can I write DSSIM+MAE loss function for model training

How can I write code DSSIM+MAE loss function from formula:
Loss = αMAE + (1-α)DSSIM
with
Mean Absolute Error(MAE) = (1/M) * ∑|yi – xi|
DSSIM = 1-SSIM
SSIM = (numerator1 * numerator2) / (denominator1 * denominator2)
numerator1 = 2 * μ12 + C1 #μ12 = μ1 * μ2
numerator2 = 2 * σ12 + C2
denominator1 = μ1_sq + μ2_sq + C1
denominator2 = σ1_sq + σ2_sq + C2
where α is a trade-off parameter between MAE and DSSIM,
M is the total number of pixels in the image,
μ is the mean value of the image,
σ is the standard variation of the image, and σx,y is the covariance of x and y two images. c1 and c2 are two variables that stabilize
the division with a weak denominator. In our implementation,
I set α = 0.75, c1 = (0.01L)^2 and c2 = (0.03L)^2
where L is the dynamic range of the pixel values in the image.
So, this is my code
def custom_loss (y_true,y_pred):
M = 512 #M = total number of pixels in the sCT image
sum = 0
y_pred = tf.cast(y_pred, tf.int32)
y_true = tf.cast(y_true, tf.int32)
print(y_pred.shape)
print(y_true.shape)
y_pred = y_pred[0]
y_true = y_true[0]
for i in range(n):
sum = sum+abs(y_true[i] - y_pred[i])
my_mae = sum / n
dssim = tf.reduce_mean((1 - tf.image.ssim(y_true,y_pred, max_val=512,
filter_size=11,filter_sigma=1.5, k1=0.01, k2=0.03)) / 2)
my_mae = tf.cast(my_mae, tf.float32)
return (0.75*my_mae) + (1 - 0.75*dssim)
it have error when I run
model.compile(optimizer='adam',loss= custom_loss,metrics=['accuracy'])
error is
Traceback (most recent call last):
File "C:/Users/CRA01/Desktop/Unet/custom loss.py", line 85, in
history = model.fit(Training_CBCT_dataset,Training_pCT_dataset,validation_split=0.2,batch_size=1, epochs=5, callbacks=[model_save_callback])
File "C:\Users\CRA01\miniconda3\envs\tf_2.9\lib\site-packages\keras\utils\traceback_utils.py", line 67, in error_handler
raise e.with_traceback(filtered_tb) from None
File "C:\Users\CRA01\AppData\Local\Temp_autograph_generated_fileqb1dimxg.py", line 15, in tf__train_function
retval = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope)
ValueError: in user code:
File "C:\Users\CRA01\miniconda3\envs\tf_2.9\lib\site-packages\keras\engine\training.py", line 1051, in train_function *
return step_function(self, iterator)
File "C:\Users\CRA01\miniconda3\envs\tf_2.9\lib\site-packages\keras\engine\training.py", line 1040, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "C:\Users\CRA01\miniconda3\envs\tf_2.9\lib\site-packages\keras\engine\training.py", line 1030, in run_step **
outputs = model.train_step(data)
File "C:\Users\CRA01\miniconda3\envs\tf_2.9\lib\site-packages\keras\engine\training.py", line 893, in train_step
self.optimizer.minimize(loss, self.trainable_variables, tape=tape)
File "C:\Users\CRA01\miniconda3\envs\tf_2.9\lib\site-packages\keras\optimizers\optimizer_v2\optimizer_v2.py", line 539, in minimize
return self.apply_gradients(grads_and_vars, name=name)
File "C:\Users\CRA01\miniconda3\envs\tf_2.9\lib\site-packages\keras\optimizers\optimizer_v2\optimizer_v2.py", line 640, in apply_gradients
grads_and_vars = optimizer_utils.filter_empty_gradients(grads_and_vars)
File "C:\Users\CRA01\miniconda3\envs\tf_2.9\lib\site-packages\keras\optimizers\optimizer_v2\utils.py", line 73, in filter_empty_gradients
raise ValueError(f"No gradients provided for any variable: {variable}. "
ValueError: No gradients provided for any variable: (['conv2d/kernel:0', 'conv2d/bias:0', 'conv2d_1/kernel:0', 'conv2d_1/bias:0', 'conv2d_2/kernel:0', 'conv2d_2/bias:0', 'conv2d_3/kernel:0', 'conv2d_3/bias:0', 'conv2d_4/kernel:0', 'conv2d_4/bias:0', 'conv2d_5/kernel:0', 'conv2d_5/bias:0', 'conv2d_6/kernel:0', 'conv2d_6/bias:0', 'conv2d_7/kernel:0', 'conv2d_7/bias:0', 'conv2d_8/kernel:0', 'conv2d_8/bias:0', 'conv2d_9/kernel:0', 'conv2d_9/bias:0', 'conv2d_10/kernel:0', 'conv2d_10/bias:0', 'conv2d_11/kernel:0', 'conv2d_11/bias:0', 'conv2d_12/kernel:0', 'conv2d_12/bias:0', 'conv2d_13/kernel:0', 'conv2d_13/bias:0', 'conv2d_14/kernel:0', 'conv2d_14/bias:0', 'conv2d_15/kernel:0', 'conv2d_15/bias:0', 'conv2d_16/kernel:0', 'conv2d_16/bias:0', 'conv2d_17/kernel:0', 'conv2d_17/bias:0', 'conv2d_18/kernel:0', 'conv2d_18/bias:0', 'conv2d_19/kernel:0', 'conv2d_19/bias:0', 'conv2d_20/kernel:0', 'conv2d_20/bias:0', 'conv2d_21/kernel:0', 'conv2d_21/bias:0', 'conv2d_22/kernel:0', 'conv2d_22/bias:0', 'conv2d_23/kernel:0', 'conv2d_23/bias:0', 'conv2d_24/kernel:0', 'conv2d_24/bias:0', 'conv2d_25/kernel:0', 'conv2d_25/bias:0', 'conv2d_26/kernel:0', 'conv2d_26/bias:0'],). Provided `grads_and_vars` is ((None, <tf.Variable 'conv2d/kernel:0' shape=(3, 3, 1, 32) dtype=float32>), (None, <tf.Variable 'conv2d/bias:0' shape=(32,) dtype=float32>), (None, <tf.Variable 'conv2d_1/kernel:0' shape=(3, 3, 32, 32) dtype=float32>), (None, <tf.Variable 'conv2d_1/bias:0' shape=(32,) dtype=float32>), (None, <tf.Variable 'conv2d_2/kernel:0' shape=(3, 3, 32, 64) dtype=float32>), (None, <tf.Variable 'conv2d_2/bias:0' shape=(64,) dtype=float32>), (None, <tf.Variable 'conv2d_3/kernel:0' shape=(3, 3, 64, 64) dtype=float32>), (None, <tf.Variable 'conv2d_3/bias:0' shape=(64,) dtype=float32>), (None, <tf.Variable 'conv2d_4/kernel:0' shape=(3, 3, 64, 128) dtype=float32>), (None, <tf.Variable 'conv2d_4/bias:0' shape=(128,) dtype=float32>), (None, <tf.Variable 'conv2d_5/kernel:0' shape=(3, 3, 128, 128) dtype=float32>), (None, <tf.Variable 'conv2d_5/bias:0' shape=(128,) dtype=float32>), (None, <tf.Variable 'conv2d_6/kernel:0' shape=(3, 3, 128, 256) dtype=float32>), (None, <tf.Variable 'conv2d_6/bias:0' shape=(256,) dtype=float32>), (None, <tf.Variable 'conv2d_7/kernel:0' shape=(3, 3, 256, 256) dtype=float32>), (None, <tf.Variable 'conv2d_7/bias:0' shape=(256,) dtype=float32>), (None, <tf.Variable 'conv2d_8/kernel:0' shape=(3, 3, 256, 512) dtype=float32>), (None, <tf.Variable 'conv2d_8/bias:0' shape=(512,) dtype=float32>), (None, <tf.Variable 'conv2d_9/kernel:0' shape=(3, 3, 512, 512) dtype=float32>), (None, <tf.Variable 'conv2d_9/bias:0' shape=(512,) dtype=float32>), (None, <tf.Variable 'conv2d_10/kernel:0' shape=(3, 3, 512, 1024) dtype=float32>), (None, <tf.Variable 'conv2d_10/bias:0' shape=(1024,) dtype=float32>), (None, <tf.Variable 'conv2d_11/kernel:0' shape=(3, 3, 1024, 1024) dtype=float32>), (None, <tf.Variable 'conv2d_11/bias:0' shape=(1024,) dtype=float32>), (None, <tf.Variable 'conv2d_12/kernel:0' shape=(3, 3, 1024, 2048) dtype=float32>), (None, <tf.Variable 'conv2d_12/bias:0' shape=(2048,) dtype=float32>), (None, <tf.Variable 'conv2d_13/kernel:0' shape=(3, 3, 2048, 2048) dtype=float32>), (None, <tf.Variable 'conv2d_13/bias:0' shape=(2048,) dtype=float32>), (None, <tf.Variable 'conv2d_14/kernel:0' shape=(3, 3, 3072, 1024) dtype=float32>), (None, <tf.Variable 'conv2d_14/bias:0' shape=(1024,) dtype=float32>), (None, <tf.Variable 'conv2d_15/kernel:0' shape=(3, 3, 1024, 1024) dtype=float32>), (None, <tf.Variable 'conv2d_15/bias:0' shape=(1024,) dtype=float32>), (None, <tf.Variable 'conv2d_16/kernel:0' shape=(3, 3, 1536, 512) dtype=float32>), (None, <tf.Variable 'conv2d_16/bias:0' shape=(512,) dtype=float32>), (None, <tf.Variable 'conv2d_17/kernel:0' shape=(3, 3, 512, 512) dtype=float32>), (None, <tf.Variable 'conv2d_17/bias:0' shape=(512,) dtype=float32>), (None, <tf.Variable 'conv2d_18/kernel:0' shape=(3, 3, 768, 256) dtype=float32>), (None, <tf.Variable 'conv2d_18/bias:0' shape=(256,) dtype=float32>), (None, <tf.Variable 'conv2d_19/kernel:0' shape=(3, 3, 256, 256) dtype=float32>), (None, <tf.Variable 'conv2d_19/bias:0' shape=(256,) dtype=float32>), (None, <tf.Variable 'conv2d_20/kernel:0' shape=(3, 3, 384, 128) dtype=float32>), (None, <tf.Variable 'conv2d_20/bias:0' shape=(128,) dtype=float32>), (None, <tf.Variable 'conv2d_21/kernel:0' shape=(3, 3, 128, 128) dtype=float32>), (None, <tf.Variable 'conv2d_21/bias:0' shape=(128,) dtype=float32>), (None, <tf.Variable 'conv2d_22/kernel:0' shape=(3, 3, 192, 64) dtype=float32>), (None, <tf.Variable 'conv2d_22/bias:0' shape=(64,) dtype=float32>), (None, <tf.Variable 'conv2d_23/kernel:0' shape=(3, 3, 64, 64) dtype=float32>), (None, <tf.Variable 'conv2d_23/bias:0' shape=(64,) dtype=float32>), (None, <tf.Variable 'conv2d_24/kernel:0' shape=(3, 3, 96, 32) dtype=float32>), (None, <tf.Variable 'conv2d_24/bias:0' shape=(32,) dtype=float32>), (None, <tf.Variable 'conv2d_25/kernel:0' shape=(3, 3, 32, 32) dtype=float32>), (None, <tf.Variable 'conv2d_25/bias:0' shape=(32,) dtype=float32>), (None, <tf.Variable 'conv2d_26/kernel:0' shape=(1, 1, 32, 1) dtype=float32>), (None, <tf.Variable 'conv2d_26/bias:0' shape=(1,) dtype=float32>)).

How to get pre relu layers in Keras Application VGG19 network?

Is it possible to get the pre activation outputs of conv4_1 layer of VGG19, before the activation function?
The VGG19 network from Keras Applications has the following layers:
Layer (type) Output Shape Param #
=================================================================
input_1 (InputLayer) (None, 224, 224, 3) 0
_________________________________________________________________
block1_conv1 (Conv2D) (None, 224, 224, 64) 1792
_________________________________________________________________
block1_conv2 (Conv2D) (None, 224, 224, 64) 36928
_________________________________________________________________
block1_pool (MaxPooling2D) (None, 112, 112, 64) 0
_________________________________________________________________
block2_conv1 (Conv2D) (None, 112, 112, 128) 73856
_________________________________________________________________
block2_conv2 (Conv2D) (None, 112, 112, 128) 147584
_________________________________________________________________
block2_pool (MaxPooling2D) (None, 56, 56, 128) 0
_________________________________________________________________
block3_conv1 (Conv2D) (None, 56, 56, 256) 295168
_________________________________________________________________
block3_conv2 (Conv2D) (None, 56, 56, 256) 590080
_________________________________________________________________
block3_conv3 (Conv2D) (None, 56, 56, 256) 590080
_________________________________________________________________
block3_conv4 (Conv2D) (None, 56, 56, 256) 590080
_________________________________________________________________
block3_pool (MaxPooling2D) (None, 28, 28, 256) 0
_________________________________________________________________
block4_conv1 (Conv2D) (None, 28, 28, 512) 1180160
_________________________________________________________________
block4_conv2 (Conv2D) (None, 28, 28, 512) 2359808
_________________________________________________________________
block4_conv3 (Conv2D) (None, 28, 28, 512) 2359808
_________________________________________________________________
block4_conv4 (Conv2D) (None, 28, 28, 512) 2359808
_________________________________________________________________
block4_pool (MaxPooling2D) (None, 14, 14, 512) 0
_________________________________________________________________
block5_conv1 (Conv2D) (None, 14, 14, 512) 2359808
_________________________________________________________________
block5_conv2 (Conv2D) (None, 14, 14, 512) 2359808
_________________________________________________________________
block5_conv3 (Conv2D) (None, 14, 14, 512) 2359808
_________________________________________________________________
block5_conv4 (Conv2D) (None, 14, 14, 512) 2359808
_________________________________________________________________
block5_pool (MaxPooling2D) (None, 7, 7, 512) 0
_________________________________________________________________
flatten (Flatten) (None, 25088) 0
_________________________________________________________________
fc1 (Dense) (None, 4096) 102764544
_________________________________________________________________
fc2 (Dense) (None, 4096) 16781312
_________________________________________________________________
predictions (Dense) (None, 1000) 4097000
=================================================================
Total params: 143,667,240
Trainable params: 143,667,240
Non-trainable params: 0
I am assuming that the outputs of these are post activation (relu).

Yes, you are right, the activation functions are built into the layers here, so you cannot access the output before activation, unless you are willing to do some work:
First, make an exact copy of the layer whose output you want to observe, just leave out the activation function. If I understand you correctly, you want block4_conv1, which is at index 12. Inspect it's config like this:
>>> vgg.layers[12].name
'block4_conv1'
>>> vgg.layers[12].filters
512
>>> vgg.layers[12].kernel_size
(3, 3)
>>> vgg.layers[12].padding
'same'
Create a copy of this layer without activation:
block4_conv1_copy = Conv2D(filters=512, kernel_size=(3, 3), padding='same')
Now, create a new model consisting of all the vgg-layers 0-11 plus the copy of layer 12:
injection_model = Sequential(vgg.layers[:12] + [block4_conv1_copy])
injection_model.summary()
This should yield
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
block1_conv1 (Conv2D) (None, 224, 224, 64) 1792
_________________________________________________________________
block1_conv2 (Conv2D) (None, 224, 224, 64) 36928
_________________________________________________________________
block1_pool (MaxPooling2D) (None, 112, 112, 64) 0
_________________________________________________________________
block2_conv1 (Conv2D) (None, 112, 112, 128) 73856
_________________________________________________________________
block2_conv2 (Conv2D) (None, 112, 112, 128) 147584
_________________________________________________________________
block2_pool (MaxPooling2D) (None, 56, 56, 128) 0
_________________________________________________________________
block3_conv1 (Conv2D) (None, 56, 56, 256) 295168
_________________________________________________________________
block3_conv2 (Conv2D) (None, 56, 56, 256) 590080
_________________________________________________________________
block3_conv3 (Conv2D) (None, 56, 56, 256) 590080
_________________________________________________________________
block3_conv4 (Conv2D) (None, 56, 56, 256) 590080
_________________________________________________________________
block3_pool (MaxPooling2D) (None, 28, 28, 256) 0
_________________________________________________________________
conv2d_2 (Conv2D) (None, 28, 28, 512) 1180160
=================================================================
Total params: 3,505,728
Trainable params: 3,505,728
Non-trainable params: 0
_________________________________________________________________
Now block4_conv1_copy knows its input shape, which is required to set the weights:
block4_conv1_copy.set_weights(vgg.layers[12].get_weights())
That should be it! Just call injection_model.predict(some_input) and you can observe the output of layer 12 before activation.

Why does fixing the network input size reduce the model's file size

I've a model, whose inputs during training are variable in size, for generalisation.
In order to quantise, I have to fix the input size, so I just recreate the model with fixed input sizes, and copy across all the weights and biases, then save the model.
For some reason though, the model size gets roughly quartered.
Note this is before quantisation or anything else, and parameters remain the same.
Two model summaries are below:
Model 1 = 4.6MB
old_model.summary(line_length=110)
______________________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==============================================================================================================
input_1 (InputLayer) (None, None, None, 4) 0
______________________________________________________________________________________________________________
gaussian_noise_1 (GaussianNoise) (None, None, None, 4) 0 input_1[0][0]
______________________________________________________________________________________________________________
conv2d_1 (Conv2D) (None, None, None, 32) 1184 gaussian_noise_1[0][0]
______________________________________________________________________________________________________________
batch_normalization_1 (BatchNormali (None, None, None, 32) 128 conv2d_1[0][0]
______________________________________________________________________________________________________________
gaussian_noise_2 (GaussianNoise) (None, None, None, 32) 0 batch_normalization_1[0][0]
______________________________________________________________________________________________________________
conv2d_2 (Conv2D) (None, None, None, 32) 9248 gaussian_noise_2[0][0]
______________________________________________________________________________________________________________
batch_normalization_2 (BatchNormali (None, None, None, 32) 128 conv2d_2[0][0]
______________________________________________________________________________________________________________
gaussian_noise_3 (GaussianNoise) (None, None, None, 32) 0 batch_normalization_2[0][0]
______________________________________________________________________________________________________________
conv2d_3 (Conv2D) (None, None, None, 64) 18496 gaussian_noise_3[0][0]
______________________________________________________________________________________________________________
batch_normalization_3 (BatchNormali (None, None, None, 64) 256 conv2d_3[0][0]
______________________________________________________________________________________________________________
gaussian_noise_4 (GaussianNoise) (None, None, None, 64) 0 batch_normalization_3[0][0]
______________________________________________________________________________________________________________
conv2d_4 (Conv2D) (None, None, None, 64) 36928 gaussian_noise_4[0][0]
______________________________________________________________________________________________________________
batch_normalization_4 (BatchNormali (None, None, None, 64) 256 conv2d_4[0][0]
______________________________________________________________________________________________________________
gaussian_noise_5 (GaussianNoise) (None, None, None, 64) 0 batch_normalization_4[0][0]
______________________________________________________________________________________________________________
up_sampling2d_1 (UpSampling2D) (None, None, None, 64) 0 gaussian_noise_5[0][0]
______________________________________________________________________________________________________________
concatenate_1 (Concatenate) (None, None, None, 96) 0 up_sampling2d_1[0][0]
batch_normalization_1[0][0]
______________________________________________________________________________________________________________
gaussian_noise_6 (GaussianNoise) (None, None, None, 96) 0 concatenate_1[0][0]
______________________________________________________________________________________________________________
conv2d_5 (Conv2D) (None, None, None, 64) 55360 gaussian_noise_6[0][0]
______________________________________________________________________________________________________________
batch_normalization_5 (BatchNormali (None, None, None, 64) 256 conv2d_5[0][0]
______________________________________________________________________________________________________________
gaussian_noise_7 (GaussianNoise) (None, None, None, 64) 0 batch_normalization_5[0][0]
______________________________________________________________________________________________________________
conv2d_6 (Conv2D) (None, None, None, 64) 36928 gaussian_noise_7[0][0]
______________________________________________________________________________________________________________
batch_normalization_6 (BatchNormali (None, None, None, 64) 256 conv2d_6[0][0]
______________________________________________________________________________________________________________
gaussian_noise_8 (GaussianNoise) (None, None, None, 64) 0 batch_normalization_6[0][0]
______________________________________________________________________________________________________________
conv2d_7 (Conv2D) (None, None, None, 64) 36928 gaussian_noise_8[0][0]
______________________________________________________________________________________________________________
batch_normalization_7 (BatchNormali (None, None, None, 64) 256 conv2d_7[0][0]
______________________________________________________________________________________________________________
gaussian_noise_9 (GaussianNoise) (None, None, None, 64) 0 batch_normalization_7[0][0]
______________________________________________________________________________________________________________
conv2d_8 (Conv2D) (None, None, None, 64) 36928 gaussian_noise_9[0][0]
______________________________________________________________________________________________________________
batch_normalization_8 (BatchNormali (None, None, None, 64) 256 conv2d_8[0][0]
______________________________________________________________________________________________________________
gaussian_noise_10 (GaussianNoise) (None, None, None, 64) 0 batch_normalization_8[0][0]
______________________________________________________________________________________________________________
conv2d_9 (Conv2D) (None, None, None, 64) 36928 gaussian_noise_10[0][0]
______________________________________________________________________________________________________________
batch_normalization_9 (BatchNormali (None, None, None, 64) 256 conv2d_9[0][0]
______________________________________________________________________________________________________________
gaussian_noise_11 (GaussianNoise) (None, None, None, 64) 0 batch_normalization_9[0][0]
______________________________________________________________________________________________________________
up_sampling2d_2 (UpSampling2D) (None, None, None, 64) 0 gaussian_noise_11[0][0]
______________________________________________________________________________________________________________
input_2 (InputLayer) (None, None, None, 3) 0
______________________________________________________________________________________________________________
concatenate_2 (Concatenate) (None, None, None, 67) 0 up_sampling2d_2[0][0]
input_2[0][0]
______________________________________________________________________________________________________________
gaussian_noise_12 (GaussianNoise) (None, None, None, 67) 0 concatenate_2[0][0]
______________________________________________________________________________________________________________
conv2d_10 (Conv2D) (None, None, None, 67) 40468 gaussian_noise_12[0][0]
______________________________________________________________________________________________________________
batch_normalization_10 (BatchNormal (None, None, None, 67) 268 conv2d_10[0][0]
______________________________________________________________________________________________________________
gaussian_noise_13 (GaussianNoise) (None, None, None, 67) 0 batch_normalization_10[0][0]
______________________________________________________________________________________________________________
conv2d_11 (Conv2D) (None, None, None, 67) 40468 gaussian_noise_13[0][0]
______________________________________________________________________________________________________________
batch_normalization_11 (BatchNormal (None, None, None, 67) 268 conv2d_11[0][0]
______________________________________________________________________________________________________________
gaussian_noise_14 (GaussianNoise) (None, None, None, 67) 0 batch_normalization_11[0][0]
______________________________________________________________________________________________________________
conv2d_12 (Conv2D) (None, None, None, 32) 19328 gaussian_noise_14[0][0]
______________________________________________________________________________________________________________
batch_normalization_12 (BatchNormal (None, None, None, 32) 128 conv2d_12[0][0]
______________________________________________________________________________________________________________
gaussian_noise_15 (GaussianNoise) (None, None, None, 32) 0 batch_normalization_12[0][0]
______________________________________________________________________________________________________________
conv2d_13 (Conv2D) (None, None, None, 3) 867 gaussian_noise_15[0][0]
==============================================================================================================
Total params: 372,771
Trainable params: 371,415
Non-trainable params: 1,356
Model 2 = 1.6MB
model.summary(line_length=110)
______________________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==============================================================================================================
input_1 (InputLayer) (None, 368, 256, 4) 0
______________________________________________________________________________________________________________
gaussian_noise_1 (GaussianNoise) (None, 368, 256, 4) 0 input_1[0][0]
______________________________________________________________________________________________________________
conv2d_1 (Conv2D) (None, 368, 256, 32) 1184 gaussian_noise_1[0][0]
______________________________________________________________________________________________________________
batch_normalization_1 (BatchNormali (None, 368, 256, 32) 128 conv2d_1[0][0]
______________________________________________________________________________________________________________
gaussian_noise_2 (GaussianNoise) (None, 368, 256, 32) 0 batch_normalization_1[0][0]
______________________________________________________________________________________________________________
conv2d_2 (Conv2D) (None, 184, 128, 32) 9248 gaussian_noise_2[0][0]
______________________________________________________________________________________________________________
batch_normalization_2 (BatchNormali (None, 184, 128, 32) 128 conv2d_2[0][0]
______________________________________________________________________________________________________________
gaussian_noise_3 (GaussianNoise) (None, 184, 128, 32) 0 batch_normalization_2[0][0]
______________________________________________________________________________________________________________
conv2d_3 (Conv2D) (None, 184, 128, 64) 18496 gaussian_noise_3[0][0]
______________________________________________________________________________________________________________
batch_normalization_3 (BatchNormali (None, 184, 128, 64) 256 conv2d_3[0][0]
______________________________________________________________________________________________________________
gaussian_noise_4 (GaussianNoise) (None, 184, 128, 64) 0 batch_normalization_3[0][0]
______________________________________________________________________________________________________________
conv2d_4 (Conv2D) (None, 184, 128, 64) 36928 gaussian_noise_4[0][0]
______________________________________________________________________________________________________________
batch_normalization_4 (BatchNormali (None, 184, 128, 64) 256 conv2d_4[0][0]
______________________________________________________________________________________________________________
gaussian_noise_5 (GaussianNoise) (None, 184, 128, 64) 0 batch_normalization_4[0][0]
______________________________________________________________________________________________________________
up_sampling2d_1 (UpSampling2D) (None, 368, 256, 64) 0 gaussian_noise_5[0][0]
______________________________________________________________________________________________________________
concatenate_1 (Concatenate) (None, 368, 256, 96) 0 up_sampling2d_1[0][0]
batch_normalization_1[0][0]
______________________________________________________________________________________________________________
gaussian_noise_6 (GaussianNoise) (None, 368, 256, 96) 0 concatenate_1[0][0]
______________________________________________________________________________________________________________
conv2d_5 (Conv2D) (None, 368, 256, 64) 55360 gaussian_noise_6[0][0]
______________________________________________________________________________________________________________
batch_normalization_5 (BatchNormali (None, 368, 256, 64) 256 conv2d_5[0][0]
______________________________________________________________________________________________________________
gaussian_noise_7 (GaussianNoise) (None, 368, 256, 64) 0 batch_normalization_5[0][0]
______________________________________________________________________________________________________________
conv2d_6 (Conv2D) (None, 368, 256, 64) 36928 gaussian_noise_7[0][0]
______________________________________________________________________________________________________________
batch_normalization_6 (BatchNormali (None, 368, 256, 64) 256 conv2d_6[0][0]
______________________________________________________________________________________________________________
gaussian_noise_8 (GaussianNoise) (None, 368, 256, 64) 0 batch_normalization_6[0][0]
______________________________________________________________________________________________________________
conv2d_7 (Conv2D) (None, 368, 256, 64) 36928 gaussian_noise_8[0][0]
______________________________________________________________________________________________________________
batch_normalization_7 (BatchNormali (None, 368, 256, 64) 256 conv2d_7[0][0]
______________________________________________________________________________________________________________
gaussian_noise_9 (GaussianNoise) (None, 368, 256, 64) 0 batch_normalization_7[0][0]
______________________________________________________________________________________________________________
conv2d_8 (Conv2D) (None, 368, 256, 64) 36928 gaussian_noise_9[0][0]
______________________________________________________________________________________________________________
batch_normalization_8 (BatchNormali (None, 368, 256, 64) 256 conv2d_8[0][0]
______________________________________________________________________________________________________________
gaussian_noise_10 (GaussianNoise) (None, 368, 256, 64) 0 batch_normalization_8[0][0]
______________________________________________________________________________________________________________
conv2d_9 (Conv2D) (None, 368, 256, 64) 36928 gaussian_noise_10[0][0]
______________________________________________________________________________________________________________
batch_normalization_9 (BatchNormali (None, 368, 256, 64) 256 conv2d_9[0][0]
______________________________________________________________________________________________________________
gaussian_noise_11 (GaussianNoise) (None, 368, 256, 64) 0 batch_normalization_9[0][0]
______________________________________________________________________________________________________________
up_sampling2d_2 (UpSampling2D) (None, 736, 512, 64) 0 gaussian_noise_11[0][0]
______________________________________________________________________________________________________________
input_2 (InputLayer) (None, 736, 512, 3) 0
______________________________________________________________________________________________________________
concatenate_2 (Concatenate) (None, 736, 512, 67) 0 up_sampling2d_2[0][0]
input_2[0][0]
______________________________________________________________________________________________________________
gaussian_noise_12 (GaussianNoise) (None, 736, 512, 67) 0 concatenate_2[0][0]
______________________________________________________________________________________________________________
conv2d_10 (Conv2D) (None, 736, 512, 67) 40468 gaussian_noise_12[0][0]
______________________________________________________________________________________________________________
batch_normalization_10 (BatchNormal (None, 736, 512, 67) 268 conv2d_10[0][0]
______________________________________________________________________________________________________________
gaussian_noise_13 (GaussianNoise) (None, 736, 512, 67) 0 batch_normalization_10[0][0]
______________________________________________________________________________________________________________
conv2d_11 (Conv2D) (None, 736, 512, 67) 40468 gaussian_noise_13[0][0]
______________________________________________________________________________________________________________
batch_normalization_11 (BatchNormal (None, 736, 512, 67) 268 conv2d_11[0][0]
______________________________________________________________________________________________________________
gaussian_noise_14 (GaussianNoise) (None, 736, 512, 67) 0 batch_normalization_11[0][0]
______________________________________________________________________________________________________________
conv2d_12 (Conv2D) (None, 736, 512, 32) 19328 gaussian_noise_14[0][0]
______________________________________________________________________________________________________________
batch_normalization_12 (BatchNormal (None, 736, 512, 32) 128 conv2d_12[0][0]
______________________________________________________________________________________________________________
gaussian_noise_15 (GaussianNoise) (None, 736, 512, 32) 0 batch_normalization_12[0][0]
______________________________________________________________________________________________________________
conv2d_13 (Conv2D) (None, 736, 512, 3) 867 gaussian_noise_15[0][0]
==============================================================================================================
Total params: 372,771
Trainable params: 371,415
Non-trainable params: 1,356
___________________________

#FCOS I think the difference is due to the fact that you haven't trained one model whereas the other model was trained.
When you save a trained model, it saves
model architecture,
weights & biases, and
optimizer's configuration
However, when you save a model that was not trained, it will not have optimizer's configuration.
To test the size difference, I created simple model with and without input size and found out that both the models have exactly same size as the number of parameters are same for both the models. Please check the model1 and model2 below.
Here is model1
import tensorflow as tf
model1 = tf.keras.models.Sequential([
tf.keras.layers.Dense(128, activation='relu',input_shape=(None, None, 784,)),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dense(512, activation='relu'),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(10, activation='softmax')
])
model1.save('mymodel1.h5',overwrite=True,include_optimizer=True)
model1.summary()
Model: "sequential_10"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_20 (Dense) (None, None, None, 128) 100480
_________________________________________________________________
dense_21 (Dense) (None, None, None, 256) 33024
_________________________________________________________________
dense_22 (Dense) (None, None, None, 512) 131584
_________________________________________________________________
dense_23 (Dense) (None, None, None, 256) 131328
_________________________________________________________________
dense_24 (Dense) (None, None, None, 128) 32896
_________________________________________________________________
dense_25 (Dense) (None, None, None, 64) 8256
_________________________________________________________________
dense_26 (Dense) (None, None, None, 10) 650
=================================================================
Total params: 438,218
Trainable params: 438,218
Non-trainable params: 0
Here is model2
model2 = tf.keras.models.Sequential([
tf.keras.layers.Dense(128, activation='relu',input_shape=(300, 300, 784,)),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dense(512, activation='relu'),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dense(128, activation='relu'),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(10, activation='softmax')
])
model2.save('mymodel2.h5',overwrite=True,include_optimizer=True)
model2.summary()
Model: "sequential_11"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_27 (Dense) (None, 300, 300, 128) 100480
_________________________________________________________________
dense_28 (Dense) (None, 300, 300, 256) 33024
_________________________________________________________________
dense_29 (Dense) (None, 300, 300, 512) 131584
_________________________________________________________________
dense_30 (Dense) (None, 300, 300, 256) 131328
_________________________________________________________________
dense_31 (Dense) (None, 300, 300, 128) 32896
_________________________________________________________________
dense_32 (Dense) (None, 300, 300, 64) 8256
_________________________________________________________________
dense_33 (Dense) (None, 300, 300, 10) 650
=================================================================
Total params: 438,218
Trainable params: 438,218
Non-trainable params: 0
_________________________________________________________________
Size of mode#1 and model#2 is same (1.7 MB).
Please let us know if you have any comments. Thanks!

Tensorflow Estimators MirrorStrategy Assertion Error

I'm using TF 1.10.0 on Ubuntu 16.04.
I'm using the Estimator API to build a language model and want to preserve the last hidden states to initialize the states for next batch. It looks something like this(Ref: https://stackoverflow.com/a/41240243/6948766):
init_state = get_state_variables(params.batch_size, lstm_cell)
# list of [batch_size, max_steps, lstm_dim]
with tf.variable_scope(direction):
_lstm_output_unpacked, final_state = tf.nn.static_rnn(
lstm_cell,
tf.unstack(lstm_input, axis=1),
initial_state=init_state,
dtype=DTYPE)
self.state_update_op.append(
get_state_update_op(init_state, final_state))
And training code looks like this:
dist_trategy = tf.contrib.distribute.MirroredStrategy()
run_config = tf.estimator.RunConfig(
train_distribute=dist_trategy)
estimator = tf.estimator.Estimator(model_fn=model.model_fn,
model_dir='model/',
params=params,
config=run_config)
But I get the following errors:
Traceback (most recent call last):
File "src/train_eval.py", line 64, in <module>
main()
File "src/train_eval.py", line 60, in main
run(p, arg.mode)
File "src/train_eval.py", line 47, in run
params, 'train'), max_steps=1000000)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 356, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1179, in _train_model
return self._train_model_distributed(input_fn, hooks, saving_listeners)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/estimator/estimator.py", line 1290, in _train_model_distributed
self.config)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/distribute.py", line 718, in call_for_each_tower
return self._call_for_each_tower(fn, *args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/distribute/python/mirrored_strategy.py", line 552, in _call_for_each_tower
return _call_for_each_tower(self, fn, *args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/distribute/python/mirrored_strategy.py", line 183, in _call_for_each_tower
coord.join(threads)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/coordinator.py", line 389, in join
six.reraise(*self._exc_info_to_raise)
File "/usr/local/lib/python3.5/dist-packages/six.py", line 693, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/coordinator.py", line 297, in stop_on_exception
yield
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/distribute/python/mirrored_strategy.py", line 166, in _call_for_each_tower
merge_args = values.regroup({t.device: t.merge_args for t in threads})
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/distribute/python/values.py", line 585, in regroup
for i in range(len(v0)))
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/distribute/python/values.py", line 585, in <genexpr>
for i in range(len(v0)))
File "/usr/local/lib/python3.5/dist-packages/tensorflow/contrib/distribute/python/values.py", line 576, in regroup
(len(v), len(v0), v, v0))
AssertionError: len(v) == 33, len(v0) == 2, v: [(<tensorflow.python.framework.ops.IndexedSlices object at 0x7fe818e9c518>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'char_embed/replica_3:0' shape=(11204, 64) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'char_embed/replica_2:0' shape=(11204, 64) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'char_embed:0' shape=(11204, 64) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'char_embed/replica_1:0' shape=(11204, 64) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_1:0' shape=(1, 1, 64, 32) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/W_cnn_0/replica_3:0' shape=(1, 1, 64, 32) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/W_cnn_0/replica_2:0' shape=(1, 1, 64, 32) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/W_cnn_0:0' shape=(1, 1, 64, 32) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/W_cnn_0/replica_1:0' shape=(1, 1, 64, 32) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_2:0' shape=(32,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/b_cnn_0/replica_3:0' shape=(32,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/b_cnn_0/replica_2:0' shape=(32,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/b_cnn_0:0' shape=(32,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/b_cnn_0/replica_1:0' shape=(32,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_3:0' shape=(1, 2, 64, 32) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/W_cnn_1/replica_3:0' shape=(1, 2, 64, 32) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/W_cnn_1/replica_2:0' shape=(1, 2, 64, 32) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/W_cnn_1:0' shape=(1, 2, 64, 32) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/W_cnn_1/replica_1:0' shape=(1, 2, 64, 32) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_4:0' shape=(32,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/b_cnn_1/replica_3:0' shape=(32,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/b_cnn_1/replica_2:0' shape=(32,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/b_cnn_1:0' shape=(32,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/b_cnn_1/replica_1:0' shape=(32,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_5:0' shape=(1, 3, 64, 64) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/W_cnn_2/replica_3:0' shape=(1, 3, 64, 64) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/W_cnn_2/replica_2:0' shape=(1, 3, 64, 64) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/W_cnn_2:0' shape=(1, 3, 64, 64) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/W_cnn_2/replica_1:0' shape=(1, 3, 64, 64) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_6:0' shape=(64,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/b_cnn_2/replica_3:0' shape=(64,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/b_cnn_2/replica_2:0' shape=(64,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/b_cnn_2:0' shape=(64,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/b_cnn_2/replica_1:0' shape=(64,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_7:0' shape=(1, 4, 64, 128) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/W_cnn_3/replica_3:0' shape=(1, 4, 64, 128) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/W_cnn_3/replica_2:0' shape=(1, 4, 64, 128) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/W_cnn_3:0' shape=(1, 4, 64, 128) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/W_cnn_3/replica_1:0' shape=(1, 4, 64, 128) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_8:0' shape=(128,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/b_cnn_3/replica_3:0' shape=(128,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/b_cnn_3/replica_2:0' shape=(128,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/b_cnn_3:0' shape=(128,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/b_cnn_3/replica_1:0' shape=(128,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_9:0' shape=(1, 5, 64, 256) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/W_cnn_4/replica_3:0' shape=(1, 5, 64, 256) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/W_cnn_4/replica_2:0' shape=(1, 5, 64, 256) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/W_cnn_4:0' shape=(1, 5, 64, 256) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/W_cnn_4/replica_1:0' shape=(1, 5, 64, 256) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_10:0' shape=(256,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/b_cnn_4/replica_3:0' shape=(256,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/b_cnn_4/replica_2:0' shape=(256,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/b_cnn_4:0' shape=(256,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/b_cnn_4/replica_1:0' shape=(256,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_11:0' shape=(1, 6, 64, 512) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/W_cnn_5/replica_3:0' shape=(1, 6, 64, 512) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/W_cnn_5/replica_2:0' shape=(1, 6, 64, 512) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/W_cnn_5:0' shape=(1, 6, 64, 512) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/W_cnn_5/replica_1:0' shape=(1, 6, 64, 512) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_12:0' shape=(512,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/b_cnn_5/replica_3:0' shape=(512,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/b_cnn_5/replica_2:0' shape=(512,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/b_cnn_5:0' shape=(512,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/b_cnn_5/replica_1:0' shape=(512,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_13:0' shape=(1, 7, 64, 1024) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/W_cnn_6/replica_3:0' shape=(1, 7, 64, 1024) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/W_cnn_6/replica_2:0' shape=(1, 7, 64, 1024) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/W_cnn_6:0' shape=(1, 7, 64, 1024) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/W_cnn_6/replica_1:0' shape=(1, 7, 64, 1024) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_14:0' shape=(1024,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN/b_cnn_6/replica_3:0' shape=(1024,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN/b_cnn_6/replica_2:0' shape=(1024,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN/b_cnn_6:0' shape=(1024,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN/b_cnn_6/replica_1:0' shape=(1024,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_15:0' shape=(2048, 512) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN_proj/W_proj/replica_3:0' shape=(2048, 512) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN_proj/W_proj/replica_2:0' shape=(2048, 512) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN_proj/W_proj:0' shape=(2048, 512) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN_proj/W_proj/replica_1:0' shape=(2048, 512) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_16:0' shape=(512,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN_proj/b_proj/replica_3:0' shape=(512,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN_proj/b_proj/replica_2:0' shape=(512,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN_proj/b_proj:0' shape=(512,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN_proj/b_proj/replica_1:0' shape=(512,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_17:0' shape=(2048, 2048) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN_high_0/W_carry/replica_3:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN_high_0/W_carry/replica_2:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN_high_0/W_carry:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN_high_0/W_carry/replica_1:0' shape=(2048, 2048) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_18:0' shape=(2048,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN_high_0/b_carry/replica_3:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN_high_0/b_carry/replica_2:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN_high_0/b_carry:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN_high_0/b_carry/replica_1:0' shape=(2048,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_19:0' shape=(2048, 2048) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN_high_0/W_transform/replica_3:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN_high_0/W_transform/replica_2:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN_high_0/W_transform:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN_high_0/W_transform/replica_1:0' shape=(2048, 2048) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_20:0' shape=(2048,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN_high_0/b_transform/replica_3:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN_high_0/b_transform/replica_2:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN_high_0/b_transform:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN_high_0/b_transform/replica_1:0' shape=(2048,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_21:0' shape=(2048, 2048) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN_high_1/W_carry/replica_3:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN_high_1/W_carry/replica_2:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN_high_1/W_carry:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN_high_1/W_carry/replica_1:0' shape=(2048, 2048) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_22:0' shape=(2048,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN_high_1/b_carry/replica_3:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN_high_1/b_carry/replica_2:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN_high_1/b_carry:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN_high_1/b_carry/replica_1:0' shape=(2048,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_23:0' shape=(2048, 2048) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN_high_1/W_transform/replica_3:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN_high_1/W_transform/replica_2:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN_high_1/W_transform:0' shape=(2048, 2048) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN_high_1/W_transform/replica_1:0' shape=(2048, 2048) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_24:0' shape=(2048,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'CNN_high_1/b_transform/replica_3:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'CNN_high_1/b_transform/replica_2:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'CNN_high_1/b_transform:0' shape=(2048,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'CNN_high_1/b_transform/replica_1:0' shape=(2048,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_25:0' shape=(1024, 16384) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/kernel/replica_3:0' shape=(1024, 16384) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/kernel/replica_2:0' shape=(1024, 16384) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/kernel:0' shape=(1024, 16384) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/kernel/replica_1:0' shape=(1024, 16384) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_26:0' shape=(16384,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/bias/replica_3:0' shape=(16384,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/bias/replica_2:0' shape=(16384,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/bias:0' shape=(16384,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/bias/replica_1:0' shape=(16384,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_27:0' shape=(4096, 512) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/projection/kernel/replica_3:0' shape=(4096, 512) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/projection/kernel/replica_2:0' shape=(4096, 512) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/projection/kernel:0' shape=(4096, 512) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_0/lstm_cell/projection/kernel/replica_1:0' shape=(4096, 512) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_28:0' shape=(1024, 16384) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/kernel/replica_3:0' shape=(1024, 16384) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/kernel/replica_2:0' shape=(1024, 16384) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/kernel:0' shape=(1024, 16384) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/kernel/replica_1:0' shape=(1024, 16384) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_29:0' shape=(16384,) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/bias/replica_3:0' shape=(16384,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/bias/replica_2:0' shape=(16384,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/bias:0' shape=(16384,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/bias/replica_1:0' shape=(16384,) dtype=float32>})), (<tf.Tensor 'clip_by_global_norm/clip_by_global_norm/_30:0' shape=(4096, 512) dtype=float32>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/projection/kernel/replica_3:0' shape=(4096, 512) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/projection/kernel/replica_2:0' shape=(4096, 512) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/projection/kernel:0' shape=(4096, 512) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'fw/rnn/multi_rnn_cell/cell_1/lstm_cell/projection/kernel/replica_1:0' shape=(4096, 512) dtype=float32>})), (<tensorflow.python.framework.ops.IndexedSlices object at 0x7fe818e9c390>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'softmax/W/replica_3:0' shape=(556527, 512) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'softmax/W/replica_2:0' shape=(556527, 512) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'softmax/W:0' shape=(556527, 512) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'softmax/W/replica_1:0' shape=(556527, 512) dtype=float32>})), (<tensorflow.python.framework.ops.IndexedSlices object at 0x7fe818e9c0b8>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'softmax/b/replica_3:0' shape=(556527,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'softmax/b/replica_2:0' shape=(556527,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'softmax/b:0' shape=(556527,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'softmax/b/replica_1:0' shape=(556527,) dtype=float32>}))], v0: [(<tensorflow.python.framework.ops.IndexedSlices object at 0x7fe8127cfe10>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'softmax/W/replica_3:0' shape=(556527, 512) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'softmax/W/replica_2:0' shape=(556527, 512) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'softmax/W:0' shape=(556527, 512) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'softmax/W/replica_1:0' shape=(556527, 512) dtype=float32>})), (<tensorflow.python.framework.ops.IndexedSlices object at 0x7fe8127cfe80>, MirroredVariable({'/replica:0/task:0/device:GPU:3': <tf.Variable 'softmax/b/replica_3:0' shape=(556527,) dtype=float32>, '/replica:0/task:0/device:GPU:2': <tf.Variable 'softmax/b/replica_2:0' shape=(556527,) dtype=float32>, '/replica:0/task:0/device:GPU:0': <tf.Variable 'softmax/b:0' shape=(556527,) dtype=float32>, '/replica:0/task:0/device:GPU:1': <tf.Variable 'softmax/b/replica_1:0' shape=(556527,) dtype=float32>}))]
Thanks in advance.