I am building a graph via a function and am trying to extract the value of a variable to add further operations. A part of the function I have written is shown below :
def build(self, save_path=None, save_name=None):
g = tf.Graph()
with g.as_default():
init_op = tf.initialize_all_variables()
images = tf.placeholder(tf.float32, shape=[None, 300, 300, 3], name='input')
with tf.variable_scope('conv1_'):
conv11 = self.conv_relu(images, kernel_shape=[3, 3, 3, 64], bias_shape=64, name='c1')
conv12 = self.conv_relu(conv11, kernel_shape=[3, 3, 64, 64], bias_shape=64, name='c2')
pool1 = tf.nn.max_pool(conv12, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool1')
with tf.variable_scope('conv2_'):
conv21 = self.conv_relu(pool1, kernel_shape=[3, 3, 64, 128], bias_shape=128, name='c1')
conv22 = self.conv_relu(conv21, kernel_shape=[3, 3, 128, 128], bias_shape=128, name='c2')
pool2 = tf.nn.max_pool(conv22, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool2')
with tf.variable_scope('conv3_'):
conv31 = self.conv_relu(pool2, kernel_shape=[3, 3, 128, 256], bias_shape=256, name='c1')
conv32 = self.conv_relu(conv31, kernel_shape=[3, 3, 256, 256], bias_shape=256, name='c2')
conv33 = self.conv_relu(conv32, kernel_shape=[3, 3, 256, 256], bias_shape=256, name='c3')
pool3 = tf.nn.max_pool(conv33, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool3')
with tf.variable_scope('conv4_'):
conv41 = self.conv_relu(pool3, kernel_shape=[3, 3, 256, 512], bias_shape=512, name='c1')
conv42 = self.conv_relu(conv41, kernel_shape=[3, 3, 512, 512], bias_shape=512, name='c2')
conv43 = self.conv_relu(conv42, kernel_shape=[3, 3, 512, 512], bias_shape=512, name='c3')
pool4 = tf.nn.max_pool(conv43, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool4')
with tf.variable_scope('conv5_'):
conv51 = self.conv_relu(pool4, kernel_shape=[3, 3, 512, 512], bias_shape=512, name='c1')
conv52 = self.conv_relu(conv51, kernel_shape=[3, 3, 512, 512], bias_shape=512, name='c2')
conv53 = self.conv_relu(conv52, kernel_shape=[3, 3, 512, 512], bias_shape=512, name='c3')
pool5 = tf.nn.max_pool(conv53, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool5')
pool5_shape = tf.shape(pool5)
pool5_reshaped = tf.reshape(pool5, shape=[pool5_shape[0], -1], name='pool5_reshaped')
weight_rows = pool5_shape[1] * pool5_shape[2] * pool5_shape[3]
sess = tf.Session(graph=g)
inp = np.zeros(shape=(2, 300, 300, 3))
print(inp.shape)
sess.run(init_op)
print(sess.run(weight_rows, feed_dict={images:inp}))
sess.close()
At the line print(sess.run(weight_rows, feed_dict={images:inp})) i get the following error :
FailedPreconditionError (see above for traceback): Attempting to use uninitialized value conv5_/biasesc3
[[Node: conv5_/biasesc3/read = Identity[T=DT_FLOAT, _class=["loc:#conv5_/biasesc3"], _device="/job:localhost/replica:0/task:0/cpu:0"](conv5_/biasesc3)]]
What is the reason for this error when I have run the init_op operation in the session before ? Exactly how does this work and what am I doing wrong over here ?
You need to define your init_op (i.e. call tf.initialize_all_variables()) after you declared all variables.
Creating a variable via tf.get_variable or tf.Variable places it in GLOBAL_VARIABLES collection (unless otherwise specified with collections kwarg). tf.initialize_all_variables() takes a look at this collection and creates an op that initializes variables listed.
To see GLOBAL_VARIABLES collection, you can use tf.get_collection with tf.GraphKeys.GLOBAL_VARIABLES as argument.
TL;DR Place init_op = tf.initialize_all_variables() after the graph was created.
Related
Question
Please confirm if the below is as designed and expected, or an issue of tf. slice, or a mistake in the usage of tf. slice. If a mistake, kindly suggest how to correct it.
Background
Introduction to tensor slicing - Extract tensor slices says Numpy-like slice syntax is an alternative of tf. slice.
Perform NumPy-like tensor slicing using tf. slice.
t1 = tf.constant([0, 1, 2, 3, 4, 5, 6, 7])
print(tf.slice(t1,
begin=[1],
size=[3]))
Alternatively, you can use a more Pythonic syntax. Note that tensor slices are evenly spaced over a start-stop range.
print(t1[1:4])
Problem
To Update the dark orange region.
TYPE = tf.int32
N = 4
D = 5
shape = (N,D)
# Target to update
Y = tf.Variable(
initial_value=tf.reshape(tf.range(N*D,dtype=TYPE), shape=shape),
trainable=True
)
print(f"Target Y: \n{Y}\n")
---
Target Y:
<tf.Variable 'Variable:0' shape=(4, 5) dtype=int32, numpy=
array([[ 0, 1, 2, 3, 4],
[ 5, 6, 7, 8, 9],
[10, 11, 12, 13, 14],
[15, 16, 17, 18, 19]], dtype=int32)>
tf. slice does not work.
# --------------------------------------------------------------------------------
# Slice region in the target to be updated
# --------------------------------------------------------------------------------
S = tf.slice( # Error "EagerTensor' object has no attribute 'assign'"
Y,
begin=[0,1], # Coordinate (n,d) as the start point
size=[3,2] # Shape (3,2) -> (n+3, n+2) as the end point
)
print(f"Slice to update S: \n{S}\n")
# Values to set
V = tf.ones(shape=tf.shape(S), dtype=TYPE)
print(f"Values to set V: \n{V}\n")
# Assing V to S region of T
S.assign(V)
---
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-17-e5692b1750c8> in <module>
24
25 # Assing V to S region of T
---> 26 S.assign(V)
AttributeError: 'tensorflow.python.framework.ops.EagerTensor' object has no attribute 'assign'
However, slice syntax works.
S = Y[
0:3, # From coordinate (n=0,d), slice rows (0,1,2) or 'size'=3 -> shape (3,?)
1:3 # From coordinate (n=0,d=1), slice columns (1,2) or 'size'=2 -> shape (3,2)
]
print(f"Slice to update S: \n{S}\n")
# Values to set
V = tf.ones(shape=tf.shape(S), dtype=TYPE)
print(f"Values to set V: \n{V}\n")
# Assing V to S region of T
S.assign(V)
---
<tf.Variable 'UnreadVariable' shape=(4, 5) dtype=int32, numpy=
array([[ 0, 1, 1, 3, 4],
[ 5, 1, 1, 8, 9],
[10, 1, 1, 13, 14],
[15, 16, 17, 18, 19]], dtype=int32)>
In my understanding, the above behavior is expected or not a bug at least. As the error said, there is no attribute called assign in tf. Tensor (EagerTensor for eager execution) but there is in tf. Variable. And generally, tf. slice returns a tensor as its output and thus it doesn't possess assign attribute.
AttributeError: 'tensorflow.python.framework.ops.EagerTensor' object has no attribute 'assign'
But when we do np like slicing and use it to modify the original tf. Variable, it seamlessly works.
Possible Solution
A workaround is to use tf.strided_slice instead of tf.slice. If we follow the source code of it, we will see, it takes the var argument which is a variable corresponding to input_
#tf_export("strided_slice")
#dispatch.add_dispatch_support
def strided_slice(input_,
begin,
end,
..........
var=None,
name=None):
And when we pass a parameter for var that basically corresponding to the input_, it then calls assign function that is defined within it
def assign(val, name=None):
"""Closure that holds all the arguments to create an assignment."""
if var is None:
raise ValueError("Sliced assignment is only supported for variables")
else:
if name is None:
name = parent_name + "_assign"
return var._strided_slice_assign(
begin=begin,
end=end,
strides=strides,
value=val,
name=name,
begin_mask=begin_mask,
end_mask=end_mask,
ellipsis_mask=ellipsis_mask,
new_axis_mask=new_axis_mask,
shrink_axis_mask=shrink_axis_mask)
So, when we pass var in the tf.strided_slice, it will return an assignable object.
Code
Here is the full working code for reference.
import tensorflow as tf
print(tf.__version__)
TYPE = tf.int32
N = 4
D = 5
shape = (N,D)
# Target to update
Y = tf.Variable(
initial_value=tf.reshape(tf.range(N*D,dtype=TYPE), shape=shape),
trainable=True
)
Y
2.4.1
<tf.Variable 'Variable:0' shape=(4, 5) dtype=int32, numpy=
array([[ 0, 1, 2, 3, 4],
[ 5, 6, 7, 8, 9],
[10, 11, 12, 13, 14],
[15, 16, 17, 18, 19]], dtype=int32)>
Now, we use tf.stried_slice instead of tf.slice.
S = tf.strided_slice(
Y,
begin = [0, 1],
end = [3, 3],
var = Y,
name ='slice_op'
)
S
<tf.Tensor: shape=(3, 2), dtype=int32, numpy=
array([[ 1, 2],
[ 6, 7],
[11, 12]], dtype=int32)>
Update the variables with no attribution error.
# Values to set
V = tf.ones(shape=tf.shape(S), dtype=TYPE)
print(V)
print()
# Assing V to S region of T
S.assign(V)
tf.Tensor(
[[1 1]
[1 1]
[1 1]], shape=(3, 2), dtype=int32)
<tf.Variable 'UnreadVariable' shape=(4, 5) dtype=int32, numpy=
array([[ 0, 1, 1, 3, 4],
[ 5, 1, 1, 8, 9],
[10, 1, 1, 13, 14],
[15, 16, 17, 18, 19]], dtype=int32)>
Using np like slicing.
# slicing
S = Y[
0:3,
1:3
]
S
<tf.Tensor: shape=(3, 2), dtype=int32, numpy=
array([[ 1, 2],
[ 6, 7],
[11, 12]], dtype=int32)>
# Values to set
V = tf.ones(shape=tf.shape(S), dtype=TYPE)
print(V)
# Assing V to S region of T
S.assign(V)
tf.Tensor(
[[1 1]
[1 1]
[1 1]], shape=(3, 2), dtype=int32)
<tf.Variable 'UnreadVariable' shape=(4, 5) dtype=int32, numpy=
array([[ 0, 1, 1, 3, 4],
[ 5, 1, 1, 8, 9],
[10, 1, 1, 13, 14],
[15, 16, 17, 18, 19]], dtype=int32)>
Materials
tf.Variable - tf.Tensor.
Suppose I have a 4D tensor x from a previous layer with shape [2, 2, 7, 7, 64] where batch = 2, depth = 2, height = 7, width = 7, and in_channels = 64.
And I'd like to upsample it to a tensor with shape [2, 4, 14, 14, 32].
Maybe next steps are transferring it with shape like [2, 8, 28, 28, 16] and [2, 16, 112, 112, 1] and so on.
I'm new to Tensorflow and I know that the implementations of transposed convolution between CAFFE and Tensorflow are different. I mean, in CAFFE, you can define the size of output by changing the strides of kernel. However, it's more complicated in tensorflow.
So how can I do that with tf.layers.conv3d_transpose or tf.nn.conv3d_transpose?
Would anyone give me a hand? Thanks!
You can do the upsampling with both tf.layers.conv3d_transpose and tf.nn.conv3d_transpose.
Lets consider your input tensor as:
input_layer = tf.placeholder(tf.float32, (2, 2, 7, 7, 64)) # batch, depth, height, width, in_channels
With tf.nn.conv3d_transpose we need to take care of the creation of the variables (weights and bias):
def conv3d_transpose(name, l_input, w, b, output_shape, stride=1):
transp_conv = tf.nn.conv3d_transpose(l_input, w, output_shape, strides=[1, stride, stride, stride, 1], padding='SAME')
return tf.nn.bias_add(transp_conv, b, name=name)
# Create variables for the operation
with tf.device('/cpu:0'):
# weights will have the shape [depth, height, width, output_channels, in_channels]
weights = tf.get_variable(name='w_transp_conv', shape=[3, 3, 3, 32, 64])
bias = tf.get_variable(name='b_transp_conv', shape=[32])
t_conv_layer = conv3d_transpose('t_conv_layer', input_layer, weights, bias,
output_shape=[2, 4, 14, 14, 32], stride=2)
print(t_conv_layer)
# Tensor("t_conv_layer:0", shape=(2, 4, 14, 14, 32), dtype=float32)
With tf.layers.conv3d_transpose, which will take care of the creation of both weights and bias, we use the same input tensor input_layer:
t_conv_layer2 = tf.layers.conv3d_transpose(input_layer, filters=32, kernel_size=[3, 3, 3],
strides=(2, 2, 2), padding='SAME', name='t_conv_layer2')
print(t_conv_layer2)
# Tensor("t_conv_layer2/Reshape_1:0", shape=(2, 4, 14, 14, 32), dtype=float32)
To get the other upsampled tensors you can repeat this procedure by changing the strides as you need:
Example with tf.layers.conv3d_transpose:
t_conv_layer3 = tf.layers.conv3d_transpose(t_conv_layer2, filters=16, kernel_size=[3, 3, 3],
strides=(2, 2, 2), padding='SAME', name='t_conv_layer3')
t_conv_layer4 = tf.layers.conv3d_transpose(t_conv_layer3, filters=8, kernel_size=[3, 3, 3],
strides=(2, 2, 2), padding='SAME', name='t_conv_layer4')
t_conv_layer5 = tf.layers.conv3d_transpose(t_conv_layer4, filters=1, kernel_size=[3, 3, 3],
strides=(1, 2, 2), padding='SAME', name='t_conv_layer5')
print(t_conv_layer5)
# Tensor("t_conv_layer5/Reshape_1:0", shape=(2, 16, 112, 112, 1), dtype=float32)
Note: since tf.nn.conv3d_transpose is actually the gradient of tf.nn.conv3d, you can make sure that the variable output_shape is correct, by considering the forward operation with tf.nn.conv3d.
def print_expected(weights, shape, stride=1):
output = tf.constant(0.1, shape=shape)
expected_layer = tf.nn.conv3d(output, weights, strides=[1, stride, stride, stride, 1], padding='SAME')
print("Expected shape of input layer when considering the output shape ({} and stride {}): {}".format(shape, stride, expected_layer.get_shape()))
Therefore, to produce a transposed convolution with shape [2, 4, 14, 14, 32], we can check, for example, strides 1 and 2:
print_expected(weights, shape=[2, 4, 14, 14, 32], stride=1)
print_expected(weights, shape=[2, 4, 14, 14, 32], stride=2)
which prints and confirms that the second option (using stride 2) is the right one to produce a tensor with our desired shape:
Expected shape of input layer when considering the output shape ([2, 4, 14, 14, 32] and stride 1): (2, 4, 14, 14, 64)
Expected shape of input layer when considering the output shape ([2, 4, 14, 14, 32] and stride 2): (2, 2, 7, 7, 64)
I am trying to make a text recognition model using CNNs and LSTMs with CTC Loss. My current model looks like this where the numbers in bracket are the shape of tensor after each layer. I have a vocabulary size of 94 and size of my input image is (64x1024). This model is improving very slowly and I need some thought if I can change some thing. Thanks :)
Input: (?, 64, 1024, 1)
cnn-1: [None, 64, 1024, 64]
bn-1: [None, 64, 1024, 64]
relu-1: [None, 64, 1024, 64]
maxpool-1: [None, 32, 512, 64]
cnn-2: [None, 32, 512, 128]
bn-2: [None, 32, 512, 128]
relu-2: [None, 32, 512, 128]
maxpool-2: [None, 16, 256, 128]
cnn-3: [None, 16, 256, 128]
bn-3: [None, 16, 256, 128]
relu-3: [None, 16, 256, 128]
maxpool-3: [None, 8, 128, 128]
cnn-4: [None, 8, 128, 256]
bn-4: [None, 8, 128, 256]
relu-4: [None, 8, 128, 256]
maxpool-4: [None, 4, 64, 256]
cnn-5: [None, 4, 64, 256]
bn-5: [None, 4, 64, 256]
relu-5: [None, 4, 64, 256]
maxpool-5: [None, 2, 32, 256]
lstm-input: [None, 32, 512]
lstm-output: [None, 32, 64]
lstm-output-reshaped: [None, 64]
fully-connected: [None, 94]
reshaped_logits: [None, None, 94]
transposed_logits: [None, None, 94]
the image size is [m,32,32,3] (m = no. of training examples)
the filter size is [3,3,3,10]
stride = 1
padding = None
if I convolve this using tensorflow.nn.conv2d then the output shape should be this, according to the formula
out ={ ( 32 - 3 + 2*(0) ) / 1 }+ 1 = 30
so the output size should be [m, 30, 30, 10]
but the output shape i am getting is [m, 32, 32, 10]
why is this happening?
# convolution layer 1
c1 = tf.nn.conv2d(x_train, w1, strides = [1,1,1,1], padding = 'SAME')
print('c1 size: ', c1.shape)
# activation function for c1: relu
r1 = tf.nn.relu(c1)
# maxpooling
p1 = tf.nn.max_pool(r1, ksize = [1,2,2,1], strides = [1,2,2,1], padding = 'SAME')
padding = "SAME" means:
input = [1, 2, 3, 4, 5, 6, 7, 8]
filter size = [1, 3]
stride = [2]
so input to filter will be [[1, 2, 3], [3, 4, 5], [5, 6, 7], [7, 8, 0]]
padding = "VALID" means:
input = [1, 2, 3, 4, 5, 6, 7, 8]
filter size = [1, 3]
stride = [2]
so input to filter will be [[1, 2, 3], [3, 4, 5], [5, 6, 7]]
Last pixel got dropped in this case.
So padding "VALID" will give you the output you expect.
I have a sobel filter
sobel_x = tf.constant([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]], tf.float32)
I want to get a depth of 64. The shape is momentarily [3,3,1], but it should result in [3,3,64].
How do to that? With the following line, I get shape errors.
tf.tile(sobel_x, [1, 1, 64])
ValueError: Shape must be rank 2 but is rank 3 for 'Tile' (op: 'Tile') with input shapes: [3,3], [3].
The reason you cannot broadcast is that the third dimension does not exist, and so you actually have a rank 2 tensor.
>>> sess.run(tf.shape(sobel_x))
array([3, 3], dtype=int32)
We can solve this problem by reshaping the tensor first.
>>> sobel_x = tf.reshape(sobel_x, [3, 3, 1])
>>> tf.tile(sobel_x, [1, 1, 64])
<tf.Tensor 'Tile_6:0' shape=(3, 3, 64) dtype=float32>
I think your issue is with sobel_x.
sobel_x.get_shape(): TensorShape([Dimension(3), Dimension(3)])
sobel_x: <tf.tensor 'Const:0' shape=(3, 3) dtype=float32
So sobel_x is a two dimension matrix and your passing a rank 3 input to tile hence the error.
Fix: Make sobel_x 3 dimensional such that the shape is shape=(3, 3, 1)
then tf.tile(sobel_x, [1, 1, 64] will output shape=(1, 1, 64)