now I'm learning TensorFlow, I wonder why numpy.swapaxes(0,3) required.
I know that result is (1, 14, 14, 5) means [ 15element[ 145element[ 145element[ 5element ] ] ] ]
and after bumpy.swapaxes(3,0) -> (5, 14, 14, 1) and 5 images.
below is my code, please save my question. thank you.
#load mnist data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
#get only 1 image & reshape it
img = mnist.train.images[0].reshape(28,28)
plt.imshow(img, cmap='gray')
sess = tf.InteractiveSession()
#reshape image to get color = 1
img = img.reshape(-1,28,28,1)
#filter 3X3, count = 5
W1 = tf.Variable(tf.random_normal([3, 3, 1, 5], stddev=0.01))
#zero-padded USE
conv2d = tf.nn.conv2d(img, W1, strides=[1, 2, 2, 1], padding='SAME')
print(conv2d)
sess.run(tf.global_variables_initializer())
#make convoultion data
conv2d_img = conv2d.eval()
#print converted images
conv2d_img = np.swapaxes(conv2d_img, 0, 3)
for i, one_img in enumerate(conv2d_img):
plt.subplot(1,5,i+1), plt.imshow(one_img.reshape(14,14), cmap='gray')
#pooling
pool = tf.nn.max_pool(conv2d, ksize=[1, 2, 2, 1], strides=[
1, 2, 2, 1], padding='SAME')
print(pool)
sess.run(tf.global_variables_initializer())
pool_img = pool.eval()
#print pooling image
pool_img = np.swapaxes(pool_img, 0, 3)
for i, one_img in enumerate(pool_img):
plt.subplot(1,5,i+1), plt.imshow(one_img.reshape(7, 7), cmap='gray')
The swapping is necessary because it changes the order of the image channel.
By default, TensorFlow uses NHWC, where C = 1 since we have a grayscale image.
Therefore, you need the number of channels (1 for a grayscale image, 3 for an RGB) to be on the last axis in your data.
In your code, you can see that the NHWC relation holds (5 for number of images == batch_size, 14 for height, 14 for width, and 1 for image channel).
Related
I have small matrix 4*4, I want to filter it with two different filters in TensorFlow (1.8.0). I have an example with one filter (my_filter):
I want to change the filter to
my_filter = tf.constant([0.2,0.5], shape=[2, 2, 3, 1])
One will be 2*2 all 0.25 other 2*2 all 0.5. But how to set the values?
This is my code:
import tensorflow as tf
import numpy as np
tf.reset_default_graph()
x_shape = [1, 4, 4, 1]
x_val = np.ones(shape=x_shape)
x_val[0,1,1,0]=5
print(x_val)
x_data = tf.placeholder(tf.float32, shape=x_shape)
my_filter = tf.constant(0.25, shape=[2, 2, 1, 1])
my_strides = [1, 2, 2, 1]
mov_avg_layer= tf.nn.conv2d(x_data, my_filter, my_strides,
padding='SAME', name='Moving_Avg_Window')
# Execute the operations
with tf.Session() as sess:
#print(x_data.eval())
result =sess.run(mov_avg_layer,feed_dict={x_data: x_val})
print("Filter: " , result)
print("Filter: " , result.shape)
sess.close()
First option
The filter can also be defined as a placeholder
filter = tf.placeholder(filter_type, filter_shape)
...
with tf.Session() as sess:
for i in range (number_filters) :
result =sess.run(mov_avg_layer,feed_dict={x_data: x_val, filter: filter_val})
Second option
define a second filter in the graph
my_filter = tf.constant(0.25, shape=[2, 2, 1, 1])
my_filter2 = tf.constant(0.5, shape=[2, 2, 1, 1])
mov_avg_layer= tf.nn.conv2d(x_data, my_filter, my_strides,
padding='SAME', name='Moving_Avg_Window')
mov_avg_laye2= tf.nn.conv2d(x_data, my_filter2, my_strides,
padding='SAME', name='Moving_Avg_Window')
...
with tf.Session() as sess:
result1, result2 =sess.run([mov_avg_layer1, mov_avg_layer2],feed_dict={x_data: x_val})
sess.close()
I have 200 images on a set, 100 identical squares and 100 identical circles. Images are 44x41 pixels and images are grayscale. I am trying to build a simple classifier to learn tensorflow.
The problem: the predictor vectors have always the same value regardless the input image.
Here's the code of my neural net:
import tensorflow as tf
import random as r
import matplotlib
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches
from PIL import Image
%matplotlib inline
#create pictures
for i in range(100):
fig1 = plt.figure(frameon = False, figsize=(1,1), dpi=32)
ax1 = fig1.add_subplot(111, aspect='equal')
posx = 0.25
posy = 0.25
ax1.add_patch(
patches.Rectangle(
(posx,posy), # (x,y)
0.5, # width
0.5, # height
)
)
ax1.axis('off')
fig1.savefig('rect' + str(i) + '.png', bbox_inches='tight')
for i in range(100):
fig1 = plt.figure(frameon = False, figsize=(1,1), dpi=32)
ax1 = fig1.add_subplot(111, aspect='equal')
posx = 0.5
posy = 0.5
ax1.add_patch(
patches.Circle(
(posx,posy), # (x,y)
0.3,
)
)
ax1.axis('off')
fig1.savefig('circ' + str(i) + '.png', bbox_inches='tight')
# create vectors
train_features = np.zeros((200,44,41,1))
train_labels = np.zeros((200,2))
for i in range(100):
#get rect
im = Image.open("rect" + str(i) + ".png")
im = im.convert(mode = "L")
xxx =list(im.getdata())
imdata = np.reshape(xxx, (44,41,1))
train_features[i] = imdata
train_labels[i] = np.array([0,1])
#get circle
im = Image.open("circ" + str(i) + ".png")
im = im.convert(mode = "L")
xxx = list(im.getdata())
imdata = np.reshape(xxx, (44,41,1))
train_features[i+100] = imdata
train_labels[i+100] = np.array([1,0])
tf.reset_default_graph()
features = tf.placeholder(tf.float32,shape=[None,44,41, 1])
labels = tf.placeholder(tf.float32,shape=[None,2])
weights = tf.Variable(tf.truncated_normal([3,3, 1, 16], stddev=0.1))
biases = tf.Variable(tf.zeros(16))
weights2 = tf.Variable(tf.truncated_normal([3,3, 16, 64], stddev=0.1))
biases2 = tf.Variable(tf.zeros(64))
conv_layer = tf.nn.conv2d(features, weights, strides=[1, 1, 1, 1], padding='SAME')
conv_layer_b = tf.nn.bias_add(conv_layer, biases)
conv_layer_relu = tf.nn.relu(conv_layer_b)
conv_layer_pool = tf.nn.max_pool(conv_layer_relu, ksize=[1, 2, 2, 1], strides=[1, 1, 1, 1], padding='SAME')
conv_layer2 = tf.nn.conv2d(conv_layer_pool, weights2, strides=[1, 1, 1, 1], padding='SAME')
conv_layer2_b = tf.nn.bias_add(conv_layer2, biases2)
conv_layer2_relu = tf.nn.relu(conv_layer2_b)
conv_layer2_pool = tf.nn.max_pool(conv_layer2_relu, ksize=[1, 2, 2, 1], strides=[1, 1, 1, 1], padding='SAME')
#fully connected layer
weights_fc = tf.Variable(tf.truncated_normal([44*41*64, 256], stddev=0.1))
biases_fc = tf.Variable(tf.zeros([256]))
fc = tf.reshape(conv_layer2_pool, [-1, weights_fc.get_shape().as_list()[0]])
fc_logit = tf.add(tf.matmul(fc, weights_fc), biases_fc)
fc_relu = tf.nn.relu(fc_logit)
#fc_drop = tf.nn.dropout(fc_relu, 0.75)
# final layer
weights_out = tf.Variable(tf.truncated_normal([256, 2], stddev=0.1))
biases_out = tf.Variable(tf.zeros([2]))
out = tf.add(tf.matmul(fc_relu, weights_out), biases_out)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=out, labels=labels))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for _ in range(100):
sess.run(optimizer, feed_dict={
features: train_features[:],
labels: train_labels[:]})
for i in range(200):
outx = sess.run(out, feed_dict={
features: [train_features[i]],
labels: [train_labels[i]]})
print(outx)
print(train_labels[i])
print('---')
Try not to give the same name to two tensors. For example, you have conv_layer that is equal to tf.nn.conv2d(features, weights, strides=[1, 1, 1, 1], padding='SAME') then rewriten to tf.nn.bias_add(conv_layer, biases), then once more then its another shape and then ....
Use this naming for example:
conv_layer = tf.nn.conv2d(features, weights, strides=[1, 1, 1, 1], padding='SAME')
conv_layer_b = tf.nn.bias_add(conv_layer, biases)
conv_layer_relu = tf.nn.relu(conv_layer_b)
conv_layer_pool = tf.nn.max_pool(conv_layer_relu, ksize=[1, 2, 2, 1], strides=[1, 1, 1, 1], padding='SAME')
The algorithm learns one image at a time. Try to feed all the images in your set if your machine can handel it: sess.run(optimizer, feed_dict={features: train_features[:], labels: train_labels[:]}). If not 100 images from both classes. Are the images shuffled or first come 100 circle and than 100 squares? Here can lie the error. You update your weights 100 times with only squares in the last loop.
Can I see the the complete program, with the part that you print the predicted vector? As a first stage I would take the dropout out; let it overfit. And then, maybe, use a smaller fc_layer (512 or 256), smaller learning rate (0.01), and I prefere tf.get_variable('w1', shape=[3,3,1,16]) instead of tf.Variable(...), initialize the biases with value 0.1.
In the api of tf.contrib.rnn.DropoutWrapper, I am trying to set variational_recurrent=True, in which case, input_size is mandatory. As explained, input_size is TensorShape objects containing the depth(s) of the input tensors.
depth(s) is confusing, what is it please? Is it just the shape of the tensor as we can get by tf.shape()? Or the number of channels for the special case of images? But my input tensor is not an image.
And I don't understand why dtype is demanded when variational_recurrent=True.
Thanks!
Inpput_size for tf.TensorShape([200, None, 300]) is just 300
Play with this example.
import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" # see TF issue #152
os.environ["CUDA_VISIBLE_DEVICES"]="1"
import tensorflow as tf
import numpy as np
n_steps = 2
n_inputs = 3
n_neurons = 5
keep_prob = 0.5
learning_rate = 0.001
X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])
X_seqs = tf.unstack(tf.transpose(X, perm=[1, 0, 2]))
basic_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_neurons)
basic_cell_drop = tf.contrib.rnn.DropoutWrapper(
basic_cell,
input_keep_prob=keep_prob,
variational_recurrent=True,
dtype=tf.float32,
input_size=n_inputs)
output_seqs, states = tf.contrib.rnn.static_rnn(
basic_cell_drop,
X_seqs,
dtype=tf.float32)
outputs = tf.transpose(tf.stack(output_seqs), perm=[1, 0, 2])
init = tf.global_variables_initializer()
X_batch = np.array([
# t = 0 t = 1
[[0, 1, 2], [9, 8, 7]], # instance 1
[[3, 4, 5], [0, 0, 0]], # instance 2
[[6, 7, 8], [6, 5, 4]], # instance 3
[[9, 0, 1], [3, 2, 1]], # instance 4
])
with tf.Session() as sess:
init.run()
outputs_val = outputs.eval(feed_dict={X: X_batch})
print(outputs_val)
See this for more details: https://github.com/tensorflow/tensorflow/issues/7927
The Problem
When I run my training, my preprocessing examples are successfully created, however my training does not start. Much weirder is the fact, that on analyzing my TensorBoard graph, I see some extra conditional nodes which do not exist in the code. I want to know where and why do these extra nodes come into picture and exactly why the training does not begin. Below is a systematic description of the situation :
TensorFlow Graph
The following TensorBoard diagram shows my graph :
The code which constructs this graph is below
def getconv2drelu(inputtensor, kernelsize, strides, padding, convname,
imagesummaries=False):
weights = tf.get_variable("weights", shape=kernelsize, dtype=tf.float32,
initializer=tf.truncated_normal_initializer(0,
0.01),
regularizer=tf.nn.l2_loss)
biases = tf.get_variable("biases", shape=kernelsize[3], dtype=tf.float32,
initializer=tf.constant_initializer(0.0))
conv = tf.nn.conv2d(input=inputtensor, filter=weights, strides=strides,
padding=padding, name=convname)
response = tf.nn.bias_add(conv, biases)
if imagesummaries:
filters = (weights - tf.reduce_min(weights)) / (tf.reduce_max(
weights) - tf.reduce_min(weights))
filters = tf.transpose(filters, [3, 0, 1, 2])
tf.summary.image(convname + " filters", filters,
max_outputs=kernelsize[3])
response = tf.nn.relu(response)
activation_summary(response)
return response
def getfullyconnected(inputtensor, numinput, numoutput):
weights = tf.get_variable("weights", shape=[numinput, numoutput],
dtype=tf.float32,
initializer=
tf.truncated_normal_initializer(0, 0.01))
biases = tf.get_variable("biases", shape=[numoutput], dtype=tf.float32,
initializer=tf.truncated_normal_initializer(
0, 0.01))
response = tf.add(tf.matmul(inputtensor, weights), biases)
response = tf.nn.relu(response)
activation_summary(response)
return response
def inference(inputs):
with tf.variable_scope("layer1"):
conv = getconv2drelu(inputtensor=inputs, kernelsize=[7, 7, 3, 96],
strides=[1, 2, 2, 1], padding="VALID",
convname="conv1", imagesummaries=True)
pool = tf.nn.max_pool(conv, [1, 3, 3, 1], strides=[1, 3, 3, 1],
padding="SAME", name="pool1")
with tf.variable_scope("layer2"):
conv = getconv2drelu(inputtensor=pool, kernelsize=[7, 7, 96, 256],
strides=[1, 1, 1, 1], padding="VALID",
convname="conv2", imagesummaries=False)
pool = tf.nn.max_pool(conv, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],
padding="SAME", name="pool2")
with tf.variable_scope("layer3"):
conv = getconv2drelu(inputtensor=pool, kernelsize=[7, 7, 256, 512],
strides=[1, 1, 1, 1], padding="SAME",
convname="conv3", imagesummaries=False)
with tf.variable_scope("layer4"):
conv = getconv2drelu(inputtensor=conv, kernelsize=[3, 3, 512, 512],
strides=[1, 1, 1, 1], padding="SAME",
convname="conv4", imagesummaries=False)
with tf.variable_scope("layer5"):
conv = getconv2drelu(inputtensor=conv, kernelsize=[3, 3, 512, 1024],
strides=[1, 1, 1, 1], padding="SAME",
convname="conv5", imagesummaries=False)
with tf.variable_scope("layer6"):
conv = getconv2drelu(inputtensor=conv, kernelsize=[3, 3, 1024, 1024],
strides=[1, 1, 1, 1], padding="SAME",
convname="conv6", imagesummaries=False)
pool = tf.nn.max_pool(conv, [1, 3, 3, 1], strides=[1, 3, 3, 1],
padding="SAME", name="pool1")
pool = tf.contrib.layers.flatten(pool)
with tf.variable_scope("fc1"):
fc = getfullyconnected(pool, 5 * 5 * 1024, 4096)
drop = tf.nn.dropout(fc, keep_prob=0.5)
with tf.variable_scope("fc2"):
fc = getfullyconnected(drop, 4096, 4096)
drop = tf.nn.dropout(fc, keep_prob=0.5)
with tf.variable_scope("fc3"):
logits = getfullyconnected(drop, 4096, 1000)
return logits
The complete TensorBoard graph is shown below :
The figure is too small, but you can see a series of pink nodes to the left. An expanded version of such a segment is shown below :
Expansion of one of the condition blocks ( all blocks are similar !!) is shown below :
I am unable to understand the presence and existence of these extra condition blocks. All my images when fed to the graph are of size [221, 221, 3].
You can also see that inside a condition block, there is an isVariableInitialized test. I do initialize my variables right after the launch of a session. So, I do not understand as to why these checks will be performed.I have figured out that these condition blocks are there due to the use tf.get_variable() which checks for the initialization Do they cause any performance difference ?
Another observation
When I decrease the batchsize, the size of my tensorboard file also decreases. But the nodes shown on the graph remain the same. Why is this so ?
My training code is as follows :
with tf.control_dependencies(putops):
train_op = tf.group(apply_gradient_op, variables_averages_op)
sess.run(train_op) # tf.Session() as been defined before sess.run()
And putops is initialized to [] and during graph construction for each GPU, it is populated as follows :
# cpu_compute_stage is appended only once since it corresponds to centralized preprocessing
cpu_compute_stage = data_flow_ops.StagingArea(
[tf.float32, tf.int32],
shapes=[images_shape, labels_shape]
)
cpu_compute_stage_op = gpu_copy_stage.put(
[host_images, host_labels])
putops.append(gpu_copy_stage_op)
# For each device the putops is further appended by gpu_compute_stage which is for each GPU since CPU-GPU copy has to take place
with tf.device('/gpu:%d' % i):
with tf.name_scope('%s_%d' % (TOWER_NAME, i)) as scope:
gpu_compute_stage = data_flow_ops.StagingArea(
[tf.float32, tf.int32],
shapes=[images_shape, labels_shape]
)
gpu_compute_stage_op = gpu_compute_stage.put(
[host_images, host_labels]
)
putops.append(gpu_compute_stage_op)
However, my code does not run despite the fact that I do initialize both global and local variables.
I am using the code below to create CNN layers.
conv1 = tf.layers.conv2d(inputs = input, filters = 20, kernel_size = [3,3],
padding = "same", activation = tf.nn.relu)
and I want to get the values of all kernels after training. It does not work it I simply do
kernels = conv1.kernel
So how should I retrieve the value of these kernels? I am also not sure what variables and method does conv2d has since tensorflow don't really tell it in conv2d class.
You can find all the variables in list returned by tf.global_variables() and easily lookup for variable you need.
If you wish to get these variables by name, declare a layer as:
conv_layer_1 = tf.layers.conv2d(activation=tf.nn.relu,
filters=10,
inputs=input_placeholder,
kernel_size=(3, 3),
name="conv1", # NOTE THE NAME
padding="same",
strides=(1, 1))
Recover the graph as:
gr = tf.get_default_graph()
Recover the kernel values as:
conv1_kernel_val = gr.get_tensor_by_name('conv1/kernel:0').eval()
Recover the bias values as:
conv1_bias_val = gr.get_tensor_by_name('conv1/bias:0').eval()
You mean you want to get the value of the weights for the conv1 layer.
You haven't actually defined the weights with conv2d, you need to do that. When I create a convolutional layer I use a function that performs all the necessary steps, here's a copy/paste of the function I use to create a each of my convolutional layers:
def _conv_layer(self, name, in_channels, filters, kernel, input_tensor, strides, dtype=tf.float32):
with tf.variable_scope(name):
w = tf.get_variable("w", shape=[kernel, kernel, in_channels, filters],
initializer=tf.contrib.layers.xavier_initializer_conv2d(), dtype=dtype)
b = tf.get_variable("b", shape=[filters], initializer=tf.constant_initializer(0.0), dtype=dtype)
c = tf.nn.conv2d(input_tensor, w, strides, padding='SAME', name=name + "c")
a = tf.nn.relu(c + b, name=name + "_a")
print name + "_a", a.get_shape().as_list(), name + "_w", w.get_shape().as_list(), \
"params", np.prod(w.get_shape().as_list()[1:]) + filters
return a, w.get_shape().as_list()
This is what I use to define 5 convolutional layers, this example is straight out of my code, so note that it's 5 convolutional layers stacked without using max pooling or anything, strides of 2 and 5x5 kernels.
conv1_a, _ = self._conv_layer("conv1", 3, 24, 5, self.imgs4d, [1, 2, 2, 1]) # 24.8 MiB/feature -> 540 x 960
conv2_a, _ = self._conv_layer("conv2", 24, 80, 5, conv1_a, [1, 2, 2, 1]) # 6.2 MiB -> 270 x 480
conv3_a, _ = self._conv_layer("conv3", 80, 256, 5, conv2_a, [1, 2, 2, 1]) # 1.5 MiB -> 135 x 240
conv4_a, _ = self._conv_layer("conv4", 256, 750, 5, conv3_a, [1, 2, 2, 1]) # 0.4 MiB -> 68 x 120
conv5_a, _ = self._conv_layer("conv5", 750, 2048, 5, conv4_a, [1, 2, 2, 1]) # 0.1 MiB -> 34 x 60
There's also a good tutorial on the tensorflow website on how to set up a convolutional network:
https://www.tensorflow.org/tutorials/deep_cnn
The direct answer to your question is that the weights for the convolutional layer are defined there as w, that's the tensor you're asking about if I understand you correctly.