I am trying to implement rotation-invariant convolution layer using tensorflow as a part of udacity deep learning course.
I tried the approach below but it doesn't because i get the following error:
grad_ys, name, colocate_gradients_with_ops, gate_gradients, aggregation_method)
424 raise LookupError(
425 "No gradient defined for operation '%s' (op type: %s)" %
--> 426 (op.name, op.type))
427 if grad_fn and any(out_grads):
428 # NOTE: If _AggregatedGrads didn't compute a value for the i'th
LookupError: No gradient defined for operation 'Reverse_1' (op type: Reverse)
Here I tried to implement the simplest approach by feeding an original and 90 degree rotated image to the same convolution kernel. After this i rotated back the output of the corresponding convolution layer.
Do you have any ideas how to fix this approach or may be you know a better way to implement at least 90 degree rotation-invariant convolution layer?
my code:
batch_size = 16
patch_size_1 = 3
patch_size_2 = 3
patch_size_3 = 2
depth = 32
num_hidden = 32
mx_pool_size_1 = 3
mx_pool_size_2 = 3
mx_pool_size_3 = 2
stride = 1
starter_learning_rate = 0.05
num_steps = 1001
graph = tf.Graph()
with graph.as_default():
# Input data.
tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size, image_size, num_channels))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
global_step = tf.Variable(0, trainable=False)
# Variables.
layer1_weights = tf.Variable(tf.truncated_normal([patch_size_1, patch_size_1, num_channels, depth], stddev=0.1))
layer1_biases = tf.Variable(tf.zeros([depth]))
layer12_weights = tf.Variable(tf.truncated_normal([patch_size_1, patch_size_1, num_channels, depth], stddev=0.1))
layer12_biases = tf.Variable(tf.zeros([depth]))
layer2_weights = tf.Variable(tf.truncated_normal([patch_size_2, patch_size_2, depth, depth], stddev=0.1))
layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))
layer3_weights = tf.Variable(tf.truncated_normal([patch_size_3, patch_size_3, depth, depth], stddev=0.1))
layer3_biases = tf.Variable(tf.constant(1.0, shape=[depth]))
layer4_weights = tf.Variable(tf.truncated_normal(
#[image_size * image_size * depth / (mx_pool_size_1 ** 2 * mx_pool_size_2 ** 2 * stride ** 4) , num_hidden],
[512 , num_hidden],
stddev=0.1))
layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))
layer5_weights = tf.Variable(tf.truncated_normal([num_hidden, num_labels], stddev=0.1))
layer5_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))
# Model.
def model(data, train = False):
conv1 = tf.nn.conv2d(data, layer1_weights, [1, stride, stride, 1], padding='SAME')
conv1 = tf.nn.max_pool(conv1,
[1, mx_pool_size_1, mx_pool_size_1, 1],
[1, mx_pool_size_1, mx_pool_size_1, 1],
padding='SAME')
hidden1 = tf.nn.relu(conv1 + layer1_biases)
hidden1 = tf.reshape(hidden1, [-1, 100, 1, depth])
data1 = tf.reverse(tf.transpose(data, [0, 2, 1, 3]), [False, True, False, False])
conv2 = tf.nn.conv2d(data, layer12_weights, [1, stride, stride, 1], padding='SAME')
conv2 = tf.nn.max_pool(conv2,
[1, mx_pool_size_1, mx_pool_size_1, 1],
[1, mx_pool_size_1, mx_pool_size_1, 1],
padding='SAME')
hidden2 = tf.nn.relu(conv2 + layer12_biases)
hidden2 = tf.reverse(tf.transpose(hidden2, [0, 2, 1, 3]), [False, True, False, False])
hidden2 = tf.reshape(hidden2, [-1, 100, 1, depth])
hidden = tf.concat(2, [hidden1, hidden2])
hidden = tf.nn.max_pool(hidden,[1, 1, 2, 1], [1, 1, 2, 1], padding='SAME')
hidden = tf.reshape(hidden, [-1, 10, 10, depth])
conv = tf.nn.conv2d(hidden,
layer2_weights,
[1, stride, stride, 1],
padding='SAME')
conv = tf.nn.max_pool(conv,
[1, mx_pool_size_2, mx_pool_size_2, 1],
[1, mx_pool_size_2, mx_pool_size_2, 1],
padding='SAME')
hidden = tf.nn.relu(conv + layer2_biases)
shape = hidden.get_shape().as_list()
reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])
hidden = tf.nn.relu(tf.matmul(reshape, layer4_weights) + layer4_biases)
return tf.matmul(hidden, layer5_weights) + layer5_biases
# Training computation.
logits = model(tf_train_dataset, True)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
learning_rate = starter_learning_rate #tf.train.exponential_decay(starter_learning_rate, global_step, batch_size , 0.999, staircase=True)
# Optimizer.
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(model(tf_valid_dataset))
test_prediction = tf.nn.softmax(model(tf_test_dataset))
The latest release of TensorFlow (0.6.0) was missing a gradient definition for tf.reverse(). It was added in a subsequent commit, and you can add the following code to the top level of your program to make use of it without upgrading:
#tf.RegisterGradient("Reverse")
def _ReverseGrad(op, grad):
reverse_dims = op.inputs[1]
return tf.reverse(grad, reverse_dims), None
Related
I am using CNN to do short text classification. I know that if overfitting may lead all neurons become zero. But it is wired that, when overfitting, all training batch accuracies are 1, which does not make sense because not all true categories are 0. I think the accuracy should be low, but not 1.
Below is the part of the code that I use:
... Define some input placeholders here ...
pooled_outputs = []
for filter_size in filter_sizes:
filter_shape = [filter_size, embed_dimen, 1, num_filters]
W_filter = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1))
b_filter = tf.Variable(tf.constant(0.1, shape=[num_filters]))
x_embed_expanded = tf.expand_dims(x_embed, -1)
conv = tf.nn.conv2d(x_embed_expanded, W_filter, strides=[1, 1, 1, 1], padding="VALID")
h = tf.nn.relu(tf.nn.bias_add(conv, b_filter), name="relu")
pooled = tf.nn.max_pool(h, ksize=[1, self.params['max_domain_segments_len'] - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1], padding='VALID')
pooled_outputs.append(pooled)
h_pool = tf.concat(pooled_outputs, axis=3)
num_filters_total = num_filters * len(filter_sizes)
output_vec = tf.reshape(h_pool, [-1, num_filters_total])
logits = tf.contrib.layers.fully_connected(output_vec, num_outputs=n_rnn_neurons, activation_fn=tf.nn.relu)
logits = tf.contrib.layers.fully_connected(logits, self.params['num_targets'], activation_fn=tf.nn.relu)
crossentropy = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=logits)
loss_mean = tf.reduce_mean(crossentropy)
optimizer = tf.train.AdamOptimizer(learning_rate=lr_rate)
training_op = optimizer.minimize(loss_mean)
prediction = tf.argmax(logits, axis=-1)
is_correct = tf.nn.in_top_k(logits, y, 1) # logits are unscaled, but here we only care the argmax
n_correct = tf.reduce_sum(tf.cast(is_correct, tf.float32))
accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))
init = tf.global_variables_initializer()
with tf.Session() as sess:
init.run()
......
here is the output of AlexNet, I mean it's the output of the third full-connection
I don't know why all the output is zero. And I try to reduce the layer, while cutting the last two layer, the output is all the same in each dimension, after training several steps. Any idea will be appreciated.
Here is the main Inference code and initial value:
import tensorflow as tf
import numpy as np
import os
import csv
import cifar10
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
IMAGE_SIZES = 32
IMAGE_CHANELS = 3
NUM_CLASSES = 10
FIRST_CONV_NUM = 64
SECOND_CONV_NUM = 192
THIRD_CONV_NUM = 384
FOURTH_CONV_NUM = 256
FIFTH_CONV_NUM = 256
MAX_POOL_SIZE = 3
BATCH_SIZE = 100
FIRST_FC_UNIT_NUM = 4096
SECOND_FC_UNIT_NUM = 1000
DROP_OUT_PRO = 0.5
THIRD_FC_UNIT_NUM = NUM_CLASSES
TRAIN_EPOCH = 10
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
NUM_EXAMPLES_PER_EPOCH_FOR_EVAL = 10000
NUM_EPOCHS_PER_DECAY = 350.0
LEARNING_RATE_DECAY_FACTOR = 0.1
INITIAL_LEARNING_RATE = 0.01
DISPLAY_STEPS = 5
def leaky_relu(x, alpha=0.0):
return tf.nn.relu(x) - alpha * tf.nn.relu(-x)
def activation(x,alpha=0.0):
if alpha > 0:
return leaky_relu(x,alpha)
else:
return tf.nn.relu(x)
def Alex_Weight(weight_name, weight_shape, weight_stddev, weight_type):
Weight = tf.truncated_normal(shape=weight_shape, stddev=weight_stddev,type=weight_type)
return tf.Variable(initial_value=Weight, trainable=True, name=weight_name)
def Alex_Bias(bias_name, bias_shape, bias_type, bias_init=0.1):
initial = tf.constant(bias_init, shape=bias_shape)
return tf.Variable(initial, trainable=True, dtype=bias_type,name=bias_name)
def Alex_AddActivationSummary(out):
tf.summary.histogram('/activations',out)
tf.summary.scalar('/sparsity',tf.nn.zero_fraction(out))
def Alex_Conv(conv_name, input, weight, bias, strides, alpha=0.1,padding="SAME", activation=activation, act_name="ReLU"):
with tf.name_scope(conv_name):
conv = tf.nn.conv2d(input, weight, [1, strides, strides, 1], padding)
pre_activation = tf.nn.bias_add(conv, bias)
with tf.name_scope(act_name):
conv = activation(pre_activation,alpha=alpha)
return conv
def Alex_Pool(conv, ksize, strides, pool_fuction=tf.nn.max_pool, padding="SAME"):
return pool_fuction(conv, [1, ksize, ksize, 1], [1, strides,strides, 1], padding)
def Alex_Fully_Connect( input, weight, bias, activation=tf.nn.relu, act_name="ReLU"):
with tf.name_scope("Wx_b"):
y = tf.add(tf.matmul(input, weight), bias)
with tf.name_scope(act_name):
fc = activation(y, act_name)
return fc
def Alex_Norm(norm_name, pool):
with tf.name_scope(norm_name):
norm = tf.nn.lrn(pool, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75,
name=norm_name)
return norm
def Alex_Inference(images):
with tf.name_scope("First_Conv"):
W1 = Alex_Weight("Fist_Conv_Weight", [5, 5, IMAGE_CHANELS, FIRST_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias1 = Alex_Bias("First_Conv_Bias", [FIRST_CONV_NUM], tf.float32,bias_init=0.0)
first_conv = Alex_Conv("First_Conv", images, W1, bias1, strides=1, padding="SAME")
Alex_AddActivationSummary(first_conv)
with tf.name_scope('lrn1') as scope:
lrn1 = tf.nn.local_response_normalization(first_conv,
alpha=1e-4,
beta=0.75,
depth_radius=2,
bias=2.0)
with tf.name_scope("First_Pool"):
first_pool = Alex_Pool(lrn1, MAX_POOL_SIZE, strides=2, padding="VALID")
with tf.name_scope("Second_Conv"):
W2 = Alex_Weight("Second_Conv_Weight", [5, 5, FIRST_CONV_NUM, SECOND_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias2 = Alex_Bias("Second_Conv_Bias", [SECOND_CONV_NUM], tf.float32,bias_init=1.0)
second_conv = Alex_Conv("Second_Conv", first_pool, W2, bias2, strides=1, padding="SAME")
Alex_AddActivationSummary(second_conv)
with tf.name_scope('lrn2') as scope:
lrn2 = tf.nn.local_response_normalization(second_conv,
alpha=1e-4,
beta=0.75,
depth_radius=2,
bias=2.0)
with tf.name_scope("Second_Pool"):
second_pool = Alex_Pool(lrn2, MAX_POOL_SIZE, strides=2, padding="VALID")
with tf.name_scope("Third_Conv"):
W3 = Alex_Weight("Third_Conv_Weight", [3, 3, SECOND_CONV_NUM, THIRD_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias3 = Alex_Bias("Third_Conv_Bias", [THIRD_CONV_NUM], tf.float32,bias_init=0.0)
third_conv = Alex_Conv("Third_Conv", second_pool, W3, bias3, strides=1, padding="SAME")
Alex_AddActivationSummary(third_conv)
with tf.name_scope("Fourth_Conv"):
W4 = Alex_Weight("Fourth_Conv_Weight", [3, 3, THIRD_CONV_NUM, FOURTH_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias4 = Alex_Bias("Fourth_Conv_Bias", [FOURTH_CONV_NUM], tf.float32,bias_init=1.0)
fourth_conv = Alex_Conv("Fourth_Conv", third_conv, W4, bias4, strides=1, padding="SAME")
Alex_AddActivationSummary(fourth_conv)
with tf.name_scope("Fifth_Conv"):
W5 = Alex_Weight("Fifth_Conv_Weight", [3, 3, FOURTH_CONV_NUM, FIFTH_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias5 = Alex_Bias("Fifth_Conv_Bias", [FIFTH_CONV_NUM], tf.float32,bias_init=1.0)
fifth_conv = Alex_Conv("Fifth_Conv", fourth_conv, W5, bias5, strides=1, padding="SAME")
Alex_AddActivationSummary(fifth_conv)
with tf.name_scope("Third_Pool"):
third_pool = Alex_Pool(fifth_conv, MAX_POOL_SIZE, strides=2, padding="VALID")
with tf.name_scope("Flatten"):
flatten = tf.reshape(third_pool, [BATCH_SIZE, -1])
flatten_dim = flatten.get_shape()[1].value
with tf.name_scope("First_Fully_Connection"):
W = Alex_Weight("Fist_FC_Weight", [flatten_dim, FIRST_FC_UNIT_NUM], weight_stddev=4e-2, weight_type=tf.float32)
bias = Alex_Bias("First_FC_Bias", [FIRST_FC_UNIT_NUM], tf.float32, bias_init=1.0)
fc1 = Alex_Fully_Connect(flatten, W, bias, activation=tf.nn.relu, act_name="ReLU")
Alex_AddActivationSummary(fc1)
with tf.name_scope("Drop_Out_1"):
drop_out_1 = tf.nn.dropout(fc1, DROP_OUT_PRO)
with tf.name_scope("Second_Fully_Connection"):
W = Alex_Weight("Second_FC_Weight", [FIRST_FC_UNIT_NUM, SECOND_FC_UNIT_NUM], weight_stddev=4e-2,
weight_type=tf.float32)
bias = Alex_Bias("Second_FC_Bias", [SECOND_FC_UNIT_NUM], tf.float32, bias_init=1.0)
fc2 = Alex_Fully_Connect(drop_out_1, W, bias, activation=tf.nn.relu, act_name="ReLU")
Alex_AddActivationSummary(fc2)
with tf.name_scope("Drop_Out_2"):
drop_out_2 = tf.nn.dropout(fc2, DROP_OUT_PRO)
with tf.name_scope("Third_Fully_Connection"):
W = Alex_Weight("Third_FC_Weight", [SECOND_FC_UNIT_NUM, THIRD_FC_UNIT_NUM], weight_stddev=1/SECOND_FC_UNIT_NUM,
weight_type=tf.float32)
bias = Alex_Bias("Third_FC_Bias", [THIRD_FC_UNIT_NUM], tf.float32,bias_init=1.0)
fc3 = Alex_Fully_Connect(drop_out_2, W, bias, activation=tf.nn.relu, act_name="ReLU")
Alex_AddActivationSummary(fc3)
return fc3
According to the documentation, the model presented in this example is similar to the following paper:
"Character-level Convolutional Networks for Text Classification"
I found that the original model (presented in the paper) contains 9 layers deep with 6 convolutional layers and 3 fully-connected layers, but the implemented example contains only two convolutional layers:
with tf.variable_scope('CNN_Layer1'):
# Apply Convolution filtering on input sequence.
conv1 = tf.contrib.layers.convolution2d(
byte_list, N_FILTERS, FILTER_SHAPE1, padding='VALID')
# Add a RELU for non linearity.
conv1 = tf.nn.relu(conv1)
# Max pooling across output of Convolution+Relu.
pool1 = tf.nn.max_pool(
conv1,
ksize=[1, POOLING_WINDOW, 1, 1],
strides=[1, POOLING_STRIDE, 1, 1],
padding='SAME')
# Transpose matrix so that n_filters from convolution becomes width.
pool1 = tf.transpose(pool1, [0, 1, 3, 2])
with tf.variable_scope('CNN_Layer2'):
# Second level of convolution filtering.
conv2 = tf.contrib.layers.convolution2d(
pool1, N_FILTERS, FILTER_SHAPE2, padding='VALID')
# Max across each filter to get useful features for classification.
pool2 = tf.squeeze(tf.reduce_max(conv2, 1), squeeze_dims=[1])
If anybody can help me to extend this model for more layers?
Similar to BVLC Caffenet :
def bvlc_caffenet(imgs,weights,biases):
# mean subtraction
mean = tf.constant([123.68, 116.779, 103.939], dtype=tf.float32, shape=[1, 1, 1, 3], name='img_mean')
images = imgs-mean
#conv1
conv1 = tf.nn.conv2d(images,weights['c1'], [1, 3, 3, 1], padding='VALID')
out1 = tf.nn.relu(tf.nn.bias_add(conv1, biases['b1']))
pool1 = tf.nn.max_pool(out1,ksize=[1,3,3,1], strides=[1,2,2,1],padding='VALID')
#conv2
conv2 = tf.nn.conv2d(pool1,weights['c2'], [1, 1, 1, 1], padding='VALID')
out2 = tf.nn.relu(tf.nn.bias_add(conv2, biases['b2']))
pool2 = tf.nn.max_pool(out2,ksize=[1,3,3,1], strides=[1,2,2,1],padding='VALID')
#conv3
conv3 = tf.nn.conv2d(pool2,weights['c3'], [1, 1, 1, 1], padding='VALID')
out3 = tf.nn.relu(tf.nn.bias_add(conv3, biases['b3']))
#conv4
conv4 = tf.nn.conv2d(out3,weights['c4'], [1, 1, 1, 1], padding='VALID')
out4 = tf.nn.relu(tf.nn.bias_add(conv4, biases['b4']))
#conv5
conv5 = tf.nn.conv2d(out4,weights['c5'], [1, 1, 1, 1], padding='VALID')
out5 = tf.nn.relu(tf.nn.bias_add(conv5, biases['b5']))
pool5 = tf.nn.max_pool(out5,ksize=[1,3,3,1], strides=[1,2,2,1],padding='VALID')
#flattening
shape = int(np.prod(pool5.get_shape()[1:]))
pool5_flat = tf.reshape(pool5, [-1, shape])
#fc6
fc6 = tf.matmul(pool5_flat,weights['f6'])
out6 = tf.nn.relu(tf.nn.bias_add(fc6,biases['b6']))
out6 = tf.nn.dropout(out6,0.5)
#fc7
fc7 = tf.matmul(out6,weights['f7'])
out7 = tf.nn.relu(tf.nn.bias_add(fc7,biases['b7']))
out7 = tf.nn.dropout(out7,0.5)
#fc8
fc8 = tf.matmul(out7,weights['f8'])
out8 = tf.nn.relu(tf.nn.bias_add(fc8,biases['b8']))
out8 = tf.nn.dropout(out8,0.5)
probs = tf.nn.softmax(out8)
return probs
Initialized Weights and Biases for the Network
weights = {
'c1': tf.Variable(tf.truncated_normal([7,7,3,96],stddev=0.1)),
'c2': tf.Variable(tf.truncated_normal([5,5,96,256],stddev=0.1)),
'c3': tf.Variable(tf.truncated_normal([3,3,256,384],stddev=0.1)),
'c4': tf.Variable(tf.truncated_normal([3,3,384,384],stddev=0.1)),
'c5': tf.Variable(tf.truncated_normal([3,3,384,256],stddev=0.1)),
'f6': tf.Variable(tf.truncated_normal([4096,2048],stddev=0.1)),
'f7': tf.Variable(tf.truncated_normal([2048,2048],stddev=0.1)),
'f8': tf.Variable(tf.truncated_normal([2048,1000],stddev=0.1))
}
biases = {
'b1' : tf.Variable(tf.constant(0.1, shape=[96])),
'b2' : tf.Variable(tf.constant(0.1, shape=[256])),
'b3' : tf.Variable(tf.constant(0.1, shape=[384])),
'b4' : tf.Variable(tf.constant(0.1, shape=[384])),
'b5' : tf.Variable(tf.constant(0.1, shape=[256])),
'b6' : tf.Variable(tf.constant(0.1, shape=[2048])),
'b7' : tf.Variable(tf.constant(0.1, shape=[2048])),
'b8' : tf.Variable(tf.constant(0.1, shape=[1000]))
}
Or
follow this (another format) : https://www.cs.toronto.edu/~frossard/vgg16/vgg16.py
Are these helpful ?
Actually, I don't know how to describe this question. It's so strange.
import tensorflow as tf
import numpy as np
import pickle
def weight_and_bias(name ,shape):
weight = tf.get_variable("W" + name, shape=shape, initializer=tf.contrib.layers.xavier_initializer())
bias = tf.get_variable("B" + name, shape=shape[-1], initializer=tf.contrib.layers.xavier_initializer())
return weight, bias
def conv2d_2x2(x, W):
return tf.nn.conv2d(x, W, strides=[1, 5, 5, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
sess = tf.InteractiveSession()
source = tf.placeholder(tf.float32, [None, None, 50, 50])
source_len = tf.placeholder(tf.int32, [None])
source_max_step = tf.shape(source)[1]
target = tf.placeholder(tf.float32, [None, None, 50, 50])
target_len = tf.placeholder(tf.int32, [None])
target_max_step = tf.shape(target)[1]
W_conv, B_conv = weight_and_bias('conv1', [5, 5, 1, 32])
source = tf.reshape(source, [-1, 50, 50], "source_reshape")
source_tmp = tf.reshape(source, [-1, 50, 50 ,1])
source_conv = tf.nn.relu(conv2d_2x2(source_tmp, W_conv) + B_conv)
source_pool = max_pool_2x2(source_conv)
source_flat = tf.reshape(source_pool, [-1, 5 * 5 * 32], "source_pool_reshape")
source = tf.reshape(source_flat, [-1, source_max_step, 5*5*32], "source_flat_reshape")
W_conv, B_conv = weight_and_bias('conv2', [5, 5, 1, 32])
target = tf.reshape(target, [-1, 50, 50], "target_reshape")
target_tmp = tf.reshape(target, [-1, 50, 50 ,1])
target_conv = tf.nn.relu(conv2d_2x2(target_tmp, W_conv) + B_conv)
target_pool = max_pool_2x2(target_conv)
target_flat = tf.reshape(target_pool, [-1, 5 * 5 * 32], "target_pool_reshape")
target = tf.reshape(target_flat, [-1, target_max_step, 5*5*32], "target_flat_reshape")
source_cell = tf.nn.rnn_cell.LSTMCell(500, initializer=tf.contrib.layers.xavier_initializer())
target_cell = tf.nn.rnn_cell.LSTMCell(500, initializer=tf.contrib.layers.xavier_initializer())
source_rnn_output, _ = tf.nn.dynamic_rnn(source_cell, source, source_len, dtype=tf.float32, scope = "source")
target_rnn_output, _ = tf.nn.dynamic_rnn(target_cell, target, target_len, dtype=tf.float32, scope = "target")
source_output = tf.transpose(source_rnn_output, [1, 0, 2])
target_output = tf.transpose(target_rnn_output, [1, 0, 2])
source_final_output = tf.gather(source_output, -1)
target_final_output = tf.gather(target_output, -1)
output = tf.concat(1, [source_final_output, target_final_output])
W_sf, B_sf = weight_and_bias('sf', [1000, 2])
predict = tf.nn.softmax(tf.matmul(output, W_sf) + B_sf)
y = tf.placeholder(tf.float32, [None, 2])
cross_entropy = -tf.reduce_sum(y * tf.log(predict))
train_step = tf.train.RMSPropOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.arg_max(predict, 1), tf.arg_max(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
with open('set', 'rb') as f:
_set = pickle.load(f)
training_set = _set[0]
training_len = _set[1]
training_label = _set[2]
sess.run(tf.global_variables_initializer())
for i in range(20000):
if i % 100 == 0:
train_accuacy = accuracy.eval(feed_dict = {source: training_set[0], target: training_set[1], source_len: training_len[0], target_len: training_len[1], y: training_label})
print("step %d, training accuracy %g"%(i, train_accuacy))
train_step.run(feed_dict = {source: training_set[0], target: training_set[1], source_len: training_len[0], target_len: training_len[1], y: training_label})
These are my whole code, I can't find any problem in it.
But a ValueError: Cannot feed value of shape (1077, 27, 50, 50) for Tensor 'source_flat_reshape:0', which has shape '(?, ?, 800)' was raised.
The error message is strange, because it seems happen at source = tf.reshape(source_flat, [-1, source_max_step, 5*5*32], "source_flat_reshape"), but how could source_flat has a shape of (1077, 27, 50, 50)? It should be (1077*77, 800)
And, sometimes another ValueError: Cannot feed value of shape (1077, 27, 50, 50) for Tensor 'Reshape:0', which has shape '(?, 50, 50)' was raised.
It is also difficult to understand, why it happened?
Hope anyone could give me a hand.
Look what happens when you use feed_dict - you reference the variables source and target. However, the python variable no longer refers to the placeholders but rather the reshape ops - hence the op is 'skipped'.
The easiest fix is renaming the placeholders to something unique. Further down in the network it is OK to reuse the same name (you could just call every layer net), it doesn't matter as long as you no longer need to reference them.
Try giving this a go?
import tensorflow as tf
import numpy as np
import pickle
def weight_and_bias(name ,shape):
weight = tf.get_variable("W" + name, shape=shape, initializer=tf.contrib.layers.xavier_initializer())
bias = tf.get_variable("B" + name, shape=shape[-1], initializer=tf.contrib.layers.xavier_initializer())
return weight, bias
def conv2d_2x2(x, W):
return tf.nn.conv2d(x, W, strides=[1, 5, 5, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
sess = tf.InteractiveSession()
source_placeholder = tf.placeholder(tf.float32, [None, None, 50, 50])
source_len = tf.placeholder(tf.int32, [None])
source_max_step = tf.shape(source)[1]
target_placeholder = tf.placeholder(tf.float32, [None, None, 50, 50])
target_len = tf.placeholder(tf.int32, [None])
target_max_step = tf.shape(target)[1]
W_conv, B_conv = weight_and_bias('conv1', [5, 5, 1, 32])
source = tf.reshape(source_placeholder, [-1, 50, 50], "source_reshape")
source_tmp = tf.reshape(source, [-1, 50, 50 ,1])
source_conv = tf.nn.relu(conv2d_2x2(source_tmp, W_conv) + B_conv)
source_pool = max_pool_2x2(source_conv)
source_flat = tf.reshape(source_pool, [-1, 5 * 5 * 32], "source_pool_reshape")
source = tf.reshape(source_flat, [-1, source_max_step, 5*5*32], "source_flat_reshape")
W_conv, B_conv = weight_and_bias('conv2', [5, 5, 1, 32])
target = tf.reshape(target_placeholder, [-1, 50, 50], "target_reshape")
target_tmp = tf.reshape(target, [-1, 50, 50 ,1])
target_conv = tf.nn.relu(conv2d_2x2(target_tmp, W_conv) + B_conv)
target_pool = max_pool_2x2(target_conv)
target_flat = tf.reshape(target_pool, [-1, 5 * 5 * 32], "target_pool_reshape")
target = tf.reshape(target_flat, [-1, target_max_step, 5*5*32], "target_flat_reshape")
source_cell = tf.nn.rnn_cell.LSTMCell(500, initializer=tf.contrib.layers.xavier_initializer())
target_cell = tf.nn.rnn_cell.LSTMCell(500, initializer=tf.contrib.layers.xavier_initializer())
source_rnn_output, _ = tf.nn.dynamic_rnn(source_cell, source, source_len, dtype=tf.float32, scope = "source")
target_rnn_output, _ = tf.nn.dynamic_rnn(target_cell, target, target_len, dtype=tf.float32, scope = "target")
source_output = tf.transpose(source_rnn_output, [1, 0, 2])
target_output = tf.transpose(target_rnn_output, [1, 0, 2])
source_final_output = tf.gather(source_output, -1)
target_final_output = tf.gather(target_output, -1)
output = tf.concat(1, [source_final_output, target_final_output])
W_sf, B_sf = weight_and_bias('sf', [1000, 2])
predict = tf.nn.softmax(tf.matmul(output, W_sf) + B_sf)
y = tf.placeholder(tf.float32, [None, 2])
cross_entropy = -tf.reduce_sum(y * tf.log(predict))
train_step = tf.train.RMSPropOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.arg_max(predict, 1), tf.arg_max(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
with open('set', 'rb') as f:
_set = pickle.load(f)
training_set = _set[0]
training_len = _set[1]
training_label = _set[2]
sess.run(tf.global_variables_initializer())
for i in range(20000):
if i % 100 == 0:
train_accuacy = accuracy.eval(feed_dict = {source_placeholder: training_set[0], target_placeholder: training_set[1], source_len: training_len[0], target_len: training_len[1], y: training_label})
print("step %d, training accuracy %g"%(i, train_accuacy))
train_step.run(feed_dict = {source_placeholder: training_set[0], target_placeholder: training_set[1], source_len: training_len[0], target_len: training_len[1], y: training_label})
Iam new to Tensorflow. I am trying to build a convnet with the below model. I am getting a NotImplementedError. The code looks fine, not sure what I am missing. Error is in line -> hidden2 = tf.nn.relu(conv2,layer2_biases)
with graph.as_default():
tf_train_dataset = tf.placeholder(
tf.float32, shape=(128, 24, 24, 3))
tf_train_labels = tf.placeholder(tf.float32, shape=(128, 10))
layer1_weights = tf.Variable(tf.truncated_normal([5,5,3,64],stddev=0.1))
layer1_biases = tf.Variable(tf.zeros([64]))
layer2_weights = tf.Variable(tf.truncated_normal([5,5,64,64],stddev=0.1))
layer2_biases = tf.Variable(tf.constant(1.0, shape=[64]))
layer3_weights = tf.Variable(tf.truncated_normal([6 * 6 * 64 ,384],stddev=0.1))
layer3_biases = tf.Variable(tf.constant(1.0, shape=[384]))
layer4_weights = tf.Variable(tf.truncated_normal([384,192],stddev=0.1))
layer4_biases = tf.Variable(tf.constant(1.0, shape=[192]))
layer5_weights = tf.Variable(tf.truncated_normal([192,10],stddev=0.1))
layer5_biases = tf.Variable(tf.constant(1.0, shape=[64]))
def model(data):
conv1 = tf.nn.conv2d(data,layer1_weights,[1,1,1,1],padding='SAME')
hidden1 = tf.nn.relu(conv1 + layer1_biases)
hidden1 = tf.nn.max_pool(hidden1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
padding='SAME', name='pool1')
conv2 = tf.nn.conv2d(hidden1,layer2_weights,[1,1,1,1],padding='SAME')
hidden2 = tf.nn.relu(conv2,layer2_biases)
hidden2 = tf.nn.max_pool(hidden2,ksize=[1,3,3,1],strides=[1,2,2,1],padding='SAME',name='pool2')
shape = hidden2.get_shape().as_list()
hidden2 = tf.reshape(hidden2, [shape[0], shape[1] * shape[2] * shape[3]])
hidden3 = tf.nn.relu(tf.matmul(hidden2, layer3_weights) + layer3_biases)
hidden4 = tf.nn.relu(tf.matmul(hidden3, layer4_weights) + layer4_biases)
return tf.matmul(hidden4,layer5_weights) + layer5_biases
logits = model(tf_train_dataset)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits,tf_train_labels))
optimizer = tf.train.GradientDescentOptimizer(0.05).minimize(loss)
train_prediction = tf.nn.softmax(logits)
tf.relu takes two arguments: features Tensor and name string and you send two tensors. It should be:
conv2 = tf.nn.bias_add(conv2, layer2_biases)
hidden2 = tf.nn.relu(conv2)
tf.bias_add is just a special case of tf.add.