Related
here is the output of AlexNet, I mean it's the output of the third full-connection
I don't know why all the output is zero. And I try to reduce the layer, while cutting the last two layer, the output is all the same in each dimension, after training several steps. Any idea will be appreciated.
Here is the main Inference code and initial value:
import tensorflow as tf
import numpy as np
import os
import csv
import cifar10
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
IMAGE_SIZES = 32
IMAGE_CHANELS = 3
NUM_CLASSES = 10
FIRST_CONV_NUM = 64
SECOND_CONV_NUM = 192
THIRD_CONV_NUM = 384
FOURTH_CONV_NUM = 256
FIFTH_CONV_NUM = 256
MAX_POOL_SIZE = 3
BATCH_SIZE = 100
FIRST_FC_UNIT_NUM = 4096
SECOND_FC_UNIT_NUM = 1000
DROP_OUT_PRO = 0.5
THIRD_FC_UNIT_NUM = NUM_CLASSES
TRAIN_EPOCH = 10
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
NUM_EXAMPLES_PER_EPOCH_FOR_EVAL = 10000
NUM_EPOCHS_PER_DECAY = 350.0
LEARNING_RATE_DECAY_FACTOR = 0.1
INITIAL_LEARNING_RATE = 0.01
DISPLAY_STEPS = 5
def leaky_relu(x, alpha=0.0):
return tf.nn.relu(x) - alpha * tf.nn.relu(-x)
def activation(x,alpha=0.0):
if alpha > 0:
return leaky_relu(x,alpha)
else:
return tf.nn.relu(x)
def Alex_Weight(weight_name, weight_shape, weight_stddev, weight_type):
Weight = tf.truncated_normal(shape=weight_shape, stddev=weight_stddev,type=weight_type)
return tf.Variable(initial_value=Weight, trainable=True, name=weight_name)
def Alex_Bias(bias_name, bias_shape, bias_type, bias_init=0.1):
initial = tf.constant(bias_init, shape=bias_shape)
return tf.Variable(initial, trainable=True, dtype=bias_type,name=bias_name)
def Alex_AddActivationSummary(out):
tf.summary.histogram('/activations',out)
tf.summary.scalar('/sparsity',tf.nn.zero_fraction(out))
def Alex_Conv(conv_name, input, weight, bias, strides, alpha=0.1,padding="SAME", activation=activation, act_name="ReLU"):
with tf.name_scope(conv_name):
conv = tf.nn.conv2d(input, weight, [1, strides, strides, 1], padding)
pre_activation = tf.nn.bias_add(conv, bias)
with tf.name_scope(act_name):
conv = activation(pre_activation,alpha=alpha)
return conv
def Alex_Pool(conv, ksize, strides, pool_fuction=tf.nn.max_pool, padding="SAME"):
return pool_fuction(conv, [1, ksize, ksize, 1], [1, strides,strides, 1], padding)
def Alex_Fully_Connect( input, weight, bias, activation=tf.nn.relu, act_name="ReLU"):
with tf.name_scope("Wx_b"):
y = tf.add(tf.matmul(input, weight), bias)
with tf.name_scope(act_name):
fc = activation(y, act_name)
return fc
def Alex_Norm(norm_name, pool):
with tf.name_scope(norm_name):
norm = tf.nn.lrn(pool, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75,
name=norm_name)
return norm
def Alex_Inference(images):
with tf.name_scope("First_Conv"):
W1 = Alex_Weight("Fist_Conv_Weight", [5, 5, IMAGE_CHANELS, FIRST_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias1 = Alex_Bias("First_Conv_Bias", [FIRST_CONV_NUM], tf.float32,bias_init=0.0)
first_conv = Alex_Conv("First_Conv", images, W1, bias1, strides=1, padding="SAME")
Alex_AddActivationSummary(first_conv)
with tf.name_scope('lrn1') as scope:
lrn1 = tf.nn.local_response_normalization(first_conv,
alpha=1e-4,
beta=0.75,
depth_radius=2,
bias=2.0)
with tf.name_scope("First_Pool"):
first_pool = Alex_Pool(lrn1, MAX_POOL_SIZE, strides=2, padding="VALID")
with tf.name_scope("Second_Conv"):
W2 = Alex_Weight("Second_Conv_Weight", [5, 5, FIRST_CONV_NUM, SECOND_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias2 = Alex_Bias("Second_Conv_Bias", [SECOND_CONV_NUM], tf.float32,bias_init=1.0)
second_conv = Alex_Conv("Second_Conv", first_pool, W2, bias2, strides=1, padding="SAME")
Alex_AddActivationSummary(second_conv)
with tf.name_scope('lrn2') as scope:
lrn2 = tf.nn.local_response_normalization(second_conv,
alpha=1e-4,
beta=0.75,
depth_radius=2,
bias=2.0)
with tf.name_scope("Second_Pool"):
second_pool = Alex_Pool(lrn2, MAX_POOL_SIZE, strides=2, padding="VALID")
with tf.name_scope("Third_Conv"):
W3 = Alex_Weight("Third_Conv_Weight", [3, 3, SECOND_CONV_NUM, THIRD_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias3 = Alex_Bias("Third_Conv_Bias", [THIRD_CONV_NUM], tf.float32,bias_init=0.0)
third_conv = Alex_Conv("Third_Conv", second_pool, W3, bias3, strides=1, padding="SAME")
Alex_AddActivationSummary(third_conv)
with tf.name_scope("Fourth_Conv"):
W4 = Alex_Weight("Fourth_Conv_Weight", [3, 3, THIRD_CONV_NUM, FOURTH_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias4 = Alex_Bias("Fourth_Conv_Bias", [FOURTH_CONV_NUM], tf.float32,bias_init=1.0)
fourth_conv = Alex_Conv("Fourth_Conv", third_conv, W4, bias4, strides=1, padding="SAME")
Alex_AddActivationSummary(fourth_conv)
with tf.name_scope("Fifth_Conv"):
W5 = Alex_Weight("Fifth_Conv_Weight", [3, 3, FOURTH_CONV_NUM, FIFTH_CONV_NUM], weight_stddev=0.01,
weight_type=tf.float32)
bias5 = Alex_Bias("Fifth_Conv_Bias", [FIFTH_CONV_NUM], tf.float32,bias_init=1.0)
fifth_conv = Alex_Conv("Fifth_Conv", fourth_conv, W5, bias5, strides=1, padding="SAME")
Alex_AddActivationSummary(fifth_conv)
with tf.name_scope("Third_Pool"):
third_pool = Alex_Pool(fifth_conv, MAX_POOL_SIZE, strides=2, padding="VALID")
with tf.name_scope("Flatten"):
flatten = tf.reshape(third_pool, [BATCH_SIZE, -1])
flatten_dim = flatten.get_shape()[1].value
with tf.name_scope("First_Fully_Connection"):
W = Alex_Weight("Fist_FC_Weight", [flatten_dim, FIRST_FC_UNIT_NUM], weight_stddev=4e-2, weight_type=tf.float32)
bias = Alex_Bias("First_FC_Bias", [FIRST_FC_UNIT_NUM], tf.float32, bias_init=1.0)
fc1 = Alex_Fully_Connect(flatten, W, bias, activation=tf.nn.relu, act_name="ReLU")
Alex_AddActivationSummary(fc1)
with tf.name_scope("Drop_Out_1"):
drop_out_1 = tf.nn.dropout(fc1, DROP_OUT_PRO)
with tf.name_scope("Second_Fully_Connection"):
W = Alex_Weight("Second_FC_Weight", [FIRST_FC_UNIT_NUM, SECOND_FC_UNIT_NUM], weight_stddev=4e-2,
weight_type=tf.float32)
bias = Alex_Bias("Second_FC_Bias", [SECOND_FC_UNIT_NUM], tf.float32, bias_init=1.0)
fc2 = Alex_Fully_Connect(drop_out_1, W, bias, activation=tf.nn.relu, act_name="ReLU")
Alex_AddActivationSummary(fc2)
with tf.name_scope("Drop_Out_2"):
drop_out_2 = tf.nn.dropout(fc2, DROP_OUT_PRO)
with tf.name_scope("Third_Fully_Connection"):
W = Alex_Weight("Third_FC_Weight", [SECOND_FC_UNIT_NUM, THIRD_FC_UNIT_NUM], weight_stddev=1/SECOND_FC_UNIT_NUM,
weight_type=tf.float32)
bias = Alex_Bias("Third_FC_Bias", [THIRD_FC_UNIT_NUM], tf.float32,bias_init=1.0)
fc3 = Alex_Fully_Connect(drop_out_2, W, bias, activation=tf.nn.relu, act_name="ReLU")
Alex_AddActivationSummary(fc3)
return fc3
Implementing a model adapted from multilayer CNN Tutorial
I am learning TensorFlow from the example at: MNIST Tutorial - Julia.jl
unlike their tutorial, I am using my own custom images(32 X 32 (RGB) ) format asimular to cifar-10. Each row in the csv is 3073 columns the last one being the label.
And I have 300 rows out of which I used array slicing to select the first 240 rows for training and the remaining for testing. So in a nut shell I have 4 arrays:
images_train = 240 X 3072 labels_train = 240 X 1 images_test = 60 X 3072 labels_test = 60 X 1 .
When I try training the network I get error,
Incompatible shapes: [16] vs. [16,9]
Actually I have nine classes with 240 training set and 60 images for testing.
The code is:
using TensorFlow
using Distributions
include("loader.jl")
session = Session(Graph())
function weight_variable(shape)
initial = map(Float32, rand(Normal(0, .001), shape...))
return Variable(initial)
end
function bias_variable(shape)
initial = fill(Float32(.1), shape...)
return Variable(initial)
end
function conv2d(x, W)
nn.conv2d(x, W, [1, 1, 1, 1], "SAME")
end
function max_pool_2x2(x)
nn.max_pool(x, [1, 2, 2, 1], [1, 2, 2, 1], "SAME")
end
x = placeholder(Float32)
y_ = placeholder(Float32)
W_conv1 = weight_variable([5, 5, 3, 32])
b_conv1 = bias_variable([32])
x_image = x #reshape(x, [-1, 32, 32, 3])
h_conv1 = nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
W_conv2 = weight_variable([5, 5, 32, 64])
b_conv2 = bias_variable([64])
h_conv2 = nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)
W_fc1 = weight_variable([8*8*64, 1024]) #
b_fc1 = bias_variable([1024])
h_pool2_flat = reshape(h_pool2, [-1, 8*8*64])
h_fc1 = nn.relu(h_pool2_flat * W_fc1 + b_fc1)
keep_prob = placeholder(Float32)
h_fc1_drop = nn.dropout(h_fc1, keep_prob)
W_fc2 = weight_variable([1024, 9])
b_fc2 = bias_variable([9])
y_conv = nn.softmax(h_fc1_drop * W_fc2 + b_fc2)
cross_entropy = reduce_mean(-reduce_sum((y_ .* log(y_conv)), axis=[2]))
train_step = train.minimize(train.AdamOptimizer(1e-4), cross_entropy)
correct_prediction = indmax(y_conv, 2) .== indmax(y_, 2)
accuracy = reduce_mean(cast(correct_prediction, Float32))
run(session, global_variables_initializer())
for i in 1:100
images_train,labels_train = batching(16) # randomly generate batches from training dataset (16,32,32,3)
if i%4 == 1
train_accuracy = run(session, accuracy, Dict(x=>images_train, y_=>labels_train, keep_prob=>1.0))
info("step $i, training accuracy $train_accuracy")
end
run(session, train_step, Dict(x=>images_train, y_=>labels_train, keep_prob=>.5))
end
images_test, labels_test = testloader() # (60, 33,32,3), (60,) Arrays
test_accuracy = run(session, accuracy, Dict(x=>images_test, y_=>labels_test, keep_prob=>1.0))
info("test accuracy $test_accuracy")
Applying
reshape(A,(16,1))
Solved the issue with the incompatibility.
I am building a binary classifier with my own image pipeline. The images are 178x65 (wxh) greyscale jpgs. It loads the images and their labels correctly, and fits all of the layer sizes correctly, but for some reason, it always outputs the same prediction for every image ([ 1. 0.]). Can anyone point me in the right direction?
# Example on how to use the tensorflow input pipelines. The explanation can be found here ischlag.github.io.
import tensorflow as tf
import random
from tensorflow.python.framework import ops
from tensorflow.python.framework import dtypes
dataset_path = "./v5-tensorflow/"
test_labels_file = "test-labels.csv"
train_labels_file = "train-labels.csv"
test_set_size = 39
IMAGE_HEIGHT = 65
IMAGE_WIDTH = 178
NUM_CHANNELS = 1
BATCH_SIZE = 39
def encode_label(label):
if label == "kalli\n":
return [0, 1]
elif label == "francisco\n":
return [1, 0]
else:
print("SOMETHING WENT WRONG")
return [0, 0]
def read_label_file(file):
f = open(file, "r")
filepaths = []
labels = []
for line in f:
filepath, label = line.split(",")
filepaths.append(filepath)
labels.append(encode_label(label))
return filepaths, labels
# reading labels and file path
train_filepaths, train_labels = read_label_file(dataset_path + train_labels_file)
test_filepaths, test_labels = read_label_file(dataset_path + test_labels_file)
# transform relative path into full path
train_filepaths = [ dataset_path + fp for fp in train_filepaths]
test_filepaths = [ dataset_path + fp for fp in test_filepaths]
# convert string into tensors
train_images = ops.convert_to_tensor(train_filepaths, dtype=dtypes.string)
test_images = ops.convert_to_tensor(test_filepaths, dtype=dtypes.string)
train_labels = ops.convert_to_tensor(train_labels, dtype=dtypes.int32)
test_labels = ops.convert_to_tensor(test_labels, dtype=dtypes.int32)
# create input queues
train_input_queue = tf.train.slice_input_producer(
[train_images, train_labels],
shuffle=True)
test_input_queue = tf.train.slice_input_producer(
[test_images, test_labels],
shuffle=True)
# process path and string tensor into an image and a label
file_content = tf.read_file(train_input_queue[0])
train_image = tf.image.decode_jpeg(file_content, channels=NUM_CHANNELS)
train_label = train_input_queue[1]
file_content = tf.read_file(test_input_queue[0])
test_image = tf.image.decode_jpeg(file_content, channels=NUM_CHANNELS)
test_label = test_input_queue[1]
# define tensor shape
train_image.set_shape([IMAGE_HEIGHT, IMAGE_WIDTH, NUM_CHANNELS])
test_image.set_shape([IMAGE_HEIGHT, IMAGE_WIDTH, NUM_CHANNELS])
# collect batches of images before processing
# train_image_batch, train_label_batch
train_batch = tf.train.batch(
[train_image, train_label],
batch_size=BATCH_SIZE
#,num_threads=1
)
# test_image_batch, test_label_batch
test_batch = tf.train.batch(
[test_image, test_label],
batch_size=BATCH_SIZE
#,num_threads=1
)
x = tf.placeholder(tf.float32, shape=[None, 65, 178, 1], name="x")
y_ = tf.placeholder(tf.float32, shape=[None, 2], name="y_")
#
def weight_variable(shape, n):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial, name=n)
def bias_variable(shape, n):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial, name=n)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
#
input_reshape = tf.reshape(x, [-1,65,178,1])
W_conv1 = weight_variable([5, 5, 1, 32], "w1")
b_conv1 = bias_variable([32], "b1")
h_conv1 = tf.nn.relu(conv2d(input_reshape, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1) # 33x89x32
#
W_conv2 = weight_variable([5, 5, 32, 64], "w2")
b_conv2 = bias_variable([64], "b2")
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2) # now 17x45x64
W_conv3 = weight_variable([5, 5, 64, 128], "w3")
b_conv3 = bias_variable([128], "b3")
h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3)
h_pool3 = max_pool_2x2(h_conv3)# 9x23x128
W_conv4 = weight_variable([5, 5, 128, 256], "w4")
b_conv4 = bias_variable([256], "b4")
h_conv4 = tf.nn.relu(conv2d(h_pool3, W_conv4) + b_conv4)
h_pool4 = max_pool_2x2(h_conv4) # 5x12x256
W_fc1 = weight_variable([5 * 12 * 256, 5120], "w5")
b_fc1 = bias_variable([5120], "b5")
h_pool4_flat = tf.reshape(h_pool4, [-1, 5*12*256])
h_fc1 = tf.nn.relu(tf.matmul(h_pool4_flat, W_fc1) + b_fc1) # 1x5120
W_fc2 = weight_variable([5120, 2], "w6")
b_fc2 = bias_variable([2], "b6") # 1x2
y_conv = tf.nn.softmax(tf.matmul(h_fc1, W_fc2) + b_fc2, name="softmax")
y_clipped = tf.clip_by_value(y_conv, 1e-10, 0.9999999)
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_clipped), reduction_indices=[1]))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_clipped, 1), tf.argmax(y_, 1), name="correct")
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name="accuracy")
saver = tf.train.Saver()
print("input pipeline ready")
with tf.Session() as sess:
# initialize the variables
sess.run(tf.global_variables_initializer())
# initialize the queue threads to start to shovel data
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
print("from the train set:")
for i in range(62):
batch = sess.run(train_batch)
if i % 3 == 0:
train_accuracy = accuracy.eval(feed_dict={
x:batch[0], y_: batch[1]
})
print("step %d, training accuracy %.2f" % (i, train_accuracy))
train_step.run(feed_dict={x: batch[0], y_: batch[1]})
saver.save(sess, "model.ckpt")
tf.train.write_graph(sess.graph_def, '', 'graph.pb')
t_batch = sess.run(test_batch)
print("test accuracy %g" % accuracy.eval(feed_dict={
x: t_batch[0], y_: t_batch[1]
}))
# stop our queue threads and properly close the session
coord.request_stop()
coord.join(threads)
sess.close()
I have a checkpoint which is trained with 11 classes. I added one class to my dataset and trying to restore it in order to retain the CNN but it gave me an error related to shape because the previous one was trained with 11 classes and actually have 12 classes, did i saved the weights and biases variable in a right way ? what should I do ? here is the code:
batch_size = 10
num_hidden = 64
num_channels = 1
depth = 32
....
graph = tf.Graph()
with graph.as_default():
# Input data.
tf_train_dataset = tf.placeholder(
tf.float32, shape=(batch_size, IMAGE_SIZE_H, IMAGE_SIZE_W, num_channels))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
w_b = {
'weight_0': tf.Variable(tf.random_normal([patch_size_1, patch_size_1, num_channels, depth],stddev=0.1)),
'weight_1': tf.Variable(tf.random_normal([patch_size_2, patch_size_2, depth, depth], stddev=0.1)),
'weight_2': tf.Variable(tf.random_normal([patch_size_3, patch_size_3, depth, depth], stddev=0.1)),
'weight_3': tf.Variable(tf.random_normal([IMAGE_SIZE_H // 32 * IMAGE_SIZE_W // 32 * depth, num_hidden], stddev=0.1)),
'weight_4': tf.Variable(tf.random_normal([num_hidden, num_labels], stddev=0.1)),
'bias_0' : tf.Variable(tf.zeros([depth])),
'bias_1' : tf.Variable(tf.constant(1.0, shape=[depth])),
'bias_2' : tf.Variable(tf.constant(1.0, shape=[depth])),
'bias_3' : tf.Variable(tf.constant(1.0, shape=[num_hidden])),
'bias_4' : tf.Variable(tf.constant(1.0, shape=[num_labels]))
}
# Model.
def model(data):
conv_1 = tf.nn.conv2d(data, w_b['weight_0'] , [1, 2, 2, 1], padding='SAME')
hidden_1 = tf.nn.relu(conv_1 + w_b['bias_0'])
pool_1 = tf.nn.max_pool(hidden_1,ksize = [1,5,5,1], strides= [1,2,2,1],padding ='SAME' )
conv_2 = tf.nn.conv2d(pool_1, w_b['weight_1'], [1, 2, 2, 1], padding='SAME')
hidden_2 = tf.nn.relu(conv_2 + w_b['bias_1'])
conv_3 = tf.nn.conv2d(hidden_2, w_b['weight_2'], [1, 2, 2, 1], padding='SAME')
hidden_3 = tf.nn.relu(conv_3 + w_b['bias_2'])
pool_2 = tf.nn.max_pool(hidden_3,ksize = [1,3,3,1], strides= [1,2,2,1],padding ='SAME' )
shape = pool_2.get_shape().as_list()
reshape = tf.reshape(pool_2, [shape[0], shape[1] * shape[2] * shape[3]])
hidden_4 = tf.nn.relu(tf.matmul(reshape, w_b['weight_3']) + w_b['bias_3'])
return tf.matmul(hidden_4, w_b['weight_4']) + w_b['bias_4']
# Training computation.
logits = model(tf_train_dataset)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
optimizer = tf.train.GradientDescentOptimizer(0.01).minimize(loss)
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(model(tf_valid_dataset))
test_prediction = tf.nn.softmax(model(tf_test_dataset))
init = tf.initialize_all_variables()
w_b_saver = tf.train.Saver(var_list = w_b)
num_steps = 1001
with tf.Session(graph=graph) as sess:
ckpt = ("/home/..../w_b_models.ckpt")
if os.path.isfile(ckpt) :
w_b_saver.restore(sess,ckpt)
print("restore complete")
print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval() , test_labels))
else:
print("Error while loading model checkpoint.")
print('Initialized')
sess.run(init)
for step in range(num_steps):
.....
accuracy(test_prediction.eval(),test_labels, force = False ))
save_path_w_b = w_b_saver.save(sess, "/home/...../w_b_models.ckpt")
print("Model saved in file: %s" % save_path_w_b)
and here is the error :
InvalidArgumentError: Assign requires shapes of both tensors to match. lhs shape= [64,12] rhs shape= [64,11]
[[Node: save/Assign_9 = Assign[T=DT_FLOAT, _class=["loc:#Variable_4"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_4, save/restore_slice_9/_12)]]
I believe the problem is you need to remove this one from w_b then save it then restore it as you're doing.
Remove this:
'weight_4': tf.Variable(tf.random_normal([num_hidden, num_labels], stddev=0.1)),
Then it should work. Main reason is that you're changing number of labels and expecting it to restore to this same variable. As a side note it's better to use tf.get_variable instead of tf.Variable.
Updated answer:
make a new variable called
w_b_to_save = {
'weight_0': tf.Variable(tf.random_normal([patch_size_1, patch_size_1, num_channels, depth],stddev=0.1)),
'weight_1': tf.Variable(tf.random_normal([patch_size_2, patch_size_2, depth, depth], stddev=0.1)),
'weight_2': tf.Variable(tf.random_normal([patch_size_3, patch_size_3, depth, depth], stddev=0.1)),
'weight_3': tf.Variable(tf.random_normal([IMAGE_SIZE_H // 32 * IMAGE_SIZE_W // 32 * depth, num_hidden], stddev=0.1)),
'bias_0' : tf.Variable(tf.zeros([depth])),
'bias_1' : tf.Variable(tf.constant(1.0, shape=[depth])),
'bias_2' : tf.Variable(tf.constant(1.0, shape=[depth])),
'bias_3' : tf.Variable(tf.constant(1.0, shape=[num_hidden])),
}
...
w_b_saver = tf.train.Saver(var_list = w_b_to_save)
now you'll be able to save just the ones you want. This is a bit excessive to create a new variable that's basically the same as the last one but it's to show the point that you can't both save the last layer, and restore it while changing it.
I am trying to implement rotation-invariant convolution layer using tensorflow as a part of udacity deep learning course.
I tried the approach below but it doesn't because i get the following error:
grad_ys, name, colocate_gradients_with_ops, gate_gradients, aggregation_method)
424 raise LookupError(
425 "No gradient defined for operation '%s' (op type: %s)" %
--> 426 (op.name, op.type))
427 if grad_fn and any(out_grads):
428 # NOTE: If _AggregatedGrads didn't compute a value for the i'th
LookupError: No gradient defined for operation 'Reverse_1' (op type: Reverse)
Here I tried to implement the simplest approach by feeding an original and 90 degree rotated image to the same convolution kernel. After this i rotated back the output of the corresponding convolution layer.
Do you have any ideas how to fix this approach or may be you know a better way to implement at least 90 degree rotation-invariant convolution layer?
my code:
batch_size = 16
patch_size_1 = 3
patch_size_2 = 3
patch_size_3 = 2
depth = 32
num_hidden = 32
mx_pool_size_1 = 3
mx_pool_size_2 = 3
mx_pool_size_3 = 2
stride = 1
starter_learning_rate = 0.05
num_steps = 1001
graph = tf.Graph()
with graph.as_default():
# Input data.
tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size, image_size, num_channels))
tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))
tf_valid_dataset = tf.constant(valid_dataset)
tf_test_dataset = tf.constant(test_dataset)
global_step = tf.Variable(0, trainable=False)
# Variables.
layer1_weights = tf.Variable(tf.truncated_normal([patch_size_1, patch_size_1, num_channels, depth], stddev=0.1))
layer1_biases = tf.Variable(tf.zeros([depth]))
layer12_weights = tf.Variable(tf.truncated_normal([patch_size_1, patch_size_1, num_channels, depth], stddev=0.1))
layer12_biases = tf.Variable(tf.zeros([depth]))
layer2_weights = tf.Variable(tf.truncated_normal([patch_size_2, patch_size_2, depth, depth], stddev=0.1))
layer2_biases = tf.Variable(tf.constant(1.0, shape=[depth]))
layer3_weights = tf.Variable(tf.truncated_normal([patch_size_3, patch_size_3, depth, depth], stddev=0.1))
layer3_biases = tf.Variable(tf.constant(1.0, shape=[depth]))
layer4_weights = tf.Variable(tf.truncated_normal(
#[image_size * image_size * depth / (mx_pool_size_1 ** 2 * mx_pool_size_2 ** 2 * stride ** 4) , num_hidden],
[512 , num_hidden],
stddev=0.1))
layer4_biases = tf.Variable(tf.constant(1.0, shape=[num_hidden]))
layer5_weights = tf.Variable(tf.truncated_normal([num_hidden, num_labels], stddev=0.1))
layer5_biases = tf.Variable(tf.constant(1.0, shape=[num_labels]))
# Model.
def model(data, train = False):
conv1 = tf.nn.conv2d(data, layer1_weights, [1, stride, stride, 1], padding='SAME')
conv1 = tf.nn.max_pool(conv1,
[1, mx_pool_size_1, mx_pool_size_1, 1],
[1, mx_pool_size_1, mx_pool_size_1, 1],
padding='SAME')
hidden1 = tf.nn.relu(conv1 + layer1_biases)
hidden1 = tf.reshape(hidden1, [-1, 100, 1, depth])
data1 = tf.reverse(tf.transpose(data, [0, 2, 1, 3]), [False, True, False, False])
conv2 = tf.nn.conv2d(data, layer12_weights, [1, stride, stride, 1], padding='SAME')
conv2 = tf.nn.max_pool(conv2,
[1, mx_pool_size_1, mx_pool_size_1, 1],
[1, mx_pool_size_1, mx_pool_size_1, 1],
padding='SAME')
hidden2 = tf.nn.relu(conv2 + layer12_biases)
hidden2 = tf.reverse(tf.transpose(hidden2, [0, 2, 1, 3]), [False, True, False, False])
hidden2 = tf.reshape(hidden2, [-1, 100, 1, depth])
hidden = tf.concat(2, [hidden1, hidden2])
hidden = tf.nn.max_pool(hidden,[1, 1, 2, 1], [1, 1, 2, 1], padding='SAME')
hidden = tf.reshape(hidden, [-1, 10, 10, depth])
conv = tf.nn.conv2d(hidden,
layer2_weights,
[1, stride, stride, 1],
padding='SAME')
conv = tf.nn.max_pool(conv,
[1, mx_pool_size_2, mx_pool_size_2, 1],
[1, mx_pool_size_2, mx_pool_size_2, 1],
padding='SAME')
hidden = tf.nn.relu(conv + layer2_biases)
shape = hidden.get_shape().as_list()
reshape = tf.reshape(hidden, [shape[0], shape[1] * shape[2] * shape[3]])
hidden = tf.nn.relu(tf.matmul(reshape, layer4_weights) + layer4_biases)
return tf.matmul(hidden, layer5_weights) + layer5_biases
# Training computation.
logits = model(tf_train_dataset, True)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))
learning_rate = starter_learning_rate #tf.train.exponential_decay(starter_learning_rate, global_step, batch_size , 0.999, staircase=True)
# Optimizer.
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
# Predictions for the training, validation, and test data.
train_prediction = tf.nn.softmax(logits)
valid_prediction = tf.nn.softmax(model(tf_valid_dataset))
test_prediction = tf.nn.softmax(model(tf_test_dataset))
The latest release of TensorFlow (0.6.0) was missing a gradient definition for tf.reverse(). It was added in a subsequent commit, and you can add the following code to the top level of your program to make use of it without upgrading:
#tf.RegisterGradient("Reverse")
def _ReverseGrad(op, grad):
reverse_dims = op.inputs[1]
return tf.reverse(grad, reverse_dims), None