Here is my code:
def conv_pooling(data, sequence_length, filter_size, embedding_size, num_filters):
filter_shape = [filter_size, embedding_size, 1, num_filters]
w = tf.Variable(tf.truncated_normal(filter_shape,stddev = 0.1),
name = "w")
b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name =
"b")
conv = tf.nn.conv2d(
item,
w,
strides = [1,1,1,1],
padding = "VALID",
name = "conv"
)
h = tf.nn.relu(tf.nn.bias_add(conv, b), name = "relu")
pooled = tf.nn.max_pool(
h,
ksize = [1,sequence_length - filter_size + 1, 1, 1],
strides = [1,1,1,1],
padding = "VALID",
name = "pool"
)
return pooled
init_op = tf.global_variables_initializer()
pooled_outputs = []
with tf.Session() as sess:
sess.run(init_op)
for i, filter_size in enumerate(filter_sizes):
pooled = sess.run(conv_pooling(data, sequence_length, filter_size, embedding_size, num_filters), feed_dict = {embedded_chars: item})
pooled_outputs.append(pooled)
This 'data' is a tf.Variable that use the global tf.placeholder 'embedded_chars', so don't worry about if it is working. The error happens because of w and b cannot be initialized.
I tried sess.run(tf.local_variables_initializer()) also, not work and return the same error. Does anyone know a way that I can initialized w and b here? As you see the size of w change in for loop.
Thank you!
See the code below. That's why #mikkola means about creating your graph before initialization.
// create your computation graph
pooled = conv_pooling(data, sequence_length, filter_size, embedding_size, num_filters)
// initialize the variables in the graph
init_op = tf.global_variables_initializer()
pooled_outputs = []
with tf.Session() as sess:
sess.run(init_op)
for i, filter_size in enumerate(filter_sizes):
// run the graph to get your output
output = sess.run([pooled], feed_dict = {embedded_chars: item})
pooled_outputs.append(output)
Related
I want to modification the value about variable 'weight1' in tf.variable_scope.
I try to modification the value by other function, but it not work follow me.
def inference(q, reuse=False):
with tf.variable_scope('layer1', reuse = reuse):
x = tf.get_variable('weight1', [1, 3], initializer = tf.truncated_normal_initializer(stddev = 0.1))
y = tf.get_variable('weight2', [3, 1], initializer = tf.constant_initializer([[1],[2],[3]]))
return tf.matmul(x, y)
def update_process(reuse=True):
with tf.variable_scope('layer1', reuse = reuse):
x = tf.get_variable('weight1',[1, 3])
update=tf.assign(x, x-1)
with tf.Session() as sess:
sess.run(init)
print(sess.run(x))
init = tf.global_variables_initializer()
z = inference(1)
with tf.Session() as sess:
sess.run(init)
for i in range(5):
update_process(reuse = True)
print(sess.run(z))
print('\n')
I want to this code output different list about sess.run(z), but the value is always same.
You need to run sess.run(update) in update_process in the same session that the inference part of the graph runs:
import tensorflow as tf
def inference(q, reuse=False):
with tf.variable_scope('layer1', reuse = reuse):
x = tf.get_variable('weight1', [1, 3], initializer = tf.truncated_normal_initializer(stddev = 0.1))
y = tf.get_variable('weight2', [3, 1], initializer = tf.constant_initializer([[1],[2],[3]]))
return tf.matmul(x, y)
def update_process(reuse=True):
with tf.variable_scope('layer1', reuse = reuse):
x = tf.get_variable('weight1',[1, 3])
update=tf.assign(x, x-1)
print(sess.run(update))
z = inference(1)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for i in range(5):
update_process(reuse = True)
print(sess.run(z))
print('\n')
I have my kernel dying each time I run this code on GPU. The problem can not be cuda or cudnn related since I can run another code on GPU.
#define shapes
weight_shapes = [(8,8,4,32), (4,4,32,64), (3,3,64,64), (3136,512),
(512,action_size), (32), (64), (64), (512), (action_size)]
def conv2d(x, W, b, strides=1):
# Conv2D wrapper, with bias and relu activation
x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='VALID')
x = tf.nn.bias_add(x, b)
#x = tf.layers.batch_normalization(x,training = True,trainable = True,epsilon = 1e-5,name = 'batch_norm1')
return tf.nn.relu(x)
def dense(x, W, b):
return tf.add(tf.matmul(x, W), b)
def policy(inputs_, W):
with tf.name_scope("conv1"):
# Input is 84x84x4
conv1 = conv2d(inputs_, W[0], W[5], strides=4)
with tf.name_scope("conv2"):
conv2 = conv2d(conv1, W[1], W[6], strides=2)
with tf.name_scope("conv3"):
conv3 = conv2d(conv2, W[2], W[7], strides=1)
## --> [7, 7, 64]
with tf.name_scope("flatten"):
flatten = tf.contrib.layers.flatten(conv3)
## --> [3136]
with tf.name_scope("fc1"):
fc1 = tf.nn.relu(dense(flatten, W[3], W[8]))
with tf.name_scope("logits"):
logits = dense(fc1, W[4], W[9])
return logits
And the main part of code is:
################################
### build the network policy####
################################
class PGNetwork:
def __init__(self, state_size, action_size, learning_rate):
self.state_size = state_size
self.action_size = action_size
self.learning_rate = learning_rate
self.weights = [
tf.get_variable('wc1', shape = weight_shapes[0], initializer=tf.contrib.layers.xavier_initializer()),
tf.get_variable('wc2', shape = weight_shapes[1], initializer=tf.contrib.layers.xavier_initializer()),
tf.get_variable('wc3', shape = weight_shapes[2], initializer=tf.contrib.layers.xavier_initializer()),
tf.get_variable('wd1', shape = weight_shapes[3], initializer=tf.contrib.layers.xavier_initializer()),
tf.get_variable('wd2', shape = weight_shapes[4], initializer=tf.contrib.layers.xavier_initializer()),
tf.get_variable('bc1', shape = weight_shapes[5], initializer=tf.contrib.layers.xavier_initializer()),
tf.get_variable('bc2', shape = weight_shapes[6], initializer=tf.contrib.layers.xavier_initializer()),
tf.get_variable('bc3', shape = weight_shapes[7], initializer=tf.contrib.layers.xavier_initializer()),
tf.get_variable('bd1', shape = weight_shapes[8], initializer=tf.contrib.layers.xavier_initializer()),
tf.get_variable('bd2', shape = weight_shapes[9], initializer=tf.contrib.layers.xavier_initializer())
]
with tf.name_scope("inputs"):
# We create the placeholders
# *state_size means that we take each elements of state_size in tuple hence is like if we wrote
# [None, 84, 84, 4] #the first argument is related to batch size
self.inputs_= tf.placeholder(tf.float32, [None, *state_size], name="inputs_")
self.actions = tf.placeholder(tf.int32, [None, action_size], name="actions")
self.discounted_episode_rewards_ = tf.placeholder(tf.float32, [None, ], name="discounted_episode_rewards_")
self.flat_multiplier_tensor = tf.placeholder(tf.float32, shape = [None])
# Add this placeholder for having this variable in tensorboard
self.mean_reward_ = tf.placeholder(tf.float32, name="mean_reward")
with tf.variable_scope('PGNetwork'):
self.logits = policy(self.inputs_, self.weights)
with tf.name_scope("softmax"):
self.action_distribution = tf.nn.softmax(self.logits)
with tf.name_scope("sample_gradient"):
self.split_inputs = tf.unstack(self.inputs_, num = batch_size_zero_padded, axis=0)
self.split_actions = tf.unstack(self.actions, num = batch_size_zero_padded, axis = 0)
self.intermediate = [tf.expand_dims(self.split_inputs[i], 0) for i in range(batch_size_zero_padded)]
and then when I try to run the following code, kernel dies: The kernel appears to have died. It will restart automatically.
# Reset the graph
tf.reset_default_graph()
# Instantiate the PGNetwork
PGNetwork = PGNetwork(state_size, action_size, learning_rate)
# Initialize Session
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
I would appreciate any idea on what might be going wrong! I am using python 3.6 with tensorflow-gpu 1.9.0.
I am new to tensorflow and trying to create a simple MLP. My model is running fine but is not giving the desired performance. I tried to create summaries but am now getting this error:
FailedPreconditionError: GetNext() failed because the iterator has not been initialized. Ensure that you have run the initializer operation for this iterator before getting the next element.
My code:
def fc_layer(input, channels_in,channels_out, name = "fc"):
with tf.name_scope(name):
W = tf.Variable(tf.zeros([channels_in, channels_out]), name="weights")
clip_op = tf.assign(W, tf.clip_by_norm(W, 1, axes = None))
b = tf.Variable(tf.zeros([channels_out]), name="biases")
act = tf.matmul(input, W) + b
tf.summary.histogram("weights", W)
tf.summary.histogram("biases", b)
tf.summary.histogram("activations", act)
return act
# Setup placeholders, and reshape the data
y = tf.placeholder(tf.float32, shape=[None,128], name = 'y')
x = tf.placeholder(tf.float32, shape=[None,256], name = 'x')
dataset = tf.data.Dataset.from_tensor_slices((y, x)).batch(batch_size).repeat()
iter = dataset.make_initializable_iterator()
input_features, output_features = iter.get_next()
fc_1 = tf.nn.relu(fc_layer(input_features, 128,512, name = "fc1"))
fc_2 = tf.nn.relu(fc_layer(fc_1, 512,256, name = "fc1"))
out_layer = fc_layer(fc_2, 256,256, name = "out")
with tf.name_scope('loss'):
loss_op =
tf.sqrt(tf.reduce_mean(tf.squared_difference(out_layer,output_features)))
tf.summary.scalar("loss", loss_op)
with tf.name_scope('train'):
train_op =
tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss_op)
#Summary writer
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter(r'C:\Users\Jaweria\Documents\Code_logs',
graph=tf.get_default_graph())
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# initialise iterator with train data
sess.run(iter.initializer, feed_dict={ y: train_data[0], x: train_data[1], batch_size: Batch_Size})
print('Training...')
for i in range(training_epochs):
tot_loss = 0
for _ in range(n_batches):
_, loss_value = sess.run([train_op, loss_op])
tot_loss += loss_value
s = sess.run(merged_summary)
writer.add_summary(s,i*n_batches+ _)
print("Iter: {}, Loss: {:.4f}".format(i, tot_loss / n_batches))
# initialise iterator with test data
sess.run(iter.initializer, feed_dict={ y: test_data[0], x: test_data[1],
batch_size: test_data[0].shape[0]})
print('Test Loss: {:4f}'.format(sess.run(loss_op)))
tf.trainable_variables() returns a list of all trainable variable objects. When an object from the list is passed to an op, such as tf.nn.l2_loss, TensorFlow is able to cast the object as a Tensor and perform the necessary calculations. However, passing the same object to a user defined function throws an error.
Consider the following two layer network to work with:
# Generate random data
x_train = np.random.rand(64, 16, 16, 8)
y_train = np.random.randint(0, 5, 64)
one_hot = np.zeros((len(y_train), 5))
one_hot[list(np.indices((len(y_train),))) + [y_train]] = 1
y_train = one_hot
# Model definition
class FeedForward(object):
def __init__(self, l2_lambda=0.01):
self.x = tf.placeholder(tf.float32, shape=[None, 16, 16, 4], name="input_x")
self.y = tf.placeholder(tf.float32, [None, 5], name="input_y")
l2_loss = tf.constant(0.0)
with tf.name_scope("conv1"):
kernel_shape=[1, 1, 4, 4]
w = tf.Variable(tf.truncated_normal(kernel_shape, stddev=0.1), name="weight")
conv1 = tf.nn.conv2d(self.x, w, strides=[1, 1, 1, 1], padding="SAME", name="conv")
with tf.name_scope("conv2"):
kernel_shape=[1, 1, 4, 2]
w = tf.Variable(tf.truncated_normal(kernel_shape, stddev=0.1), name="weight")
conv2 = tf.nn.conv2d(conv1, w, strides=[1, 1, 1, 1], padding="SAME", name="conv")
out = tf.contrib.layers.flatten(conv2)
with tf.name_scope("output"):
kernel_shape=[out.get_shape()[1].value, 5]
w = tf.Variable(tf.truncated_normal(kernel_shape, stddev=0.1), name="weight")
self.scores = tf.matmul(out, w, name="scores")
predictions = tf.argmax(self.scores, axis=1, name="predictions")
# L2 Regularizer
if l2_reg_lambda > 0.:
l2_loss = tf.add_n([self.some_norm(var) for var in tf.trainable_variables() if ("weight" in var.name)])
losses = tf.nn.softmax_cross_entropy_with_logits(logits=self.scores, labels=self.y)
self.loss = tf.reduce_mean(losses) + (l2_lambda * l2_loss)
correct_predictions = tf.equal(predictions, tf.argmax(self.y, axis=1))
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
def some_norm(w):
# operate on w and return scalar
# (only) for example
return (1 / tf.nn.l2_loss(w))
with tf.Graph().as_default():
sess = tf.Session()
with sess.as_default():
ffn = FeedForward()
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=1e-2)
grads_and_vars = optimizer.compute_gradients(ffn.loss)
sess.run(tf.global_variables_initializer())
def train_step(x_batch, y_batch):
feed_dict = {
ffn.x: x_batch,
ffn.y: y_batch,
}
_, step, loss, accuracy = sess.run([train_op, global_step, ffn.loss, ffn.accuracy], feed_dict)
print("step {}, loss {:g}, acc {:g}".format(step, loss, accuracy))
batch_size = 32
n_epochs = 4
s_idx = - batch_size
for batch_index in range(n_epochs):
s_idx += batch_size
e_idx = s_idx + batch_size
x_batch = x_train[s_idx:e_idx]
y_batch = y_train[s_idx:e_idx]
train_step(x_batch, y_batch)
current_step = tf.train.global_step(sess, global_step)
The problem here is that on passing the trainable variable to some_norm(), it is passed as an object and can not be operated on. The related error message encountered at the first line inside some_norm() is:
Failed to convert object of type <class '__main__.FeedForward'> to Tensor.
Contents: <__main__.FeedForward object at 0x7fefde7e97b8>.
Consider casting elements to a supported type.
Is there a way to cast the object returned by tf.trainable_variables() as a tensor or is there a possible workaround such as passing a reference?
How is using the above different from using l2_loss = tf.add_n([tf.nn.l2_loss(var) for var in tf.trainable_variables()...]) which works just fine?
You forgot the self argument in your some_norm implementation def some_norm(w):, so it tries to convert your instance of the class (self) to a tensor.
I am new to Tensorflow and Machine Learning and trying out CNN using Tensorflow with my custom input data. But I am getting the error attached below.
The Data or Image Size is 28x28 with 15 Labels.
I am not getting the numpy reshape thing in this script or the error.
Help is highly appreciated.
import tensorflow as tf
import os
import skimage.data
import numpy as np
import random
def load_data(data_directory):
directories = [d for d in os.listdir(data_directory)
if os.path.isdir(os.path.join(data_directory, d))]
labels = []
images = []
for d in directories:
label_directory = os.path.join(data_directory, d)
file_names = [os.path.join(label_directory, f)
for f in os.listdir(label_directory)
if f.endswith(".jpg")]
for f in file_names:
images.append(skimage.data.imread(f))
labels.append(d)
print(str(d)+' Completed')
return images, labels
ROOT_PATH = "H:\Testing\TrainingData"
train_data_directory = os.path.join(ROOT_PATH, "Training")
test_data_directory = os.path.join(ROOT_PATH, "Testing")
print('Loading Data...')
images, labels = load_data(train_data_directory)
print('Data has been Loaded')
n_classes = 15
training_examples = 10500
test_examples = 4500
batch_size = 128
x = tf.placeholder('float', [None, 784])
y = tf.placeholder('float')
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME')
def maxpool2d(x):
return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')
def neural_network_model(x):
weights = {'W_Conv1':tf.Variable(tf.random_normal([5,5,1,32])),
'W_Conv2':tf.Variable(tf.random_normal([5,5,32,64])),
'W_FC':tf.Variable(tf.random_normal([7*7*64, 1024])),
'Output':tf.Variable(tf.random_normal([1024, n_classes]))}
biases = {'B_Conv1':tf.Variable(tf.random_normal([32])),
'B_Conv2':tf.Variable(tf.random_normal([64])),
'B_FC':tf.Variable(tf.random_normal([1024])),
'Output':tf.Variable(tf.random_normal([n_classes]))}
x = tf.reshape(x, shape=[-1,28,28,1])
conv1 = conv2d(x, weights['W_Conv1'])
conv1 = maxpool2d(conv1)
conv2 = conv2d(conv1, weights['W_Conv2'])
conv2 = maxpool2d(conv2)
fc = tf.reshape(conv2, [-1, 7*7*64])
fc = tf.nn.relu(tf.matmul(fc, weights['W_FC'])+biases['B_FC'])
output = tf.matmul(fc, weights['Output'])+biases['Output']
return output
def next_batch(num, data, labels):
idx = np.arange(0 , len(data))
np.random.shuffle(idx)
idx = idx[:num]
data_shuffle = [data[ i] for i in idx]
labels_shuffle = [labels[ i] for i in idx]
return np.asarray(data_shuffle), np.asarray(labels_shuffle)
def train_neural_network(x):
prediction = neural_network_model(x)
cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y) )
optimizer = tf.train.AdamOptimizer().minimize(cost)
hm_epochs = 10
with tf.Session() as sess:
# OLD:
#sess.run(tf.initialize_all_variables())
# NEW:
sess.run(tf.global_variables_initializer())
for epoch in range(hm_epochs):
epoch_loss = 0
for _ in range(int(training_examples/batch_size)):
epoch_x, epoch_y = next_batch(batch_size, images, labels)
_, c = sess.run([optimizer, cost], feed_dict={x: epoch_x, y: epoch_y})
epoch_loss += c
print('Epoch', epoch, 'completed out of',hm_epochs,'loss:',epoch_loss)
correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
print('Accuracy:',accuracy.eval({x: images, y: labels}))
print('Training Neural Network...')
train_neural_network(x)
What am I doing wrong? What is needed to be fixed and how do I fix the shape of numpy array?
If you look closely, you'll see that you have two x placeholders:
x = tf.placeholder('float', [None, 784]) # global
...
x = tf.reshape(x, shape=[-1,28,28,1]) # in neural_network_model
One of them is in the function scope, hence not visible in train_neural_network, so tensorflow takes the one with [?, 784] shape. You should get rid of one of them.
Also note that your training data has the rank 3, i.e. [batch_size, 28, 28], so it's not directly compatible with any of those placeholders.
To feed it into the first x, take epoch_x.reshape([-1, 784]). For the second placeholder (once you make it visible), take epoch_x.reshape([-1, 28, 28, 1]).