Related
I am building an LSTM net using the Dataset API.
The input tensor (named x in code) has different shapes for the train and the val sets and the iterator is defined without specifying an output shape.
The problem is that when tf.nn.dynamic_rnn graph_op is defined the shape of x is unknown and the following error is raised:
ValueError: as_list() is not defined on an unknown TensorShape.
Using tf.nn.dynamic_rnn without the Dataset API works as expected.
How can this error be fixed?
TF version: 1.4
import tensorflow as tf
import numpy as np
"""
1d: Number of examples per epoch
2d: Time steps size
3d: Batch size e.g. number of independent time series
4d: Number of points that are given as input in the lstm each time step
Batch size is usually smaller in val set because we use most of data for training.
Time steps size is bigger in val set because we want to speed up inference.
"""
x_train = np.random.rand(100, 8, 12, 2).astype(np.float32)
x_val = np.random.rand(8, 100, 4, 2).astype(np.float32)
use_dataset_api = True
with tf.device('/gpu:0'):
tf.reset_default_graph()
if not use_dataset_api:
batch_size_pl = tf.placeholder(shape=[], dtype=tf.int32)
x_pl = tf.placeholder(shape=[None, None, 2], dtype=tf.float32)
cell = tf.contrib.rnn.LSTMCell(num_units=11)
init_state = cell.zero_state(batch_size=batch_size_pl, dtype=tf.float32)
rnn_outputs, current_state = tf.nn.dynamic_rnn(cell, x_pl, initial_state=init_state,
time_major=True, dtype=tf.float32)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Use first example of train set
rnn_outputs_, current_state_ = sess.run([rnn_outputs, current_state],
feed_dict={batch_size_pl: 12, x_pl: x_train[0]})
# Use first example of val set
rnn_outputs_, current_state_ = sess.run([rnn_outputs, current_state],
feed_dict={batch_size_pl: 4, x_pl: x_val[0]})
else:
batch_size_pl = tf.placeholder(shape=[], dtype=tf.int32)
train_set = tf.data.Dataset.from_tensor_slices((x_train))
val_set = tf.data.Dataset.from_tensor_slices((x_val))
iterator = tf.data.Iterator.from_structure(train_set.output_types) # , train_set.output_shapes)
train_init_op = iterator.make_initializer(train_set)
val_init_op = iterator.make_initializer(val_set)
x = iterator.get_next()
cell = tf.contrib.rnn.LSTMCell(num_units=11)
init_state = cell.zero_state(batch_size=batch_size_pl, dtype=tf.float32)
# Raises error for tensor x: as_list() is not defined on an unknown TensorShape.
rnn_outputs, current_state = tf.nn.dynamic_rnn(cell, x, initial_state=init_state,
time_major=True, dtype=tf.float32)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Use first example of train set
sess.run(train_init_op)
rnn_outputs_, current_state_ = sess.run([rnn_outputs, current_state],
feed_dict={batch_size_pl: 12})
# Use first example of val set
sess.run(val_init_op)
rnn_outputs_, current_state_ = sess.run([rnn_outputs, current_state],
feed_dict={batch_size_pl: 4})
The solution is to change the following line:
iterator = tf.data.Iterator.from_structure(train_set.output_types)
with:
iterator = tf.data.Iterator.from_structure(train_set.output_types, [None, None, 2])
tf.trainable_variables() returns a list of all trainable variable objects. When an object from the list is passed to an op, such as tf.nn.l2_loss, TensorFlow is able to cast the object as a Tensor and perform the necessary calculations. However, passing the same object to a user defined function throws an error.
Consider the following two layer network to work with:
# Generate random data
x_train = np.random.rand(64, 16, 16, 8)
y_train = np.random.randint(0, 5, 64)
one_hot = np.zeros((len(y_train), 5))
one_hot[list(np.indices((len(y_train),))) + [y_train]] = 1
y_train = one_hot
# Model definition
class FeedForward(object):
def __init__(self, l2_lambda=0.01):
self.x = tf.placeholder(tf.float32, shape=[None, 16, 16, 4], name="input_x")
self.y = tf.placeholder(tf.float32, [None, 5], name="input_y")
l2_loss = tf.constant(0.0)
with tf.name_scope("conv1"):
kernel_shape=[1, 1, 4, 4]
w = tf.Variable(tf.truncated_normal(kernel_shape, stddev=0.1), name="weight")
conv1 = tf.nn.conv2d(self.x, w, strides=[1, 1, 1, 1], padding="SAME", name="conv")
with tf.name_scope("conv2"):
kernel_shape=[1, 1, 4, 2]
w = tf.Variable(tf.truncated_normal(kernel_shape, stddev=0.1), name="weight")
conv2 = tf.nn.conv2d(conv1, w, strides=[1, 1, 1, 1], padding="SAME", name="conv")
out = tf.contrib.layers.flatten(conv2)
with tf.name_scope("output"):
kernel_shape=[out.get_shape()[1].value, 5]
w = tf.Variable(tf.truncated_normal(kernel_shape, stddev=0.1), name="weight")
self.scores = tf.matmul(out, w, name="scores")
predictions = tf.argmax(self.scores, axis=1, name="predictions")
# L2 Regularizer
if l2_reg_lambda > 0.:
l2_loss = tf.add_n([self.some_norm(var) for var in tf.trainable_variables() if ("weight" in var.name)])
losses = tf.nn.softmax_cross_entropy_with_logits(logits=self.scores, labels=self.y)
self.loss = tf.reduce_mean(losses) + (l2_lambda * l2_loss)
correct_predictions = tf.equal(predictions, tf.argmax(self.y, axis=1))
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
def some_norm(w):
# operate on w and return scalar
# (only) for example
return (1 / tf.nn.l2_loss(w))
with tf.Graph().as_default():
sess = tf.Session()
with sess.as_default():
ffn = FeedForward()
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=1e-2)
grads_and_vars = optimizer.compute_gradients(ffn.loss)
sess.run(tf.global_variables_initializer())
def train_step(x_batch, y_batch):
feed_dict = {
ffn.x: x_batch,
ffn.y: y_batch,
}
_, step, loss, accuracy = sess.run([train_op, global_step, ffn.loss, ffn.accuracy], feed_dict)
print("step {}, loss {:g}, acc {:g}".format(step, loss, accuracy))
batch_size = 32
n_epochs = 4
s_idx = - batch_size
for batch_index in range(n_epochs):
s_idx += batch_size
e_idx = s_idx + batch_size
x_batch = x_train[s_idx:e_idx]
y_batch = y_train[s_idx:e_idx]
train_step(x_batch, y_batch)
current_step = tf.train.global_step(sess, global_step)
The problem here is that on passing the trainable variable to some_norm(), it is passed as an object and can not be operated on. The related error message encountered at the first line inside some_norm() is:
Failed to convert object of type <class '__main__.FeedForward'> to Tensor.
Contents: <__main__.FeedForward object at 0x7fefde7e97b8>.
Consider casting elements to a supported type.
Is there a way to cast the object returned by tf.trainable_variables() as a tensor or is there a possible workaround such as passing a reference?
How is using the above different from using l2_loss = tf.add_n([tf.nn.l2_loss(var) for var in tf.trainable_variables()...]) which works just fine?
You forgot the self argument in your some_norm implementation def some_norm(w):, so it tries to convert your instance of the class (self) to a tensor.
Here is my code:
def conv_pooling(data, sequence_length, filter_size, embedding_size, num_filters):
filter_shape = [filter_size, embedding_size, 1, num_filters]
w = tf.Variable(tf.truncated_normal(filter_shape,stddev = 0.1),
name = "w")
b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name =
"b")
conv = tf.nn.conv2d(
item,
w,
strides = [1,1,1,1],
padding = "VALID",
name = "conv"
)
h = tf.nn.relu(tf.nn.bias_add(conv, b), name = "relu")
pooled = tf.nn.max_pool(
h,
ksize = [1,sequence_length - filter_size + 1, 1, 1],
strides = [1,1,1,1],
padding = "VALID",
name = "pool"
)
return pooled
init_op = tf.global_variables_initializer()
pooled_outputs = []
with tf.Session() as sess:
sess.run(init_op)
for i, filter_size in enumerate(filter_sizes):
pooled = sess.run(conv_pooling(data, sequence_length, filter_size, embedding_size, num_filters), feed_dict = {embedded_chars: item})
pooled_outputs.append(pooled)
This 'data' is a tf.Variable that use the global tf.placeholder 'embedded_chars', so don't worry about if it is working. The error happens because of w and b cannot be initialized.
I tried sess.run(tf.local_variables_initializer()) also, not work and return the same error. Does anyone know a way that I can initialized w and b here? As you see the size of w change in for loop.
Thank you!
See the code below. That's why #mikkola means about creating your graph before initialization.
// create your computation graph
pooled = conv_pooling(data, sequence_length, filter_size, embedding_size, num_filters)
// initialize the variables in the graph
init_op = tf.global_variables_initializer()
pooled_outputs = []
with tf.Session() as sess:
sess.run(init_op)
for i, filter_size in enumerate(filter_sizes):
// run the graph to get your output
output = sess.run([pooled], feed_dict = {embedded_chars: item})
pooled_outputs.append(output)
I am new to Tensorflow and Machine Learning and trying out CNN using Tensorflow with my custom input data. But I am getting the error attached below.
The Data or Image Size is 28x28 with 15 Labels.
I am not getting the numpy reshape thing in this script or the error.
Help is highly appreciated.
import tensorflow as tf
import os
import skimage.data
import numpy as np
import random
def load_data(data_directory):
directories = [d for d in os.listdir(data_directory)
if os.path.isdir(os.path.join(data_directory, d))]
labels = []
images = []
for d in directories:
label_directory = os.path.join(data_directory, d)
file_names = [os.path.join(label_directory, f)
for f in os.listdir(label_directory)
if f.endswith(".jpg")]
for f in file_names:
images.append(skimage.data.imread(f))
labels.append(d)
print(str(d)+' Completed')
return images, labels
ROOT_PATH = "H:\Testing\TrainingData"
train_data_directory = os.path.join(ROOT_PATH, "Training")
test_data_directory = os.path.join(ROOT_PATH, "Testing")
print('Loading Data...')
images, labels = load_data(train_data_directory)
print('Data has been Loaded')
n_classes = 15
training_examples = 10500
test_examples = 4500
batch_size = 128
x = tf.placeholder('float', [None, 784])
y = tf.placeholder('float')
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME')
def maxpool2d(x):
return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')
def neural_network_model(x):
weights = {'W_Conv1':tf.Variable(tf.random_normal([5,5,1,32])),
'W_Conv2':tf.Variable(tf.random_normal([5,5,32,64])),
'W_FC':tf.Variable(tf.random_normal([7*7*64, 1024])),
'Output':tf.Variable(tf.random_normal([1024, n_classes]))}
biases = {'B_Conv1':tf.Variable(tf.random_normal([32])),
'B_Conv2':tf.Variable(tf.random_normal([64])),
'B_FC':tf.Variable(tf.random_normal([1024])),
'Output':tf.Variable(tf.random_normal([n_classes]))}
x = tf.reshape(x, shape=[-1,28,28,1])
conv1 = conv2d(x, weights['W_Conv1'])
conv1 = maxpool2d(conv1)
conv2 = conv2d(conv1, weights['W_Conv2'])
conv2 = maxpool2d(conv2)
fc = tf.reshape(conv2, [-1, 7*7*64])
fc = tf.nn.relu(tf.matmul(fc, weights['W_FC'])+biases['B_FC'])
output = tf.matmul(fc, weights['Output'])+biases['Output']
return output
def next_batch(num, data, labels):
idx = np.arange(0 , len(data))
np.random.shuffle(idx)
idx = idx[:num]
data_shuffle = [data[ i] for i in idx]
labels_shuffle = [labels[ i] for i in idx]
return np.asarray(data_shuffle), np.asarray(labels_shuffle)
def train_neural_network(x):
prediction = neural_network_model(x)
cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y) )
optimizer = tf.train.AdamOptimizer().minimize(cost)
hm_epochs = 10
with tf.Session() as sess:
# OLD:
#sess.run(tf.initialize_all_variables())
# NEW:
sess.run(tf.global_variables_initializer())
for epoch in range(hm_epochs):
epoch_loss = 0
for _ in range(int(training_examples/batch_size)):
epoch_x, epoch_y = next_batch(batch_size, images, labels)
_, c = sess.run([optimizer, cost], feed_dict={x: epoch_x, y: epoch_y})
epoch_loss += c
print('Epoch', epoch, 'completed out of',hm_epochs,'loss:',epoch_loss)
correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
print('Accuracy:',accuracy.eval({x: images, y: labels}))
print('Training Neural Network...')
train_neural_network(x)
What am I doing wrong? What is needed to be fixed and how do I fix the shape of numpy array?
If you look closely, you'll see that you have two x placeholders:
x = tf.placeholder('float', [None, 784]) # global
...
x = tf.reshape(x, shape=[-1,28,28,1]) # in neural_network_model
One of them is in the function scope, hence not visible in train_neural_network, so tensorflow takes the one with [?, 784] shape. You should get rid of one of them.
Also note that your training data has the rank 3, i.e. [batch_size, 28, 28], so it's not directly compatible with any of those placeholders.
To feed it into the first x, take epoch_x.reshape([-1, 784]). For the second placeholder (once you make it visible), take epoch_x.reshape([-1, 28, 28, 1]).
Basically, I have a function that expects a tensor x and two placeholders z and c.
def error_robust(x,z,c):
zz = tf.reshape(z, [-1, 28, 28, 1])
var = tf.reduce_mean(x-zz)
out = tf.cond( tf.abs(var) <= c, lambda: (c*c/6.0)*(1 - tf.pow(1-tf.pow(var/c,2),3)), lambda: tf.Variable(c*c/6.0) )
return out
I define the placeholders and tensors that I am gonna use:
# TENSORFLOW PLACEHOLDERS
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
flat_mnist_data = tf.placeholder(tf.float32, [None, 28*28])
dropout_keep_prob = tf.placeholder(tf.float32)
param_robust = tf.placeholder(tf.float32, shape=())
Calling the defined function does not generate any errors:
error_r = error_robust(layer1_b.reconstruction, flat_mnist_data, param_robust)
This generates an error:
sess.run(tf.global_variables_initializer())
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'Placeholder' with dtype float
[[Node: Placeholder = Placeholderdtype=DT_FLOAT, shape=[], _device="/job:localhost/replica:0/task:0/gpu:0"]]
I don't really understand why it happens. Any ideas on how to solve this one?
Ok, I got it. I was first expecting c to be a simple scalar. So I was using tf.Variable as the second argument of the tf.cond.
Updating the error_robust function solves it:
def error_robust(x,z,c):
zz = tf.reshape(z, [-1, 28, 28, 1])
var = tf.reduce_mean(x-zz)
out = tf.cond( tf.abs(var) <= c, lambda: (c*c/6.0)*(1 - tf.pow(1-tf.pow(var/c,2),3)), lambda: c*c/6.0 )
return out