Below codes show me "inputs must be a list". at this.
outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)
When I define placeholder for input x. I have already set a shape as [None,None]. I think this shape is 2-dimensional array. However, the code continuously requires list type of x.
Below, I have attached all of my codes before training. And this codes are inserted into function of class.
x = tf.placeholder("float",[None,None])
y = tf.placeholder("float",[None])
lstm_cell = rnn_cell.BasicLSTMCell(self.n_hidden, forget_bias=1.0)
outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)
pred = tf.matmul(outpus[-1], self.weights['out']) + self.biases['out']
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred,y))
optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(cost)
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
init = tf.initialize_all_variables()
self.sess = tf.Session()
self.sess.run(init)
Additionally, practical inputs will be float of word sequence and float of label formed as x=[["aaa","aaa","aaa"],["bbb","bbb"]], y=["c1","c2"].
At that, the first element array of x is labeled with "c1" and the second is "c2". Especially, size of each element array of x cannot be deterministic.
As stated by the documentation, the argument inputs of the function tf.nn.rnn() is:
inputs: A length T list of inputs, each a Tensor of shape [batch_size, input_size], or a nested tuple of such elements.
In your code, the argument inputs is x, a Tensor placeholder of shape [None, None]. In order for your code to work, x must be a list of T tensors of shape [None, input_lenght].
The following code generates a list of tensors as inputs and therefore the function tf.nn.rnn works.
import tensorflow as tf
x = tf.placeholder("float",[None,16])
y = tf.placeholder("float",[None])
lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(256, forget_bias=1.0)
inputs = []
for t in range(10):
inputs.append(x)
print(len(inputs))
outputs, states = tf.nn.rnn(lstm_cell, inputs, dtype=tf.float32)
pred = tf.matmul(outputs[-1], self.weights['out']) + self.biases['out']
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred,y))
optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate).minimize(cost)
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
init = tf.initialize_all_variables()
self.sess = tf.Session()
self.sess.run(init)
Note how the placeholder x has a defined shape of [None, input_shape]. It won't work with a shape [None, None] because the first dimensions is the batch_size, which can be None, but the second dimension is the size of each item in the input sequence, and that value can't be None.
Related
I'm trying to build an RNN with LSTM on TensorFlow. Both the input and output are 5000 by 2 matrices, where the columns represent the features. Those matrices are then fed to the batchX and batchY placeholders which enable the backpropagation. The main definition of the code is at the bottom. I am getting the following error :
"Rank mismatch: Rank of labels (received 2) should equal rank of logits minus 1 (received 2)."
I have checked both logits_series and labels_series and they seem to both contain backpropagation amount of tensors of the shape of [batch_size, num_features]
The thing I am confused about is the following: since logits are predictions of labels, shouldn't they have the same dimensions?
'''
RNN definitions
input_dimensions = [batch_size, truncated_backprop_length, num_features_input]
output_dimensions = [batch_size, truncated_backprop_length, num_features_output]
state_dimensions = [batch_size, state_size]
'''
batchX_placeholder = tf.placeholder(tf.float32, (batch_size, truncated_backprop_length, num_features_input))
batchY_placeholder = tf.placeholder(tf.int32, (batch_size, truncated_backprop_length, num_features_output))
init_state = tf.placeholder(tf.float32, (batch_size, state_size))
inputs_series = tf.unstack(batchX_placeholder, axis=1)
labels_series = tf.unstack(batchY_placeholder, axis=1)
w = tf.Variable(np.random.rand(num_features_input+state_size,state_size), dtype = tf.float32)
b = tf.Variable(np.zeros((batch_size, state_size)), dtype = tf.float32)
w2 = tf.Variable(np.random.rand(state_size, num_features_output), dtype = tf.float32)
b2 = tf.Variable(np.zeros((batch_size, num_features_output)), dtype=tf.float32)
#calculate state and output variables
state_series = []
output_series = []
current_state = init_state
#iterate over each truncated_backprop_length
for current_input in inputs_series:
current_input = tf.reshape(current_input,[batch_size, num_features_input])
input_and_state_concatenated = tf.concat([current_input,current_state], 1)
next_state = tf.tanh(tf.matmul(input_and_state_concatenated, w) + b)
state_series.append(next_state)
current_state = next_state
output = tf.matmul(current_state, w2)+b2
output_series.append(output)
#calculate expected output for each state
logits_series = [tf.matmul(state, w2) + b2 for state in state_series]
#print(logits_series)
predictions_series = [tf.nn.softmax(logits) for logits in logits_series]
'''
batchY_placeholder = np.zeros((batch_size,truncated_backprop_length))
for i in range(batch_size):
for j in range(truncated_backprop_length):
batchY_placeholder[i,j] = batchY1_placeholder[j, i, 0]+batchY1_placeholder[j, i, 1]
'''
print("logits_series", logits_series)
print("labels_series", labels_series)
#calculate losses given each actual and calculated output
losses = [tf.nn.sparse_softmax_cross_entropy_with_logits(logits = logits, labels = labels) for logits, labels in zip(logits_series,labels_series)]
total_loss = tf.reduce_mean(losses)
Thanks to Maosi Chen, I found the issue. It was because the
tf.nn.sparse_softmax_cross_entropy_with_logits
Requires labels to have one less dimension than logits. Specifically, the labels argument takes values of the shape [batch_size] and the dtype int32 or int64
I solved the issue by enumerating the one hot encoded labels I had, reducing the dimension
However, it was also possible to use
tf.nn.softmax_cross_entropy_with_logits
Which does not have the dimension reduction requirement, as it takes labels values with shape [batch_size, num_classes] and dtype float32 or float64.
I have lately been vexed by the following error message:
ValueError: Cannot feed value of shape (2455040,) for Tensor 'Placeholder:0', which has shape '(2455040, ?)'
Which is being produced from running the following code:
NUMCLASSES=16
NUMPIXELS=959*640*4
# set up to feed an array of images [images, size_of_image]
x = tf.placeholder(tf.float32, [NUMPIXELS,None])
....deletia....
# Define loss and optimizer..why is this 2d?
y_ = tf.placeholder(tf.float32, [None,NUMCLASSES])
sess = tf.InteractiveSession()
tf.global_variables_initializer().run(session=sess)
tl = get_tensor_list()
for f, n in tl:
str = '/users/me/downloads/train/' + f
mm = Image.open(str)
mm = mm.convert('F')
mma=np.array(mm)
i = mma.flatten() #now this is an array of floats of size NUMPIXELS
sess.run(train_step, feed_dict={x: i, y_: n}) # <<DEATH
Somehow, that array is getting a shape that tf does not like [(x,) when it wants (x,?)]. How to satisfy the tensorgods in this case? The tensor must be what it must be for other mathematical reasons not discussed.
reshaping the array might help.
i = mma.flatten().reshape((NUMPIXELS,1))
The error happens because the two tensors have different ranks: tensor with shape (2455040,) has rank 1, while tensor with shape (2455040,?) has rank 2.
You can do this:
x = tf.placeholder(tf.float32, [None])
x = tf.reshape(x, [NUMPIXELS,-1])
Currently I have the following code:
init_state = tf.Variable(tf.zeros([batch_partition_length, state_size])) # -> [16, 1024].
final_state = tf.Variable(tf.zeros([batch_partition_length, state_size]))
And inside my inference method that is responsible producing the output, I have the following:
def inference(frames):
# Note that I write the final_state as a global valriable to avoid the shadowing issue, since it is referenced at the dynamic_rnn line.
global final_state
# .... Here we have some conv layers and so on...
# Now the RNN cell
with tf.variable_scope('local1') as scope:
# Move everything into depth so we can perform a single matrix multiply.
shape_d = pool3.get_shape()
shape = shape_d[1] * shape_d[2] * shape_d[3]
# tf_shape = tf.stack(shape)
tf_shape = 1024
print("shape:", shape, shape_d[1], shape_d[2], shape_d[3])
# So note that tf_shape = 1024, this means that we have 1024 features are fed into the network. And
# the batch size = 1024. Therefore, the aim is to divide the batch_size into num_steps so that
reshape = tf.reshape(pool3, [-1, tf_shape])
# Now we need to reshape/divide the batch_size into num_steps so that we would be feeding a sequence
rnn_inputs = tf.reshape(reshape, [batch_partition_length, step_size, tf_shape])
print('RNN inputs shape: ', rnn_inputs.get_shape()) # -> (16, 64, 1024).
cell = tf.contrib.rnn.BasicRNNCell(state_size)
# note that rnn_outputs are the outputs but not multiplied by W.
rnn_outputs, final_state = tf.nn.dynamic_rnn(cell, rnn_inputs, initial_state=init_state)
# linear Wx + b
with tf.variable_scope('softmax_linear') as scope:
weight_softmax = \
tf.Variable(
tf.truncated_normal([state_size, n_classes], stddev=1 / state_size, dtype=tf.float32, name='weight_softmax'))
bias_softmax = tf.constant(0.0, tf.float32, [n_classes], name='bias_softmax')
softmax_linear = tf.reshape(
tf.matmul(tf.reshape(rnn_outputs, [-1, state_size]), weight_softmax) + bias_softmax,
[batch_size, n_classes])
print('Output shape:', softmax_linear.get_shape())
return softmax_linear
# Here we define the loss, accuracy and the optimzer.
# now run the graph:
with tf.Session() as sess:
_, accuracy_train, loss_train, summary = \
sess.run([optimizer, accuracy, cost_scalar, merged], feed_dict={x: image_batch,
y_valence: valences,
confidence_holder: confidences})
....
Problem: How I would be able to assign initial_state the value stored in final_state? That is, how to more update a Variable value given the other?
I have used the following:
tf.assign(init_state, final_state.eval())
under session after running the sess.run command. But, this is throwing an error:
You must feed a value for placeholder tensor 'inputs' with dtype float
Where tf.Variable: "input" is declared as follows:
x = tf.placeholder(tf.float32, [None, 112, 112, 3], name='inputs')
And the feeding is done after reading the images from the tfRecords through the following command:
example = tf.train.Example()
example.ParseFromString(string_record)
height = int(example.features.feature['height']
.int64_list
.value[0])
width = int(example.features.feature['width']
.int64_list
.value[0])
img_string = (example.features.feature['image_raw']
.bytes_list
.value[0])
img_1d = np.fromstring(img_string, dtype=np.uint8)
reconstructed_img = img_1d.reshape((height, width, -1)) # Where this is added to the image_batch list, which is fed into the placeholder.
And if tried the following:
img_1d = np.fromstring(img_string, dtype=np.float32)
This will produce the following error:
ValueError: cannot reshape array of size 9408 into shape (112,112,newaxis)
Any help is much appreciated!!
So here are the mistakes that I have done so far. After doing some revision I figured out the following:
I shouldn't create the final_state as a tf.Variable. Since tf.nn.dynamic_rnn return tensors as ndarray, then, I should not instantiate the final_state int the beginning. And I should not use the global final_state under the function definition.
In order to assign the initial state the final_state, I used:
tf.assign(intial_state, final_state)
And things work out.
Note: in tensorflow, an operation returns the data as numpy array in python and as tensorflow::Tensor in C and C++.
Have a look at https://www.tensorflow.org/versions/r0.10/get_started/basic_usage for more informaiton.
I am trying to implement an LSTM Model as a model_fn input to an Estimator. My X is only a .txt with a time series of prices. Before going into my first hidden layer, I try to define the lstm cell as:
def lstm_cell():
return tf.contrib.rnn.BasicLSTMCell(
size, forget_bias=0.0, state_is_tuple=True)
attn_cell = lstm_cell
if is_training and keep_prob < 1:
def attn_cell():
return tf.contrib.rnn.DropoutWrapper(
lstm_cell(), output_keep_prob=keep_prob)
cell = tf.contrib.rnn.MultiRNNCell([attn_cell() for _ in range(num_layers)], state_is_tuple=True)
initial_state = cell.zero_state(batch_size, data_type())
inputs = tf.unstack(X, num=num_steps, axis=0)
outputs = []
outputs, state = tf.nn.dynamic_rnn(cell, inputs,
initial_state=initial_state)
This then is supposed to go into:
first_hidden_layer = tf.contrib.layers.relu(outputs, 1000)
Unfortunately, it throws an error idicating that "ValueError: Dimension must be 1 but is 3 for 'transpose' (op: 'Transpose') with input shapes: [1], [3]."
I gather that my problem is the "inputs" tensor. In its description, the inputs variable is supposed to be a tensor with form [batch_size,max_time,...], but Ihave no idea how to translate this into above structure since, through the estimator, only input values X and target values y are fed to the system. So my question would be how to create a tensor that can serve as an inputs variable to the dynamic_rnn class.
Thanks a lot.
I believe you don't need the line:
inputs = tf.unstack(X, num=num_steps, axis=0)
you can supply X directly to dynamic_rnn since dynamic_rnn doesn't take a list of tensors; It takes one tensor where the time axis is dimension 0 (if time_major == True) or dimension 1 (if time_major == False).
Actually, it seems that X has 2 dimensions only, since inputs is list of 1 dimensional tensors (as indicated by the error message). so you should replace the unstack line with:
inputs = tf.expand_dims(X, axis=2)
This will add a 3rd dimension of size 1 that is needed by dynamic_rnn
I want to stack two LSTMs without using MultiRNN wrapper. However, following code results with ValueError: Shapes (3,) and (2,) are not compatible because of inputs=states_fw_1 in the second LSTM. How can I pass hidden state of the first LSTM as input to the second?
LSTM 1
with tf.name_scope("BiLSTM_1"):
with tf.variable_scope('forward_1'):
cell_fw_1 = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size, state_is_tuple=True)
with tf.variable_scope('backward_srl'):
cell_bw_srl = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size, state_is_tuple=True)
outputs_1, states_1 = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw_1,
cell_bw=cell_bw_1,
dtype=tf.float64,
sequence_length=self.input_seq_len,
inputs=self.embedded_input_layer,
scope='BiLSTM_1')
State is tuple
states_fw_1, states_bw_1 = states
LSTM 2
with tf.name_scope("BiLSTM_2"):
with tf.variable_scope('forward'):
cell_fw = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size, state_is_tuple=True)
with tf.variable_scope('backward'):
cell_bw = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size, state_is_tuple=True)
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
dtype=tf.float64,
sequence_length=self.input_seq_len,
inputs=states_fw_1,
scope="BiLSTM_extraction")
I'm learning TF 2 days (so I'm not pro-guy) and I found this problem to be interested to resolve.
Here are my findings:
You want to do thing which is not possible to obtain using 'LSTMCell' implementation. Here is why:
You want to feed the "states_fw_1 to the next BI-LSTM. So, first question should be: What are dimensions of "states_fw_1"? For any RNN implementation you need [batch_size, seq_len, input_size]. For "states_fw_1" it is [batch_size, hidden_size] (I have just check the size of "states_fw_1" running below code). So you can see that your output does not fit to RNN requirements. It is because model output just one the last state of LSTM cell, not all the history (see the documentation). And you are not interested in last state, because you want feed state[t-step] to the layer above.The 'state_fw_1' is useful when you want to classify the sequence (not each element in sequence)
Edit: 'state_fw_1' contain the last "hidden_state" and last "memory_cell". For classification only "hidden_state" will be usefull, I think.
So you just need to use the merged output (from forward and backward pass) . And 'LSTMCell' cell output have size [batch_size, seq_len, hidden_size*2], (*2 as forward and backward) so it is right for next stacked RNN (output come from each time-step, not like the state).
Here is the code which I was testing:
import tensorflow as tf
import numpy as np
hidden_size = 21
seq_len = tf.placeholder(tf.int32, [None])
inputs = tf.placeholder(tf.float32, [None, None, 32])
with tf.variable_scope('BiLSTM_1'):
with tf.variable_scope('forward_1'):
cell_fw_1 = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size, state_is_tuple=True)
with tf.variable_scope('backward_srl'):
cell_bw_1 = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size, state_is_tuple=True)
outputs_1, states_1 = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw_1,
cell_bw=cell_bw_1,
dtype=tf.float32,
sequence_length=seq_len,
inputs=inputs,
scope='BiLSTM_1')
# Merge Output tensor from forward and backward pass. It size is [batch_size, seq_len, 2*hidden_size]
outputs_1 = tf.concat(outputs_1, 2)
with tf.name_scope("BiLSTM_2"):
with tf.variable_scope('forward'):
cell_fw = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size, state_is_tuple=True)
with tf.variable_scope('backward'):
cell_bw = tf.nn.rnn_cell.LSTMCell(num_units=hidden_size, state_is_tuple=True)
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
dtype=tf.float32,
sequence_length=seq_len,
inputs=outputs_1,
scope="BiLSTM_2")
# Initializate the weights and biases
init = tf.initialize_all_variables()
batch_size = 5
seq_len_val = 10
train_inputs = np.zeros((batch_size, seq_len_val, 32))
train_seq_len = np.ones(batch_size) * seq_len_val
with tf.Session() as session:
session.run(init)
feed = {inputs: train_inputs, seq_len: train_seq_len}
out,state,state_1 = session.run([outputs,states, states_1],feed)
print ("State size: ", state_1[0].c.shape, " Out Size: ", out[0][0].shape)
print ("Batch_size: ", batch_size, " Sequence Len: ", seq_len_val, " Hidden Size: ", hidden_size)
'outputs_1' returned by LSTM 1 is a tuple containing 'outputs_fw' and 'outputs_bw'.
'outputs_fw' and 'outputs_bw' will be of dimension: [batch_size, sequence_length, hidden_size].
You have to concatenate 'outputs_fw' and 'outputs_bw' hidden states (us tf.concat with axis=2) and pass that as input to LSTM 2 instead of passing 'states_fw_1' as input to LSTM 2.