I'm trying to construct a simple one-hot converter. It takes a batch of data vectors as input, and for each data vector, converts it to a one-hot vector. The one-hots have 1s at the original data vectors' argmaxes. (e.g. [[2.3, -4.1, 0.4], [-0.1, -3.1, 2.1]] -> [[1.0, 0.0, 0.0], [0.0, 0.0, 1.0]])
I'm doing this with tf.sparse_to_dense().
import random
import tensorflow as tf
batch_size = 10
data_size = 3
data = []
for i in range(batch_size):
data.append([])
for j in range(data_size):
data[i].append(random.random())
with tf.Graph().as_default(), tf.Session() as sess:
indices = tf.reshape(tf.range(0, limit=batch_size, delta=1), [1, -1])
hot_ids = tf.reshape(tf.cast(tf.argmax(data, 1), tf.int32), [1, -1])
sparse_indices = tf.concat(0, [indices, hot_ids])
output_shape = tf.pack([batch_size, data_size])
result = tf.sparse_to_dense(sparse_indices, output_shape, 1.0, 0.0)
tf.initialize_all_variables().run()
print(data)
print(sparse_indices.eval(session=sess))
print(output_shape.eval(session=sess))
print(result.eval(session=sess))
The first three printouts happen correctly. The last printout triggers this error:
W tensorflow/core/common_runtime/executor.cc:1102] 0x7fb0e5903560 Compute status: Invalid argument: output_shape has incorrect number of elements: 2 should be: 10
[[Node: SparseToDense = SparseToDense[T=DT_FLOAT, Tindices=DT_INT32, validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](concat, pack, SparseToDense/sparse_values, SparseToDense/default_value)]]
Traceback (most recent call last):
File "one-hot_simple", line 21, in <module>
print(result.eval(session=sess))
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 465, in eval
return _eval_using_default_session(self, feed_dict, self.graph, session)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 3097, in _eval_using_default_session
return session.run(tensors, feed_dict)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 315, in run
return self._run(None, fetches, feed_dict)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 511, in _run
feed_dict_string)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 564, in _do_run
target_list)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 586, in _do_call
e.code)
tensorflow.python.framework.errors.InvalidArgumentError: output_shape has incorrect number of elements: 2 should be: 10
[[Node: SparseToDense = SparseToDense[T=DT_FLOAT, Tindices=DT_INT32, validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](concat, pack, SparseToDense/sparse_values, SparseToDense/default_value)]]
Caused by op u'SparseToDense', defined at:
File "one-hot_simple", line 16, in <module>
result = tf.sparse_to_dense(sparse_indices, output_shape, 1.0, 0.0)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/sparse_ops.py", line 358, in sparse_to_dense
name=name)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/gen_sparse_ops.py", line 322, in _sparse_to_dense
validate_indices=validate_indices, name=name)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/ops/op_def_library.py", line 655, in apply_op
op_def=op_def)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2040, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1087, in __init__
self._traceback = _extract_stack()
I don't understand why output_shape should have 10 elements or why this error is happening... Please help!
The issue seems to arise from the fact that your sparse_indices matrix is a 2 x 10 matrix, whereas it expects a num_elems x num_dims (i.e. 10 x 2) matrix. You should change the code that computes this matrix as follows:
indices = tf.reshape(tf.range(0, limit=batch_size, delta=1), [-1, 1])
hot_ids = tf.reshape(tf.cast(tf.argmax(data, 1), tf.int32), [-1, 1])
sparse_indices = tf.concat(1, [indices, hot_ids])
You might also find the recently added tf.one_hot() op useful.
Related
I'm trying to implement the following custom loss function from this SO post; however, I've had to make some minor changes to suit my model. For some context, I'm using multi labels with 5 classes (below is an example of how they're encoded).
0 => [1, 0, 0, 0, 0]
1 => [1, 1, 0, 0, 0]
2 => [1, 1, 1, 0, 0]
3 => [1, 1, 1, 1, 0]
4 => [1, 1, 1, 1, 1]
My custom loss function
def _cohen_kappa(y_true, y_pred, num_classes=5, weights=None, metrics_collections=None, updates_collections=None, name=None):
kappa, update_op = tf.contrib.metrics.cohen_kappa(y_true, y_pred, num_classes, weights, metrics_collections, updates_collections, name)
kappa = K.cast(kappa, 'float32')
K.get_session().run(tf.local_variables_initializer())
with tf.control_dependencies([update_op]):
kappa = tf.identity(kappa)
return kappa
def cohen_kappa_loss(num_classes=5, weights=None, metrics_collections=None, updates_collections=None, name=None):
def cohen_kappa(y_true, y_pred):
y_true = K.cast(y_true, 'int32')
y_pred = K.cast(y_pred + 0.5, 'int32')
y_true = tf.subtract(K.sum(y_true, axis=1), tf.constant(1))
y_pred = tf.subtract(K.sum(y_pred, axis=1), tf.constant(1))
return -_cohen_kappa(y_true, y_pred, num_classes, weights, metrics_collections, updates_collections, name)
return cohen_kappa
This is how I'm attempting to use my loss function:
model_cohen_kappa = cohen_kappa_loss(num_classes=5)
model.compile(loss=model_cohen_kappa,
optimizer=optimizers.SGD(lr=0.0001, momentum=0.9),
metrics=['accuracy'])
Unfortunately, I get the following error, which is confusing since my loss function doesn't contain K.argmax, K.round, K.eval., which are mentioned in the error message as operations that are non-differentiable. Is there another non-differentiable operation in my custom loss function that I'm not noticing that is giving me this error?
Traceback (most recent call last):
File "small_test.py", line 106, in <module>
main()
File "small_test.py", line 101, in main
max_queue_size=2
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\keras\legacy\interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\keras\engine\training.py", line 1418, in fit_generator
initial_epoch=initial_epoch)
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\keras\engine\training_generator.py", line 40, in fit_generator
model._make_train_function()
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\keras\engine\training.py", line 509, in _make_train_function
loss=self.total_loss)
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\keras\legacy\interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\keras\optimizers.py", line 184, in get_updates
grads = self.get_gradients(loss, params)
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\keras\optimizers.py", line 91, in get_gradients
raise ValueError('An operation has `None` for gradient. '
ValueError: An operation has `None` for gradient. Please make sure that all of your ops have a gradient defined (i.e. are differentiable). Common ops without gradient: K.argmax, K.round, K.eval.
While I suspect K.cast is non-differentiable, removing the below snippet from my loss function results in the following error:
kappa = K.cast(kappa, 'float32')
Error
Traceback (most recent call last):
File "small_test.py", line 106, in <module>
main()
File "small_test.py", line 91, in main
metrics=['accuracy'])
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\keras\engine\training.py", line 342, in compile
sample_weight, mask)
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\keras\engine\training_utils.py", line 421, in weighted
score_array *= weights
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\tensorflow\python\ops\math_ops.py", line 884, in binary_op_wrapper
return func(x, y, name=name)
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\tensorflow\python\ops\math_ops.py", line 1180, in _mul_dispatch
return gen_math_ops.mul(x, y, name=name)
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 6879, in mul
"Mul", x=x, y=y, name=name)
File "C:\Users\Anaconda3\envs\tensor\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 563, in _apply_op_helper
inferred_from[input_arg.type_attr]))
TypeError: Input 'y' of 'Mul' Op has type float32 that does not match type float64 of argument 'x'.
Tensorflow 1.7 when using dynamic_rnn.It runs fine at first , but at the 32th(it changes when i run the code) step , the error appears. When i used smaller batch , it seems the code can run longer , however the error still poped up .Just cannt figure out what's wrong.
from mapping import *
def my_input_fn(features, targets, batch_size=20, shuffle=True, num_epochs=None, sequece_lenth=None):
ds = tf.data.Dataset.from_tensor_slices(
(features, targets, sequece_lenth)) # warning: 2GB limit
ds = ds.batch(batch_size).repeat(num_epochs)
if shuffle:
ds = ds.shuffle(10000)
features, labels, sequence = ds.make_one_shot_iterator().get_next()
return features, labels, sequence
def lstm_cell(lstm_size=50):
return tf.contrib.rnn.BasicLSTMCell(lstm_size)
class RnnModel:
def __init__(self,
batch_size,
hidden_units,
time_steps,
num_features
):
self.batch_size = batch_size
self.hidden_units = hidden_units
stacked_lstm = tf.contrib.rnn.MultiRNNCell(
[lstm_cell(i) for i in self.hidden_units])
self.initial_state = stacked_lstm.zero_state(batch_size, tf.float32)
self.model = stacked_lstm
self.state = self.initial_state
self.time_steps = time_steps
self.num_features = num_features
def loss_mean_squre(self, outputs, targets):
pos = tf.add(outputs, tf.ones(self.batch_size))
eve = tf.div(pos, 2)
error = tf.subtract(eve,
targets)
return tf.reduce_mean(tf.square(error))
def train(self,
num_steps,
learningRate,
input_fn,
inputs,
targets,
sequenceLenth):
periods = 10
step_per_periods = int(num_steps / periods)
input, target, sequence = input_fn(inputs, targets, self.batch_size, shuffle=True, sequece_lenth=sequenceLenth)
initial_state = self.model.zero_state(self.batch_size, tf.float32)
outputs, state = tf.nn.dynamic_rnn(self.model, input, initial_state=initial_state)
loss = self.loss_mean_squre(tf.reshape(outputs, [self.time_steps, self.batch_size])[-1], target)
optimizer = tf.train.AdamOptimizer(learning_rate=learningRate)
grads_and_vars = optimizer.compute_gradients(loss, self.model.variables)
optimizer.apply_gradients(grads_and_vars)
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
for i in range(num_steps):
sess.run(init_op)
state2, current_loss= sess.run([state, loss])
if i % step_per_periods == 0:
print("period " + str(int(i / step_per_periods)) + ":" + str(current_loss))
return self.model, self.state
def processFeature(df):
df = df.drop('class', 1)
features = []
for i in range(len(df["vecs"])):
features.append(df["vecs"][i])
aa = pd.Series(features).tolist() # tramsform into list
featuresList = []
for i in features:
p1 = []
for k in i:
p1.append(list(k))
featuresList.append(p1)
return featuresList
def processTargets(df):
selected_features = df[
"class"]
processed_features = selected_features.copy()
return tf.convert_to_tensor(processed_features.astype(float).tolist())
if __name__ == '__main__':
dividNumber = 30
"""
some code here to modify my data to input
it looks like this:
inputs before use input function : [fullLenth, charactorLenth, embeddinglenth]
"""
model = RnnModel(15, [100, 80, 80, 1], time_steps=dividNumber, num_features=25)
model.train(5000, 0.0001, my_input_fn, training_examples, training_targets, sequenceLenth=trainSequenceL)
And error is under here
Traceback (most recent call last):
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\client\session.py", line 1330, in _do_call
return fn(*args)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\client\session.py", line 1315, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\client\session.py", line 1423, in _call_tf_sessionrun
status, run_metadata)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\framework\errors_impl.py", line 516, in __exit__
c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.InvalidArgumentError: ConcatOp : Dimensions of inputs should match: shape[0] = [20,25] vs. shape[1] = [30,100]
[[Node: rnn/while/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](rnn/while/TensorArrayReadV3, rnn/while/Switch_4:1, rnn/while/rnn/multi_rnn_cell/cell_3/basic_lstm_cell/Const)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:/programming/mlwords/dnn_gragh.py", line 198, in <module>
model.train(5000, 0.0001, my_input_fn, training_examples, training_targets, sequenceLenth=trainSequenceL)
File "D:/programming/mlwords/dnn_gragh.py", line 124, in train
state2, current_loss, nowAccuracy = sess.run([state, loss, accuracy])
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\client\session.py", line 908, in run
run_metadata_ptr)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\client\session.py", line 1143, in _run
feed_dict_tensor, options, run_metadata)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\client\session.py", line 1324, in _do_run
run_metadata)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\client\session.py", line 1343, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: ConcatOp : Dimensions of inputs should match: shape[0] = [20,25] vs. shape[1] = [30,100]
[[Node: rnn/while/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](rnn/while/TensorArrayReadV3, rnn/while/Switch_4:1, rnn/while/rnn/multi_rnn_cell/cell_3/basic_lstm_cell/Const)]]
Caused by op 'rnn/while/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/concat', defined at:
File "D:/programming/mlwords/dnn_gragh.py", line 198, in <module>
model.train(5000, 0.0001, my_input_fn, training_examples, training_targets, sequenceLenth=trainSequenceL)
File "D:/programming/mlwords/dnn_gragh.py", line 95, in train
outputs, state = tf.nn.dynamic_rnn(self.model, input, initial_state=initial_state)#,sequence_length=sequence
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\rnn.py", line 627, in dynamic_rnn
dtype=dtype)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\rnn.py", line 824, in _dynamic_rnn_loop
swap_memory=swap_memory)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 3205, in while_loop
result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2943, in BuildLoop
pred, body, original_loop_vars, loop_vars, shape_invariants)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 2880, in _BuildLoop
body_result = body(*packed_vars_for_body)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\control_flow_ops.py", line 3181, in <lambda>
body = lambda i, lv: (i + 1, orig_body(*lv))
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\rnn.py", line 795, in _time_step
(output, new_state) = call_cell()
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\rnn.py", line 781, in <lambda>
call_cell = lambda: cell(input_t, state)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 232, in __call__
return super(RNNCell, self).__call__(inputs, state)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\layers\base.py", line 714, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 1283, in call
cur_inp, new_state = cell(cur_inp, cur_state)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 339, in __call__
*args, **kwargs)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\layers\base.py", line 714, in __call__
outputs = self.call(inputs, *args, **kwargs)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\rnn_cell_impl.py", line 620, in call
array_ops.concat([inputs, h], 1), self._kernel)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\array_ops.py", line 1181, in concat
return gen_array_ops.concat_v2(values=values, axis=axis, name=name)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\ops\gen_array_ops.py", line 1101, in concat_v2
"ConcatV2", values=values, axis=axis, name=name)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\framework\ops.py", line 3309, in create_op
op_def=op_def)
File "D:\Anaconda3\envs\tensorflow-cpu\lib\site-packages\tensorflow\python\framework\ops.py", line 1669, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): ConcatOp : Dimensions of inputs should match: shape[0] = [20,25] vs. shape[1] = [30,100]
[[Node: rnn/while/rnn/multi_rnn_cell/cell_0/basic_lstm_cell/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](rnn/while/TensorArrayReadV3, rnn/while/Switch_4:1, rnn/while/rnn/multi_rnn_cell/cell_3/basic_lstm_cell/Const)]]
this is my code used to check my input
def checkData(inputs, targets, sequencelence):
batch_size = 20
features, target, sequece = my_input_fn(inputs, targets, batch_size=batch_size, shuffle=True, num_epochs=None,
sequece_lenth=sequencelence)
with tf.Session() as sess:
for i in range(1000):
features1, target1, sequece1 = sess.run([features, target, sequece])
assert len(features1) == batch_size
for sentence in features1 :
assert len(sentence) == 30
for word in sentence:
assert len(word) == 25
assert len(target1) == batch_size
assert len(sequece1) == batch_size
print(target1)
print("OK")
The error is coming from LSTMCell.call call method. There we are trying to tf.concat([inputs, h], 1) meaning that we want to concatenate the next input with the current hidden state before matmul'ing with the kernel variables matrix. The error is saying that you can't do it because the batch (0th) dimensions don't match up - your input is shaped [20,25] and your hidden state is shaped [30,100].
For some reason on your 32nd iteration, or whenever you see the error, the input is not batched to 30, but only to 20. This usually happens at the end of your training data when the total number of training examples does not evenly divide your batch size. This hypothesis is also consistent with "When i used smaller batch , it seems the code can run longer" statement.
I had the same issue. When I corrected the image input size to match the input shape, it ran without errors.
Here is the traceback:
Traceback (most recent call last):
File "test.py", line 39, in <module>
hess = tf.hessians(loss, wrt_variables)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/gradients_impl.py", line 970, in hessians
_gradients = array_ops.unstack(_gradients)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/ops/array_ops.py", line 952, in unstack
value = ops.convert_to_tensor(value)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/ops.py", line 639, in convert_to_tensor
as_ref=False)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/ops.py", line 704, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/constant_op.py", line 113, in _constant_tensor_conversion_function
return constant(v, dtype=dtype, name=name)
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/constant_op.py", line 102, in constant
tensor_util.make_tensor_proto(value, dtype=dtype, shape=shape, verify_shape=verify_shape))
File "/usr/local/lib/python3.4/dist-packages/tensorflow/python/framework/tensor_util.py", line 360, in make_tensor_proto
raise ValueError("None values not supported.")
ValueError: None values not supported.
Variables:
import tensorflow as tf
data_x = [0., 1., 2.]
data_y = [-1., 1., 3.]
batch_size = len(data_x)
x = tf.placeholder(shape=[batch_size], dtype=tf.float32, name="x")
y = tf.placeholder(shape=[batch_size], dtype=tf.float32, name="y")
W = tf.Variable(tf.ones(shape=[1]), dtype=tf.float32, name="W")
b = tf.Variable(tf.zeros(shape=[1]), dtype=tf.float32, name="b")
pred = x * W + b
loss = tf.reduce_mean(0.5 * (y - pred)**2)
Then, following up with this code would work:
wrt_variables = [W, b]
hess = tf.hessians(loss, wrt_variables)
But this fails:
wrt_variables = tf.concat([W, b], axis=0)
hess = tf.hessians(loss, wrt_variables)
And this would also fail:
wrt_variables = [tf.concat([W, b], axis=0)]
hess = tf.hessians(loss, wrt_variables)
It also fails for reshape operations.
The complete version of this code and with comments can be seen here:
https://gist.github.com/guillaume-chevalier/6b01c4e43a123abf8db69fa97532993f
Thanks!
This is because in your graph, the node loss does not depend on the node tf.concat([W,b], axis=0). There is no backpropagation of one onto the other, and therefore no derivative.
Tensorflow is not a formal calculus engine, it can only estimate derivatives of a node by another node if the former is downstream of the later. So for example even
tf.hessian(loss, 2*W)
will fail for the same reasons (2*W is a new node and loss does not depend on it) even though the relationship to tf.hessian(loss, W) is straightfoward.
Note that the sitatuation is the same with tf.gradients, even though it fails differently: it returns Nones rather than throwing an exception.
My TensorFlow version is 1.0.
When I run the following code:
train_file='~/tf_code/train'
filename_queue = tf.train.string_input_producer([train_file],num_epochs=None)
reader = tf.TFRecordReader()
_, ex = reader.read(filename_queue)
sequence_features = {
"x":tf.FixedLenSequenceFeature([], dtype = tf.int64),
"tomatch_indices_1D":tf.FixedLenSequenceFeature([], dtype = tf.int64)
}
context_parsed, sequence_parsed = tf.parse_single_sequence_example(
serialized=ex,
context_features={},
sequence_features=sequence_features
)
indices = tf.cast(sequence_parsed['tomatch_indices_1D'],tf.int64)
indices = tf.reshape(indices, (-1,3))
x = sequence_parsed['x']
lens = tf.shape(x)[0]
tomatch_sparse = tf.SparseTensor(indices, tf.ones((tf.shape(indices)[0],)),
dense_shape=(lens,lens,lens))
tomatch = tf.sparse_tensor_to_dense(tomatch_sparse, validate_indices=False)
print(tomatch)
Then I got this error on tf.SparseTensor():
Traceback (most recent call last):
File "/Users/qingping/tf_code/SequenceExample/example_test.py", line 284, in <module>
stack_test()
File "/Users/qingping/tf_code/SequenceExample/example_test.py", line 276, in stack_test
tomatch_sparse = tf.SparseTensor(indices, tf.ones((tf.shape(indices)[0],)), dense_shape=(lens,lens,lens))
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/sparse_tensor.py", line 127, in __init__
dense_shape, name="dense_shape", dtype=dtypes.int64)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 637, in convert_to_tensor
as_ref=False)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 702, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/constant_op.py", line 110, in _constant_tensor_conversion_function
return constant(v, dtype=dtype, name=name)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/constant_op.py", line 99, in constant
tensor_util.make_tensor_proto(value, dtype=dtype, shape=shape, verify_shape=verify_shape))
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/tensor_util.py", line 367, in make_tensor_proto
_AssertCompatible(values, dtype)
File "/usr/local/lib/python2.7/site-packages/tensorflow/python/framework/tensor_util.py", line 302, in _AssertCompatible
(dtype.name, repr(mismatch), type(mismatch).__name__))
TypeError: Expected int64, got list containing Tensors of type '_Message' instead.
If I want to build SparseTensor by read data(indices) from file, and the dense_shape of SparseTensor is varied, what should I do? Thanks!
I think this bad error message arises when TensorFlow tries to convert the tuple of tf.int32 tensors (lens, lens, lens) to a single tf.int64 tensor as the dense_shape argument of the tf.SparseTensor.
The default return value of tf.shape() is tf.int32. You can solve this problem by adding an explicit out_type argument when you compute lens, as follows:
lens = tf.shape(x, out_type=tf.int64)[0]
I'm following these tutorials:
https://www.youtube.com/watch?v=wuo4JdG3SvU&list=PL9Hr9sNUjfsmEu1ZniY0XpHSzl5uihcXZ
and prettytensor is introduced in tutorial 4.
Following the tutorial, i wrote this code to run a small neural network:
import tensorflow as tf
# Use PrettyTensor to simplify Neural Network construction.
import prettytensor as pt
from tensorflow.examples.tutorials.mnist import input_data
data = input_data.read_data_sets('../data/MNIST/', one_hot=True)
# We know that MNIST images are 28 pixels in each dimension.
img_size = 28
# Images are stored in one-dimensional arrays of this length.
img_size_flat = img_size * img_size
# Tuple with height and width of images used to reshape arrays.
img_shape = (img_size, img_size)
# Number of colour channels for the images: 1 channel for gray-scale.
num_channels = 1
# Number of classes, one class for each of 10 digits.
num_classes = 10
# the placeholders
x = tf.placeholder(tf.float32, shape=[None, img_size_flat], name='x')
x_image = tf.reshape(x, [-1, img_size, img_size, num_channels])
y_true = tf.placeholder(tf.float32, shape=[None, 10], name='y_true')
# use prettyTensor to build the model
# this will give us the predictions and the loss functions
x_pretty = pt.wrap(x_image)
with pt.defaults_scope(activation_fn=tf.nn.relu):
y_pred, loss = x_pretty.\
conv2d(kernel=5, depth=16, name='layer_conv1').\
max_pool(kernel=2, stride=2).\
conv2d(kernel=5, depth=36, name='layer_conv2').\
max_pool(kernel=2, stride=2).\
flatten().\
fully_connected(size=128, name='layer_fc1').\
softmax_classifier(class_count=10, labels=y_true)
# the model optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=1e-4).minimize(loss)
# the model testing
correct_prediction = tf.equal(tf.argmax(y_pred,1), tf.argmax(y_true,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# start the session
session = tf.InteractiveSession()
# Start the training
tf.global_variables_initializer().run(session = session)
train_batch_size = 64
for i in range(1000):
print("training batch ",i)
x_batch, y_true_batch = data.train.next_batch(train_batch_size)
session.run(optimizer, feed_dict={x:x_batch, y_true:y_true_batch})
When i tried to run it, I got the following error:
tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value layer_conv1/bias
[[Node: layer_conv1/bias/read = Identity[T=DT_FLOAT, _class=["loc:#layer_conv1/bias"], _device="/job:localhost/replica:0/task:0/cpu:0"](layer_conv1/bias)]]
Caused by op u'layer_conv1/bias/read', defined at:
File "/home/gal/Documents/Workspace/EclipseWorkspace/Melanoma Classification!/tutorial4/tutorial4Test.py", line 31, in <module>
the full error trace:
Traceback (most recent call last):
File "/home/gal/Documents/Workspace/EclipseWorkspace/Melanoma Classification!/tutorial4/tutorial4Test.py", line 55, in <module>
session.run(optimizer, feed_dict={x:x_batch, y_true:y_true_batch})
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 766, in run
run_metadata_ptr)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 964, in _run
feed_dict_string, options, run_metadata)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1014, in _do_run
target_list, options, run_metadata)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1034, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.FailedPreconditionError: Attempting to use uninitialized value layer_conv1/bias
[[Node: layer_conv1/bias/read = Identity[T=DT_FLOAT, _class=["loc:#layer_conv1/bias"], _device="/job:localhost/replica:0/task:0/cpu:0"](layer_conv1/bias)]]
Caused by op u'layer_conv1/bias/read', defined at:
File "/home/gal/Documents/Workspace/EclipseWorkspace/Melanoma Classification!/tutorial4/tutorial4Test.py", line 31, in <module>
conv2d(kernel=5, depth=16, name='layer_conv1').\
File "/home/gal/anaconda2/lib/python2.7/site-packages/prettytensor/pretty_tensor_class.py", line 1981, in method
result = func(non_seq_layer, *args, **kwargs)
File "/home/gal/anaconda2/lib/python2.7/site-packages/prettytensor/pretty_tensor_image_methods.py", line 163, in __call__
y += self.variable('bias', [size[-1]], bias_init, dt=dtype)
File "/home/gal/anaconda2/lib/python2.7/site-packages/prettytensor/pretty_tensor_class.py", line 1695, in variable
collections=variable_collections)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 1024, in get_variable
custom_getter=custom_getter)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 850, in get_variable
custom_getter=custom_getter)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 346, in get_variable
validate_shape=validate_shape)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 331, in _true_getter
caching_device=caching_device, validate_shape=validate_shape)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.py", line 677, in _get_single_variable
expected_shape=shape)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 224, in __init__
expected_shape=expected_shape)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variables.py", line 370, in _init_from_args
self._snapshot = array_ops.identity(self._variable, name="read")
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 1424, in identity
result = _op_def_lib.apply_op("Identity", input=input, name=name)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 759, in apply_op
op_def=op_def)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2240, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/gal/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1128, in __init__
self._traceback = _extract_stack()
FailedPreconditionError (see above for traceback): Attempting to use uninitialized value layer_conv1/bias
[[Node: layer_conv1/bias/read = Identity[T=DT_FLOAT, _class=["loc:#layer_conv1/bias"], _device="/job:localhost/replica:0/task:0/cpu:0"](layer_conv1/bias)]]
So my question is, How can i solve this error?
This problem is caused by a bug in the 0.12rc0 release candidate of TensorFlow, and the fact that Pretty Tensor uses a deprecated TensorFlow API (for which I've opened an issue).
Until this bug is fixed, the best workaround I can think of is a hack. Add the following line at the top of your program, after import tensorflow as tf:
tf.GraphKeys.VARIABLES = tf.GraphKeys.GLOBAL_VARIABLES