Tensorflo- InvalidArgumentError (see above for traceback): Input must be at least rank 1, got 0 - tensorflow

I try to run the blow code on the tensorflow :
import tensorflow as tf
import numpy as np
def nuclear_norm(inputs,lamda,beta):
sigma,U,V=tf.svd(inputs)
rank=tf.count_nonzero(sigma)
svp=tf.reduce_sum(tf.cast(sigma>2*(lamda/beta), tf.int32))
def case1(sigma,lamda=1e-2):
svp1=svp
sigma1=sigma[0:svp]-2*tf.to_double(tf.constant(lamda))
return svp1,sigma1
def case2(sigma,lamda=1e-2):
svp1=tf.constant(1)
sigma1=tf.to_double(tf.constant(0))
return svp1,sigma1
svp,sigma=tf.cond(svp<tf.constant(1),lambda:case2(sigma,lamda),lambda:case1(sigma,lamda))
return rank,tf.matmul(tf.matmul(U[:,0:svp],tf.diag(sigma)),tf.transpose(V[:,0:svp]))
x=np.random.randn(2,3)
xs=tf.placeholder(tf.float64,[2,3])
lamda=1
beta=1
init = tf.global_variables_initializer()
sess = tf.Session()
a,b=nuclear_norm(xs,lamda,beta)
b=sess.run(b,feed_dict={xs:x})
when i run the code,the IDLE display:InvalidArgumentError: Input must be at least rank 1, got 0. The detail information as blow:
Traceback (most recent call last):
File "C:\Users\yql\Desktop\3.py", line 31, in <module>
sess.run(b,feed_dict={xs:x})
File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\client\session.py", line 905, in run
run_metadata_ptr)
File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1140, in _run
feed_dict_tensor, options, run_metadata)
File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1321, in _do_run
run_metadata)
File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1340, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Input must be at least rank 1, got 0
[[Node: Diag = Diag[T=DT_DOUBLE, _device="/job:localhost/replica:0/task:0/device:CPU:0"](cond/Merge_1)]]
Caused by op 'Diag', defined at:
File "<string>", line 1, in <module>
File "C:\Program Files\Python36\lib\idlelib\run.py", line 144, in main
ret = method(*args, **kwargs)
File "C:\Program Files\Python36\lib\idlelib\run.py", line 474, in runcode
exec(code, self.locals)
File "C:\Users\yql\Desktop\3.py", line 29, in <module>
a,b=nuclear_norm(xs,lamda,beta)
File "C:\Users\yql\Desktop\3.py", line 20, in nuclear_norm
return rank,tf.matmul(tf.matmul(U[:,0:svp],tf.diag(sigma)),tf.transpose(V[:,0:svp]))
File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\ops\gen_array_ops.py", line 1882, in diag
"Diag", diagonal=diagonal, name=name)
File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 3290, in create_op
op_def=op_def)
File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 1654, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InvalidArgumentError (see above for traceback): Input must be at least rank 1, got 0
[[Node: Diag = Diag[T=DT_DOUBLE, _device="/job:localhost/replica:0/task:0/device:CPU:0"](cond/Merge_1)]]
Sometimes it can be run,and sometimes not.what's wrong? and what should i fix the error?

The problem is with
tf.diag(sigma)
in the line segment.
return rank, tf.matmul(tf.matmul(U[:, 0:svp], tf.diag(sigma)), tf.transpose(V[:, 0:svp]))
You are passing sigma, which has a rank of 0 to the method tf.dig.
The error is easy to understand,
When you pass a set of diagonal elements to the tf.diag method, it returns a diagonal tensor with the given diagonal values.
For example:
# 'diagonal' is [1, 2, 3, 4]
tf.diag(diagonal) ==> [[1, 0, 0, 0]
[0, 2, 0, 0]
[0, 0, 3, 0]
[0, 0, 0, 4]]
However, since you are passing only a single element, for example just 1, the tf.diag method cannot return the diagonal tensor.
Hope this helps.

Related

tensorflow v1 GradientTape: AttributeError: 'NoneType' object has no attribute 'eval'

I want to compute the gradient of the distance between the NSynth WaveNet encoding of two sine waves.
This is tensorflow v1.
I am working with code based upon https://github.com/magenta/magenta/blob/master/magenta/models/nsynth/wavenet/fastgen.py
A minimal example of my bug is in this colab notebook: https://colab.research.google.com/drive/1oTEU8QAaOs0K1A0KHrAdt7kA7MkadNDr?usp=sharing
Here is the code:
# Commented out IPython magic to ensure Python compatibility.
# %tensorflow_version 1.x
!pip3 install -q magenta
!wget -c http://download.magenta.tensorflow.org/models/nsynth/wavenet-ckpt.tar && tar xvf wavenet-ckpt.tar
checkpoint_path = './wavenet-ckpt/model.ckpt-200000'
import math
from magenta.models.nsynth.wavenet import fastgen
import tensorflow as tf
session_config = tf.ConfigProto(allow_soft_placement=True)
session_config.gpu_options.allow_growth = True
sess = tf.Session(config=session_config)
pi = 3.1415926535897
SR = 16000
sample_length = 64000
DURATION_SECONDS = sample_length / SR
def sine(hz):
time = tf.linspace(0.0, DURATION_SECONDS, sample_length)
return tf.constant(0.5) * tf.cos(2.0 * pi * time * hz)
net = fastgen.load_nsynth(batch_size=2, sample_length=sample_length)
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
"""We have two sine waves at 440 and 660 Hz. We use the encoder to generate two (125, 16) encodings:"""
twosines = tf.stack([sine(440), sine(660)]).eval(session=sess)
print(sess.run(net["encoding"], feed_dict={net["X"]: twosines}).shape)
"""Compute the distance between the two sine waves"""
distencode = tf.reduce_mean(tf.abs(net["encoding"][0] - net["encoding"][1]))
print(sess.run(distencode, feed_dict={net["X"]: twosines}))
"""I don't know why the following code doesn't work, but if I did I could solve the real task....
"""
net["X"] = twosines
distencode.eval(session=sess)
"""Here is the code that I need to work. I want to compute the gradient of the distance between the NSynth encoding of two sine waves:"""
fp = tf.constant(660.0)
newsines = tf.stack([sine(440), sine(fp)])
with tf.GradientTape() as g:
g.watch(fp)
dd_dfp = g.gradient(distencode, fp)
print(dd_dfp.eval(session=sess))
The last block, which I want to evaluate, gets the following error:
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-12-b5b8cdd00b24> in <module>()
4 g.watch(fp)
5 dd_dfp = g.gradient(distencode, fp)
----> 6 print(dd_dfp.eval(session=sess))
AttributeError: 'NoneType' object has no attribute 'eval'
I believe I need to define the operations to be executed within this block. However, I am using a pretrained model that I am just computing the distance over, so I am not sure how to define execution in that block.
The second-to-last block, which would help me fix the last block, gives the following error:
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-10-c3411dcbfa2c> in <module>()
3 with tf.GradientTape() as g:
4 g.watch(fp)
----> 5 dd_dfp = g.gradient(distencode, g)
6 print(dd_dfp.eval(session=sess))
/tensorflow-1.15.2/python3.6/tensorflow_core/python/eager/backprop.py in gradient(self, target, sources, output_gradients, unconnected_gradients)
997 flat_sources = [_handle_or_self(x) for x in flat_sources]
998 for t in flat_sources_raw:
--> 999 if not t.dtype.is_floating:
1000 logging.vlog(
1001 logging.WARN, "The dtype of the source tensor must be "
AttributeError: 'GradientTape' object has no attribute 'dtype'
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py in _do_call(self, fn, *args)
1364 try:
-> 1365 return fn(*args)
1366 except errors.OpError as e:
8 frames
InvalidArgumentError: 2 root error(s) found.
(0) Invalid argument: You must feed a value for placeholder tensor 'Placeholder' with dtype float and shape [2,64000]
[[{{node Placeholder}}]]
[[Mean/_759]]
(1) Invalid argument: You must feed a value for placeholder tensor 'Placeholder' with dtype float and shape [2,64000]
[[{{node Placeholder}}]]
0 successful operations.
0 derived errors ignored.
During handling of the above exception, another exception occurred:
InvalidArgumentError Traceback (most recent call last)
/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py in _do_call(self, fn, *args)
1382 '\nsession_config.graph_options.rewrite_options.'
1383 'disable_meta_optimizer = True')
-> 1384 raise type(e)(node_def, op, message)
1385
1386 def _extend_graph(self):
InvalidArgumentError: 2 root error(s) found.
(0) Invalid argument: You must feed a value for placeholder tensor 'Placeholder' with dtype float and shape [2,64000]
[[node Placeholder (defined at /tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py:1748) ]]
[[Mean/_759]]
(1) Invalid argument: You must feed a value for placeholder tensor 'Placeholder' with dtype float and shape [2,64000]
[[node Placeholder (defined at /tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py:1748) ]]
0 successful operations.
0 derived errors ignored.
Original stack trace for 'Placeholder':
File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/usr/local/lib/python3.6/dist-packages/traitlets/config/application.py", line 664, in launch_instance
app.start()
File "/usr/local/lib/python3.6/dist-packages/ipykernel/kernelapp.py", line 499, in start
self.io_loop.start()
File "/usr/local/lib/python3.6/dist-packages/tornado/platform/asyncio.py", line 132, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.6/asyncio/base_events.py", line 438, in run_forever
self._run_once()
File "/usr/lib/python3.6/asyncio/base_events.py", line 1451, in _run_once
handle._run()
File "/usr/lib/python3.6/asyncio/events.py", line 145, in _run
self._callback(*self._args)
File "/usr/local/lib/python3.6/dist-packages/tornado/ioloop.py", line 758, in _run_callback
ret = callback()
File "/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py", line 300, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py", line 548, in <lambda>
self.io_loop.add_callback(lambda : self._handle_events(self.socket, 0))
File "/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py", line 462, in _handle_events
self._handle_recv()
File "/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py", line 492, in _handle_recv
self._run_callback(callback, msg)
File "/usr/local/lib/python3.6/dist-packages/zmq/eventloop/zmqstream.py", line 444, in _run_callback
callback(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/tornado/stack_context.py", line 300, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "/usr/local/lib/python3.6/dist-packages/ipykernel/kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "/usr/local/lib/python3.6/dist-packages/ipykernel/ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.6/dist-packages/ipykernel/zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py", line 2718, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py", line 2822, in run_ast_nodes
if self.run_code(code, result):
File "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py", line 2882, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-5-5120c8282e75>", line 1, in <module>
net = fastgen.load_nsynth(batch_size=2, sample_length=sample_length)
File "/tensorflow-1.15.2/python3.6/magenta/models/nsynth/wavenet/fastgen.py", line 64, in load_nsynth
x = tf.placeholder(tf.float32, shape=[batch_size, sample_length])
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/array_ops.py", line 2619, in placeholder
return gen_array_ops.placeholder(dtype=dtype, shape=shape, name=name)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/gen_array_ops.py", line 6669, in placeholder
"Placeholder", dtype=dtype, shape=shape, name=name)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/op_def_library.py", line 794, in _apply_op_helper
op_def=op_def)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py", line 3357, in create_op
attrs, op_def, compute_device)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py", line 3426, in _create_op_internal
op_def=op_def)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py", line 1748, in __init__
self._traceback = tf_stack.extract_stack()
Thank you.

tf.clip_by_norm(grad, 1.0) throws InvalidArgumentError Shapes must be equal rank, but are 2 and 1

can someone explain why tensorflow is giving me trouble when running the following code.
import tensorflow as tf
x = tf.keras.layers.Input(shape=(1,))
y = tf.keras.layers.Dense(1, activation=tf.nn.relu)(x)
loss = tf.losses.mean_squared_error(x,y)
grad = tf.gradients(loss, tf.trainable_variables())
# !!! GIVES ME TROUBLE !!!
clipped_grad = tf.clip_by_norm(grad, 1.0)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(y, feed_dict={x: [[1.0], [2.0], [3.0]]})
The error I get:
Traceback (most recent call last):
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\framework\ops.py", line 1589, in _create_c_op
c_op = c_api.TF_FinishOperation(op_desc)
tensorflow.python.framework.errors_impl.InvalidArgumentError: Shapes must be equal rank, but are 2 and 1
From merging shape 0 with other shapes. for 'clip_by_norm/t' (op: 'Pack') with input shapes: [1,1], [1].
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/TObs/.PyCharmCE2018.1/config/scratches/scratch.py", line 11, in <module>
clipped_grad = tf.clip_by_norm(grad, 1.0)
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\ops\clip_ops.py", line 140, in clip_by_norm
t = ops.convert_to_tensor(t, name="t")
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\framework\ops.py", line 1011, in convert_to_tensor
as_ref=False)
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\framework\ops.py", line 1107, in internal_convert_to_tensor
ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\ops\array_ops.py", line 960, in _autopacking_conversion_function
return _autopacking_helper(v, inferred_dtype, name or "packed")
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\ops\array_ops.py", line 923, in _autopacking_helper
return gen_array_ops.pack(elems_as_tensors, name=scope)
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\ops\gen_array_ops.py", line 5532, in pack
"Pack", values=values, axis=axis, name=name)
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\framework\ops.py", line 3414, in create_op
op_def=op_def)
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\framework\ops.py", line 1756, in __init__
control_input_ops)
File "D:\Program Files\Python\Python_3_6_2\lib\site-packages\tensorflow\python\framework\ops.py", line 1592, in _create_c_op
raise ValueError(str(e))
ValueError: Shapes must be equal rank, but are 2 and 1
From merging shape 0 with other shapes. for 'clip_by_norm/t' (op: 'Pack') with input shapes: [1,1], [1].
Any thoughts? Im running on a Windows10 machine, with tensorflow-gpu 1.9.0, NVidia GTX 1080.
Help would be much appreciated :)
Cheers,
Tobs.
so after tinkering around I found out that one have to apply the tf.clip_by_norm seperatly for each value in the gradients tensor liek so:
clipped_gradients = [tf.clip_by_norm(g, grad_norm_clip) for g in tf.gradients(loss, tf.trainable_variables())]
I guess, thats the right way to do, right?
Cheers,
Tobs.

no kernel image is available for execution on the device

I training maskrcnn ,use tf-1.2 can train, but I use tf-1.5 it not training
The error is as follows:
Caused by op u'pyramid_1/AssignGTBoxes/Where_6', defined at:
File "/home/zhouzd2/letrain/applications/letrain.py", line 349, in <module>
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 124, in run
_sys.exit(main(argv))
File "/home/zhouzd2/letrain/applications/letrain.py", line 346, in main
LeTrain().model_train(user_mode)
File "/home/zhouzd2/letrain/platform/base_train.py", line 1228, in model_train
cluster=self.cluster_spec)
File "/home/zhouzd2/letrain/platform/deployment/model_deploy.py", line 226, in create_clones
outputs, feed_ops,verify_model_loss = model_fn(*args, **kwargs)
File "/home/zhouzd2/letrain/platform/base_train.py", line 1195, in clone_fn
model_loss, end_points, feed_ops = network_fn(data_direct, data_batch, int_network_fn)
File "/home/zhouzd2/letrain/applications/letrain.py", line 214, in get_loss
FLAGS.batch_size)
File "/home/zhouzd2/letrain/applications/fmrcnn/get_fmrcnn_loss.py", line 23, in model_fn
loss_weights=[0.2, 0.2, 1.0, 0.2, 1.0])
File "/home/zhouzd2/letrain/applications/fmrcnn/libs/nets/pyramid_network.py", line 580, in build
is_training=is_training, gt_boxes=gt_boxes)
File "/home/zhouzd2/letrain/applications/fmrcnn/libs/nets/pyramid_network.py", line 263, in build_heads
assign_boxes(rois, [rois, batch_inds], [2, 3, 4, 5])
File "/home/zhouzd2/letrain/applications/fmrcnn/libs/layers/wrapper.py", line 173, in assign_boxes
inds = tf.where(tf.equal(assigned_layers, l))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_ops.py", line 2538, in where
return gen_array_ops.where(condition=condition, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.py", line 6087, in where
"Where", input=condition, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 3160, in create_op
op_def=op_def)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1625, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
InternalError (see above for traceback): WhereOp: Could not launch cub::DeviceReduce::Sum to count number of true / nonzero indices. temp_storage_bytes: 1, status: no kernel image is available for execution on the device
[[Node: pyramid_1/AssignGTBoxes/Where_6 = Where[T=DT_BOOL, _device="/job:worker/replica:0/task:0/device:GPU:0"](pyramid_1/AssignGTBoxes/Equal_6_S9493)]]
[[Node: pyramid_1/AssignGTBoxes/Reshape_8_G1028 = _Recv[client_terminated=false, recv_device="/job:worker/replica:0/task:0/device:CPU:0", send_device="/job:worker/replica:0/task:0/device:GPU:0", send_device_incarnation=5407481677180697062, tensor_name="edge_1349_pyramid_1/AssignGTBoxes/Reshape_8", tensor_type=DT_INT64, _device="/job:worker/replica:0/task:0/device:CPU:0"]()]]
No problem when loading calculation graphs, error is reported in sess.run()。
Does anyone know how to solve this problem? Or does anyone know what function can replace tf.where?
Thank you!
If you are using Visual Studio:
Right click on the project > Properies > Cuda C/C++ > Device
and add the following to Code Generation field
compute_30,sm_30;compute_35,sm_35;compute_37,sm_37;compute_50,sm_50;compute_52,sm_52;compute_60,sm_60;compute_61,sm_61;compute_70,sm_70;compute_75,sm_75;

InvalidArgumentError (see above for traceback): slice index 15 of dimension 0 out of bounds

In a task to implement the minimum risk training for a neural machine translation system I need to sample sentences and gather the respective logits for the sampled word IDs. The step of gathering looks like this:
for i in range(1,self._num_of_samples):
logits, _, _, sampled_ids = self.decoder._decoding_loop(train_mode=False,sample=True)
ind=[[[tf.constant(i),tf.constant(j),sampled_ids[i][j]] for j in range(self.batch_size)] for i in range(self.decoder.max_output_len)]
gathered_logits = tf.gather_nd(logits,ind)
sentence_sum_logit = tf.reduce_sum(gathered_logits,0)
self.sample_sen_ids = self.sample_sen_ids.write(steps[i],sampled_ids)
self.sample_logits = self.sample_logits.write(steps[i], sentence_sum_logit)
self.sample_sen_ids = tf.transpose(self.sample_sen_ids.stack())
self.sample_logits = tf.transpose(self.sample_logits.stack())
But I dont get it why after some batches I get this:
Traceback (most recent call last):
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1022, in _do_call
return fn(*args)
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1004, in _run_fn
status, run_metadata)
File "/usr/lib/python3.5/contextlib.py", line 66, in __exit__
next(self.gen)
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py", line 466, in raise_exception_on_not_ok_status
pywrap_tensorflow.TF_GetCode(status))
tensorflow.python.framework.errors_impl.InvalidArgumentError: slice index 49 of dimension 0 out of bounds.
[[Node: sampling/strided_slice_4900 = StridedSlice[Index=DT_INT32, T=DT_INT32, begin_mask=0, ellipsis_mask=0, end_mask=0, new_axis_mask=0, shrink_axis_mask=1, _device="/job:localhost/replica:0/task:0/cpu:0"](sampling/TensorArrayStack_3/TensorArrayGatherV3, sampling/strided_slice_4900/stack, sampling/strided_slice_4900/stack_1, sampling/strided_slice_4900/stack_2)]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "bin/neuralmonkey-train", line 6, in <module>
main()
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/train.py", line 211, in main
initial_variables=cfg.model.initial_variables)
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/learning_utils.py", line 185, in training_loop
results, meta=tf_manager.execute(batch_dataset, [trainer],train=True, summaries=False)
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/tf_manager.py", line 217, in execute
for sess in self.sessions]
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/tf_manager.py", line 217, in <listcomp>
for sess in self.sessions]
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 767, in run
run_metadata_ptr)
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 965, in _run
feed_dict_string, options, run_metadata)
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1015, in _do_run
target_list, options, run_metadata)
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/client/session.py", line 1035, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: slice index 49 of dimension 0 out of bounds.
[[Node: sampling/strided_slice_4900 = StridedSlice[Index=DT_INT32, T=DT_INT32, begin_mask=0, ellipsis_mask=0, end_mask=0, new_axis_mask=0, shrink_axis_mask=1, _device="/job:localhost/replica:0/task:0/cpu:0"](sampling/TensorArrayStack_3/TensorArrayGatherV3, sampling/strided_slice_4900/stack, sampling/strided_slice_4900/stack_1, sampling/strided_slice_4900/stack_2)]]
Caused by op 'sampling/strided_slice_4900', defined at:
File "bin/neuralmonkey-train", line 6, in <module>
main()
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/train.py", line 170, in main
cfg.build_model(warn_unused=True)
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/config/configuration.py", line 86, in build_model
model = build_config(self.config_dict, self.ignored, warn_unused)
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/config/builder.py", line 198, in build_config
value, config_dicts, existing_objects, 0)
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/config/builder.py", line 109, in build_object
obj = instantiate_class(value[7:], all_dicts, existing_objects, depth)
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/config/builder.py", line 165, in instantiate_class
obj = clazz(*bounded_params.args, **bounded_params.kwargs)
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/trainers/mrt_trainer.py", line 80, in __init__
ind=[[[tf.constant(i),tf.constant(j),sampled_ids[i][j]] for j in range(self.batch_size)] for i in range(self.decoder.max_output_len)]
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/trainers/mrt_trainer.py", line 80, in <listcomp>
ind=[[[tf.constant(i),tf.constant(j),sampled_ids[i][j]] for j in range(self.batch_size)] for i in range(self.decoder.max_output_len)]
File "/home/stoyan/neuralmonkey/bin/neuralmonkey/trainers/mrt_trainer.py", line 80, in <listcomp>
ind=[[[tf.constant(i),tf.constant(j),sampled_ids[i][j]] for j in range(self.batch_size)] for i in range(self.decoder.max_output_len)]
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/ops/array_ops.py", line 495, in _SliceHelper
name=name)
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/ops/array_ops.py", line 653, in strided_slice
shrink_axis_mask=shrink_axis_mask)
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/ops/gen_array_ops.py", line 3688, in strided_slice
shrink_axis_mask=shrink_axis_mask, name=name)
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
op_def=op_def)
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 2327, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/stoyan/neurmon/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 1226, in __init__
self._traceback = _extract_stack()
InvalidArgumentError (see above for traceback): slice index 49 of dimension 0 out of bounds.
[[Node: sampling/strided_slice_4900 = StridedSlice[Index=DT_INT32, T=DT_INT32, begin_mask=0, ellipsis_mask=0, end_mask=0, new_axis_mask=0, shrink_axis_mask=1, _device="/job:localhost/replica:0/task:0/cpu:0"](sampling/TensorArrayStack_3/TensorArrayGatherV3, sampling/strided_slice_4900/stack, sampling/strided_slice_4900/stack_1, sampling/strided_slice_4900/stack_2)]]
What should this InvalidArgumentError refer to and what goes wrong?
Best,
Stoyan
According to the stack trace, the error comes from this expression in your code:
sampled_ids[i][j]
...but it's hard to tell without context whether it comes from taking the [i] slice or the [j] slice. Presumably one of the tensors in this structure has fewer than 15 (or 49 in the error message) elements in the 0th dimension. Often this can happen if your input data includes word IDs that are not present in the vocabulary used for training the model.

Error in 'ValidationMonitor' when passing 'metrics' parameter

I'm using the following code to log accuracy as the validation measure (TensorFlow 0.10):
validation_metrics = {"accuracy": tf.contrib.metrics.streaming_accuracy}
validation_monitor = tf.contrib.learn.monitors.ValidationMonitor(
input_fn=input_fn_eval,
every_n_steps=FLAGS.eval_every,
# metrics=validation_metrics,
early_stopping_rounds=500,
early_stopping_metric="loss",
early_stopping_metric_minimize=True)
After running, in 'every_n_steps', I see the following lines in the output:
INFO:tensorflow:Validation (step 1000): loss = 1.04875, global_step = 900
The problem is that when 'metrics=validation_metrics' parameter uncomment in the above code, I get the following error in the validation phase:
INFO:tensorflow:Error reported to Coordinator: <type 'exceptions.TypeError'>, Input 'y' of 'Equal' Op has type int64 that does not match type float32 of argument 'x'.
E tensorflow/core/client/tensor_c_api.cc:485] Enqueue operation was cancelled
[[Node: read_batch_features_train/file_name_queue/file_name_queue_EnqueueMany = QueueEnqueueMany[Tcomponents=[DT_STRING], _class=["loc:#read_batch_features_train/file_name_queue"], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](read_batch_features_train/file_name_queue, read_batch_features_train/file_name_queue/RandomShuffle)]]
E tensorflow/core/client/tensor_c_api.cc:485] Enqueue operation was cancelled
[[Node: read_batch_features_train/random_shuffle_queue_EnqueueMany = QueueEnqueueMany[Tcomponents=[DT_STRING, DT_STRING], _class=["loc:#read_batch_features_train/random_shuffle_queue"], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](read_batch_features_train/random_shuffle_queue, read_batch_features_train/read/ReaderReadUpTo, read_batch_features_train/read/ReaderReadUpTo:1)]]
Traceback (most recent call last):
File "udc_train.py", line 74, in <module>
tf.app.run()
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/platform/app.py", line 30, in run
sys.exit(main(sys.argv))
File "udc_train.py", line 70, in main
estimator.fit(input_fn=input_fn_train, steps=None, monitors=[validation_monitor])
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 240, in fit
max_steps=max_steps)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 578, in _train_model
max_steps=max_steps)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/graph_actions.py", line 280, in _supervised_train
None)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/supervised_session.py", line 270, in run
run_metadata=run_metadata)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/recoverable_session.py", line 54, in run
run_metadata=run_metadata)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/coordinated_session.py", line 70, in run
self._coord.join(self._coordinated_threads_to_join)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/training/coordinator.py", line 357, in join
six.reraise(*self._exc_info_to_raise)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/coordinated_session.py", line 66, in run
return self._sess.run(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/monitored_session.py", line 107, in run
induce_stop = monitor.step_end(monitors_step, monitor_outputs)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 396, in step_end
return self.every_n_step_end(step, output)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/monitors.py", line 687, in every_n_step_end
steps=self.eval_steps, metrics=self.metrics, name=self.name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 356, in evaluate
name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 630, in _evaluate_model
eval_dict = self._get_eval_ops(features, targets, metrics)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 877, in _get_eval_ops
result[name] = metric(predictions, targets)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/metrics/python/ops/metric_ops.py", line 432, in streaming_accuracy
is_correct = math_ops.to_float(math_ops.equal(predictions, labels))
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_math_ops.py", line 708, in equal
result = _op_def_lib.apply_op("Equal", x=x, y=y, name=name)
File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 468, in apply_op
inferred_from[input_arg.type_attr]))
TypeError: Input 'y' of 'Equal' Op has type int64 that does not match type float32 of argument 'x'.
This looks like a problem with your input_fn and your estimator, which are returning different types for the label.