I'm getting a TypeError when trying to convert a keras .h5 file to tflite.
The new layer is a gaussian kernel (Radial Basis Layer).
To be able to save and load the keras model I defined also the get_config() method in the custom layer. So I'm able to save and load the model correctly.
class RBFLayer(Layer):
def __init__(self, output_dim, centers=None, tol = 1e-6, gamma=0, **kwargs):
super(RBFLayer, self).__init__(**kwargs)
self.centers_ = centers
self.output_dim= output_dim
self.gamma_ = gamma
self.tol_ = tol
def build(self, input_shape):
self.mu = K.variable(self.centers_, name='centers')
self.gamma = K.variable(self.gamma_, name='gamma')
self.tol = K.constant(self.tol_,name='tol')
super(RBFLayer, self).build(input_shape)
def call(self, inputs): #Kernel radial
a,b = self.mu.shape
diff = K.reshape( K.tile(inputs,(1,a))-K.reshape(self.mu,(1,-1)), (-1,a,b))
l2 = K.sum(K.pow(diff, 2), axis=-1)
res = K.exp(-1 * self.gamma * l2)
mask = K.greater( res, self.tol)
return K.switch(mask, res, K.zeros_like(res))
def compute_output_shape(self, input_shape):
return (input_shape[0], self.output_dim)
def get_config(self):
config = {
'output_dim': self.output_dim,
'centers': self.centers_,
'gamma': self.gamma_,
'tol': self.tol_
}
base_config = super(RBFLayer, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
Now I want to save the model to tflite. I use TFLiteConverter from keras file including also 'custom_objects'.
def save_tflite(self, base_name):
file =base_name +'.h5'
converter = tf.lite.TFLiteConverter.from_keras_model_file(file, custom_objects={'RBFLayer':RBFLayer})
tflite_model = converter.convert()
open(base_name+".tflite", "wb").write(tflite_model)
I expect to get the tflite model file including the K.variables used while training the complete model (centers, tol, gamma).
When converting I get these error messages:
airgorbn.save_tflite( base_name)
Traceback (most recent call last):
File "<ipython-input-7-cdaa1ec46233>", line 1, in <module>
airgorbn.save_tflite( base_name)
File "C:/Users/AIRFI/Hospital/keras_RadialBasis.py", line 158, in save_tflite
converter = tf.lite.TFLiteConverter.from_keras_model_file(file, custom_objects={'RBFLayer':RBFLayer})
File "C:\Users\AIRFI\Anaconda3\envs\tf\lib\site-packages\tensorflow\lite\python\lite.py", line 747, in from_keras_model_file
keras_model = _keras.models.load_model(model_file, custom_objects)
File "C:\Users\AIRFI\Anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\saving\save.py", line 146, in load_model
return hdf5_format.load_model_from_hdf5(filepath, custom_objects, compile)
File "C:\Users\AIRFI\Anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\saving\hdf5_format.py", line 212, in load_model_from_hdf5
custom_objects=custom_objects)
File "C:\Users\AIRFI\Anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\saving\model_config.py", line 55, in model_from_config
return deserialize(config, custom_objects=custom_objects)
File "C:\Users\AIRFI\Anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\layers\serialization.py", line 89, in deserialize
printable_module_name='layer')
File "C:\Users\AIRFI\Anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\utils\generic_utils.py", line 192, in deserialize_keras_object
list(custom_objects.items())))
File "C:\Users\AIRFI\Anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\engine\sequential.py", line 353, in from_config
model.add(layer)
File "C:\Users\AIRFI\Anaconda3\envs\tf\lib\site-packages\tensorflow\python\training\tracking\base.py", line 457, in _method_wrapper
result = method(self, *args, **kwargs)
File "C:\Users\AIRFI\Anaconda3\envs\tf\lib\site-packages\tensorflow\python\keras\engine\sequential.py", line 154, in add
'Found: ' + str(layer))
TypeError: The added layer must be an instance of class Layer. Found: <__main__.RBFLayer object at 0x0000017D3A75AC50>
you need to define that layer as a custom op.
refer to this https://www.tensorflow.org/lite/guide/ops_custom
Related
So, I have done some modifications with the VGG16 neural network to make a linear-regression and classification model.
And at last I am writing the following code to compile and fit my data into the model to initiate training->
class Facetracker(Model):
# The initialization function-
def __init__(self,eyetracker,**kwargs):
super().__init__(**kwargs)
self.model = eyetracker #Instantiating the model
def compile(self,opt,classlosss,localization_loss,**kwargs):
super().compile(**kwargs)
self.classloss = class_loss
self.localization_loss = regress_loss
self.opt = optimizer
# Defining the training step
def train_step(self,batch,**kwargs):
X,y = batch #unpacking our data
with tf.GradientTape() as tape:
classes,coords = self.model(X,training=True)
batch_classloss = self.classloss(y[0],classes)
batch_localloss = self.localization_loss(tf.cast(y[1],tf.float32),coords)
# calculating total loss-
total_loss = batch_localloss+0.5*batch_classloss
grad = tape.gradient(total_loss,self.model.trainable_variables)
optimizer.apply_gradients(zip(grad,self.model.trainable_variables))
return{
"total_loss":total_loss,
"class_loss":batch_classloss,
"localilzation_loss":batch_localloss
}
def test_step(self,batch):
X,y = batch
classes,coords = self.model(X,training=False)
batch_classloss = self.classloss(y[0],classes)
batch_localloss = self.localization_loss(tf.cast(y[1],tf.float32),coords)
total_loss = batch_localloss+0.5*batch_classloss
return{
"total_loss": total_loss,
"class_loss": batch_classloss,
"localilzation_loss": batch_localloss
}
# def call(self, X, **kwargs):
# return self.model(X,**kwargs)
# Replacing the call function with a lambda function
lambda self,X,**kwargs: self.model(X,**kwargs)
# Subclassing our model-
print("Subclassing.....")
model = Facetracker(facetracker)
print("Compiling......")
model.compile(optimizer,classlosss=class_loss,localization_loss=localization_loss)
# Preparing the log directory
logdir="logdir"
tensorboard_callbacks = tf.keras.callbacks.TensorBoard(log_dir=logdir)
print("Fitting the model")
hist = model.fit(train.take(80),
epochs=16,
initial_epoch =8,
validation_data=val,
validation_steps =8,
validation_freq=2,
callbacks = [[tensorboard_callbacks]])
This includes the class prepared for training the model and the last few lines are subclassing the model and fitting the prepared data into the model.
The error I now get seems pretty hefty to me and it goes like this->
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\main.py", line 535, in <module>
hist = model.fit(train.take(80),
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\utils\traceback_utils.py", line 67, in error_handler
raise e.with_traceback(filtered_tb) from None
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\tensorflow\python\eager\execute.py", line 54, in quick_execute
tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
tensorflow.python.framework.errors_impl.InvalidArgumentError: Graph execution error:
Detected at node 'gradient_tape/sub_2/BroadcastGradientArgs' defined at (most recent call last):
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\main.py", line 535, in <module>
hist = model.fit(train.take(80),
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\utils\traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\engine\training.py", line 1409, in fit
tmp_logs = self.train_function(iterator)
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\engine\training.py", line 1051, in train_function
return step_function(self, iterator)
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\engine\training.py", line 1040, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\MarkATT\lib\site-packages\keras\engine\training.py", line 1030, in run_step
outputs = model.train_step(data)
File "C:\Users\Radhe Krishna\OneDrive\Documents\MarkATT\main.py", line 498, in train_step
grad = tape.gradient(total_loss,self.model.trainable_variables)
Node: 'gradient_tape/sub_2/BroadcastGradientArgs'
Incompatible shapes: [2,4] vs. [8]
[[{{node gradient_tape/sub_2/BroadcastGradientArgs}}]] [Op:__inference_train_function_22026]
It would be a great help if someone could examine this problem and help me get out of it. I have been quite struggling with it now.
Thanks in advance!!!
I has planned to initiate training with the code provided, but I firstly received an error for the call function and as I resolved it by converting it into lambda function- This issue of incompatible shapes came up. I tries to adjust the integers in the input data and batch related fields but nothing worked.
i use the tape.gradient(g_loss, aa_mutator.trainable_variables) to calculate the gradient of a model called aa_mutator and got the error
File "/home/tialan/Data/gan/code/g.py", line 297, in <module>
grads_g = tape.gradient(g_loss, aa_mutator.trainable_variables)
File "/home/tialan/tf/lib/python3.7/site-packages/tensorflow/python/eager/backprop.py", line 1086, in gradient
unconnected_gradients=unconnected_gradients)
File "/home/tialan/tf/lib/python3.7/site-packages/tensorflow/python/eager/imperative_grad.py", line 77, in imperative_grad
compat.as_str(unconnected_gradients.value))
File "/home/tialan/tf/lib/python3.7/site-packages/tensorflow/python/eager/backprop.py", line 162, in _gradient_function
return grad_fn(mock_op, *out_grads)
File "/home/tialan/tf/lib/python3.7/site-packages/tensorflow/python/ops/array_grad.py", line 782, in _ReshapeGrad
_IndexedSlicesToTensorNoWarning(grad), array_ops.shape(op.inputs[0])),
File "/home/tialan/tf/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py", line 201, in wrapper
return target(*args, **kwargs)
File "/home/tialan/tf/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py", line 195, in reshape
result = gen_array_ops.reshape(tensor, shape, name)
File "/home/tialan/tf/lib/python3.7/site-packages/tensorflow/python/ops/gen_array_ops.py", line 8368, in reshape
_ops.raise_from_not_ok_status(e, name)
File "/home/tialan/tf/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 6862, in raise_from_not_ok_status
six.raise_from(core._status_to_exception(e.code, message), None)
File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: Input to reshape is a tensor with 1012 values, but the requested shape has 20240 [Op:Reshape]
within the aa_mutator model i build a customized keras layer
class mutate_func_layer(layers.Layer):
def __init__(self):
super(mutate_func_layer, self).__init__()
def call(self, inputs):
Mut_pos_layer_out, input_pre, Mutation_3 = inputs
where = where_func(Mut_pos_layer_out)
return mutate_func(Mutation_3, where, input_pre)
with mutate_func defined as
#tf.custom_gradient
def mutate_func(x, where, input_pre):#(Mutation_3, where, input_pre): ## x = mutation 3
print('m3_in_mutate_func')
print(x.shape)
aa_aft = gather_nd_func(x, where)
print('m3_in_mutate_func')
print(aa_aft.shape)
aa_aft = K.argmax(aa_aft, axis=-1)
print('m3_in_mutate_func')
print(aa_aft.shape)
aa_aft = tf.reshape(aa_aft, [-1])
print('m3_in_mutate_func')
print(aa_aft.shape)
aa_aft = tf.cast(aa_aft, dtype=tf.float32)
print('m3_in_mutate_func')
print(aa_aft.shape)
aa_seq_out = tf.tensor_scatter_nd_update(input_pre, [where], [aa_aft])
print('m3_in_mutate_func')
print(aa_seq_out.shape)
def grad(upstream):
return upstream*1, upstream*1, upstream*1
the shape for layers in the mutate_func are printed as
m3_in_mutate_func
(1, 1012, 20)
m3_in_mutate_func
(675, 20)
m3_in_mutate_func
(675,)
m3_in_mutate_func
(675,)
m3_in_mutate_func
(675,)
m3_in_mutate_func
(1, 1012)
the model is able to predict given the input. just for fitting the error shows at the stage of tape.gradient. is the error raised due to the customized layer? Thanks for any help or suggestion
import tensorflow as tf
from ..models.encoder import encoder_build
from ..models.decoder import decoder_build
def compute_attention_maps(inputs,name,upsample=False):
attMap = tf.reduce_sum(tf.square(inputs),axis=-1,keepdims=True,name= str(name)+"reducSum")
if upsample:
attMap = tf.keras.layers.UpSampling2D(size=(2, 2),
interpolation='bilinear',
name = str(name)+"bilinear")(attMap)
attMap = tf.squeeze(attMap,axis=-1,name = str(name)+"squeeze")
attMap = tf.reshape(attMap,
(tf.shape(attMap)[0],tf.shape(attMap)[1]*tf.shape(attMap)[2]),
name = str(name)+"reshape")
attMap = tf.nn.softmax(attMap,
axis=-1,
name = str(name)+"spatialSoftmax")
return attMap
def compute_mse(x,y,name):
diff = tf.math.squared_difference(x,y,name = str(name)+"squError")
diff = tf.reduce_mean(diff,axis=0, name = str(name)+"mean")
diff = tf.reduce_sum(diff, name = str(name)+"sum")
return diff
def compute_distillation(attention_inputs):
inp1,inp2,inp3,inp4 = attention_inputs
attMap1 = compute_attention_maps(inp1,"attmap1_")
attMap2_upsample = compute_attention_maps(inp2,"attmap2UP_",upsample=True)
attMap2 = compute_attention_maps(inp2,"attmap2_")
attMap3_upsample = compute_attention_maps(inp3,"attmap3UP_",upsample=True)
attMap3 = compute_attention_maps(inp3,"attmap3_")
attMap4 = compute_attention_maps(inp4,"attmap4_")
distillation1 = compute_mse(attMap1,attMap2_upsample,"distil1_")
distillation2 = compute_mse(attMap2,attMap3_upsample,"distil2_")
distillation3 = compute_mse(attMap3,attMap4,"distil3_")
return tf.math.add_n([distillation1,distillation2,distillation3], name="distill_loss")
if __name__ == '__main__':
inputs = tf.keras.layers.Input(shape=(None, None, 3), name='image')
encoderTuple = encoder_build(inputs) # import from encoder.py file
attention_inputs = encoderTuple[1]
outputs = decoder_build(encoderTuple) # import from decoder.py file
model = tf.keras.models.Model(inputs=inputs, outputs=outputs)
model.add_loss(compute_distillation(attention_inputs))
model.summary()
model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=0.00001, clipnorm=0.001),
loss='binary_crossentropy',
metrics=['accuracy'])
model.fit(x = train_generator,
epochs=epochs,
verbose=1,
callbacks=callbacks,
validation_data=validation_generator,
shuffle=True)
i have created keras segmentation model for lane detection (https://arxiv.org/pdf/1908.00821.pdf). I'm able to compile, start training and save models for each epoch without any errors. But if i add my custom loss to model model.add_loss(compute_distillation(attention_inputs)) model gets trained for 1 epoch, after that model is not saving and displaying below error. How to resolve this error?
374/375 [============================>.] - ETA: 0s - loss: 4.4717 - acc: 0.9781Epoch 1/50
78/78[============================>.] - ETA: 37:38 - val_loss: 4.5855 - val_acc: 0.9758
Epoch 00001: saving model to /workspace/work/enet_sad_naiveresize/snapshot/enetNRSAD_Tusimple_L_4.4718_VL_4.5855.h5
Traceback (most recent call last):
File "/workspace/work/enet_sad_naiveresize/bin/train.py", line 82, in <module>
shuffle=True)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training.py", line 727, in fit
use_multiprocessing=use_multiprocessing)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_generator.py", line 603, in fit
steps_name='steps_per_epoch')
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/training_generator.py", line 332, in model_iteration
callbacks.on_epoch_end(epoch, epoch_logs)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/callbacks.py", line 299, in on_epoch_end
callback.on_epoch_end(epoch, logs)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/callbacks.py", line 968, in on_epoch_end
self._save_model(epoch=epoch, logs=logs)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/callbacks.py", line 1015, in _save_model
self.model.save(filepath, overwrite=True)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/engine/network.py", line 1171, in save
signatures)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/saving/save.py", line 109, in save_model
model, filepath, overwrite, include_optimizer)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/saving/hdf5_format.py", line 103, in save_model_to_hdf5
save_weights_to_hdf5_group(model_weights_group, model_layers)
File "/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/keras/saving/hdf5_format.py", line 619, in save_weights_to_hdf5_group
g = f.create_group(layer.name)
File "/usr/local/lib/python3.6/dist-packages/h5py/_hl/group.py", line 68, in create_group
gid = h5g.create(self.id, name, lcpl=lcpl, gcpl=gcpl)
File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
File "h5py/h5g.pyx", line 161, in h5py.h5g.create
ValueError: Unable to create group (name already exists)
The issue is because you are stacking layers (and naming them wrongly) in compute_distillation function, by calling other functions such as compute_attention_maps and compute_mse.
You would have got a similar layer if you hadn't named also and the fact that the error persists even after you named them is because the h5 models expects names in a certain format as explained here https://github.com/keras-team/keras/issues/12195.
A good solution would be to use keras lambda layers in compute_distilation function to create attMap1, attMap2, etc. or define your own custom AttentionMap layer as shown below.
class AttentionMaps(tf.keras.layers.Layer):
def __init__(self, upsample=False):
super(AttentionMaps, self).__init__()
self.upsample = upsample
def call(self, inputs):
attMap = tf.reduce_sum(
tf.square(inputs),
axis=-1,
keepdims=True
)
if self.upsample:
attMap = tf.keras.layers.UpSampling2D(
size=(2, 2),
interpolation='bilinear'
)(attMap)
attMap = tf.squeeze(attMap,axis=-1)
attMap = tf.reshape(
attMap,
(tf.shape(attMap)[0],tf.shape(attMap)[1]*tf.shape(attMap)[2]))
attMap = tf.nn.softmax(attMap,
axis=-1,)
return attMap
This custom layer can then be added to your model as per the example below. The names ofthe layers are no longer required, so I removed them.
def compute_distillation(attention_inputs):
inp1,inp2,inp3,inp4 = attention_inputs
attention_layer_1 = AttentionMaps()
attMap1 = attention_layer_1(inp1)
attention_layer_2 = AttentionMaps(upsample=True)
attMap2_upsample = attention_layer_2(inp2)
attention_layer_3 = AttentionMaps()
attMap2 = attention_layer_3(inp2)
attention_layer_4 = AttentionMaps(upsample=True)
attMap3_upsample = attention_layer_4(inp3)
attention_layer_5 = AttentionMaps()
attMap3 = attention_layer_5(inp3)
attention_layer_6 = AttentionMaps(upsample=True)
attMap4_upsample = attention_layer_6(inp4)
distillation1 = compute_mse(attMap1,attMap2_upsample)
distillation2 = compute_mse(attMap2,attMap3_upsample)
distillation3 = compute_mse(attMap3,attMap4_upsample)
return tf.math.add_n([distillation1,distillation2,distillation3], name="distill_loss")
There are a few issues opened in the keras github related to this.
https://github.com/keras-team/keras/issues/6005
https://github.com/keras-team/keras/issues/12195
This issue is not due to the custom loss function but due to the way the model is defined.
You could try out the solutions provided in the above mentioned links such as saving the model weights as a tf file rather than h5 or avoid adding the same instance of activation layer at multiple places in the model. If that doesn't resolve your issue please update the question to include the models.
I'm trying to build a simple model and save the untrained layers. (I'll later want to train it). I'm trying to use tensorflow core API's without relying on Keras layers so I can more directly control what I use and maximize compatibility with TFLite.
import numpy as np
import tensorflow as tf
class BasicModel(tf.Module):
def __init__(self):
self.const = None
#tf.function(input_signature=[
tf.TensorSpec(shape=[None,20],dtype=tf.int32),
])
def rnn(self, captions):
# ENCODER
weights = tf.Variable(tf.random.normal([10000, 724]))#, shape=[vocab_size,embedding_dimension], name="embedding_weights")
embedding_output = tf.nn.embedding_lookup(weights,captions)
#activation is tanh for GRUCell
sequence = tf.unstack(embedding_output,num=20, axis=1)
cell = tf.compat.v1.nn.rnn_cell.GRUCell(20)
print(sequence)
gru_layer = tf.compat.v1.nn.static_rnn(cell, sequence, dtype=tf.float32)
return gru_layer
root = BasicModel()
concrete_function = root.rnn.get_concrete_function()
tf.saved_model.save(root,"model",concrete_function)
I expect to have an untrained model that saves but instead I get an error:
Traceback (most recent call last):
File "model_tensorflow_2.py", line 24, in <module>
concrete_function = root.rnn.get_concrete_function()#tf.constant(images), tf.constant(captions), tf.constant(cap_lens))
File "/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py", line 782, in get_concrete_function
return self._stateless_fn.get_concrete_function(*args, **kwargs)
File "/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 1891, in get_concrete_function
graph_function, args, kwargs = self._maybe_define_function(args, kwargs)
File "/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 2150, in _maybe_define_function
graph_function = self._create_graph_function(args, kwargs)
File "/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 2041, in _create_graph_function
capture_by_value=self._capture_by_value),
File "/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/func_graph.py", line 915, in func_graph_from_py_func
func_outputs = python_func(*func_args, **func_kwargs)
File "/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py", line 358, in wrapped_fn
return weak_wrapped_fn().__wrapped__(*args, **kwds)
File "/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 2658, in bound_method_wrapper
return wrapped_fn(*args, **kwargs)
File "/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/framework/func_graph.py", line 905, in wrapper
raise e.ag_error_metadata.to_exception(e)
ValueError: in converted code:
model_tensorflow_2.py:13 rnn *
weights = tf.Variable(tf.random.normal([10000, 724]))#, shape=[vocab_size,embedding_dimension], name="embedding_weights")
/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/variables.py:260 __call__
return cls._variable_v2_call(*args, **kwargs)
/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/variables.py:254 _variable_v2_call
shape=shape)
/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/ops/variables.py:65 getter
return captured_getter(captured_previous, **kwargs)
/Users/t.capes/miniconda3/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py:413 invalid_creator_scope
"tf.function-decorated function tried to create "
ValueError: tf.function-decorated function tried to create variables on non-first call.
tf.function does not allow to create variables on non-first call, because the semantics of that are not clear: should the variables be re-created on each call? should they be implicitly cached? (see this bit in the "tf.function and AutoGraph" talk from TF Summit 2019).
A common workaround is to have a helper function creating the variables and ensure that it's called at most once for each instance:
class BasicModel(tf.Module):
# ...
def _create_parameters(self, ...):
self._weights = tf.Variable(...)
self._parameters_created = True
def rnn(self, ...):
if not self._parameters_created:
self._create_parameters(...)
...
I'm trying to learn how to use the Estimator API, using input_fn to provide Dataset backed input to a feature_column generated input layer.
My code looks like
import tensorflow as tf import random
tf.logging.set_verbosity(tf.logging.DEBUG)
def input_fn():
def gen():
for i in range(100000):
for j in range(10):
yield {"in": str(j)}, [str(j+1)]
data = tf.data.Dataset.from_generator(gen, ({"in": tf.string}, tf.string))
data = data.batch(10)
iterator = data.make_one_shot_iterator()
return iterator.get_next()
vocabulary_feature_column = tf.feature_column.categorical_column_with_vocabulary_list(
key="in",
vocabulary_list=map(lambda i: str(i), range(11)))
embedding_column = tf.feature_column.embedding_column(
categorical_column=vocabulary_feature_column,
dimension=2)
with tf.Session() as sess:
print(sess.run(input_fn()))
classifier = tf.estimator.DNNClassifier(
feature_columns = [embedding_column],
hidden_units = [5,5],
n_classes = 11,
model_dir = '/tmp/predict/snap')
classifier.train(
input_fn=input_fn)
but running it I get
Traceback (most recent call last):
File "predict.py", line 33, in
input_fn=input_fn)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 302, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 711, in _train_model
features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/estimator.py", line 694, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/canned/dnn.py", line 334, in _model_fn
config=config)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/canned/dnn.py", line 190, in _dnn_model_fn
logits = logit_fn(features=features, mode=mode)
File "/usr/lib/python2.7/site-packages/tensorflow/python/estimator/canned/dnn.py", line 89, in dnn_logit_fn
features=features, feature_columns=feature_columns)
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 230, in input_layer
trainable=trainable)
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 1837, in _get_dense_tensor
inputs, weight_collections=weight_collections, trainable=trainable)
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 2123, in _get_sparse_tensors
return _CategoricalColumn.IdWeightPair(inputs.get(self), None)
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 1533, in get
transformed = column._transform_feature(self) # pylint: disable=protected-access
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 2091, in _transform_feature
input_tensor = _to_sparse_input(inputs.get(self.key))
File "/usr/lib/python2.7/site-packages/tensorflow/python/feature_column/feature_column.py", line 1631, in _to_sparse_input
raise ValueError('Undefined input_tensor shape.')
ValueError: Undefined input_tensor shape.
Looking at the tf sources I get the impression I that the categorical_column_with_vocabulary_list expects a tensor as output instead of a string, but I have a hard time understanding how to make my input_fn provide that the right way.
Does anyone have any idea what I'm doing wrong here?
As a comparison, the following code works perfectly fine: https://pastebin.com/28QUNAjA
EDIT
I noticed that replacing tf.data.Dataset.from_generator with tf.data.Dataset.from_tensor_slices makes the code run.
I.e. the following actually works:
import tensorflow as tf
import random
tf.logging.set_verbosity(tf.logging.DEBUG)
def input_fn():
data = tf.data.Dataset.from_tensor_slices(({"in": map(lambda i: str(i), range(10))}, range(1,11)))
data = data.repeat(1000)
data = data.batch(10)
iterator = data.make_one_shot_iterator()
return iterator.get_next()
vocabulary_feature_column = tf.feature_column.categorical_column_with_vocabulary_list(
key="in",
vocabulary_list=map(lambda i: str(i), range(11)))
embedding_column = tf.feature_column.embedding_column(
categorical_column=vocabulary_feature_column,
dimension=2)
with tf.Session() as sess:
print(sess.run(input_fn()))
classifier = tf.estimator.DNNClassifier(
feature_columns = [embedding_column],
hidden_units = [5,5],
n_classes = 11,
model_dir = '/usr/local/google/home/zond/tmp/predict/snap')
classifier.train(
input_fn=input_fn)
This ought to be a bug, so I created https://github.com/tensorflow/tensorflow/issues/15178.