An error is being generated while training a federated model that uses hub.KerasLayer. The details of error and stack trace is given below. The complete code is available of gist https://gist.github.com/aksingh2411/60796ee58c88e0c3f074c8909b17b5a1. Help and suggestion in this regard would be appreciated. Thanks.
from tensorflow import keras
def create_keras_model():
encoder = hub.load("https://tfhub.dev/google/tf2-preview/gnews-swivel-20dim/1")
return tf.keras.models.Sequential([
hub.KerasLayer(encoder, input_shape=[],dtype=tf.string,trainable=True),
keras.layers.Dense(32, activation='relu'),
keras.layers.Dense(16, activation='relu'),
keras.layers.Dense(1, activation='sigmoid'),
])
def model_fn():
# We _must_ create a new model here, and _not_ capture it from an external
# scope. TFF will call this within different graph contexts.
keras_model = create_keras_model()
return tff.learning.from_keras_model(
keras_model,
input_spec=preprocessed_example_dataset.element_spec,
loss=tf.keras.losses.BinaryCrossentropy(),
metrics=[tf.keras.metrics.Accuracy()])
# Building the Federated Averaging Process
iterative_process = tff.learning.build_federated_averaging_process(
model_fn,
client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.02),
server_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=1.0))
str(iterative_process.initialize.type_signature)
state = iterative_process.initialize()
state, metrics = iterative_process.next(state, federated_train_data)
print('round 1, metrics={}'.format(metrics))
UnimplementedError Traceback (most recent call last)
<ipython-input-80-39d62fa827ea> in <module>()
----> 1 state, metrics = iterative_process.next(state, federated_train_data)
2 print('round 1, metrics={}'.format(metrics))
119 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py in
quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
58 ctx.ensure_initialized()
59 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 60 inputs, attrs, num_outputs)
61 except core._NotOkStatusException as e:
62 if name is not None:
UnimplementedError: Cast string to float is not supported
[[{{node StatefulPartitionedCall_1/StatefulPartitionedCall/Cast_1}}]]
[[StatefulPartitionedCall_1]]
[[import/StatefulPartitionedCall_3/ReduceDataset]] [Op:__inference_wrapped_function_65986]
Function call stack:
wrapped_function -> wrapped_function -> wrapped_function
The issues has now been resolved. The error was thrown because 'label' was getting passed as tf.string instead of tf.int32. Explicit casting resolved this.
Related
I'm trying to load a Keras sequential model and get some error of unexpected keyword reduction.
Here's the code:
I've trained the following Sequential model:
model = Sequential([
tf.keras.Input(shape=(in_dim,)),
layers.Dense(
units=(in_dim+1),
activation=layers.LeakyReLU(alpha=.01)
),
layers.Dropout(rate=.05),
layers.Dense(
units=(in_dim),
activation=layers.LeakyReLU(alpha=.01)
),
layers.Dropout(rate=.05),
layers.Dense(units=1, activation="sigmoid")
])
and compile and fit it by Keras's tutorial:
def compile_and_fit(
model,
name,
X_y_train,
X_y_val,
optimizer=None,
max_epochs=10000,
batch_size=BATCH_SIZE,
):
X_train, y_train = X_y_train
steps_per_epoch = len(X_train) // batch_size
X_val, y_val = X_y_val
steps_per_epoch_val = len(X_val) // batch_size
if optimizer is None:
optimizer = get_optimizer(steps_per_epoch)
model.compile(
optimizer=optimizer,
loss=tf.keras.losses.BinaryCrossentropy(),
metrics=[
tf.keras.losses.BinaryCrossentropy(name="binary_crossentropy"),
tf.keras.metrics.Precision(name="precision"),
"accuracy",
],
)
model.summary()
history = model.fit(
x=X_train,
y=y_train,
steps_per_epoch=steps_per_epoch,
batch_size=batch_size,
epochs=max_epochs,
validation_data=X_y_val,
validation_steps=steps_per_epoch_val,
callbacks=get_callbacks(name),
verbose=1,
)
return history
And at the end saving the model:
model.save(f"./saved_model/my_model", save_format="tf")
But When I tried to load it:
model = tf.keras.models.load_model("./saved_model/my_model")
And got the following error:
TypeError Traceback (most recent call last)
Cell In [25], line 2
----> 2 model = tf.keras.models.load_model("./saved_model/my_model")
File /mnt/c/Code/venv/lib/python3.9/site-packages/keras/utils/traceback_utils.py:70, in filter_traceback.<locals>.error_handler(*args, **kwargs)
67 filtered_tb = _process_traceback_frames(e.__traceback__)
68 # To get the full stack trace, call:
69 # `tf.debugging.disable_traceback_filtering()`
---> 70 raise e.with_traceback(filtered_tb) from None
71 finally:
72 del filtered_tb
File /mnt/c/Code/venv/lib/python3.9/site-packages/keras/dtensor/utils.py:144, in inject_mesh.<locals>._wrap_function(instance, *args, **kwargs)
142 if mesh is not None:
143 instance._mesh = mesh
--> 144 init_method(instance, *args, **kwargs)
TypeError: __init__() got an unexpected keyword argument 'reduction'
Any ideas on how to fix it?
mixed_precision.set_global_policy(policy="mixed_float16") gives an error when I add this line
error =
TypeError Traceback (most recent call
last) in
5 #mixed_precision.set_global_policy(policy="float32")
6 input_shape = (224, 224, 3)
----> 7 base_model = tf.keras.applications.EfficientNetB0(include_top=False)
8 base_model.trainable = False # freeze base model layers
9
4 frames
/usr/local/lib/python3.7/dist-packages/keras/applications/efficientnet.py
in EfficientNetB0(include_top, weights, input_tensor, input_shape,
pooling, classes, classifier_activation, **kwargs)
559 classes=classes,
560 classifier_activation=classifier_activation,
--> 561 **kwargs)
562
563
/usr/local/lib/python3.7/dist-packages/keras/applications/efficientnet.py
in EfficientNet(width_coefficient, depth_coefficient, default_size,
dropout_rate, drop_connect_rate, depth_divisor, activation,
blocks_args, model_name, include_top, weights, input_tensor,
input_shape, pooling, classes, classifier_activation)
332 # original implementation.
333 # See https://github.com/tensorflow/tensorflow/issues/49930 for more details
--> 334 x = x / tf.math.sqrt(IMAGENET_STDDEV_RGB)
335
336 x = layers.ZeroPadding2D(
/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/traceback_utils.py
in error_handler(*args, **kwargs)
151 except Exception as e:
152 filtered_tb = _process_traceback_frames(e.traceback)
--> 153 raise e.with_traceback(filtered_tb) from None
154 finally:
155 del filtered_tb
/usr/local/lib/python3.7/dist-packages/keras/layers/core/tf_op_layer.py
in handle(self, op, args, kwargs)
105 isinstance(x, keras_tensor.KerasTensor)
106 for x in tf.nest.flatten([args, kwargs])):
--> 107 return TFOpLambda(op)(*args, **kwargs)
108 else:
109 return self.NOT_SUPPORTED
/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py
in error_handler(*args, **kwargs)
65 except Exception as e: # pylint: disable=broad-except
66 filtered_tb = _process_traceback_frames(e.traceback)
---> 67 raise e.with_traceback(filtered_tb) from None
68 finally:
69 del filtered_tb
TypeError: Exception encountered when calling layer
"tf.math.truediv_3" (type TFOpLambda).
x and y must have the same dtype, got tf.float16 != tf.float32.
Call arguments received by layer "tf.math.truediv_3" (type
TFOpLambda): • x=tf.Tensor(shape=(None, None, None, 3),
dtype=float16) • y=tf.Tensor(shape=(3,), dtype=float32) •
name=None
this is code =
from tensorflow.keras import layers
# Create base model
mixed_precision.set_global_policy(policy="mixed_float16")
input_shape = (224, 224, 3)
base_model = tf.keras.applications.EfficientNetB0(include_top=False)
base_model.trainable = False # freeze base model layers
# Create Functional model
inputs = layers.Input(shape=input_shape, name="input_layer")
# Note: EfficientNetBX models have rescaling built-in but if your model didn't you could have a layer like below
# x = layers.Rescaling(1./255)(x)
x = base_model(inputs, training=False) # set base_model to inference mode only
x = layers.GlobalAveragePooling2D(name="pooling_layer")(x)
x = layers.Dense(len(class_names))(x) # want one output neuron per class
# Separate activation of output layer so we can output float32 activations
outputs = layers.Activation("softmax", dtype=tf.float32, name="softmax_float32")(x)
model = tf.keras.Model(inputs, outputs)
# Compile the model
model.compile(loss="sparse_categorical_crossentropy", # Use sparse_categorical_crossentropy when labels are *not* one-hot
optimizer=tf.keras.optimizers.Adam(),
metrics=["accuracy"])
When I change this line with float32 instead of mixed_float16,like
this mixed_precision.set_global_policy(policy="float32") the
error goes away. I want to use Mixed_precision, how can I do it?
i was trying to use DirectML for usage of my amd rx580 graphics card in tensorflow, but i'm having a real hard time to pull this up.
i'm getting this error:
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
Input In [73], in <cell line: 24>()
23 # fit network
24 for i in range(n_epoch):
---> 25 model_LSTM_peso.fit(X, y, epochs=1, batch_size=n_batch, verbose=1, shuffle=False)
26 model_LSTM_peso.reset_states()
File ~\anaconda3\envs\tfdml_plugin\lib\site-packages\keras\utils\traceback_utils.py:67, in filter_traceback.<locals>.error_handler(*args, **kwargs)
65 except Exception as e: # pylint: disable=broad-except
66 filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67 raise e.with_traceback(filtered_tb) from None
68 finally:
69 del filtered_tb
File ~\anaconda3\envs\tfdml_plugin\lib\site-packages\tensorflow\python\eager\execute.py:54, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
52 try:
53 ctx.ensure_initialized()
---> 54 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
55 inputs, attrs, num_outputs)
56 except core._NotOkStatusException as e:
57 if name is not None:
InvalidArgumentError: Graph execution error:
No OpKernel was registered to support Op 'CudnnRNN' used by {{node CudnnRNN}} with these attrs: [seed=0, dropout=0, T=DT_FLOAT, input_mode="linear_input", direction="unidirectional", rnn_mode="lstm", is_training=true, seed2=0]
Registered devices: [CPU, GPU]
Registered kernels:
<no registered kernels>
[[CudnnRNN]]
[[sequential_11/lstm_3/PartitionedCall]] [Op:__inference_train_function_491977]
I have running another code that fits with the model for embedding. So the rest of the code is running, but seems to be that the problem that i have is just for use LSTM
from pandas import DataFrame
from pandas import concat
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
values = input_emb_peso
X, y = values, input_peso
X = X.reshape(X.shape[0],X.shape[1], 1)
# configure network
n_batch = 1
n_epoch = 1 #X.shape[0] #len(grupo_carrera_train) #19485 carreras de entrenamiento
n_neurons = 80
# design network
model_LSTM_peso = Sequential()
model_LSTM_peso.add(LSTM(n_neurons, batch_input_shape=(n_batch,X.shape[1], X.shape[2]), stateful=True))
model_LSTM_peso.add(Dense(80))
model_LSTM_peso.add(Dense(80))
model_LSTM_peso.add(Dense(1))
model_LSTM_peso.compile(loss='mean_squared_error', optimizer='adam')
# fit network
for i in range(n_epoch):
model_LSTM_peso.fit(X, y, epochs=1, batch_size=n_batch, verbose=1, shuffle=False)
model_LSTM_peso.reset_states()
I use CRNN (CNN + RNN + CTC Loss) for my model on OCR. I'm using Tensorflow Keras
here's my code [from CTC Loss]:
labels = Input(name='the_labels', shape=[max_label_len], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
def ctc_lambda_func(args):
y_pred, labels, input_length, label_length = args
return K.ctc_batch_cost(labels, y_pred, input_length, label_length)
loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([outputs, labels, input_length, label_length])
#model to be used at training time
model = Model(inputs=[inputs, labels, input_length, label_length], outputs=loss_out)
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer = 'adam')
filepath="best_model.hdf5"
checkpoint = ModelCheckpoint(filepath=filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='auto')
callbacks_list = [checkpoint]
training_img = np.array(training_img)
train_input_length = np.array(train_input_length)
train_label_length = np.array(train_label_length)
valid_img = np.array(valid_img)
valid_input_length = np.array(valid_input_length)
valid_label_length = np.array(valid_label_length)
Error here while training:
batch_size = 256
epochs = 10
model.fit(x=[training_img, train_padded_txt, train_input_length, train_label_length], y=np.zeros(len(training_img)),
batch_size=batch_size, epochs = epochs,
validation_data = ([valid_img, valid_padded_txt, valid_input_length, valid_label_length], [np.zeros(len(valid_img))]),
verbose = 1, callbacks = callbacks_list)
ERROR RESULT:
Train on 448 samples, validate on 49 samples
Epoch 1/10
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
<ipython-input-15-1322212af569> in <module>()
4 batch_size=batch_size, epochs = epochs,
5 validation_data = ([valid_img, valid_padded_txt, valid_input_length, valid_label_length], [np.zeros(len(valid_img))]),
----> 6 verbose = 1, callbacks = callbacks_list)
7 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
58 ctx.ensure_initialized()
59 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 60 inputs, attrs, num_outputs)
61 except core._NotOkStatusException as e:
62 if name is not None:
InvalidArgumentError: sequence_length(0) <= 18
[[node ctc/CTCLoss (defined at /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3009) ]] [Op:__inference_keras_scratch_graph_12073]
Function call stack:
keras_scratch_graph
My CRNN architecture is inspired by VGG-16, I'm using 13 conv layers and 3 bi-directional LSTM Layer. I am using CTC Loss and then I got error.
My data is 1000 text-image contains 4-8 words (700 for training&valid, 300 for testing)
if you want to view my code: here's my code using google colab.
https://colab.research.google.com/drive/1nMRNUsLDNrpgeTxPFQ4mhobnFdpbmwUx
I fixed this error. It's because of this!
Before:
# split the 700 data into validation and training dataset as 10% and 90% respectively
if i%10 == 0:
valid_orig_txt.append(txt)
valid_label_length.append(len(txt))
valid_input_length.append(31)
valid_img.append(img)
valid_txt.append(encode_to_labels(txt))
else:
orig_txt.append(txt)
train_label_length.append(len(txt))
train_input_length.append(31)
training_img.append(img)
training_txt.append(encode_to_labels(txt))
After:
# split the 700 data into validation and training dataset as 10% and 90% respectively
if i%10 == 0:
valid_orig_txt.append(txt)
valid_label_length.append(len(txt))
valid_input_length.append(18)
valid_img.append(img)
valid_txt.append(encode_to_labels(txt))
else:
orig_txt.append(txt)
train_label_length.append(len(txt))
train_input_length.append(18)
training_img.append(img)
training_txt.append(encode_to_labels(txt))
I'm running tensorflow 2.1 and tensorflow_probability 0.9. I'd like to parameterize a multivariatenormal distribution with placeholders and sample from it. Here's what I've tried
import tensorflow as tf
import tensorflow_probability as tfp
#tf.function()
def sample_vae(dist):
return dist.sample()
vae_mu = tf.keras.layers.Input(shape=(5), dtype=tf.float16)
vae_logvar = tf.keras.layers.Input(shape=(5), dtype=tf.float16)
dist = tfp.distributions.MultivariateNormalDiag(loc=vae_mu, scale_diag=tf.exp(vae_logvar))
z = sample_vae(dist)
The above gives me the following error
TypeError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
60 op_name, inputs, attrs,
---> 61 num_outputs)
62 except core._NotOkStatusException as e:
TypeError: An op outside of the function building code is being passed
a "Graph" tensor. It is possible to have Graph tensors
leak out of the function building context by including a
tf.init_scope in your function building code.
For example, the following function will fail:
#tf.function
def has_init_scope():
my_constant = tf.constant(1.)
with tf.init_scope():
added = my_constant * 2
The graph tensor has name: BroadcastArgs_3:0
During handling of the above exception, another exception occurred:
_SymbolicException Traceback (most recent call last)
6 frames
/usr/local/lib/python3.6/dist-packages/tensorflow_core/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
73 raise core._SymbolicException(
74 "Inputs to eager execution function cannot be Keras symbolic "
---> 75 "tensors, but found {}".format(keras_symbolic_tensors))
76 raise e
77 # pylint: enable=protected-access
_SymbolicException: Inputs to eager execution function cannot be Keras symbolic tensors, but found [<tf.Tensor 'BroadcastArgs_3:0' shape=(1,) dtype=int32>, <tf.Tensor 'Exp_3:0' shape=(None, 5) dtype=float16>, <tf.Tensor 'input_7:0' shape=(None, 5) dtype=float16>]
I was able to get around this using DistributionLambda
vae_mu = tf.keras.layers.Input(shape=(1, 5), dtype=tf.float16)
vae_logvar = tf.keras.layers.Input(shape=(1, 5), dtype=tf.float16)
mu_logvar = tf.concat([vae_mu, vae_logvar], axis=1)
l = vae_mu.shape[1]
dist = tfp.layers.DistributionLambda(lambda theta: tfp.distributions.MultivariateNormalDiag(loc=theta[:, :l], scale_diag=theta[:, l:]))
z = dist(mu_logvar)