I have trained a model with keras layers and weight_normalization layer from tensorflow_addons. This is the model I trained and saved in tensorflow file format:
import tensorflow as tf
import tensorflow.keras as tk
import tensorflow_addons as tfa
model = tf.keras.Sequential([
tf.keras.layers.Input((X_train.shape[1]-1,)),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1048, activation="relu")),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid")),
])
(and it has no custom metrics)
from keras.callbacks import ModelCheckpoint, EarlyStopping
# autosave best Model
best_model = ModelCheckpoint("model", monitor='val_accuracy', mode='max',verbose=0, save_best_only=True)
earlystop = EarlyStopping(monitor = 'val_accuracy',
patience = 15,
mode = 'max',
verbose = 1,
restore_best_weights = True)
callbacks = [best_model, earlystop]
model.compile(loss= 'binary_crossentropy',optimizer= 'Adam',metrics= ['accuracy'])
history = model.fit(X_res, y_res, epochs=100, verbose= 2, validation_data=(X_val[X_val.columns[1:]],y_val[y_val.columns[1:]]), callbacks=callbacks)
But when I load the model it returns an error:
model = tk.models.load_model("../input/model")
--------------------------------------------------------------------------- KeyError Traceback (most recent call
last) in
2 return
3
----> 4 model = tk.models.load_model("../input/model-custom", custom_objects={'__inference_dense_layer_call_fn_1126407':f1})
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/saving/save.py
in load_model(filepath, custom_objects, compile, options)
185 if isinstance(filepath, six.string_types):
186 loader_impl.parse_saved_model(filepath)
--> 187 return saved_model_load.load(filepath, compile, options)
188
189 raise IOError(
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/saving/saved_model/load.py
in load(path, compile, options)
119
120 model = tf_load.load_internal(
--> 121 path, options=options, loader_cls=KerasObjectLoader)
122
123 # pylint: disable=protected-access
/opt/conda/lib/python3.7/site-packages/tensorflow/python/saved_model/load.py
in load_internal(export_dir, tags, options, loader_cls)
631 try:
632 loader = loader_cls(object_graph_proto, saved_model_proto, export_dir,
--> 633 ckpt_options)
634 except errors.NotFoundError as err:
635 raise FileNotFoundError(
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/saving/saved_model/load.py
in init(self, *args, **kwargs)
192 self._models_to_reconstruct = []
193
--> 194 super(KerasObjectLoader, self).init(*args, **kwargs)
195
196 # Now that the node object has been fully loaded, and the checkpoint has
/opt/conda/lib/python3.7/site-packages/tensorflow/python/saved_model/load.py
in init(self, object_graph_proto, saved_model_proto, export_dir,
ckpt_options)
128 self._concrete_functions[name] = _WrapperFunction(concrete_function)
129
--> 130 self._load_all()
131 self._restore_checkpoint()
132
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/saving/saved_model/load.py
in _load_all(self)
216
217 # Load all other nodes and functions.
--> 218 super(KerasObjectLoader, self)._load_all()
219
220 # Finish setting up layers and models. See function docstring for more info.
/opt/conda/lib/python3.7/site-packages/tensorflow/python/saved_model/load.py
in _load_all(self)
139 def _load_all(self):
140 """Loads all nodes and functions from the SavedModel and their edges."""
--> 141 self._load_nodes()
142 self._load_edges()
143 # TODO(b/124045874): There are limitations with functions whose captures
/opt/conda/lib/python3.7/site-packages/tensorflow/python/saved_model/load.py
in _load_nodes(self)
281 # interface.
282 continue
--> 283 node, setter = self._recreate(proto, node_id)
284 nodes[node_id] = node
285 node_setters[node_id] = setter
/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/saving/saved_model/load.py
in _recreate(self, proto, node_id)
237 obj._handle_name = proto.variable.name + ':0' # pylint: disable=protected-access
238 else:
--> 239 obj, setter = super(KerasObjectLoader, self)._recreate(proto, node_id)
240 return obj, setter
241
/opt/conda/lib/python3.7/site-packages/tensorflow/python/saved_model/load.py
in _recreate(self, proto, node_id)
391 if kind not in factory:
392 raise ValueError("Unknown SavedObject type: %r" % kind)
--> 393 return factorykind
394
395 def _recreate_user_object(self, proto, node_id):
/opt/conda/lib/python3.7/site-packages/tensorflow/python/saved_model/load.py
in ()
380 lambda: self._recreate_user_object(proto.user_object, node_id)),
381 "asset": lambda: self._recreate_asset(proto.asset),
--> 382 "function": lambda: self._recreate_function(proto.function),
383 "bare_concrete_function": functools.partial(
384 self._recreate_bare_concrete_function,
/opt/conda/lib/python3.7/site-packages/tensorflow/python/saved_model/load.py
in _recreate_function(self, proto)
419 def _recreate_function(self, proto):
420 return function_deserialization.recreate_function(
--> 421 proto, self._concrete_functions), setattr
422
423 def _recreate_bare_concrete_function(self, proto):
/opt/conda/lib/python3.7/site-packages/tensorflow/python/saved_model/function_deserialization.py
in recreate_function(saved_function, concrete_functions)
259 concrete_function_objects = []
260 for concrete_function_name in saved_function.concrete_functions:
--> 261 concrete_function_objects.append(concrete_functions[concrete_function_name])
262
263 for cf in concrete_function_objects:
KeyError: '__inference_dense_layer_call_fn_1126407'
Can you please help me load the model correctly.. Thanks
I suspect that you have both keras and tensorflow installed separately; I have worked with tfa and never had problems with regard to such a loading matter;
In fact, here you import everything via tensorflow:
import tensorflow as tf
import tensorflow.keras as tk
import tensorflow_addons as tfa
But here you load the callbacks via plain keras:
from keras.callbacks import ModelCheckpoint, EarlyStopping
In order to first ensure that you do have a loading model problem situation, please make sure that every import is done via tensorflow.keras (I expect the problem to disappear altogether once you do this).
Replace
from keras.callbacks import ModelCheckpoint, EarlyStopping
with:
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
To sum up, retrain from scratch with the new imports (all from tensorflow.keras) and then check if the problem is reproduced.
Related
Question
GridSearchCV via KerasClassifier causes the error when Keras Normalization has been adapted to data. Without the adapted Normalization, it works. The reason why using Normalization is because it gave better result than simply divide by 255.0.
PicklingError: Could not pickle the task to send it to the workers.
Workaround
By setting n_jobs=1 not to multi-thread, it works but perhaps not much use to run single thread.
Environment
Python 3.9.13
TensorFlow version: 2.10.0
Eager execution is: True
Keras version: 2.10.0
sklearn version: 1.1.3
Code
import numpy as np
import tensorflow as tf
from keras.layers import (
Dense,
Flatten,
Normalization,
Conv2D,
MaxPooling2D,
)
from keras.models import (
Sequential
)
from scikeras.wrappers import (
KerasClassifier,
)
from sklearn.model_selection import (
GridSearchCV
)
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()
# max_value = float(np.max(x_train))
# x_train, x_test = x_train/max_value, x_test/max_value
input_shape = x_train[0].shape
number_of_classes = 10
# Data Normalization
normalization = Normalization(
name="norm",
input_shape=input_shape, # (32, 32, 3)
axis=-1 # Regard each pixel as a feature
)
normalization.adapt(x_train)
def create_model():
model = Sequential([
# Without the adapted Normalization layer, it works.
normalization,
Conv2D(
name="conv",
filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
activation='relu',
input_shape=input_shape
),
MaxPooling2D(
name="maxpool",
pool_size=(2, 2)
),
Flatten(),
Dense(
name="full",
units=100,
activation="relu"
),
Dense(
name="label",
units=number_of_classes,
activation="softmax"
)
])
model.compile(
loss=tf.keras.losses.sparse_categorical_crossentropy,
optimizer='adam',
metrics=['accuracy']
)
return model
model = KerasClassifier(model=create_model, verbose=2)
batch_size = [32]
epochs = [2, 3]
param_grid = dict(batch_size=batch_size, epochs=epochs)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)
grid_result = grid.fit(x_train, y_train)
Log
The above exception was the direct cause of the following exception:
PicklingError Traceback (most recent call last)
Cell In [28], line 7
4 param_grid = dict(batch_size=batch_size, epochs=epochs)
6 grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)
----> 7 grid_result = grid.fit(x_train, y_train)
File ~/venv/ml/lib/python3.9/site-packages/sklearn/model_selection/_search.py:875, in BaseSearchCV.fit(self, X, y, groups, **fit_params)
869 results = self._format_results(
870 all_candidate_params, n_splits, all_out, all_more_results
871 )
873 return results
--> 875 self._run_search(evaluate_candidates)
877 # multimetric is determined here because in the case of a callable
878 # self.scoring the return type is only known after calling
879 first_test_score = all_out[0]["test_scores"]
File ~/venv/ml/lib/python3.9/site-packages/sklearn/model_selection/_search.py:1379, in GridSearchCV._run_search(self, evaluate_candidates)
1377 def _run_search(self, evaluate_candidates):
1378 """Search all candidates in param_grid"""
-> 1379 evaluate_candidates(ParameterGrid(self.param_grid))
File ~/venv/ml/lib/python3.9/site-packages/sklearn/model_selection/_search.py:822, in BaseSearchCV.fit.<locals>.evaluate_candidates(candidate_params, cv, more_results)
814 if self.verbose > 0:
815 print(
816 "Fitting {0} folds for each of {1} candidates,"
817 " totalling {2} fits".format(
818 n_splits, n_candidates, n_candidates * n_splits
819 )
820 )
--> 822 out = parallel(
823 delayed(_fit_and_score)(
824 clone(base_estimator),
825 X,
826 y,
827 train=train,
828 test=test,
829 parameters=parameters,
830 split_progress=(split_idx, n_splits),
831 candidate_progress=(cand_idx, n_candidates),
832 **fit_and_score_kwargs,
833 )
834 for (cand_idx, parameters), (split_idx, (train, test)) in product(
835 enumerate(candidate_params), enumerate(cv.split(X, y, groups))
836 )
837 )
839 if len(out) < 1:
840 raise ValueError(
841 "No fits were performed. "
842 "Was the CV iterator empty? "
843 "Were there no candidates?"
844 )
File ~/venv/ml/lib/python3.9/site-packages/joblib/parallel.py:1098, in Parallel.__call__(self, iterable)
1095 self._iterating = False
1097 with self._backend.retrieval_context():
-> 1098 self.retrieve()
1099 # Make sure that we get a last message telling us we are done
1100 elapsed_time = time.time() - self._start_time
File ~/venv/ml/lib/python3.9/site-packages/joblib/parallel.py:975, in Parallel.retrieve(self)
973 try:
974 if getattr(self._backend, 'supports_timeout', False):
--> 975 self._output.extend(job.get(timeout=self.timeout))
976 else:
977 self._output.extend(job.get())
File ~/venv/ml/lib/python3.9/site-packages/joblib/_parallel_backends.py:567, in LokyBackend.wrap_future_result(future, timeout)
564 """Wrapper for Future.result to implement the same behaviour as
565 AsyncResults.get from multiprocessing."""
566 try:
--> 567 return future.result(timeout=timeout)
568 except CfTimeoutError as e:
569 raise TimeoutError from e
File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:446, in Future.result(self, timeout)
444 raise CancelledError()
445 elif self._state == FINISHED:
--> 446 return self.__get_result()
447 else:
448 raise TimeoutError()
File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:391, in Future.__get_result(self)
389 if self._exception:
390 try:
--> 391 raise self._exception
392 finally:
393 # Break a reference cycle with the exception in self._exception
394 self = None
PicklingError: Could not pickle the task to send it to the workers.
Research
Keras KerasClassifier gridsearch TypeError: can't pickle _thread.lock objects told Keras did not support pickle was the cause. However, as the code works if the adapted Normalization is not used, not relevant.
GPU can cause the issue but there is no GPU in my environment.
References
SciKeras Basic usage¶
How to Grid Search Hyperparameters for Deep Learning Models in Python with Keras
I get above error in colab TPU from the code below. Original model had epochs, steps_per_epoch and batch but removed that while debugging. Not sure what the issue is as I do not see a string.
None TPU version of code works. Most of the code is stock code with some modifications made. I tested the code to ensure images loaded properly.
import tensorflow as tf
from tensorflow.keras import backend as K
import os
import PIL
import csv
import shutil
import numpy as np
import sys
from PIL import Image
from tensorflow.keras import backend as K
import gc
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import layers
from tensorflow.keras import Model
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.layers import Dense, Activation, Flatten
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
tf.config.experimental_connect_to_cluster(resolver)
# This is the TPU initialization code that has to be at the beginning.
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.experimental.TPUStrategy(resolver)
list_ds = tf.data.Dataset.list_files(str(gcs_pattern))
# Reads an image from a file, decodes it into a dense tensor, and resizes it
# to a fixed shape.
def parse_image(filename):
parts = tf.strings.split(filename, os.sep)
label = parts[-2]
image = tf.io.read_file(filename)
image = tf.image.decode_jpeg(image)
image = tf.image.convert_image_dtype(image, tf.float32)
image = tf.image.resize(image, [400, 400])
return image, label
list_ds = list_ds.map(parse_image)
def create_model():
pre_trained_model = InceptionV3(input_shape = (400, 400,3), include_top = False, weights = 'imagenet')
input_tensor=None, input_shape=(1024, 1024,3))
for layer in pre_trained_model.layers:
if layer.name == 'mixed1':
break
layer.trainable = False
last_layer = pre_trained_model.get_layer('mixed7')
last_output = last_layer.output
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras import regularizers
x = Flatten()(last_output)
x = layers.Dense(1024, activation= 'relu')(x)
x = layers.Dropout(.2)(x)
x = layers.Dense(4, activation= 'softmax')(x)
modelin = Model(pre_trained_model.input, x)
return modelin
def get_callbacks(name_weights, patience_lr):
mcp_save = ModelCheckpoint(name_weights, save_best_only=True, monitor='val_acc', mode='max')
# reduce_lr_loss = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=patience_lr, verbose=1, epsilon=1e-4, mode='min')
return [mcp_save] #, reduce_lr_loss]
batch_size = 16 * strategy.num_replicas_in_sync
for i in range(5):
dataset = list_ds.shuffle(buffer_size = 2280)
dataset = dataset.cache()
val = dataset.skip(i*456).take(456).batch(batch_size, drop_remainder=True).prefetch(4)
train = dataset.skip(i*456+456).take(1824).concatenate(dataset.take(456*i)).batch(batch_size, drop_remainder=True).prefetch(15)
name_weights = "/content/drive/My Drive/Plant/final_model_fold_D512_I400_mix_1_7_" + str(i) + ".{epoch:02d}-{val_acc:.2f}.h5"
# callbacks = get_callbacks(name_weights = name_weights, patience_lr=10)
with strategy.scope():
modelinc = create_model()
modelinc.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['acc'])
modelinc.fit(
train,
epochs=5)
print(modelinc.evaluate(val))
K.clear_session()
del name_weights
del callbacks
gc.collect()
'''
-
Error:
Epoch 1/5 --------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call last)
<ipython-input-4-bbe01274450b> in <module>()
31 modelinc.fit(
32 train,
---> 33 epochs=5)
34
35
10 frames /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
64 def _method_wrapper(self, *args, **kwargs):
65 if not self._in_multi_worker_mode(): # pylint: disable=protected-access ---> 66 return method(self, *args,
**kwargs)
67
68 # Running inside `run_distribute_coordinator` already.
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
853 context.async_wait()
854 logs = tmp_logs # No error, now safe to assign to logs. --> 855 callbacks.on_train_batch_end(step, logs)
856 epoch_logs = copy.copy(logs)
857
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py in on_train_batch_end(self, batch, logs)
387 """
388 if self._should_call_train_batch_hooks: --> 389 logs = self._process_logs(logs)
390 self._call_batch_hook(ModeKeys.TRAIN, 'end', batch, logs=logs)
391
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py in _process_logs(self, logs)
263 """Turns tensors into numpy arrays or Python scalars."""
264 if logs: --> 265 return tf_utils.to_numpy_or_python_type(logs)
266 return {}
267
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/tf_utils.py in to_numpy_or_python_type(tensors)
521 return t # Don't turn ragged or sparse tensors to NumPy.
522 --> 523 return nest.map_structure(_to_single_numpy_or_python_type, tensors)
524
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/nest.py in map_structure(func, *structure, **kwargs)
615
616 return pack_sequence_as( --> 617 structure[0], [func(*x) for x in entries],
618 expand_composites=expand_composites)
619
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/nest.py in <listcomp>(.0)
615
616 return pack_sequence_as( --> 617 structure[0], [func(*x) for x in entries],
618 expand_composites=expand_composites)
619
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/tf_utils.py in _to_single_numpy_or_python_type(t)
517 def _to_single_numpy_or_python_type(t):
518 if isinstance(t, ops.Tensor): --> 519 x = t.numpy()
520 return x.item() if np.ndim(x) == 0 else x
521 return t # Don't turn ragged or sparse tensors to NumPy.
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in numpy(self)
959 """
960 # TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors. --> 961 maybe_arr = self._numpy() # pylint: disable=protected-access
962 return maybe_arr.copy() if isinstance(maybe_arr, np.ndarray) else maybe_arr
963
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in _numpy(self)
927 return self._numpy_internal()
928 except core._NotOkStatusException as e: --> 929 six.raise_from(core._status_to_exception(e.code, e.message), None)
930
931 #property
/usr/local/lib/python3.6/dist-packages/six.py in raise_from(value, from_value)
InvalidArgumentError: Unsupported data type for TPU: string, caused by output cond_8/Identity_1:0
I've written a model some time ago that uses a few custom layer definitions and has been trained using TF 1.12 and standalone Keras 2.2.4. I've updated the version of TF to 1.14 and switched over to tf.keras. Using a custom load function, my model builds, loads weights and generates predictions.
Now, I'm trying to convert my keras model to a TF Estimator that I could use for inference and I'm having all sorts of issues. I believe it stems from the get_config() method in my Lambda layers. I currently define them like this:
class NamedLambda(Lambda):
def __init__(self, name=None):
Lambda.__init__(self, self.fn, name=name)
#classmethod
def invoke(cls, args, **kw):
return cls(**kw)(args)
def __repr__(self):
return '%s(%s)' % (self.__class__.__name__, self.name)
class L2Normalize(NamedLambda):
def fn(self, x):
return K.l2_normalize(x, axis=-1)
When I check, the get_config method is working just fine:
custom_objects['l2_normalize'].get_config()
{'arguments': DictWrapper({}),
'dtype': 'float32',
'function': 'fn',
'function_type': 'function',
'module': 'grademachine.utils',
'name': 'l2_normalize',
'output_shape': None,
'output_shape_module': None,
'output_shape_type': 'raw',
'trainable': True}
Below is some example code and the traceback that has me stumped. Any help would be much appreciated.
Python version: 3.6.2
TensorFlow version: 1.14.0
Keras version: 2.2.4-tf
model = load_model(model_dir,
options_fn='model123_options',
weights_fn='model123_weights')
model
<tensorflow.python.keras.engine.training.Model at 0x7fe3d43d8e10>
est = tf.keras.estimator.model_to_estimator(keras_model=model)
I've also tried adding my custom layers as follows, which yields a slightly different traceback, but ultimately ends up in the same place. The traceback below is from the version with custom_objects defined:
# custom_layer_names is a list of names of each of the custom layers in the trained model
custom_objects = {l.name: l for l in model.layers if l.name in custom_layer_names}
est = tf.keras.estimator.model_to_estimator(keras_model=model,
custom_objects=custom_objects)
INFO:tensorflow:Using default config.
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpyujm6s99
INFO:tensorflow:Using the Keras model provided.
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-13-512a382c338c> in <module>()
13 est = tf.keras.estimator.model_to_estimator(keras_model=model,
14 model_dir='saved_estimator/',
---> 15 custom_objects=custom_objects)
~/anaconda2/envs/berttf114/lib/python3.6/site-packages/tensorflow/python/keras/estimator/__init__.py in model_to_estimator(keras_model, keras_model_path, custom_objects, model_dir, config)
71 custom_objects=custom_objects,
72 model_dir=model_dir,
---> 73 config=config)
74
75 # LINT.ThenChange(//tensorflow_estimator/python/estimator/keras.py)
~/anaconda2/envs/berttf114/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/keras.py in model_to_estimator(keras_model, keras_model_path, custom_objects, model_dir, config)
448 if keras_model._is_graph_network:
449 warm_start_path = _save_first_checkpoint(keras_model, custom_objects,
--> 450 config)
451 elif keras_model.built:
452 logging.warning('You are creating an Estimator from a Keras model manually '
~/anaconda2/envs/berttf114/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/keras.py in _save_first_checkpoint(keras_model, custom_objects, config)
316 training_util.create_global_step()
317 model = _clone_and_build_model(ModeKeys.TRAIN, keras_model,
--> 318 custom_objects)
319 # save to checkpoint
320 with session.Session(config=config.session_config) as sess:
~/anaconda2/envs/berttf114/lib/python3.6/site-packages/tensorflow_estimator/python/estimator/keras.py in _clone_and_build_model(mode, keras_model, custom_objects, features, labels)
199 compile_clone=compile_clone,
200 in_place_reset=(not keras_model._is_graph_network),
--> 201 optimizer_iterations=global_step)
202
203 return clone
~/anaconda2/envs/berttf114/lib/python3.6/site-packages/tensorflow/python/keras/models.py in clone_and_build_model(model, input_tensors, target_tensors, custom_objects, compile_clone, in_place_reset, optimizer_iterations, optimizer_config)
534 if custom_objects:
535 with CustomObjectScope(custom_objects):
--> 536 clone = clone_model(model, input_tensors=input_tensors)
537 else:
538 clone = clone_model(model, input_tensors=input_tensors)
~/anaconda2/envs/berttf114/lib/python3.6/site-packages/tensorflow/python/keras/models.py in clone_model(model, input_tensors, clone_function)
324 else:
325 return _clone_functional_model(
--> 326 model, input_tensors=input_tensors, layer_fn=clone_function)
327
328
~/anaconda2/envs/berttf114/lib/python3.6/site-packages/tensorflow/python/keras/models.py in _clone_functional_model(model, input_tensors, layer_fn)
152 # Get or create layer.
153 if layer not in layer_map:
--> 154 new_layer = layer_fn(layer)
155 layer_map[layer] = new_layer
156 layer = new_layer
~/anaconda2/envs/berttf114/lib/python3.6/site-packages/tensorflow/python/keras/models.py in _clone_layer(layer)
52
53 def _clone_layer(layer):
---> 54 return layer.__class__.from_config(layer.get_config())
55
56
~/repos/grademachine/grademachine/utils.py in from_config(cls, config, custom_objects)
850 config = config.copy()
851 function = cls._parse_function_from_config(
--> 852 config, custom_objects, 'function', 'module', 'function_type')
853
854 output_shape = cls._parse_function_from_config(
~/repos/grademachine/grademachine/utils.py in _parse_function_from_config(cls, config, custom_objects, func_attr_name, module_attr_name, func_type_attr_name)
898 config[func_attr_name],
899 custom_objects=custom_objects,
--> 900 printable_module_name='function in Lambda layer')
901 elif function_type == 'lambda':
902 # Unsafe deserialization from bytecode
~/anaconda2/envs/berttf114/lib/python3.6/site-packages/tensorflow/python/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
207 obj = _GLOBAL_CUSTOM_OBJECTS[object_name]
208 else:
--> 209 obj = module_objects.get(object_name)
210 if obj is None:
211 raise ValueError('Unknown ' + printable_module_name + ':' + object_name)
AttributeError: 'NoneType' object has no attribute 'get'
I am trying to save a model which uses USE from tf-hub as its embedding layer and has a few FFN stacked upon it. The model seems to work fine, but I am facing a problem in saving and loading the model.
disable_eager_execution()
embed = hub.Module(module_url)
def UniversalEmbedding(x):
return embed(tf.squeeze(tf.cast(x, tf.string)))
input_text = Input(shape=[], dtype=tf.string)
response_text = Input(shape=[], dtype=tf.string)
text_embedding = Lambda(UniversalEmbedding, output_shape=(512, ))(input_text)
response_embedding = Lambda(UniversalEmbedding, output_shape=(512, ))(response_text)
response_embedding = Dense(512, activation='relu')(response_embedding)
response_embedding = Dense(512, activation='relu')(response_embedding)
score = Dot(axes=1, normalize=True)([text_embedding, response_embedding])
pred = Dense(2, activation='softmax')(score)
text_encoder = Model(inputs=[input_text], outputs=text_embedding)
response_encoder = Model(inputs=[response_text], outputs=response_embedding)
model = Model(inputs=[input_text, response_text], outputs=pred)
The code above is how I built my model (its a dual encoder model with USE as its encoder).
I had to disable eager execution because USE seems to be not working in eager execution environment yet. If not, and if there is a workaround for that, I'd really appreciate any help for this too :)
The model is trained and saved via the following code :
with tf.compat.v1.Session() as session:
K.set_session(session)
session.run(tf.compat.v1.global_variables_initializer())
session.run(tf.compat.v1.tables_initializer())
history = model.fit_generator(generator=train_neg_sample_generator,
validation_data=val_neg_sample_generator, epochs=20,
callbacks=[checkpointer, earlystopper], verbose=0)
and the model is loaded with no error when the weights in the checkpoints (saved in hdf5 files) are loaded to the model defined in the code above. So the code below works fine, only because the architecture 'model' is already defined above.
with tf.compat.v1.Session() as session:
K.set_session(session)
session.run(tf.compat.v1.global_variables_initializer())
session.run(tf.compat.v1.tables_initializer())
model.load_weights('./saved_models/weights.03-0.29.hdf5')
tf.keras.models.save_model(model, 'test_model2.hdf5')
predicts = model.predict([["how are you?", "how are you?", 'hi', 'my two favorites in one pic!'], ["i'm fine", "what the heck", 'hi', 'same!']])
print(predicts)
print(np.argmax(predicts, axis=1))
Then I tried 2 things. First, I tried to save the architecture in json format, load the model architecture and then load the weights, but it did not work. Then I tried to save the whole model via keras.models.save_model, but it did not work either.
In both cases, they returned
AttributeError: module 'tensorflow' has no attribute 'placeholder'
How can I save/load the whole model (if not at once, loading architecture / weight separately is fine too) ?
Here is the whole error log
AttributeError Traceback (most recent call last)
<ipython-input-31-47468f2533ad> in <module>()
1 from keras.models import load_model
2
----> 3 model2 = load_model('testest.h5')
13 frames
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in load_wrapper(*args, **kwargs)
456 os.remove(tmp_filepath)
457 return res
--> 458 return load_function(*args, **kwargs)
459
460 return load_wrapper
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in load_model(filepath, custom_objects, compile)
548 if H5Dict.is_supported_type(filepath):
549 with H5Dict(filepath, mode='r') as h5dict:
--> 550 model = _deserialize_model(h5dict, custom_objects, compile)
551 elif hasattr(filepath, 'write') and callable(filepath.write):
552 def load_function(h5file):
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in _deserialize_model(h5dict, custom_objects, compile)
241 raise ValueError('No model found in config.')
242 model_config = json.loads(model_config.decode('utf-8'))
--> 243 model = model_from_config(model_config, custom_objects=custom_objects)
244 model_weights_group = h5dict['model_weights']
245
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in model_from_config(config, custom_objects)
591 '`Sequential.from_config(config)`?')
592 from ..layers import deserialize
--> 593 return deserialize(config, custom_objects=custom_objects)
594
595
/usr/local/lib/python3.6/dist-packages/keras/layers/__init__.py in deserialize(config, custom_objects)
166 module_objects=globs,
167 custom_objects=custom_objects,
--> 168 printable_module_name='layer')
/usr/local/lib/python3.6/dist-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
145 config['config'],
146 custom_objects=dict(list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 147 list(custom_objects.items())))
148 with CustomObjectScope(custom_objects):
149 return cls.from_config(config['config'])
/usr/local/lib/python3.6/dist-packages/keras/engine/network.py in from_config(cls, config, custom_objects)
1041 # First, we create all layers and enqueue nodes to be processed
1042 for layer_data in config['layers']:
-> 1043 process_layer(layer_data)
1044
1045 # Then we process nodes in order of layer depth.
/usr/local/lib/python3.6/dist-packages/keras/engine/network.py in process_layer(layer_data)
1027
1028 layer = deserialize_layer(layer_data,
-> 1029 custom_objects=custom_objects)
1030 created_layers[layer_name] = layer
1031
/usr/local/lib/python3.6/dist-packages/keras/layers/__init__.py in deserialize(config, custom_objects)
166 module_objects=globs,
167 custom_objects=custom_objects,
--> 168 printable_module_name='layer')
/usr/local/lib/python3.6/dist-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
147 list(custom_objects.items())))
148 with CustomObjectScope(custom_objects):
--> 149 return cls.from_config(config['config'])
150 else:
151 # Then `cls` may be a function returning a class.
/usr/local/lib/python3.6/dist-packages/keras/engine/base_layer.py in from_config(cls, config)
1101 A layer instance.
1102 """
-> 1103 return cls(**config)
1104
1105 def count_params(self):
/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name + '` call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
/usr/local/lib/python3.6/dist-packages/keras/engine/input_layer.py in __init__(self, input_shape, batch_size, batch_input_shape, dtype, input_tensor, sparse, name)
85 dtype=dtype,
86 sparse=self.sparse,
---> 87 name=self.name)
88 else:
89 self.is_placeholder = False
/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py in placeholder(shape, ndim, dtype, sparse, name)
539 x = tf.sparse_placeholder(dtype, shape=shape, name=name)
540 else:
--> 541 x = tf.placeholder(dtype, shape=shape, name=name)
542 x._keras_shape = shape
543 x._uses_learning_phase = False
AttributeError: module 'tensorflow' has no attribute 'placeholder'
Provide the whole error log, not just a part of it.
If it's really error because of saving then what about model.save('model.h5')?
Not using the module from tf.keras.models but call a methomd from the Model class itself.
But why these code?
with tf.compat.v1.Session() as session:
K.set_session(session)
session.run(tf.compat.v1.global_variables_initializer())
session.run(tf.compat.v1.tables_initializer())
I believe you could juat call model.fit right away, and your tf is version2 right? Why call compat.v1?
Tensorflow 2 doesn't have Placeholder so I think it's maybe about this.
Worked fine with tensorflow version 1.15
Looking forward for tf-hub to become completely compatible with tensorflow 2.0 and keras...
Keras version 2.2.4,
tensorflow version 1.13.1,
I'm using colab notebooks
I'm trying to make a custom initializer and save the model using model.save() but when I load the model again I get the following error:
TypeError: myInit() missing 1 required positional argument: 'input_shape'
I have the following code:
import numpy as np
import tensorflow as tf
import keras
from google.colab import drive
from keras.models import Sequential, load_model
from keras.layers import Dense, Dropout, Flatten, Lambda, Reshape, Activation
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras import backend as K
K.set_image_data_format('channels_first')
K.backend()
# the output should be 'tensorflow'
'tensorflow'
def myInit( input_shape, dtype=None):
weights = np.full( input_shape, 2019 )
return K.variable( weights, dtype=dtype )
This initializer is given an input_shape and returns a keras tensor like in the docs: https://keras.io/initializers/
model = Sequential()
model.add(
Dense( 40, input_shape=(784,) )
)
model.add(
Dense( 30, kernel_initializer=myInit )
)
model.add(
Dense( 5 )
)
model.build()
The weights are initialized correctly because when I call model.layers[1].get_weights() I get an array full of 2019.
I save the model using model.save:
model.save(somepath)
In a different notebook I then call
model = load_model(somepath,
custom_objects={
'tf' : tf,
'myInit' : myInit
}
)
In this notebook myInit and all the imports are defined as well.
When I call load_model I get the following error:
TypeError: myInit() missing 1 required positional argument: 'input_shape'
So it seems when the model is loaded, the input_shape is not passed to myInit. Does anyone have any idea?
Full trace:
TypeError Traceback (most recent call last)
<ipython-input-25-544d137de03f> in <module>()
2 custom_objects={
3 'tf' : tf,
----> 4 'myInit' : myInit
5 }
6 )
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in load_model(filepath, custom_objects, compile)
417 f = h5dict(filepath, 'r')
418 try:
--> 419 model = _deserialize_model(f, custom_objects, compile)
420 finally:
421 if opened_new_file:
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in _deserialize_model(f, custom_objects, compile)
223 raise ValueError('No model found in config.')
224 model_config = json.loads(model_config.decode('utf-8'))
--> 225 model = model_from_config(model_config, custom_objects=custom_objects)
226 model_weights_group = f['model_weights']
227
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in model_from_config(config, custom_objects)
456 '`Sequential.from_config(config)`?')
457 from ..layers import deserialize
--> 458 return deserialize(config, custom_objects=custom_objects)
459
460
/usr/local/lib/python3.6/dist-packages/keras/layers/__init__.py in deserialize(config, custom_objects)
53 module_objects=globs,
54 custom_objects=custom_objects,
---> 55 printable_module_name='layer')
/usr/local/lib/python3.6/dist-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
143 config['config'],
144 custom_objects=dict(list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 145 list(custom_objects.items())))
146 with CustomObjectScope(custom_objects):
147 return cls.from_config(config['config'])
/usr/local/lib/python3.6/dist-packages/keras/engine/sequential.py in from_config(cls, config, custom_objects)
298 for conf in layer_configs:
299 layer = layer_module.deserialize(conf,
--> 300 custom_objects=custom_objects)
301 model.add(layer)
302 if not model.inputs and build_input_shape:
/usr/local/lib/python3.6/dist-packages/keras/layers/__init__.py in deserialize(config, custom_objects)
53 module_objects=globs,
54 custom_objects=custom_objects,
---> 55 printable_module_name='layer')
/usr/local/lib/python3.6/dist-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
145 list(custom_objects.items())))
146 with CustomObjectScope(custom_objects):
--> 147 return cls.from_config(config['config'])
148 else:
149 # Then `cls` may be a function returning a class.
/usr/local/lib/python3.6/dist-packages/keras/engine/base_layer.py in from_config(cls, config)
1107 A layer instance.
1108 """
-> 1109 return cls(**config)
1110
1111 def count_params(self):
/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name + '` call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
/usr/local/lib/python3.6/dist-packages/keras/layers/core.py in __init__(self, units, activation, use_bias, kernel_initializer, bias_initializer, kernel_regularizer, bias_regularizer, activity_regularizer, kernel_constraint, bias_constraint, **kwargs)
846 self.activation = activations.get(activation)
847 self.use_bias = use_bias
--> 848 self.kernel_initializer = initializers.get(kernel_initializer)
849 self.bias_initializer = initializers.get(bias_initializer)
850 self.kernel_regularizer = regularizers.get(kernel_regularizer)
/usr/local/lib/python3.6/dist-packages/keras/initializers.py in get(identifier)
509 elif isinstance(identifier, six.string_types):
510 config = {'class_name': str(identifier), 'config': {}}
--> 511 return deserialize(config)
512 elif callable(identifier):
513 return identifier
/usr/local/lib/python3.6/dist-packages/keras/initializers.py in deserialize(config, custom_objects)
501 module_objects=globals(),
502 custom_objects=custom_objects,
--> 503 printable_module_name='initializer')
504
505
/usr/local/lib/python3.6/dist-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
152 custom_objects = custom_objects or {}
153 with CustomObjectScope(custom_objects):
--> 154 return cls(**config['config'])
155 elif isinstance(identifier, six.string_types):
156 function_name = identifier
TypeError: myInit() missing 1 required positional argument: 'input_shape'
Note I also posted this on https://github.com/keras-team/keras/issues/12452 but I figured this would be a better place for this.
After viewing the source code I got the following working code, which should be the proper way to define an initializer (especially when loading a model with load_model):
import numpy as np
import tensorflow as tf
import keras
from google.colab import drive
from keras.models import Sequential, load_model
from keras.layers import Dense
from keras import backend as K
from keras.initializers import Initializer
K.backend()
# the output should be 'tensorflow'
class myInit( Initializer ):
def __init__(self, myParameter):
self.myParameter = myParameter
def __call__(self, shape, dtype=None):
# array filled entirely with 'myParameter'
weights = np.full( shape, self.myParameter )
return K.variable( weights, dtype=dtype )
def get_config(self):
return {
'myParameter' : self.myParameter
}
Building the model:
model = Sequential()
model.add(
Dense( 2, input_shape=(784,) )
)
model.add(
Dense( 3, kernel_initializer=myInit( 2019 ) )
)
model.add(
Dense( 5 )
)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
Save the model:
model.save( somepath )
Now we can load the model in a different notebook. The imports from the other notebook should be imported here as well and myInit should also be defined in this notebook.
model = load_model( somepath,
custom_objects={
'tf' : tf,
'myInit' : myInit
}
)