I get above error in colab TPU from the code below. Original model had epochs, steps_per_epoch and batch but removed that while debugging. Not sure what the issue is as I do not see a string.
None TPU version of code works. Most of the code is stock code with some modifications made. I tested the code to ensure images loaded properly.
import tensorflow as tf
from tensorflow.keras import backend as K
import os
import PIL
import csv
import shutil
import numpy as np
import sys
from PIL import Image
from tensorflow.keras import backend as K
import gc
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import layers
from tensorflow.keras import Model
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.layers import Dense, Activation, Flatten
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
tf.config.experimental_connect_to_cluster(resolver)
# This is the TPU initialization code that has to be at the beginning.
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.experimental.TPUStrategy(resolver)
list_ds = tf.data.Dataset.list_files(str(gcs_pattern))
# Reads an image from a file, decodes it into a dense tensor, and resizes it
# to a fixed shape.
def parse_image(filename):
parts = tf.strings.split(filename, os.sep)
label = parts[-2]
image = tf.io.read_file(filename)
image = tf.image.decode_jpeg(image)
image = tf.image.convert_image_dtype(image, tf.float32)
image = tf.image.resize(image, [400, 400])
return image, label
list_ds = list_ds.map(parse_image)
def create_model():
pre_trained_model = InceptionV3(input_shape = (400, 400,3), include_top = False, weights = 'imagenet')
input_tensor=None, input_shape=(1024, 1024,3))
for layer in pre_trained_model.layers:
if layer.name == 'mixed1':
break
layer.trainable = False
last_layer = pre_trained_model.get_layer('mixed7')
last_output = last_layer.output
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras import regularizers
x = Flatten()(last_output)
x = layers.Dense(1024, activation= 'relu')(x)
x = layers.Dropout(.2)(x)
x = layers.Dense(4, activation= 'softmax')(x)
modelin = Model(pre_trained_model.input, x)
return modelin
def get_callbacks(name_weights, patience_lr):
mcp_save = ModelCheckpoint(name_weights, save_best_only=True, monitor='val_acc', mode='max')
# reduce_lr_loss = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=patience_lr, verbose=1, epsilon=1e-4, mode='min')
return [mcp_save] #, reduce_lr_loss]
batch_size = 16 * strategy.num_replicas_in_sync
for i in range(5):
dataset = list_ds.shuffle(buffer_size = 2280)
dataset = dataset.cache()
val = dataset.skip(i*456).take(456).batch(batch_size, drop_remainder=True).prefetch(4)
train = dataset.skip(i*456+456).take(1824).concatenate(dataset.take(456*i)).batch(batch_size, drop_remainder=True).prefetch(15)
name_weights = "/content/drive/My Drive/Plant/final_model_fold_D512_I400_mix_1_7_" + str(i) + ".{epoch:02d}-{val_acc:.2f}.h5"
# callbacks = get_callbacks(name_weights = name_weights, patience_lr=10)
with strategy.scope():
modelinc = create_model()
modelinc.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['acc'])
modelinc.fit(
train,
epochs=5)
print(modelinc.evaluate(val))
K.clear_session()
del name_weights
del callbacks
gc.collect()
'''
-
Error:
Epoch 1/5 --------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call last)
<ipython-input-4-bbe01274450b> in <module>()
31 modelinc.fit(
32 train,
---> 33 epochs=5)
34
35
10 frames /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
64 def _method_wrapper(self, *args, **kwargs):
65 if not self._in_multi_worker_mode(): # pylint: disable=protected-access ---> 66 return method(self, *args,
**kwargs)
67
68 # Running inside `run_distribute_coordinator` already.
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
853 context.async_wait()
854 logs = tmp_logs # No error, now safe to assign to logs. --> 855 callbacks.on_train_batch_end(step, logs)
856 epoch_logs = copy.copy(logs)
857
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py in on_train_batch_end(self, batch, logs)
387 """
388 if self._should_call_train_batch_hooks: --> 389 logs = self._process_logs(logs)
390 self._call_batch_hook(ModeKeys.TRAIN, 'end', batch, logs=logs)
391
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py in _process_logs(self, logs)
263 """Turns tensors into numpy arrays or Python scalars."""
264 if logs: --> 265 return tf_utils.to_numpy_or_python_type(logs)
266 return {}
267
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/tf_utils.py in to_numpy_or_python_type(tensors)
521 return t # Don't turn ragged or sparse tensors to NumPy.
522 --> 523 return nest.map_structure(_to_single_numpy_or_python_type, tensors)
524
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/nest.py in map_structure(func, *structure, **kwargs)
615
616 return pack_sequence_as( --> 617 structure[0], [func(*x) for x in entries],
618 expand_composites=expand_composites)
619
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/nest.py in <listcomp>(.0)
615
616 return pack_sequence_as( --> 617 structure[0], [func(*x) for x in entries],
618 expand_composites=expand_composites)
619
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/tf_utils.py in _to_single_numpy_or_python_type(t)
517 def _to_single_numpy_or_python_type(t):
518 if isinstance(t, ops.Tensor): --> 519 x = t.numpy()
520 return x.item() if np.ndim(x) == 0 else x
521 return t # Don't turn ragged or sparse tensors to NumPy.
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in numpy(self)
959 """
960 # TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors. --> 961 maybe_arr = self._numpy() # pylint: disable=protected-access
962 return maybe_arr.copy() if isinstance(maybe_arr, np.ndarray) else maybe_arr
963
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in _numpy(self)
927 return self._numpy_internal()
928 except core._NotOkStatusException as e: --> 929 six.raise_from(core._status_to_exception(e.code, e.message), None)
930
931 #property
/usr/local/lib/python3.6/dist-packages/six.py in raise_from(value, from_value)
InvalidArgumentError: Unsupported data type for TPU: string, caused by output cond_8/Identity_1:0
Related
Question
GridSearchCV via KerasClassifier causes the error when Keras Normalization has been adapted to data. Without the adapted Normalization, it works. The reason why using Normalization is because it gave better result than simply divide by 255.0.
PicklingError: Could not pickle the task to send it to the workers.
Workaround
By setting n_jobs=1 not to multi-thread, it works but perhaps not much use to run single thread.
Environment
Python 3.9.13
TensorFlow version: 2.10.0
Eager execution is: True
Keras version: 2.10.0
sklearn version: 1.1.3
Code
import numpy as np
import tensorflow as tf
from keras.layers import (
Dense,
Flatten,
Normalization,
Conv2D,
MaxPooling2D,
)
from keras.models import (
Sequential
)
from scikeras.wrappers import (
KerasClassifier,
)
from sklearn.model_selection import (
GridSearchCV
)
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()
# max_value = float(np.max(x_train))
# x_train, x_test = x_train/max_value, x_test/max_value
input_shape = x_train[0].shape
number_of_classes = 10
# Data Normalization
normalization = Normalization(
name="norm",
input_shape=input_shape, # (32, 32, 3)
axis=-1 # Regard each pixel as a feature
)
normalization.adapt(x_train)
def create_model():
model = Sequential([
# Without the adapted Normalization layer, it works.
normalization,
Conv2D(
name="conv",
filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
activation='relu',
input_shape=input_shape
),
MaxPooling2D(
name="maxpool",
pool_size=(2, 2)
),
Flatten(),
Dense(
name="full",
units=100,
activation="relu"
),
Dense(
name="label",
units=number_of_classes,
activation="softmax"
)
])
model.compile(
loss=tf.keras.losses.sparse_categorical_crossentropy,
optimizer='adam',
metrics=['accuracy']
)
return model
model = KerasClassifier(model=create_model, verbose=2)
batch_size = [32]
epochs = [2, 3]
param_grid = dict(batch_size=batch_size, epochs=epochs)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)
grid_result = grid.fit(x_train, y_train)
Log
The above exception was the direct cause of the following exception:
PicklingError Traceback (most recent call last)
Cell In [28], line 7
4 param_grid = dict(batch_size=batch_size, epochs=epochs)
6 grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)
----> 7 grid_result = grid.fit(x_train, y_train)
File ~/venv/ml/lib/python3.9/site-packages/sklearn/model_selection/_search.py:875, in BaseSearchCV.fit(self, X, y, groups, **fit_params)
869 results = self._format_results(
870 all_candidate_params, n_splits, all_out, all_more_results
871 )
873 return results
--> 875 self._run_search(evaluate_candidates)
877 # multimetric is determined here because in the case of a callable
878 # self.scoring the return type is only known after calling
879 first_test_score = all_out[0]["test_scores"]
File ~/venv/ml/lib/python3.9/site-packages/sklearn/model_selection/_search.py:1379, in GridSearchCV._run_search(self, evaluate_candidates)
1377 def _run_search(self, evaluate_candidates):
1378 """Search all candidates in param_grid"""
-> 1379 evaluate_candidates(ParameterGrid(self.param_grid))
File ~/venv/ml/lib/python3.9/site-packages/sklearn/model_selection/_search.py:822, in BaseSearchCV.fit.<locals>.evaluate_candidates(candidate_params, cv, more_results)
814 if self.verbose > 0:
815 print(
816 "Fitting {0} folds for each of {1} candidates,"
817 " totalling {2} fits".format(
818 n_splits, n_candidates, n_candidates * n_splits
819 )
820 )
--> 822 out = parallel(
823 delayed(_fit_and_score)(
824 clone(base_estimator),
825 X,
826 y,
827 train=train,
828 test=test,
829 parameters=parameters,
830 split_progress=(split_idx, n_splits),
831 candidate_progress=(cand_idx, n_candidates),
832 **fit_and_score_kwargs,
833 )
834 for (cand_idx, parameters), (split_idx, (train, test)) in product(
835 enumerate(candidate_params), enumerate(cv.split(X, y, groups))
836 )
837 )
839 if len(out) < 1:
840 raise ValueError(
841 "No fits were performed. "
842 "Was the CV iterator empty? "
843 "Were there no candidates?"
844 )
File ~/venv/ml/lib/python3.9/site-packages/joblib/parallel.py:1098, in Parallel.__call__(self, iterable)
1095 self._iterating = False
1097 with self._backend.retrieval_context():
-> 1098 self.retrieve()
1099 # Make sure that we get a last message telling us we are done
1100 elapsed_time = time.time() - self._start_time
File ~/venv/ml/lib/python3.9/site-packages/joblib/parallel.py:975, in Parallel.retrieve(self)
973 try:
974 if getattr(self._backend, 'supports_timeout', False):
--> 975 self._output.extend(job.get(timeout=self.timeout))
976 else:
977 self._output.extend(job.get())
File ~/venv/ml/lib/python3.9/site-packages/joblib/_parallel_backends.py:567, in LokyBackend.wrap_future_result(future, timeout)
564 """Wrapper for Future.result to implement the same behaviour as
565 AsyncResults.get from multiprocessing."""
566 try:
--> 567 return future.result(timeout=timeout)
568 except CfTimeoutError as e:
569 raise TimeoutError from e
File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:446, in Future.result(self, timeout)
444 raise CancelledError()
445 elif self._state == FINISHED:
--> 446 return self.__get_result()
447 else:
448 raise TimeoutError()
File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:391, in Future.__get_result(self)
389 if self._exception:
390 try:
--> 391 raise self._exception
392 finally:
393 # Break a reference cycle with the exception in self._exception
394 self = None
PicklingError: Could not pickle the task to send it to the workers.
Research
Keras KerasClassifier gridsearch TypeError: can't pickle _thread.lock objects told Keras did not support pickle was the cause. However, as the code works if the adapted Normalization is not used, not relevant.
GPU can cause the issue but there is no GPU in my environment.
References
SciKeras Basic usage¶
How to Grid Search Hyperparameters for Deep Learning Models in Python with Keras
I am fairly new to ML, I am trying to fit some data on my NB-classifier.
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import LinearSVC
# Naïve Bayes:
text_clf_nb = Pipeline([('tfidf', TfidfVectorizer()),
('clf', MultinomialNB()),
])
# Linear SVC:
text_clf_lsvc = Pipeline([('tfidf', TfidfVectorizer()),
('clf', LinearSVC()),
])
Code for fitting the data:
text_clf_nb.fit(X_train, y_train)
The shape of my training & test data is
X_train.shape, X_test.shape, y_train.shape, y_test.shape : ((169, 1), (84,), (169, 1), (84,))
But keep getting :'numpy.ndarray' object has no attribute 'lower'
Here is full trace to the error:
AttributeError Traceback (most recent call last)
<ipython-input-57-139757126594> in <module>
----> 1 text_clf_nb.fit(X_train, y_train)
~\miniconda3\envs\nlp_course\lib\site-packages\sklearn\pipeline.py in fit(self, X, y, **fit_params)
263 This estimator
264 """
--> 265 Xt, fit_params = self._fit(X, y, **fit_params)
266 if self._final_estimator is not None:
267 self._final_estimator.fit(Xt, y, **fit_params)
~\miniconda3\envs\nlp_course\lib\site-packages\sklearn\pipeline.py in _fit(self, X, y, **fit_params)
228 Xt, fitted_transformer = fit_transform_one_cached(
229 cloned_transformer, Xt, y, None,
--> 230 **fit_params_steps[name])
231 # Replace the transformer of the step with the fitted
232 # transformer. This is necessary when loading the transformer
~\miniconda3\envs\nlp_course\lib\site-packages\sklearn\externals\joblib\memory.py in __call__(self, *args, **kwargs)
340
341 def __call__(self, *args, **kwargs):
--> 342 return self.func(*args, **kwargs)
343
344 def call_and_shelve(self, *args, **kwargs):
You have checked the shape of arrays, but have you tried something like:
data = vectorizer.fit_transform(array.ravel())
This should do the trick for you
Keras version 2.2.4,
tensorflow version 1.13.1,
I'm using colab notebooks
I'm trying to make a custom initializer and save the model using model.save() but when I load the model again I get the following error:
TypeError: myInit() missing 1 required positional argument: 'input_shape'
I have the following code:
import numpy as np
import tensorflow as tf
import keras
from google.colab import drive
from keras.models import Sequential, load_model
from keras.layers import Dense, Dropout, Flatten, Lambda, Reshape, Activation
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras import backend as K
K.set_image_data_format('channels_first')
K.backend()
# the output should be 'tensorflow'
'tensorflow'
def myInit( input_shape, dtype=None):
weights = np.full( input_shape, 2019 )
return K.variable( weights, dtype=dtype )
This initializer is given an input_shape and returns a keras tensor like in the docs: https://keras.io/initializers/
model = Sequential()
model.add(
Dense( 40, input_shape=(784,) )
)
model.add(
Dense( 30, kernel_initializer=myInit )
)
model.add(
Dense( 5 )
)
model.build()
The weights are initialized correctly because when I call model.layers[1].get_weights() I get an array full of 2019.
I save the model using model.save:
model.save(somepath)
In a different notebook I then call
model = load_model(somepath,
custom_objects={
'tf' : tf,
'myInit' : myInit
}
)
In this notebook myInit and all the imports are defined as well.
When I call load_model I get the following error:
TypeError: myInit() missing 1 required positional argument: 'input_shape'
So it seems when the model is loaded, the input_shape is not passed to myInit. Does anyone have any idea?
Full trace:
TypeError Traceback (most recent call last)
<ipython-input-25-544d137de03f> in <module>()
2 custom_objects={
3 'tf' : tf,
----> 4 'myInit' : myInit
5 }
6 )
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in load_model(filepath, custom_objects, compile)
417 f = h5dict(filepath, 'r')
418 try:
--> 419 model = _deserialize_model(f, custom_objects, compile)
420 finally:
421 if opened_new_file:
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in _deserialize_model(f, custom_objects, compile)
223 raise ValueError('No model found in config.')
224 model_config = json.loads(model_config.decode('utf-8'))
--> 225 model = model_from_config(model_config, custom_objects=custom_objects)
226 model_weights_group = f['model_weights']
227
/usr/local/lib/python3.6/dist-packages/keras/engine/saving.py in model_from_config(config, custom_objects)
456 '`Sequential.from_config(config)`?')
457 from ..layers import deserialize
--> 458 return deserialize(config, custom_objects=custom_objects)
459
460
/usr/local/lib/python3.6/dist-packages/keras/layers/__init__.py in deserialize(config, custom_objects)
53 module_objects=globs,
54 custom_objects=custom_objects,
---> 55 printable_module_name='layer')
/usr/local/lib/python3.6/dist-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
143 config['config'],
144 custom_objects=dict(list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 145 list(custom_objects.items())))
146 with CustomObjectScope(custom_objects):
147 return cls.from_config(config['config'])
/usr/local/lib/python3.6/dist-packages/keras/engine/sequential.py in from_config(cls, config, custom_objects)
298 for conf in layer_configs:
299 layer = layer_module.deserialize(conf,
--> 300 custom_objects=custom_objects)
301 model.add(layer)
302 if not model.inputs and build_input_shape:
/usr/local/lib/python3.6/dist-packages/keras/layers/__init__.py in deserialize(config, custom_objects)
53 module_objects=globs,
54 custom_objects=custom_objects,
---> 55 printable_module_name='layer')
/usr/local/lib/python3.6/dist-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
145 list(custom_objects.items())))
146 with CustomObjectScope(custom_objects):
--> 147 return cls.from_config(config['config'])
148 else:
149 # Then `cls` may be a function returning a class.
/usr/local/lib/python3.6/dist-packages/keras/engine/base_layer.py in from_config(cls, config)
1107 A layer instance.
1108 """
-> 1109 return cls(**config)
1110
1111 def count_params(self):
/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name + '` call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
/usr/local/lib/python3.6/dist-packages/keras/layers/core.py in __init__(self, units, activation, use_bias, kernel_initializer, bias_initializer, kernel_regularizer, bias_regularizer, activity_regularizer, kernel_constraint, bias_constraint, **kwargs)
846 self.activation = activations.get(activation)
847 self.use_bias = use_bias
--> 848 self.kernel_initializer = initializers.get(kernel_initializer)
849 self.bias_initializer = initializers.get(bias_initializer)
850 self.kernel_regularizer = regularizers.get(kernel_regularizer)
/usr/local/lib/python3.6/dist-packages/keras/initializers.py in get(identifier)
509 elif isinstance(identifier, six.string_types):
510 config = {'class_name': str(identifier), 'config': {}}
--> 511 return deserialize(config)
512 elif callable(identifier):
513 return identifier
/usr/local/lib/python3.6/dist-packages/keras/initializers.py in deserialize(config, custom_objects)
501 module_objects=globals(),
502 custom_objects=custom_objects,
--> 503 printable_module_name='initializer')
504
505
/usr/local/lib/python3.6/dist-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
152 custom_objects = custom_objects or {}
153 with CustomObjectScope(custom_objects):
--> 154 return cls(**config['config'])
155 elif isinstance(identifier, six.string_types):
156 function_name = identifier
TypeError: myInit() missing 1 required positional argument: 'input_shape'
Note I also posted this on https://github.com/keras-team/keras/issues/12452 but I figured this would be a better place for this.
After viewing the source code I got the following working code, which should be the proper way to define an initializer (especially when loading a model with load_model):
import numpy as np
import tensorflow as tf
import keras
from google.colab import drive
from keras.models import Sequential, load_model
from keras.layers import Dense
from keras import backend as K
from keras.initializers import Initializer
K.backend()
# the output should be 'tensorflow'
class myInit( Initializer ):
def __init__(self, myParameter):
self.myParameter = myParameter
def __call__(self, shape, dtype=None):
# array filled entirely with 'myParameter'
weights = np.full( shape, self.myParameter )
return K.variable( weights, dtype=dtype )
def get_config(self):
return {
'myParameter' : self.myParameter
}
Building the model:
model = Sequential()
model.add(
Dense( 2, input_shape=(784,) )
)
model.add(
Dense( 3, kernel_initializer=myInit( 2019 ) )
)
model.add(
Dense( 5 )
)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
Save the model:
model.save( somepath )
Now we can load the model in a different notebook. The imports from the other notebook should be imported here as well and myInit should also be defined in this notebook.
model = load_model( somepath,
custom_objects={
'tf' : tf,
'myInit' : myInit
}
)
I try to build up a document classification model with attention using keras(part of model in paper Hierarchical Attention Networks for Document Classification). The following codes are the test codes. I create a birnn and a custom attention layer refer to https://github.com/person-lee/LSTM_ATTENTION_CLASSIFY/blob/master/utils.py and https://github.com/richliao/textClassifier/blob/master/textClassifierHATT.py. But I got an error(see details below).
The codes are:
from keras.models import Model
from keras.layers import Input
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import GRU
from keras.layers.wrappers import Bidirectional, TimeDistributed
from keras.layers.core import Dropout, Dense, Lambda, Masking
from keras.layers import merge
from keras.engine.topology import Layer
from keras import backend as K
from keras import initializers
import keras
class AttentionLayer(Layer):
'''
Attention layer.
'''
def __init__(self, init='glorot_uniform', **kwargs):
super(AttentionLayer, self).__init__(**kwargs)
self.supports_masking = True
self.init = initializers.get(init)
def build(self, input_shape):
input_dim = input_shape[-1]
self.Uw = self.init((input_dim, ))
self.trainable_weights = [self.Uw]
super(AttentionLayer, self).build(input_shape)
def compute_mask(self, input, mask):
return mask
def call(self, x, mask=None):
eij = K.tanh(K.squeeze(K.dot(x, K.expand_dims(self.Uw)), axis=-1))
ai = K.exp(eij)
weights = ai/K.expand_dims(K.sum(ai, axis=1),1)
weighted_input = x*K.expand_dims(weights,2)
return K.sum(weighted_input, axis=1)
def get_output_shape_for(self, input_shape):
newShape = list(input_shape)
newShape[-1] = 1
return tuple(newShape)
sentence_input = Input(shape=(None,5))
# embedded_sequences = embedding_layer(sentence_input)
l_lstm = Bidirectional(GRU(10, return_sequences=True),merge_mode='concat')(sentence_input)
# l_dense = TimeDistributed(Dense(200))(l_lstm)
l_att = AttentionLayer()(l_lstm)
cls = Dense(10, activation='softmax')(l_att)
sentEncoder = Model(sentence_input, cls)
sentEncoder.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
import numpy as np
x_train = np.array([[1,2,3,4,5],
[1,2,3,4,5],
[1,2,3,4,5],
[1,2,3,4,5],
[1,2,3,4,5],
[1,2,3,4,5],
[1,2,3,4,5],
[1,2,3,4,5],
[1,2,3,4,5],
[1,2,3,4,5]])
y_train = np.array([1,2,3,4,5,6,7,8,9,0])
y_train = keras.utils.to_categorical(y_train, 10)
x_train = np.expand_dims(x_train,0)
y_train = np.expand_dims(y_train,0)
sentEncoder.fit(x=x_train,y=y_train,validation_split=0.1)
And got the following errors:
AttributeError Traceback (most recent call last)
<ipython-input-13-3f6bb30d8618> in <module>()
----> 1 sentEncoder.fit(x=x_train,y=y_train,validation_split=0.1)
~/.conda/envs/21/lib/python3.6/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1011 else:
1012 ins = x + y + sample_weights
-> 1013 self._make_train_function()
1014 f = self.train_function
1015
~/.conda/envs/21/lib/python3.6/site-packages/keras/engine/training.py in _make_train_function(self)
495 training_updates = self.optimizer.get_updates(
496 params=self._collected_trainable_weights,
--> 497 loss=self.total_loss)
498 updates = (self.updates +
499 training_updates +
~/.conda/envs/21/lib/python3.6/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name +
90 '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
~/.conda/envs/21/lib/python3.6/site-packages/keras/optimizers.py in get_updates(self, loss, params)
262 new_p = p.constraint(new_p)
263
--> 264 self.updates.append(K.update(p, new_p))
265 return self.updates
266
~/.conda/envs/21/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in update(x, new_x)
968 The variable `x` updated.
969 """
--> 970 return tf.assign(x, new_x)
971
972
~/.conda/envs/21/lib/python3.6/site-packages/tensorflow/python/ops/state_ops.py in assign(ref, value, validate_shape, use_locking, name)
282 ref, value, use_locking=use_locking, name=name,
283 validate_shape=validate_shape)
--> 284 return ref.assign(value, name=name)
285
286
AttributeError: 'Tensor' object has no attribute 'assign'
I have no idea what is wrong. I googled and asked people good at this but did not figure it out. Is it because of the bidirectional? Does anybody know what is going wrong?
I guess it is the problem of shapes of the dataset and labels.
i meet the same problem and i solved it. the reason is K.update(p, new_p), 'p' type shouldn't tensor type, when you use K.update(p, new_p), 'p' type should be tf.Variable and 'new_p' type should be Tensor type, i hope it can solve your problem.
I am trying to do a large multiclass classification(Actually a translation).
I am trying to use tensorflow nce_loss in keras, but not able to make it work. Any help here?
I am not sure how can I pass weights,num_class and bias from previous layer to nce_loss.
I am getting following error:
import tensorflow as tf
from attention_decoder import AttentionDecoder
from keras.layers import Dropout,Masking,Embedding
def keras_nce_loss(tgt, pred):
return tf.nn.nce_loss(labels=tgt,inputs=pred,num_sampled=100)
model2 = Sequential()
model2.add(Embedding(input_features, input_embed_dimension, input_length=n_timesteps_in,mask_zero=True))
model2.add(Dropout(0.2))
model2.add(LSTM(LSTM_Unitsize,return_sequences=True,activation='relu'))
model2.add(Masking(mask_value=0.))
model2.add(AttentionDecoder(LSTM_Unitsize, n_features))
model2.compile(loss=keras_nce_loss, optimizer='adam', metrics=['acc'])
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-157-0d76d4053a42> in <module>()
11 model2.add(Masking(mask_value=0.))
12 model2.add(AttentionDecoder(LSTM_Unitsize, n_features))
---> 13 model2.compile(loss=keras_nce_loss, optimizer='adam', metrics=['acc'])
14 #model2.save("model2_compiled.hd5")
/usr/local/lib/python3.6/dist-packages/keras/models.py in compile(self, optimizer, loss, metrics, sample_weight_mode, **kwargs)
786 metrics=metrics,
787 sample_weight_mode=sample_weight_mode,
--> 788 **kwargs)
789 self.optimizer = self.model.optimizer
790 self.loss = self.model.loss
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in compile(self, optimizer, loss, metrics, loss_weights, sample_weight_mode, **kwargs)
909 loss_weight = loss_weights_list[i]
910 output_loss = weighted_loss(y_true, y_pred,
--> 911 sample_weight, mask)
912 if len(self.outputs) > 1:
913 self.metrics_tensors.append(output_loss)
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in weighted(y_true, y_pred, weights, mask)
434 """
435 # score_array has ndim >= 2
--> 436 score_array = fn(y_true, y_pred)
437 if mask is not None:
438 # Cast the mask to floatX to avoid float64 upcasting in theano
<ipython-input-155-ec20de882530> in keras_nce_loss(tgt, pred)
2
3 def keras_nce_loss(tgt, pred):
----> 4 return tf.nn.nce_loss(labels=tgt,inputs=pred,num_sampled=100)
TypeError: nce_loss() missing 3 required positional arguments: 'weights', 'biases', and 'num_classes'