I am trying to do a large multiclass classification(Actually a translation).
I am trying to use tensorflow nce_loss in keras, but not able to make it work. Any help here?
I am not sure how can I pass weights,num_class and bias from previous layer to nce_loss.
I am getting following error:
import tensorflow as tf
from attention_decoder import AttentionDecoder
from keras.layers import Dropout,Masking,Embedding
def keras_nce_loss(tgt, pred):
return tf.nn.nce_loss(labels=tgt,inputs=pred,num_sampled=100)
model2 = Sequential()
model2.add(Embedding(input_features, input_embed_dimension, input_length=n_timesteps_in,mask_zero=True))
model2.add(Dropout(0.2))
model2.add(LSTM(LSTM_Unitsize,return_sequences=True,activation='relu'))
model2.add(Masking(mask_value=0.))
model2.add(AttentionDecoder(LSTM_Unitsize, n_features))
model2.compile(loss=keras_nce_loss, optimizer='adam', metrics=['acc'])
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-157-0d76d4053a42> in <module>()
11 model2.add(Masking(mask_value=0.))
12 model2.add(AttentionDecoder(LSTM_Unitsize, n_features))
---> 13 model2.compile(loss=keras_nce_loss, optimizer='adam', metrics=['acc'])
14 #model2.save("model2_compiled.hd5")
/usr/local/lib/python3.6/dist-packages/keras/models.py in compile(self, optimizer, loss, metrics, sample_weight_mode, **kwargs)
786 metrics=metrics,
787 sample_weight_mode=sample_weight_mode,
--> 788 **kwargs)
789 self.optimizer = self.model.optimizer
790 self.loss = self.model.loss
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in compile(self, optimizer, loss, metrics, loss_weights, sample_weight_mode, **kwargs)
909 loss_weight = loss_weights_list[i]
910 output_loss = weighted_loss(y_true, y_pred,
--> 911 sample_weight, mask)
912 if len(self.outputs) > 1:
913 self.metrics_tensors.append(output_loss)
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in weighted(y_true, y_pred, weights, mask)
434 """
435 # score_array has ndim >= 2
--> 436 score_array = fn(y_true, y_pred)
437 if mask is not None:
438 # Cast the mask to floatX to avoid float64 upcasting in theano
<ipython-input-155-ec20de882530> in keras_nce_loss(tgt, pred)
2
3 def keras_nce_loss(tgt, pred):
----> 4 return tf.nn.nce_loss(labels=tgt,inputs=pred,num_sampled=100)
TypeError: nce_loss() missing 3 required positional arguments: 'weights', 'biases', and 'num_classes'
Related
Question
GridSearchCV via KerasClassifier causes the error when Keras Normalization has been adapted to data. Without the adapted Normalization, it works. The reason why using Normalization is because it gave better result than simply divide by 255.0.
PicklingError: Could not pickle the task to send it to the workers.
Workaround
By setting n_jobs=1 not to multi-thread, it works but perhaps not much use to run single thread.
Environment
Python 3.9.13
TensorFlow version: 2.10.0
Eager execution is: True
Keras version: 2.10.0
sklearn version: 1.1.3
Code
import numpy as np
import tensorflow as tf
from keras.layers import (
Dense,
Flatten,
Normalization,
Conv2D,
MaxPooling2D,
)
from keras.models import (
Sequential
)
from scikeras.wrappers import (
KerasClassifier,
)
from sklearn.model_selection import (
GridSearchCV
)
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()
# max_value = float(np.max(x_train))
# x_train, x_test = x_train/max_value, x_test/max_value
input_shape = x_train[0].shape
number_of_classes = 10
# Data Normalization
normalization = Normalization(
name="norm",
input_shape=input_shape, # (32, 32, 3)
axis=-1 # Regard each pixel as a feature
)
normalization.adapt(x_train)
def create_model():
model = Sequential([
# Without the adapted Normalization layer, it works.
normalization,
Conv2D(
name="conv",
filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
activation='relu',
input_shape=input_shape
),
MaxPooling2D(
name="maxpool",
pool_size=(2, 2)
),
Flatten(),
Dense(
name="full",
units=100,
activation="relu"
),
Dense(
name="label",
units=number_of_classes,
activation="softmax"
)
])
model.compile(
loss=tf.keras.losses.sparse_categorical_crossentropy,
optimizer='adam',
metrics=['accuracy']
)
return model
model = KerasClassifier(model=create_model, verbose=2)
batch_size = [32]
epochs = [2, 3]
param_grid = dict(batch_size=batch_size, epochs=epochs)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)
grid_result = grid.fit(x_train, y_train)
Log
The above exception was the direct cause of the following exception:
PicklingError Traceback (most recent call last)
Cell In [28], line 7
4 param_grid = dict(batch_size=batch_size, epochs=epochs)
6 grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)
----> 7 grid_result = grid.fit(x_train, y_train)
File ~/venv/ml/lib/python3.9/site-packages/sklearn/model_selection/_search.py:875, in BaseSearchCV.fit(self, X, y, groups, **fit_params)
869 results = self._format_results(
870 all_candidate_params, n_splits, all_out, all_more_results
871 )
873 return results
--> 875 self._run_search(evaluate_candidates)
877 # multimetric is determined here because in the case of a callable
878 # self.scoring the return type is only known after calling
879 first_test_score = all_out[0]["test_scores"]
File ~/venv/ml/lib/python3.9/site-packages/sklearn/model_selection/_search.py:1379, in GridSearchCV._run_search(self, evaluate_candidates)
1377 def _run_search(self, evaluate_candidates):
1378 """Search all candidates in param_grid"""
-> 1379 evaluate_candidates(ParameterGrid(self.param_grid))
File ~/venv/ml/lib/python3.9/site-packages/sklearn/model_selection/_search.py:822, in BaseSearchCV.fit.<locals>.evaluate_candidates(candidate_params, cv, more_results)
814 if self.verbose > 0:
815 print(
816 "Fitting {0} folds for each of {1} candidates,"
817 " totalling {2} fits".format(
818 n_splits, n_candidates, n_candidates * n_splits
819 )
820 )
--> 822 out = parallel(
823 delayed(_fit_and_score)(
824 clone(base_estimator),
825 X,
826 y,
827 train=train,
828 test=test,
829 parameters=parameters,
830 split_progress=(split_idx, n_splits),
831 candidate_progress=(cand_idx, n_candidates),
832 **fit_and_score_kwargs,
833 )
834 for (cand_idx, parameters), (split_idx, (train, test)) in product(
835 enumerate(candidate_params), enumerate(cv.split(X, y, groups))
836 )
837 )
839 if len(out) < 1:
840 raise ValueError(
841 "No fits were performed. "
842 "Was the CV iterator empty? "
843 "Were there no candidates?"
844 )
File ~/venv/ml/lib/python3.9/site-packages/joblib/parallel.py:1098, in Parallel.__call__(self, iterable)
1095 self._iterating = False
1097 with self._backend.retrieval_context():
-> 1098 self.retrieve()
1099 # Make sure that we get a last message telling us we are done
1100 elapsed_time = time.time() - self._start_time
File ~/venv/ml/lib/python3.9/site-packages/joblib/parallel.py:975, in Parallel.retrieve(self)
973 try:
974 if getattr(self._backend, 'supports_timeout', False):
--> 975 self._output.extend(job.get(timeout=self.timeout))
976 else:
977 self._output.extend(job.get())
File ~/venv/ml/lib/python3.9/site-packages/joblib/_parallel_backends.py:567, in LokyBackend.wrap_future_result(future, timeout)
564 """Wrapper for Future.result to implement the same behaviour as
565 AsyncResults.get from multiprocessing."""
566 try:
--> 567 return future.result(timeout=timeout)
568 except CfTimeoutError as e:
569 raise TimeoutError from e
File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:446, in Future.result(self, timeout)
444 raise CancelledError()
445 elif self._state == FINISHED:
--> 446 return self.__get_result()
447 else:
448 raise TimeoutError()
File /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:391, in Future.__get_result(self)
389 if self._exception:
390 try:
--> 391 raise self._exception
392 finally:
393 # Break a reference cycle with the exception in self._exception
394 self = None
PicklingError: Could not pickle the task to send it to the workers.
Research
Keras KerasClassifier gridsearch TypeError: can't pickle _thread.lock objects told Keras did not support pickle was the cause. However, as the code works if the adapted Normalization is not used, not relevant.
GPU can cause the issue but there is no GPU in my environment.
References
SciKeras Basic usage¶
How to Grid Search Hyperparameters for Deep Learning Models in Python with Keras
I get above error in colab TPU from the code below. Original model had epochs, steps_per_epoch and batch but removed that while debugging. Not sure what the issue is as I do not see a string.
None TPU version of code works. Most of the code is stock code with some modifications made. I tested the code to ensure images loaded properly.
import tensorflow as tf
from tensorflow.keras import backend as K
import os
import PIL
import csv
import shutil
import numpy as np
import sys
from PIL import Image
from tensorflow.keras import backend as K
import gc
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import layers
from tensorflow.keras import Model
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.layers import Dense, Activation, Flatten
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
tf.config.experimental_connect_to_cluster(resolver)
# This is the TPU initialization code that has to be at the beginning.
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.experimental.TPUStrategy(resolver)
list_ds = tf.data.Dataset.list_files(str(gcs_pattern))
# Reads an image from a file, decodes it into a dense tensor, and resizes it
# to a fixed shape.
def parse_image(filename):
parts = tf.strings.split(filename, os.sep)
label = parts[-2]
image = tf.io.read_file(filename)
image = tf.image.decode_jpeg(image)
image = tf.image.convert_image_dtype(image, tf.float32)
image = tf.image.resize(image, [400, 400])
return image, label
list_ds = list_ds.map(parse_image)
def create_model():
pre_trained_model = InceptionV3(input_shape = (400, 400,3), include_top = False, weights = 'imagenet')
input_tensor=None, input_shape=(1024, 1024,3))
for layer in pre_trained_model.layers:
if layer.name == 'mixed1':
break
layer.trainable = False
last_layer = pre_trained_model.get_layer('mixed7')
last_output = last_layer.output
from tensorflow.keras.optimizers import RMSprop
from tensorflow.keras import regularizers
x = Flatten()(last_output)
x = layers.Dense(1024, activation= 'relu')(x)
x = layers.Dropout(.2)(x)
x = layers.Dense(4, activation= 'softmax')(x)
modelin = Model(pre_trained_model.input, x)
return modelin
def get_callbacks(name_weights, patience_lr):
mcp_save = ModelCheckpoint(name_weights, save_best_only=True, monitor='val_acc', mode='max')
# reduce_lr_loss = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=patience_lr, verbose=1, epsilon=1e-4, mode='min')
return [mcp_save] #, reduce_lr_loss]
batch_size = 16 * strategy.num_replicas_in_sync
for i in range(5):
dataset = list_ds.shuffle(buffer_size = 2280)
dataset = dataset.cache()
val = dataset.skip(i*456).take(456).batch(batch_size, drop_remainder=True).prefetch(4)
train = dataset.skip(i*456+456).take(1824).concatenate(dataset.take(456*i)).batch(batch_size, drop_remainder=True).prefetch(15)
name_weights = "/content/drive/My Drive/Plant/final_model_fold_D512_I400_mix_1_7_" + str(i) + ".{epoch:02d}-{val_acc:.2f}.h5"
# callbacks = get_callbacks(name_weights = name_weights, patience_lr=10)
with strategy.scope():
modelinc = create_model()
modelinc.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['acc'])
modelinc.fit(
train,
epochs=5)
print(modelinc.evaluate(val))
K.clear_session()
del name_weights
del callbacks
gc.collect()
'''
-
Error:
Epoch 1/5 --------------------------------------------------------------------------- InvalidArgumentError Traceback (most recent call last)
<ipython-input-4-bbe01274450b> in <module>()
31 modelinc.fit(
32 train,
---> 33 epochs=5)
34
35
10 frames /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
64 def _method_wrapper(self, *args, **kwargs):
65 if not self._in_multi_worker_mode(): # pylint: disable=protected-access ---> 66 return method(self, *args,
**kwargs)
67
68 # Running inside `run_distribute_coordinator` already.
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
853 context.async_wait()
854 logs = tmp_logs # No error, now safe to assign to logs. --> 855 callbacks.on_train_batch_end(step, logs)
856 epoch_logs = copy.copy(logs)
857
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py in on_train_batch_end(self, batch, logs)
387 """
388 if self._should_call_train_batch_hooks: --> 389 logs = self._process_logs(logs)
390 self._call_batch_hook(ModeKeys.TRAIN, 'end', batch, logs=logs)
391
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py in _process_logs(self, logs)
263 """Turns tensors into numpy arrays or Python scalars."""
264 if logs: --> 265 return tf_utils.to_numpy_or_python_type(logs)
266 return {}
267
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/tf_utils.py in to_numpy_or_python_type(tensors)
521 return t # Don't turn ragged or sparse tensors to NumPy.
522 --> 523 return nest.map_structure(_to_single_numpy_or_python_type, tensors)
524
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/nest.py in map_structure(func, *structure, **kwargs)
615
616 return pack_sequence_as( --> 617 structure[0], [func(*x) for x in entries],
618 expand_composites=expand_composites)
619
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/nest.py in <listcomp>(.0)
615
616 return pack_sequence_as( --> 617 structure[0], [func(*x) for x in entries],
618 expand_composites=expand_composites)
619
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/tf_utils.py in _to_single_numpy_or_python_type(t)
517 def _to_single_numpy_or_python_type(t):
518 if isinstance(t, ops.Tensor): --> 519 x = t.numpy()
520 return x.item() if np.ndim(x) == 0 else x
521 return t # Don't turn ragged or sparse tensors to NumPy.
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in numpy(self)
959 """
960 # TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors. --> 961 maybe_arr = self._numpy() # pylint: disable=protected-access
962 return maybe_arr.copy() if isinstance(maybe_arr, np.ndarray) else maybe_arr
963
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in _numpy(self)
927 return self._numpy_internal()
928 except core._NotOkStatusException as e: --> 929 six.raise_from(core._status_to_exception(e.code, e.message), None)
930
931 #property
/usr/local/lib/python3.6/dist-packages/six.py in raise_from(value, from_value)
InvalidArgumentError: Unsupported data type for TPU: string, caused by output cond_8/Identity_1:0
I want for training a CNN with Early Stopping and want to use the f1-metric as stopping criterion.
When I compile the code for the CNN model I get the a TypeError as error message.
I'm still using Tensorflow 1.4 would like to avoid an upgrade to 2.0, because I have in mind that my previous code doesn't work anymore.
The error message is as follows:
TypeError Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py in _num_samples(x)
158 try:
--> 159 return len(x)
160 except TypeError:
14 frames
/usr/local/lib/python3.6/dist-
packages/tensorflow_core/python/framework/ops.py in __len__(self)
740 "Please call `x.shape` rather than `len(x)` for "
--> 741 "shape information.".format(self.name))
742
TypeError: len is not well defined for symbolic Tensors. (dense_16_target:0) Please call `x.shape` rather than `len(x)` for shape information.
During handling of the above exception, another exception occurred:
TypeError Traceback (most recent call last)
<ipython-input-44-cd3da16e057c> in <module>()
----> 1 model = model_cnn(False,False, False,True,6, 0.2, 0.5)
2 X_train, X_val, y_train, y_val = split_data(X_train, y_train,1)
3 cnn, ep = train_model_es(model, X_train, y_train, X_val, y_val, X_test, y_test, 50, 500,1)
<ipython-input-42-d275d9c69c03> in model_cnn(spat, extra_pool, avg_pool, cw, numb_conv, drop_conv, drop_dense)
36 if cw == True:
37 print("sparse categorical crossentropy")
---> 38 model.compile(loss="sparse_categorical_crossentropy", optimizer=Adam(), metrics=['accuracy', f1_metric])
39 #model.compile(loss="sparse_categorical_crossentropy", optimizer=Adam(), metrics=['accuracy'])
40 print("nothing")
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in compile(self, optimizer, loss, metrics, loss_weights, sample_weight_mode, weighted_metrics, target_tensors, **kwargs)
452 output_metrics = nested_metrics[i]
453 output_weighted_metrics = nested_weighted_metrics[i]
--> 454 handle_metrics(output_metrics)
455 handle_metrics(output_weighted_metrics, weights=weights)
456
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in handle_metrics(metrics, weights)
421 metric_result = weighted_metric_fn(y_true, y_pred,
422 weights=weights,
--> 423 mask=masks[i])
424
425 # Append to self.metrics_names, self.metric_tensors,
/usr/local/lib/python3.6/dist-packages/keras/engine/training_utils.py in weighted(y_true, y_pred, weights, mask)
426 """
427 # score_array has ndim >= 2
--> 428 score_array = fn(y_true, y_pred)
429 if mask is not None:
430 # Cast the mask to floatX to avoid float64 upcasting in Theano
<ipython-input-9-b21dc3bd89a6> in f1_metric(y_test, y_pred)
1 def f1_metric(y_test, y_pred):
----> 2 f1 = f1_score(y_test, y_pred, average='macro')
3 return f1
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py in f1_score(y_true, y_pred, labels, pos_label, average, sample_weight, zero_division)
1097 pos_label=pos_label, average=average,
1098 sample_weight=sample_weight,
-> 1099 zero_division=zero_division)
1100
1101
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py in fbeta_score(y_true, y_pred, beta, labels, pos_label, average, sample_weight, zero_division)
1224 warn_for=('f-score',),
1225 sample_weight=sample_weight,
-> 1226 zero_division=zero_division)
1227 return f
1228
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py in precision_recall_fscore_support(y_true, y_pred, beta, labels, pos_label, average, warn_for, sample_weight, zero_division)
1482 raise ValueError("beta should be >=0 in the F-beta score")
1483 labels = _check_set_wise_labels(y_true, y_pred, average, labels,
-> 1484 pos_label)
1485
1486 # Calculate tp_sum, pred_sum, true_sum ###
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py in _check_set_wise_labels(y_true, y_pred, average, labels, pos_label)
1299 str(average_options))
1300
-> 1301 y_type, y_true, y_pred = _check_targets(y_true, y_pred)
1302 present_labels = unique_labels(y_true, y_pred)
1303 if average == 'binary':
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_classification.py in _check_targets(y_true, y_pred)
78 y_pred : array or indicator matrix
79 """
---> 80 check_consistent_length(y_true, y_pred)
81 type_true = type_of_target(y_true)
82 type_pred = type_of_target(y_pred)
/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py in check_consistent_length(*arrays)
206 """
207
--> 208 lengths = [_num_samples(X) for X in arrays if X is not None]
209 uniques = np.unique(lengths)
210 if len(uniques) > 1:
/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py in <listcomp>(.0)
206 """
207
--> 208 lengths = [_num_samples(X) for X in arrays if X is not None]
209 uniques = np.unique(lengths)
210 if len(uniques) > 1:
/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py in _num_samples(x)
159 return len(x)
160 except TypeError:
--> 161 raise TypeError(message)
162
163
TypeError: Expected sequence or array-like, got <class 'tensorflow.python.framework.ops.Tensor'>
And here is the relevant code:
def f1_metric(y_test, y_pred):
f1 = f1_score(y_test, y_pred, average='macro')
return f1
def train_model_es(model, X, y, X_val, y_val, X_test, y_test):
es = EarlyStopping(monitor='f1_metric', mode='max', patience=20, restore_best_weights=True)
y = np.argmax(y, axis=1)
y_val = np.argmax(y_val, axis=1)
y_test = np.argmax(y_test, axis=1)
class_weights = class_weight.compute_class_weight('balanced', np.unique(y), y)
class_weights = dict(enumerate(class_weights))
history = model.fit(X, y, class_weight=class_weights, batch_size=32,epochs=20, verbose=1,
validation_data=(X_val, y_val), callbacks=[es])
def model_cnn():
model = Sequential()
model.add(Conv2D(32, kernel_size=(3,3), input_shape=(28,28,1),padding='same'))
model.add(BatchNormalization())
model.add(ELU())
model.add(Conv2D(32, kernel_size=(3,3), padding='same'))
model.add(BatchNormalization())
model.add(ELU())
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(256))
model.add(BatchNormalization())
model.add(ELU())
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.compile(loss="sparse_categorical_crossentroy",optimizer=Adam(),metrics=["accuracy",f1_metric])
return model
Does anyone have a tip on how to fix this error message.
Many thanks for every hint
As the error message suggests,
Error 1) You are doing a len() operation on symbolic tensor. You cannot do that operation on symbolic tensor. You can find difference between a variable tensor and symbolic tensor here.
Error 2) You are using a tensor for the operation which expects array as input. Can you please convert y_true and y_pred from tensor to array and use in the f1_score and other operations.
Example - To convert tensor to array
%tensorflow_version 1.x
print(tf.__version__)
import tensorflow as tf
import numpy as np
x = tf.constant([1,2,3,4,5,6])
print("Type of x:",x)
with tf.Session() as sess:
y = np.array(x.eval())
print("Type of y:",y.shape,y)
Output -
1.15.2
Type of x: Tensor("Const_24:0", shape=(6,), dtype=int32)
Type of y: (6,) [1 2 3 4 5 6]
I'm trying to write a custom activation function for use with Keras. I can not write it with tensorflow primitives as it does properly compute the derivative. I followed How to make a custom activation function with only Python in Tensorflow? and it works very we in creating a tensorflow function. However, when I tried putting it into Keras as an activation function for the classic MNIST demo. I got errors. I also tried the tf_spiky function from the above reference.
Here is the sample code
tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf_spiky),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)])
Here's my entire error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-48-73a57f81db19> in <module>
3 tf.keras.layers.Dense(512, activation=tf_spiky),
4 tf.keras.layers.Dropout(0.2),
----> 5 tf.keras.layers.Dense(10, activation=tf.nn.softmax)])
6 x=tf.keras.layers.Activation(tf_spiky)
7 y=tf.keras.layers.Flatten(input_shape=(28, 28))
/opt/conda/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py in _method_wrapper(self, *args, **kwargs)
472 self._setattr_tracking = False # pylint: disable=protected-access
473 try:
--> 474 method(self, *args, **kwargs)
475 finally:
476 self._setattr_tracking = previous_value # pylint: disable=protected-access
/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/sequential.py in __init__(self, layers, name)
106 if layers:
107 for layer in layers:
--> 108 self.add(layer)
109
110 #property
/opt/conda/lib/python3.6/site-packages/tensorflow/python/training/checkpointable/base.py in _method_wrapper(self, *args, **kwargs)
472 self._setattr_tracking = False # pylint: disable=protected-access
473 try:
--> 474 method(self, *args, **kwargs)
475 finally:
476 self._setattr_tracking = previous_value # pylint: disable=protected-access
/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/sequential.py in add(self, layer)
173 # If the model is being built continuously on top of an input layer:
174 # refresh its output.
--> 175 output_tensor = layer(self.outputs[0])
176 if isinstance(output_tensor, list):
177 raise TypeError('All layers in a Sequential model '
/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, inputs, *args, **kwargs)
728
729 # Check input assumptions set before layer building, e.g. input rank.
--> 730 self._assert_input_compatibility(inputs)
731 if input_list and self._dtype is None:
732 try:
/opt/conda/lib/python3.6/site-packages/tensorflow/python/keras/engine/base_layer.py in _assert_input_compatibility(self, inputs)
1463 if x.shape.ndims is None:
1464 raise ValueError('Input ' + str(input_index) + ' of layer ' +
-> 1465 self.name + ' is incompatible with the layer: '
1466 'its rank is undefined, but the layer requires a '
1467 'defined rank.')
ValueError: Input 0 of layer dense_1 is incompatible with the layer: its rank is undefined, but the layer requires a defined rank.
From this I gather the last Dense layer is unable to get the dimensions of the output after the activation function or something to that. I did see in the tensorflow code that many activation functions register a shape. But either I'm not doing that correctly or I'm going in the wrong direction. But I'm guessing something needs to be done to the tensorflow function to make it an activation function that Keras can use.
I would appreciate any help you can give.
As requested here is the sample codes for tf_spiky, it works as described in the above reference. However, once put into Keras I get the errors shown. This is pretty much as shown in the *How to make a custom activation function with only Python in Tensorflow?" stackoverflow article.
def spiky(x):
print(x)
r = x % 1
if r <= 0.5:
return r
else:
return 0
def d_spiky(x):
r = x % 1
if r <= 0.5:
return 1
else:
return 0
np_spiky = np.vectorize(spiky)
np_d_spiky = np.vectorize(d_spiky)
np_d_spiky_32 = lambda x: np_d_spiky(x).astype(np.float32)
import tensorflow as tf
from tensorflow.python.framework import ops
def tf_d_spiky(x,name=None):
with tf.name_scope(name, "d_spiky", [x]) as name:
y = tf.py_func(np_d_spiky_32,
[x],
[tf.float32],
name=name,
stateful=False)
return y[0]
def py_func(func, inp, Tout, stateful=True, name=None, grad=None):
# Need to generate a unique name to avoid duplicates:
rnd_name = 'PyFuncGrad' + str(np.random.randint(0, 1E+8))
tf.RegisterGradient(rnd_name)(grad) # see _MySquareGrad for grad example
g = tf.get_default_graph()
with g.gradient_override_map({"PyFunc": rnd_name}):
return tf.py_func(func, inp, Tout, stateful=stateful, name=name)
def spikygrad(op, grad):
x = op.inputs[0]
n_gr = tf_d_spiky(x)
return grad * n_gr
np_spiky_32 = lambda x: np_spiky(x).astype(np.float32)
def tf_spiky(x, name=None):
with tf.name_scope(name, "spiky", [x]) as name:
y = py_func(np_spiky_32,
[x],
[tf.float32],
name=name,
grad=spikygrad) # <-- here's the call to the gradient
return y[0]
The solution is in this post Output from TensorFlow `py_func` has unknown rank/shape
The easiest fix is to add y[0].set_shape(x.get_shape()) before the return statement in the definition of tf_spiky.
Perhaps someone out there knows how to properly work with tensorflow shape functions. Digging around I found a unchanged_shape shape function in tensorflow.python.framework.common_shapes, which be appropriate here, but I don't know how to attach it to the tf_spiky function. Seems a python decorator is in order here. It would probably be a service to others to explain customizing tensorflow functions with shape functions.
I'm going over the Book Deep Learning with Python from F. Chollet.
https://www.manning.com/books/deep-learning-with-python
I'm trying to follow along with the code examples. I just installed keras, and I am getting this error when trying to run this:
from this notebook:
https://github.com/fchollet/deep-learning-with-python-notebooks/blob/master/2.1-a-first-look-at-a-neural-network.ipynb
from keras import models
from keras import layers
network = models.Sequential()
network.add(layers.Dense(512, activation='relu', input_shape=(28 * 28,)))
network.add(layers.Dense(10, activation='softmax'))
TypeError Traceback (most recent call
last) in ()
4 network = models.Sequential()
5 network.add(layers.Dense(512, activation='relu', input_shape=(28 * 28,)))
----> 6 network.add(layers.Dense(10, activation='softmax'))
~/anaconda3/lib/python3.6/site-packages/keras/engine/sequential.py in
add(self, layer)
179 self.inputs = network.get_source_inputs(self.outputs[0])
180 elif self.outputs:
--> 181 output_tensor = layer(self.outputs[0])
182 if isinstance(output_tensor, list):
183 raise TypeError('All layers in a Sequential model '
~/anaconda3/lib/python3.6/site-packages/keras/engine/base_layer.py in
call(self, inputs, **kwargs)
455 # Actually call the layer,
456 # collecting output(s), mask(s), and shape(s).
--> 457 output = self.call(inputs, **kwargs)
458 output_mask = self.compute_mask(inputs, previous_mask)
459
~/anaconda3/lib/python3.6/site-packages/keras/layers/core.py in
call(self, inputs)
881 output = K.bias_add(output, self.bias, data_format='channels_last')
882 if self.activation is not None:
--> 883 output = self.activation(output)
884 return output
885
~/anaconda3/lib/python3.6/site-packages/keras/activations.py in
softmax(x, axis)
29 raise ValueError('Cannot apply softmax to a tensor that is 1D')
30 elif ndim == 2:
---> 31 return K.softmax(x)
32 elif ndim > 2:
33 e = K.exp(x - K.max(x, axis=axis, keepdims=True))
~/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py
in softmax(x, axis) 3229 A tensor. 3230 """
-> 3231 return tf.nn.softmax(x, axis=axis) 3232 3233
TypeError: softmax() got an unexpected keyword argument 'axis'
I'm wondering if there's something off with my installation?
keras.__version__
2.2.4
If anyone could give me a clue of what to look into.
Seems like you have an incompatible Tensorflow version (which Keras is using as a backend). For details look here