Related
I want to use Keras Tuner to pick my model's optimal set of hyperparameters. My code raised:
ValueError": Input 0 of layer dense is incompatible with the layer:
expected axis -1 of input shape to have value 867000 but received input with shape (None, 100)
I also tried with model.add(layers.Flatten(input_shape=X.shape[1:])) and it raised the same error.
Label encode
encoder = LabelEncoder()
df["subtype"] = encoder.fit_transform(df[["subtype"]])
Retrieve features and target
X = df.iloc[:,7:1007] # 1000 features
y = df[["subtype"]]
Train-test-val split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, random_state=1) # 0.25 x 0.8 = 0.2
Define model
# Define Sequential model
def get_model(hp):
model = keras.Sequential()
model.add(layers.Flatten(input_shape=X.shape))
# Tune the number of units in the first Dense layer
hp_units = hp.Int('units', min_value=30, max_value=len(X.T), step=32)
model.add(keras.layers.Dense(units=hp_units, activation='relu'))
model.add(keras.layers.Dense(10))
# Tune the learning rate for the optimizer
# Choose an optimal value from 0.01, 0.001, or 0.0001
hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])
model.compile(optimizer=Adam(learning_rate=hp_learning_rate),
loss=SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
return model
Early stopping
es_callback = keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)
Feature selection
selector = SelectKBest(f_classif, k=100) # Retrieve 100 best features
selected_features_subtype = selector.fit_transform(X_train, y_train.values.ravel())
Instantiate the tuner
tuner = kt.Hyperband(get_model,
objective='val_accuracy',
max_epochs=10,
factor=3)
Hyperparameter search:
tuner.search(selected_features_subtype, y_train, validation_split=0.5, batch_size=batch_size, callbacks=[es_callback])
Traceback:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/tmp/ipykernel_17/1635591259.py in <module>
----> 1 tuner.search(selected_features_subtype, y_train, validation_split=0.5, batch_size=batch_size, callbacks=[es_callback])
2
3 # Get the optimal hyperparameters
4 best_hps=tuner.get_best_hyperparameters(num_trials=1)[0]
5
/opt/conda/lib/python3.7/site-packages/keras_tuner/engine/base_tuner.py in search(self, *fit_args, **fit_kwargs)
177
178 self.on_trial_begin(trial)
--> 179 results = self.run_trial(trial, *fit_args, **fit_kwargs)
180 # `results` is None indicates user updated oracle in `run_trial()`.
181 if results is None:
/opt/conda/lib/python3.7/site-packages/keras_tuner/tuners/hyperband.py in run_trial(self, trial, *fit_args, **fit_kwargs)
382 fit_kwargs["epochs"] = hp.values["tuner/epochs"]
383 fit_kwargs["initial_epoch"] = hp.values["tuner/initial_epoch"]
--> 384 return super(Hyperband, self).run_trial(trial, *fit_args, **fit_kwargs)
385
386 def _build_model(self, hp):
/opt/conda/lib/python3.7/site-packages/keras_tuner/engine/tuner.py in run_trial(self, trial, *args, **kwargs)
292 callbacks.append(model_checkpoint)
293 copied_kwargs["callbacks"] = callbacks
--> 294 obj_value = self._build_and_fit_model(trial, *args, **copied_kwargs)
295
296 histories.append(obj_value)
/opt/conda/lib/python3.7/site-packages/keras_tuner/engine/tuner.py in _build_and_fit_model(self, trial, *args, **kwargs)
220 hp = trial.hyperparameters
221 model = self._try_build(hp)
--> 222 results = self.hypermodel.fit(hp, model, *args, **kwargs)
223 return tuner_utils.convert_to_metrics_dict(
224 results, self.oracle.objective, "HyperModel.fit()"
/opt/conda/lib/python3.7/site-packages/keras_tuner/engine/hypermodel.py in fit(self, hp, model, *args, **kwargs)
135 If return a float, it should be the `objective` value.
136 """
--> 137 return model.fit(*args, **kwargs)
138
139
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
1182 _r=1):
1183 callbacks.on_train_batch_begin(step)
-> 1184 tmp_logs = self.train_function(iterator)
1185 if data_handler.should_sync:
1186 context.async_wait()
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)
883
884 with OptionalXlaContext(self._jit_compile):
--> 885 result = self._call(*args, **kwds)
886
887 new_tracing_count = self.experimental_get_tracing_count()
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)
931 # This is the first call of __call__, so we have to initialize.
932 initializers = []
--> 933 self._initialize(args, kwds, add_initializers_to=initializers)
934 finally:
935 # At this point we know that the initialization is complete (or less
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)
758 self._concrete_stateful_fn = (
759 self._stateful_fn._get_concrete_function_internal_garbage_collected( # pylint: disable=protected-access
--> 760 *args, **kwds))
761
762 def invalid_creator_scope(*unused_args, **unused_kwds):
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _get_concrete_function_internal_garbage_collected(self, *args, **kwargs)
3064 args, kwargs = None, None
3065 with self._lock:
-> 3066 graph_function, _ = self._maybe_define_function(args, kwargs)
3067 return graph_function
3068
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)
3461
3462 self._function_cache.missed.add(call_context_key)
-> 3463 graph_function = self._create_graph_function(args, kwargs)
3464 self._function_cache.primary[cache_key] = graph_function
3465
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)
3306 arg_names=arg_names,
3307 override_flat_arg_shapes=override_flat_arg_shapes,
-> 3308 capture_by_value=self._capture_by_value),
3309 self._function_attributes,
3310 function_spec=self.function_spec,
/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes, acd_record_initial_resource_uses)
1005 _, original_func = tf_decorator.unwrap(python_func)
1006
-> 1007 func_outputs = python_func(*func_args, **func_kwargs)
1008
1009 # invariant: `func_outputs` contains only Tensors, CompositeTensors,
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, **kwds)
666 # the function a weak reference to itself to avoid a reference cycle.
667 with OptionalXlaContext(compile_with_xla):
--> 668 out = weak_wrapped_fn().__wrapped__(*args, **kwds)
669 return out
670
/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
992 except Exception as e: # pylint:disable=broad-except
993 if hasattr(e, "ag_error_metadata"):
--> 994 raise e.ag_error_metadata.to_exception(e)
995 else:
996 raise
ValueError: in user code:
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py:853 train_function *
return step_function(self, iterator)
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py:842 step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py:1286 run
return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py:2849 call_for_each_replica
return self._call_for_each_replica(fn, args, kwargs)
/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py:3632 _call_for_each_replica
return fn(*args, **kwargs)
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py:835 run_step **
outputs = model.train_step(data)
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py:787 train_step
y_pred = self(x, training=True)
/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py:1037 __call__
outputs = call_fn(inputs, *args, **kwargs)
/opt/conda/lib/python3.7/site-packages/keras/engine/sequential.py:369 call
return super(Sequential, self).call(inputs, training=training, mask=mask)
/opt/conda/lib/python3.7/site-packages/keras/engine/functional.py:415 call
inputs, training=training, mask=mask)
/opt/conda/lib/python3.7/site-packages/keras/engine/functional.py:550 _run_internal_graph
outputs = node.layer(*args, **kwargs)
/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py:1020 __call__
input_spec.assert_input_compatibility(self.input_spec, inputs, self.name)
/opt/conda/lib/python3.7/site-packages/keras/engine/input_spec.py:254 assert_input_compatibility
' but received input with shape ' + display_shape(x.shape))
ValueError: Input 0 of layer dense is incompatible with the layer: expected axis -1 of input shape to have value 867000 but received input with shape (None, 100)
Sample data (as dict):
{'cg00000289': {'TCGA-5P-A9JW-01A': 0.619047033443068,
'TCGA-5P-A9JY-01A': 0.662345356057447,
'TCGA-5P-A9JZ-01A': 0.699464419990523,
'TCGA-5P-A9K0-01A': 0.581701228463189,
'TCGA-5P-A9K2-01A': 0.673198201701496,
'TCGA-5P-A9K3-01A': 0.626858745753386,
'TCGA-5P-A9K4-01A': 0.68070511240185,
'TCGA-5P-A9K6-01A': 0.736978843263676,
'TCGA-5P-A9K8-01A': 0.520786654021559,
'TCGA-5P-A9K9-01A': 0.630167782543268,
'TCGA-5P-A9KA-01A': 0.626926559177059,
'TCGA-5P-A9KC-01A': 0.630977089669812,
'TCGA-5P-A9KE-01A': 0.67563913188817,
'TCGA-5P-A9KF-01A': 0.645760654461395,
'TCGA-5P-A9KH-01A': 0.743286554209644,
'TCGA-6D-AA2E-01A': 0.664370540359405,
'TCGA-A3-3357-01A': 0.604326417072586,
'TCGA-A3-3358-01A': 0.458291671200214,
'TCGA-A3-3367-01A': 0.643363591881443,
'TCGA-A3-3370-01A': 0.75808536817831},
'cg00000292': {'TCGA-5P-A9JW-01A': 0.833751448211989,
'TCGA-5P-A9JY-01A': 0.761479643484317,
'TCGA-5P-A9JZ-01A': 0.555219387831784,
'TCGA-5P-A9K0-01A': 0.911434986725676,
'TCGA-5P-A9K2-01A': 0.592839496707225,
'TCGA-5P-A9K3-01A': 0.728724740061591,
'TCGA-5P-A9K4-01A': 0.871647081931081,
'TCGA-5P-A9K6-01A': 0.687137365325043,
'TCGA-5P-A9K8-01A': 0.377068349756215,
'TCGA-5P-A9K9-01A': 0.885089826740071,
'TCGA-5P-A9KA-01A': 0.678749255227915,
'TCGA-5P-A9KC-01A': 0.82812139328519,
'TCGA-5P-A9KE-01A': 0.864590733749797,
'TCGA-5P-A9KF-01A': 0.858070865799283,
'TCGA-5P-A9KH-01A': 0.914435928566657,
'TCGA-6D-AA2E-01A': 0.502154665104261,
'TCGA-A3-3357-01A': 0.662047197870235,
'TCGA-A3-3358-01A': 0.837059251611538,
'TCGA-A3-3367-01A': 0.519939858249351,
'TCGA-A3-3370-01A': 0.515743234863198},
'cg00000321': {'TCGA-5P-A9JW-01A': 0.489674207394165,
'TCGA-5P-A9JY-01A': 0.558997284357574,
'TCGA-5P-A9JZ-01A': 0.169654991100549,
'TCGA-5P-A9K0-01A': 0.524017780921585,
'TCGA-5P-A9K2-01A': 0.613973121455874,
'TCGA-5P-A9K3-01A': 0.695670625292722,
'TCGA-5P-A9K4-01A': 0.54705053331032,
'TCGA-5P-A9K6-01A': 0.430048300391044,
'TCGA-5P-A9K8-01A': 0.198107812192402,
'TCGA-5P-A9K9-01A': 0.688004412660976,
'TCGA-5P-A9KA-01A': 0.463728628068311,
'TCGA-5P-A9KC-01A': 0.195610858899826,
'TCGA-5P-A9KE-01A': 0.550900635535007,
'TCGA-5P-A9KF-01A': 0.183061803083559,
'TCGA-5P-A9KH-01A': 0.0687041391027568,
'TCGA-6D-AA2E-01A': 0.570614767869701,
'TCGA-A3-3357-01A': 0.680216582417148,
'TCGA-A3-3358-01A': 0.288408070866453,
'TCGA-A3-3367-01A': 0.68957300320214,
'TCGA-A3-3370-01A': 0.432450706527388},
'cg00000363': {'TCGA-5P-A9JW-01A': 0.276273359465442,
'TCGA-5P-A9JY-01A': 0.25703867804956,
'TCGA-5P-A9JZ-01A': 0.0981746197111535,
'TCGA-5P-A9K0-01A': 0.569434380074143,
'TCGA-5P-A9K2-01A': 0.229164840583894,
'TCGA-5P-A9K3-01A': 0.819181728250669,
'TCGA-5P-A9K4-01A': 0.385400157144987,
'TCGA-5P-A9K6-01A': 0.121114850921845,
'TCGA-5P-A9K8-01A': 0.0913002964111161,
'TCGA-5P-A9K9-01A': 0.157709640291589,
'TCGA-5P-A9KA-01A': 0.182786461024344,
'TCGA-5P-A9KC-01A': 0.413992919815649,
'TCGA-5P-A9KE-01A': 0.461011690059099,
'TCGA-5P-A9KF-01A': 0.206300754004666,
'TCGA-5P-A9KH-01A': 0.105913974360403,
'TCGA-6D-AA2E-01A': 0.414195666341229,
'TCGA-A3-3357-01A': 0.236700230821645,
'TCGA-A3-3358-01A': 0.26841943012007,
'TCGA-A3-3367-01A': 0.324092383012077,
'TCGA-A3-3370-01A': 0.20675761437022},
'cg00000622': {'TCGA-5P-A9JW-01A': 0.0127307580597565,
'TCGA-5P-A9JY-01A': 0.0120665526567922,
'TCGA-5P-A9JZ-01A': 0.0129414663486043,
'TCGA-5P-A9K0-01A': 0.0151053469374601,
'TCGA-5P-A9K2-01A': 0.0147191458229104,
'TCGA-5P-A9K3-01A': 0.0128586680833482,
'TCGA-5P-A9K4-01A': 0.0131839246367822,
'TCGA-5P-A9K6-01A': 0.0145337257462313,
'TCGA-5P-A9K8-01A': 0.0131548295351121,
'TCGA-5P-A9K9-01A': 0.0164371867221858,
'TCGA-5P-A9KA-01A': 0.0153209540816947,
'TCGA-5P-A9KC-01A': 0.0146341900882511,
'TCGA-5P-A9KE-01A': 0.0138002584809048,
'TCGA-5P-A9KF-01A': 0.012958575912875,
'TCGA-5P-A9KH-01A': 0.0142346121115625,
'TCGA-6D-AA2E-01A': 0.0139666701044385,
'TCGA-A3-3357-01A': 0.0082183731354049,
'TCGA-A3-3358-01A': 0.0143527756424356,
'TCGA-A3-3367-01A': 0.0100636145864037,
'TCGA-A3-3370-01A': 0.0108528842825329},
'type': {'TCGA-5P-A9JW-01A': 'tumor',
'TCGA-5P-A9JY-01A': 'tumor',
'TCGA-5P-A9JZ-01A': 'tumor',
'TCGA-5P-A9K0-01A': 'tumor',
'TCGA-5P-A9K2-01A': 'tumor',
'TCGA-5P-A9K3-01A': 'tumor',
'TCGA-5P-A9K4-01A': 'tumor',
'TCGA-5P-A9K6-01A': 'tumor',
'TCGA-5P-A9K8-01A': 'tumor',
'TCGA-5P-A9K9-01A': 'tumor',
'TCGA-5P-A9KA-01A': 'tumor',
'TCGA-5P-A9KC-01A': 'tumor',
'TCGA-5P-A9KE-01A': 'tumor',
'TCGA-5P-A9KF-01A': 'tumor',
'TCGA-5P-A9KH-01A': 'tumor',
'TCGA-6D-AA2E-01A': 'tumor',
'TCGA-A3-3357-01A': 'tumor',
'TCGA-A3-3358-01A': 'tumor',
'TCGA-A3-3367-01A': 'tumor',
'TCGA-A3-3370-01A': 'tumor'},
'subtype': {'TCGA-5P-A9JW-01A': 'KIRP',
'TCGA-5P-A9JY-01A': 'KIRP',
'TCGA-5P-A9JZ-01A': 'KIRP',
'TCGA-5P-A9K0-01A': 'KIRP',
'TCGA-5P-A9K2-01A': 'KIRP',
'TCGA-5P-A9K3-01A': 'KIRP',
'TCGA-5P-A9K4-01A': 'KIRP',
'TCGA-5P-A9K6-01A': 'KIRP',
'TCGA-5P-A9K8-01A': 'KIRP',
'TCGA-5P-A9K9-01A': 'KIRP',
'TCGA-5P-A9KA-01A': 'KIRP',
'TCGA-5P-A9KC-01A': 'KIRP',
'TCGA-5P-A9KE-01A': 'KIRP',
'TCGA-5P-A9KF-01A': 'KIRP',
'TCGA-5P-A9KH-01A': 'KIRP',
'TCGA-6D-AA2E-01A': 'KIRC',
'TCGA-A3-3357-01A': 'KIRC',
'TCGA-A3-3358-01A': 'KIRC',
'TCGA-A3-3367-01A': 'KIRC',
'TCGA-A3-3370-01A': 'KIRC'}}
Make sure that units=hp_units in the last (output layer) is an int number, also change the activation='relu' to softmax. Hope this can work
I am building a RNN model using Tensorflow Keras, and I want to reduce learning rate whenever an validation accuracy does not increase. However, I have received an error message indicating container does not exist. I have tried to explicitly initialize the learning rate; however, the same issue still happens. Does anyone who what is going on? Below are my code of my model and learning rate reduction function.
lr_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=5,
verbose=1,
factor=0.5,
min_lr=0.000001)
model_1 = tf.keras.Sequential([
tf.keras.layers.Embedding(vocal_size, embedding_dim,
input_length=max_length),
tf.keras.layers.GlobalAveragePooling1D(),
tf.keras.layers.Dense(24, activation='relu'),
tf.keras.layers.Dropout(0.1),
tf.keras.layers.Dense(1, activation='sigmoid')
])
model_1.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model_1.summary()
epochs = 15
model_1_hist = model_1.fit(train_padded_sequence, train_target, epochs=epochs,
validation_data = (valid_padded_sequence, valid_target),
verbose=1,
callbacks=[lr_reduction]
)
Train on 6090 samples, validate on 1523 samples
Epoch 1/15
5888/6090 [============================>.] - ETA: 0s - loss: 0.5245 - acc: 0.7328
---------------------------------------------------------------------------
FailedPreconditionError Traceback (most recent call last)
/usr/local/lib/python3.7/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1355 try:
-> 1356 return fn(*args)
1357 except errors.OpError as e:
/usr/local/lib/python3.7/site-packages/tensorflow/python/client/session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1340 return self._call_tf_sessionrun(
-> 1341 options, feed_dict, fetch_list, target_list, run_metadata)
1342
/usr/local/lib/python3.7/site-packages/tensorflow/python/client/session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1428 self._session, options, feed_dict, fetch_list, target_list,
-> 1429 run_metadata)
1430
FailedPreconditionError: Error while reading resource variable Adam_2/learning_rate from Container: localhost. This could mean that the variable was uninitialized. Not found: Container localhost does not exist. (Could not find resource: localhost/Adam_2/learning_rate)
[[{{node Adam_2/learning_rate/Read/ReadVariableOp}}]]
During handling of the above exception, another exception occurred:
FailedPreconditionError Traceback (most recent call last)
<ipython-input-212-62874f053345> in <module>
3 validation_data = (valid_padded_sequence, valid_target),
4 verbose=1,
----> 5 callbacks=[lr_reduction]
6 )
/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)
778 validation_steps=validation_steps,
779 validation_freq=validation_freq,
--> 780 steps_name='steps_per_epoch')
781
782 def evaluate(self,
/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py in model_iteration(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq, mode, validation_in_fit, prepared_feed_values_from_dataset, steps_name, **kwargs)
417 if mode == ModeKeys.TRAIN:
418 # Epochs only apply to `fit`.
--> 419 callbacks.on_epoch_end(epoch, epoch_logs)
420 progbar.on_epoch_end(epoch, epoch_logs)
421
/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/callbacks.py in on_epoch_end(self, epoch, logs)
309 logs = logs or {}
310 for callback in self.callbacks:
--> 311 callback.on_epoch_end(epoch, logs)
312
313 def on_train_batch_begin(self, batch, logs=None):
/usr/local/lib/python3.7/site-packages/keras/callbacks.py in on_epoch_end(self, epoch, logs)
1371 def on_epoch_end(self, epoch, logs=None):
1372 logs = logs or {}
-> 1373 logs['lr'] = K.get_value(self.model.optimizer.lr)
1374 current = logs.get(self.monitor)
1375 if current is None:
/usr/local/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py in get_value(x)
2667 A Numpy array.
2668 """
-> 2669 return x.eval(session=get_session())
2670
2671
/usr/local/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py in eval(self, session)
898 if context.executing_eagerly():
899 raise RuntimeError("Trying to eval in EAGER mode")
--> 900 return self._graph_element.eval(session=session)
901
902 def numpy(self):
/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/ops.py in eval(self, feed_dict, session)
729
730 """
--> 731 return _eval_using_default_session(self, feed_dict, self.graph, session)
732
733
/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/ops.py in _eval_using_default_session(tensors, feed_dict, graph, session)
5577 "the tensor's graph is different from the session's "
5578 "graph.")
-> 5579 return session.run(tensors, feed_dict)
5580
5581
/usr/local/lib/python3.7/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
948 try:
949 result = self._run(None, fetches, feed_dict, options_ptr,
--> 950 run_metadata_ptr)
951 if run_metadata:
952 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/usr/local/lib/python3.7/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1171 if final_fetches or final_targets or (handle and feed_dict_tensor):
1172 results = self._do_run(handle, final_targets, final_fetches,
-> 1173 feed_dict_tensor, options, run_metadata)
1174 else:
1175 results = []
/usr/local/lib/python3.7/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1348 if handle is None:
1349 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1350 run_metadata)
1351 else:
1352 return self._do_call(_prun_fn, handle, feeds, fetches)
/usr/local/lib/python3.7/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1368 pass
1369 message = error_interpolation.interpolate(message, self._graph)
-> 1370 raise type(e)(node_def, op, message)
1371
1372 def _extend_graph(self):
FailedPreconditionError: Error while reading resource variable Adam_2/learning_rate from Container: localhost. This could mean that the variable was uninitialized. Not found: Container localhost does not exist. (Could not find resource: localhost/Adam_2/learning_rate)
[[node Adam_2/learning_rate/Read/ReadVariableOp (defined at <ipython-input-212-62874f053345>:5) ]]
Original stack trace for 'Adam_2/learning_rate/Read/ReadVariableOp':
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/usr/local/lib/python3.7/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/usr/local/lib/python3.7/site-packages/ipykernel/kernelapp.py", line 563, in start
self.io_loop.start()
File "/usr/local/lib/python3.7/site-packages/tornado/platform/asyncio.py", line 148, in start
self.asyncio_loop.run_forever()
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_events.py", line 534, in run_forever
self._run_once()
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/base_events.py", line 1771, in _run_once
handle._run()
File "/usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.7/site-packages/tornado/ioloop.py", line 690, in <lambda>
lambda f: self._run_callback(functools.partial(callback, future))
File "/usr/local/lib/python3.7/site-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 787, in inner
self.run()
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 748, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.7/site-packages/ipykernel/kernelbase.py", line 365, in process_one
yield gen.maybe_future(dispatch(*args))
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.7/site-packages/ipykernel/kernelbase.py", line 272, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.7/site-packages/ipykernel/kernelbase.py", line 542, in execute_request
user_expressions, allow_stdin,
File "/usr/local/lib/python3.7/site-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.7/site-packages/ipykernel/ipkernel.py", line 294, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.7/site-packages/ipykernel/zmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 2855, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/usr/local/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 2881, in _run_cell
return runner(coro)
File "/usr/local/lib/python3.7/site-packages/IPython/core/async_helpers.py", line 68, in _pseudo_sync_runner
coro.send(None)
File "/usr/local/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3058, in run_cell_async
interactivity=interactivity, compiler=compiler, result=result)
File "/usr/local/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3249, in run_ast_nodes
if (await self.run_code(code, result, async_=asy)):
File "/usr/local/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3326, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-212-62874f053345>", line 5, in <module>
callbacks=[lr_reduction]
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 780, in fit
steps_name='steps_per_epoch')
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 157, in model_iteration
f = _make_execution_function(model, mode)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_arrays.py", line 532, in _make_execution_function
return model._make_execution_function(mode)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 2276, in _make_execution_function
self._make_train_function()
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 2219, in _make_train_function
params=self._collected_trainable_weights, loss=self.total_loss)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py", line 497, in get_updates
return [self.apply_gradients(grads_and_vars)]
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py", line 434, in apply_gradients
self._create_hypers()
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py", line 608, in _create_hypers
aggregation=tf_variables.VariableAggregation.ONLY_FIRST_REPLICA)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py", line 770, in add_weight
aggregation=aggregation)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/training/tracking/base.py", line 663, in _add_variable_with_custom_getter
**kwargs_for_getter)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer_utils.py", line 155, in make_variable
shape=variable_shape if variable_shape.rank else None)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 259, in __call__
return cls._variable_v1_call(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 220, in _variable_v1_call
shape=shape)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 198, in <lambda>
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py", line 2495, in default_variable_creator
shape=shape)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/ops/variables.py", line 263, in __call__
return super(VariableMetaclass, cls).__call__(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py", line 460, in __init__
shape=shape)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py", line 649, in _init_from_args
value = self._read_variable_op()
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py", line 935, in _read_variable_op
self._dtype)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/ops/gen_resource_variable_ops.py", line 587, in read_variable_op
"ReadVariableOp", resource=resource, dtype=dtype, name=name)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py", line 788, in _apply_op_helper
op_def=op_def)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 3616, in create_op
op_def=op_def)
File "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 2005, in __init__
self._traceback = tf_stack.extract_stack()
did you import Adam as in
from tensorflow.keras.optimizers import Adam
When I monitor the Redis using monitor command I observe there are lots of info command is being executed. Is this executed by Redis server itself or by the client applications?
The problem is, when I check the output of the client list command on the server, there are lots of open connection objects on info command which is causing the server to crash with below error.
ReplyError: Ready check failed: ERR max number of clients reached
id=3066244 addr=127.0.0.1:47496 fd=440 name= age=101 idle=101 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066245 addr=127.0.0.1:47498 fd=441 name= age=101 idle=100 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066246 addr=127.0.0.1:47500 fd=442 name= age=101 idle=101 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066247 addr=127.0.0.1:47502 fd=443 name= age=101 idle=100 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066248 addr=127.0.0.1:47504 fd=444 name= age=101 idle=101 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066249 addr=127.0.0.1:47506 fd=445 name= age=101 idle=100 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066250 addr=127.0.0.1:47508 fd=446 name= age=101 idle=101 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066251 addr=127.0.0.1:47510 fd=447 name= age=101 idle=101 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066252 addr=127.0.0.1:47512 fd=448 name= age=101 idle=100 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066253 addr=127.0.0.1:47514 fd=449 name= age=101 idle=100 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066254 addr=127.0.0.1:47516 fd=450 name= age=101 idle=100 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066255 addr=127.0.0.1:47518 fd=451 name= age=101 idle=100 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066256 addr=127.0.0.1:47520 fd=452 name= age=101 idle=100 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
id=3066257 addr=127.0.0.1:47522 fd=454 name= age=101 idle=100 flags=N db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=0 omem=0 events=r cmd=info
I am trying to train a model in keras. I have 4 gpus and 1 cpu. When I start training keras/tensorflow tries to use cpu 1 device which is non existant. I am running keras 2.1.6 and the latest tensorflow release was built from source.
with tf.device("/cpu:0"):
model = Model(inputs=inputs, outputs=x)
p_model = multi_gpu_model(model , 4)
p_model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
batch_size = 32
p_model.fit(x=train_x,
y=train_y,
batch_size=batch_size,
epochs=50,
verbose=1,
validation_data=(test_x, test_y),
shuffle=True)
I got the following error when I try to start training. I am not sure why it's trying to use cpu 1 even though its non existant.
InvalidArgumentError Traceback (most recent call last)
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1321 try:
-> 1322 return fn(*args)
1323 except errors.OpError as e:
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1306 return self._call_tf_sessionrun(
-> 1307 options, feed_dict, fetch_list, target_list, run_metadata)
1308
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1408 self._session, options, feed_dict, fetch_list, target_list,
-> 1409 run_metadata)
1410 else:
InvalidArgumentError: Creating a partition for /device:CPU:1 which doesn't exist in the list of available devices. Available devices: /device:CPU:0,/device:GPU:0,/device:GPU:1,/device:GPU:2,/device:GPU:3
During handling of the above exception, another exception occurred:
InvalidArgumentError Traceback (most recent call last)
<ipython-input-19-ad177e58b8dc> in <module>()
19 verbose=1,
20 validation_data=(test_x, test_y),
---> 21 shuffle=True)
/usr/local/lib/python3.5/dist-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1703 initial_epoch=initial_epoch,
1704 steps_per_epoch=steps_per_epoch,
-> 1705 validation_steps=validation_steps)
1706
1707 def evaluate(self, x=None, y=None,
/usr/local/lib/python3.5/dist-packages/keras/engine/training.py in _fit_loop(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
1234 ins_batch[i] = ins_batch[i].toarray()
1235
-> 1236 outs = f(ins_batch)
1237 if not isinstance(outs, list):
1238 outs = [outs]
/usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs)
2480 session = get_session()
2481 updated = session.run(fetches=fetches, feed_dict=feed_dict,
-> 2482 **self.session_kwargs)
2483 return updated[:len(self.outputs)]
2484
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
898 try:
899 result = self._run(None, fetches, feed_dict, options_ptr,
--> 900 run_metadata_ptr)
901 if run_metadata:
902 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1133 if final_fetches or final_targets or (handle and feed_dict_tensor):
1134 results = self._do_run(handle, final_targets, final_fetches,
-> 1135 feed_dict_tensor, options, run_metadata)
1136 else:
1137 results = []
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1314 if handle is None:
1315 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1316 run_metadata)
1317 else:
1318 return self._do_call(_prun_fn, handle, feeds, fetches)
/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1333 except KeyError:
1334 pass
-> 1335 raise type(e)(node_def, op, message)
1336
1337 def _extend_graph(self):
InvalidArgumentError: Creating a partition for /device:CPU:1 which doesn't exist in the list of available devices. Available devices: /device:CPU:0,/device:GPU:0,/device:GPU:1,/device:GPU:2,/device:GPU:3
There was an issue with my backend(tensorflow) installation.
I am getting this error when the flow comes to 'Execute a batch element' processor.
java.lang.ClassNotFoundException: org.apache.commons.csv.CSVFormat
I have nothing to do with CSV, and I am getting this error sometimes not all the time in the same flow.
When I added the dependency for commons-csv
I get the below exception
Message : Could not serialize object (org.mule.api.serialization.SerializationException).
Root Exception stack trace:
java.util.ConcurrentModificationException
at java.util.Vector$Itr.checkForComodification(Vector.java:1184)
at java.util.Vector$Itr.next(Vector.java:1137)
at com.esotericsoftware.kryo.serializers.CollectionSerializer.write(CollectionSerializer.java:92)
at com.esotericsoftware.kryo.serializers.CollectionSerializer.write(CollectionSerializer.java:40)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:628)
at com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:366)
at com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:307)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeObjectOrNull(Kryo.java:606)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:87)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:628)
at com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:366)
at com.esotericsoftware.kryo.serializers.DefaultArraySerializers$ObjectArraySerializer.write(DefaultArraySerializers.java:307)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:628)
at com.esotericsoftware.kryo.serializers.MapSerializer.write(MapSerializer.java:113)
at com.esotericsoftware.kryo.serializers.MapSerializer.write(MapSerializer.java:39)
at com.esotericsoftware.kryo.Kryo.writeObjectOrNull(Kryo.java:579)
at com.mulesoft.module.serialization.kryo.internal.CopyOnWriteCaseInsensitiveMapSerializer.write(CopyOnWriteCaseInsensitiveMapSerializer.java:30)
at com.mulesoft.module.serialization.kryo.internal.CopyOnWriteCaseInsensitiveMapSerializer.write(CopyOnWriteCaseInsensitiveMapSerializer.java:1)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.mulesoft.module.serialization.kryo.internal.MuleEventKryoSerializer.write(MuleEventKryoSerializer.java:36)
at com.mulesoft.module.serialization.kryo.internal.MuleEventKryoSerializer.write(MuleEventKryoSerializer.java:1)
at com.esotericsoftware.kryo.Kryo.writeObject(Kryo.java:552)
at com.esotericsoftware.kryo.serializers.ObjectField.write(ObjectField.java:80)
at com.esotericsoftware.kryo.serializers.FieldSerializer.write(FieldSerializer.java:518)
at com.esotericsoftware.kryo.Kryo.writeClassAndObject(Kryo.java:628)
at com.mulesoft.module.serialization.kryo.internal.KryoObjectSerializer.doSerialize(KryoObjectSerializer.java:105)
at com.mulesoft.module.serialization.kryo.internal.KryoObjectSerializer.doSerialize(KryoObjectSerializer.java:97)
at org.mule.serialization.internal.AbstractObjectSerializer.serialize(AbstractObjectSerializer.java:64)
at com.mulesoft.module.batch.engine.DefaultBatchJobInstanceStore.doStore(DefaultBatchJobInstanceStore.java:351)
at com.mulesoft.module.batch.engine.DefaultBatchJobInstanceStore.store(DefaultBatchJobInstanceStore.java:96)
at com.mulesoft.module.batch.engine.DefaultBatchEngine.createNewJobInstance(DefaultBatchEngine.java:226)
at com.mulesoft.module.batch.DefaultBatchJob.execute(DefaultBatchJob.java:371)
at com.mulesoft.module.batch.processor.BatchExecuteMessageProcessor.process(BatchExecuteMessageProcessor.java:49)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:88)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:98)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.processor.AsyncInterceptingMessageProcessor.process(AsyncInterceptingMessageProcessor.java:108)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:88)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:98)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.construct.DynamicPipelineMessageProcessor.process(DynamicPipelineMessageProcessor.java:55)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:88)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:98)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.interceptor.AbstractEnvelopeInterceptor.processBlocking(AbstractEnvelopeInterceptor.java:58)
at org.mule.processor.AbstractRequestResponseMessageProcessor.process(AbstractRequestResponseMessageProcessor.java:47)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:98)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.processor.AbstractFilteringMessageProcessor.process(AbstractFilteringMessageProcessor.java:52)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:98)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.processor.AbstractRequestResponseMessageProcessor.processBlocking(AbstractRequestResponseMessageProcessor.java:56)
at org.mule.processor.AbstractRequestResponseMessageProcessor.process(AbstractRequestResponseMessageProcessor.java:47)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:88)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.construct.Flow$2.process(Flow.java:138)
at org.mule.construct.Flow$2.process(Flow.java:133)
at org.mule.execution.ExecuteCallbackInterceptor.execute(ExecuteCallbackInterceptor.java:16)
at org.mule.execution.CommitTransactionInterceptor.execute(CommitTransactionInterceptor.java:35)
at org.mule.execution.CommitTransactionInterceptor.execute(CommitTransactionInterceptor.java:22)
at org.mule.execution.HandleExceptionInterceptor.execute(HandleExceptionInterceptor.java:30)
at org.mule.execution.HandleExceptionInterceptor.execute(HandleExceptionInterceptor.java:14)
at org.mule.execution.BeginAndResolveTransactionInterceptor.execute(BeginAndResolveTransactionInterceptor.java:67)
at org.mule.execution.SuspendXaTransactionInterceptor.execute(SuspendXaTransactionInterceptor.java:50)
at org.mule.execution.RethrowExceptionInterceptor.execute(RethrowExceptionInterceptor.java:28)
at org.mule.execution.RethrowExceptionInterceptor.execute(RethrowExceptionInterceptor.java:13)
at org.mule.execution.ErrorHandlingExecutionTemplate.execute(ErrorHandlingExecutionTemplate.java:60)
at org.mule.execution.ErrorHandlingExecutionTemplate.execute(ErrorHandlingExecutionTemplate.java:30)
at org.mule.construct.Flow.process(Flow.java:132)
at org.mule.module.apikit.AbstractRouter.processBlockingRequest(AbstractRouter.java:108)
at org.mule.module.apikit.AbstractRouter.processBlocking(AbstractRouter.java:98)
at org.mule.module.apikit.AbstractRouter.process(AbstractRouter.java:80)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:88)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:98)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.processor.AsyncInterceptingMessageProcessor.process(AsyncInterceptingMessageProcessor.java:108)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:98)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.construct.DynamicPipelineMessageProcessor.process(DynamicPipelineMessageProcessor.java:55)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:88)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:98)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.interceptor.AbstractEnvelopeInterceptor.processBlocking(AbstractEnvelopeInterceptor.java:58)
at org.mule.processor.AbstractRequestResponseMessageProcessor.process(AbstractRequestResponseMessageProcessor.java:47)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:98)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.processor.AbstractFilteringMessageProcessor.process(AbstractFilteringMessageProcessor.java:52)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:98)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.processor.AbstractRequestResponseMessageProcessor.processBlocking(AbstractRequestResponseMessageProcessor.java:56)
at org.mule.processor.AbstractRequestResponseMessageProcessor.process(AbstractRequestResponseMessageProcessor.java:47)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.processor.BlockingProcessorExecutor.executeNext(BlockingProcessorExecutor.java:88)
at org.mule.processor.BlockingProcessorExecutor.execute(BlockingProcessorExecutor.java:59)
at org.mule.execution.ExceptionToMessagingExceptionExecutionInterceptor.execute(ExceptionToMessagingExceptionExecutionInterceptor.java:27)
at org.mule.execution.MessageProcessorNotificationExecutionInterceptor.execute(MessageProcessorNotificationExecutionInterceptor.java:108)
at org.mule.execution.MessageProcessorExecutionTemplate.execute(MessageProcessorExecutionTemplate.java:44)
at org.mule.construct.AbstractPipeline$3.process(AbstractPipeline.java:232)
at org.mule.module.http.internal.listener.HttpMessageProcessorTemplate.routeEvent(HttpMessageProcessorTemplate.java:73)
at org.mule.execution.AsyncResponseFlowProcessingPhase$1.process(AsyncResponseFlowProcessingPhase.java:72)
at org.mule.execution.AsyncResponseFlowProcessingPhase$1.process(AsyncResponseFlowProcessingPhase.java:59)
at org.mule.execution.ExecuteCallbackInterceptor.execute(ExecuteCallbackInterceptor.java:16)
at org.mule.execution.CommitTransactionInterceptor.execute(CommitTransactionInterceptor.java:35)
at org.mule.execution.CommitTransactionInterceptor.execute(CommitTransactionInterceptor.java:22)
at org.mule.execution.HandleExceptionInterceptor.execute(HandleExceptionInterceptor.java:30)
at org.mule.execution.HandleExceptionInterceptor.execute(HandleExceptionInterceptor.java:14)
at org.mule.execution.BeginAndResolveTransactionInterceptor.execute(BeginAndResolveTransactionInterceptor.java:67)
at org.mule.execution.ResolvePreviousTransactionInterceptor.execute(ResolvePreviousTransactionInterceptor.java:44)
at org.mule.execution.SuspendXaTransactionInterceptor.execute(SuspendXaTransactionInterceptor.java:50)
at org.mule.execution.ValidateTransactionalStateInterceptor.execute(ValidateTransactionalStateInterceptor.java:40)
at org.mule.execution.IsolateCurrentTransactionInterceptor.execute(IsolateCurrentTransactionInterceptor.java:41)
at org.mule.execution.ExternalTransactionInterceptor.execute(ExternalTransactionInterceptor.java:48)
at org.mule.execution.RethrowExceptionInterceptor.execute(RethrowExceptionInterceptor.java:28)
at org.mule.execution.RethrowExceptionInterceptor.execute(RethrowExceptionInterceptor.java:13)
at org.mule.execution.TransactionalErrorHandlingExecutionTemplate.execute(TransactionalErrorHandlingExecutionTemplate.java:110)
at org.mule.execution.AsyncResponseFlowProcessingPhase.runPhase(AsyncResponseFlowProcessingPhase.java:58)
at org.mule.execution.AsyncResponseFlowProcessingPhase.runPhase(AsyncResponseFlowProcessingPhase.java:35)
at org.mule.execution.PhaseExecutionEngine$InternalPhaseExecutionEngine.phaseSuccessfully(PhaseExecutionEngine.java:65)
at org.mule.execution.PhaseExecutionEngine$InternalPhaseExecutionEngine.phaseSuccessfully(PhaseExecutionEngine.java:69)
at com.mulesoft.mule.throttling.ThrottlingPhase.runPhase(ThrottlingPhase.java:187)
at com.mulesoft.mule.throttling.ThrottlingPhase.runPhase(ThrottlingPhase.java:58)
at org.mule.execution.PhaseExecutionEngine$InternalPhaseExecutionEngine.phaseSuccessfully(PhaseExecutionEngine.java:65)
at com.mulesoft.gateway.http.phases.GatewayValidationPhase.runPhase(GatewayValidationPhase.java:93)
at com.mulesoft.gateway.http.phases.GatewayValidationPhase.runPhase(GatewayValidationPhase.java:49)
at org.mule.execution.PhaseExecutionEngine$InternalPhaseExecutionEngine.phaseSuccessfully(PhaseExecutionEngine.java:65)
at org.mule.modules.cors.CorsPhase.runPhase(CorsPhase.java:112)
at org.mule.modules.cors.CorsPhase.runPhase(CorsPhase.java:39)
at org.mule.execution.PhaseExecutionEngine$InternalPhaseExecutionEngine.process(PhaseExecutionEngine.java:114)
at org.mule.execution.PhaseExecutionEngine.process(PhaseExecutionEngine.java:41)
at org.mule.execution.MuleMessageProcessingManager.processMessage(MuleMessageProcessingManager.java:32)
at org.mule.module.http.internal.listener.DefaultHttpListener$1.handleRequest(DefaultHttpListener.java:133)
at org.mule.module.http.internal.listener.grizzly.GrizzlyRequestDispatcherFilter.handleRead(GrizzlyRequestDispatcherFilter.java:83)
at org.glassfish.grizzly.filterchain.ExecutorResolver$9.execute(ExecutorResolver.java:119)
at org.glassfish.grizzly.filterchain.DefaultFilterChain.executeFilter(DefaultFilterChain.java:284)
at org.glassfish.grizzly.filterchain.DefaultFilterChain.executeChainPart(DefaultFilterChain.java:201)
at org.glassfish.grizzly.filterchain.DefaultFilterChain.execute(DefaultFilterChain.java:133)
at org.glassfish.grizzly.filterchain.DefaultFilterChain.process(DefaultFilterChain.java:112)
at org.glassfish.grizzly.ProcessorExecutor.execute(ProcessorExecutor.java:77)
at org.glassfish.grizzly.nio.transport.TCPNIOTransport.fireIOEvent(TCPNIOTransport.java:526)
at org.glassfish.grizzly.strategies.AbstractIOStrategy.fireIOEvent(AbstractIOStrategy.java:112)
at org.mule.module.http.internal.listener.grizzly.ExecutorPerServerAddressIOStrategy.run0(ExecutorPerServerAddressIOStrategy.java:102)
at org.mule.module.http.internal.listener.grizzly.ExecutorPerServerAddressIOStrategy.access$100(ExecutorPerServerAddressIOStrategy.java:30)
at org.mule.module.http.internal.listener.grizzly.ExecutorPerServerAddressIOStrategy$WorkerThreadRunnable.run(ExecutorPerServerAddressIOStrategy.java:125)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
********************************************************************************
.
The flow calling the batch is very simple, basically just call the batch process. The error is coming from input phase of the batch where I have 'set variable's, 'database connector's, collection splitter-aggregator which makes calculations and modifies records.
#mCeviker, can you please share your flow?
Normally, you will have to add the commons-csv dependency. Although the batch execute element does not require that dependency.
#mCeviker, you are probably trying to iterate over a collection (with a foreach) an try to modify it (remove elements) in the same loop. You should use an iterator instead. Maybe this link could help you.