Using Keras Tuner for time series split - tensorflow

Question

Is it possible to use Keras tuner for tuning a NN using Time Series Split , similar to sklearn.model_selection.TimeSeriesSplit in sklearn.
For example consider a sample tuner class from https://towardsdatascience.com/hyperparameter-tuning-with-keras-tuner-283474fbfbe
from kerastuner import HyperModel
class SampleModel(HyperModel):
def __init__(self, input_shape):
self.input_shape = input_shape
def build(self, hp):
model = Sequential()
model.add(
layers.Dense(
units=hp.Int('units', 8, 64, 4, default=8),
activation=hp.Choice(
'dense_activation',
values=['relu', 'tanh', 'sigmoid'],
default='relu'),
input_shape=input_shape
)
)
model.add(layers.Dense(1))
model.compile(
optimizer='rmsprop',loss='mse',metrics=['mse']
)
return model
tuner:
tuner_rs = RandomSearch(
hypermodel,
objective='mse',
seed=42,
max_trials=10,
executions_per_trial=2)
tuner_rs.search(x_train_scaled, y_train, epochs=10, validation_split=0.2, verbose=0)
So instead of validation_split = 0.2, in the above line is it possible to do the following
from sklearn.model_selection import TimeSeriesSplit
#defining a time series split object
tscv = TimeSeriesSplit(n_splits = 5)
#using that in Keras Tuner
tuner_rs.search(x_train, y_train, epochs=10, validation_split=tscv, verbose=0)

Answer 1

I solved in this way:
First I have istanciated a class that allows to perform Blocking Time Series Split. I found out that it might be better to use this time series split rather than Sklearn TimeSeriesSplit because we won't make our model train on instances with already seen data. As you can see from the picture, if number of splits is 5, BTSS will divide your training data in 5 parts with only the validation data in common across the splits. (Since StackOverflow doesn't allow me to upload images i'll post a reference link: https://hub.packtpub.com/cross-validation-strategies-for-time-series-forecasting-tutorial/)
class BlockingTimeSeriesSplit():
def __init__(self, n_splits):
self.n_splits = n_splits
def get_n_splits(self, X, y, groups):
return self.n_splits
def split(self, X, y=None, groups=None):
n_samples = len(X)
k_fold_size = n_samples // self.n_splits
indices = np.arange(n_samples)
margin = 0
for i in range(self.n_splits):
start = i * k_fold_size
stop = start + k_fold_size
mid = int(0.8 * (stop - start)) + start
yield indices[start: mid], indices[mid + margin: stop]
Then you will proceed by creating your own model:
def build_model(hp):
pass
Finally you can create your CVtuner as a class which will call back BlockingTimeSeriesSplit.
class CVTuner(kt.engine.tuner.Tuner):
def run_trial(self, trial, x, y, *args, **kwargs):
cv = BlockingTimeSeriesSplit(n_splits=5)
val_accuracy_list = []
batch_size = trial.hyperparameters.Int('batch_size', 0, 64, step=8)
epochs = trial.hyperparameters.Int('epochs', 10, 100, step=10)
for train_indices, test_indices in cv.split(x):
x_train, x_test = x[train_indices], x[test_indices]
y_train, y_test = y[train_indices], y[test_indices]
model = self.hypermodel.build(trial.hyperparameters)
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs)
val_loss, val_accuracy, val_auc = model.evaluate(x_test, y_test)
val_accuracy_list.append(val_accuracy)
self.oracle.update_trial(trial.trial_id, {'val_accuracy': np.mean(val_accuracy_list)})
self.save_model(trial.trial_id, model)
tuner = CVTuner(oracle=kt.oracles.BayesianOptimization(objective='val_accuracy',max_trials=1), hypermodel=create_model)
stop_early = tf.keras.callbacks.EarlyStopping(monitor='accuracy', patience=10)
tuner.search(X, Y, callbacks=[stop_early])
best_model = tuner.get_best_models()[0]
best_model.summary()
best_model.evaluate(x_out_of_sample, y_out_of_sample)