We are currently facing a problem with a Prophet model which stops unexpectedly. We've used optuna as our HyperParameter model searching framework, and gave 400 trials per each TimeSeries (we are traininig Prophet on ~ 1000 Timeseries, 1 by 1, not all at once). For cross validation, we've used the methode from the Prophet class called cross_validation.
Here is the code we've used:
def get_best_hp_prophet_cv(df_for_cv, weeks_to_forecast):
logging.getLogger('prophet').setLevel(logging.ERROR)
logging.getLogger('fbprophet').setLevel(logging.ERROR)
# print('DF entered for search: ', df_for_cv)
cutoffs = generate_cutoffs(df_for_cv, weeks_to_forecast)
def objective(trial):
# print(cutoffs)
# print(df.tail(30))
param_grid = {
"changepoint_prior_scale": trial.suggest_categorical(
"changepoint_prior_scale", [0.001, 0.01, 0.1, 0.5, 0.05, 0.8, 0.9]
),
"seasonality_prior_scale": trial.suggest_categorical(
"seasonality_prior_scale", [0.01, 0.05, 0.1, 0.5, 1.0, 10]
),
"seasonality_mode": trial.suggest_categorical(
"seasonality_mode", ["multiplicative", "additive"]
),
"growth": trial.suggest_categorical("growth", ["linear"]),
"yearly_seasonality": trial.suggest_categorical(
"yearly_seasonality", [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16]
),
"daily_seasonality": trial.suggest_categorical("daily_seasonality",[False]),
"weekly_seasonality": trial.suggest_categorical("weekly_seasonality",[False]),
"uncertainty_samples": trial.suggest_categorical("uncertainty_samples",[0]),
}
prior_scale_month = trial.suggest_categorical('prior_scale_month', [0.001, 0.01, 0.1, 0.5, 0.05, 0.9])
prior_scale_week_num = trial.suggest_categorical('prior_scale_week_num', [0.001, 0.01, 0.1, 0.5, 0.05, 0.9])
prior_scale_avg_month_qty_over_df = trial.suggest_categorical('prior_scale_avg_month_qty_over_df', [0.001, 0.01, 0.1, 0.5, 0.05, 0.9])
prior_scale_avg_week_nr_qty_over_df = trial.suggest_categorical('prior_scale_avg_week_nr_qty_over_df', [0.001, 0.01, 0.1, 0.5, 0.05, 0.9])
# I ve used this only for testing to see if everything works fine
# param_grid = {
# 'changepoint_prior_scale': trial.suggest_categorical('changepoint_prior_scale', [0.001]),
# 'seasonality_prior_scale': trial.suggest_categorical('seasonality_prior_scale',[0.01, 0.1]),
# 'seasonality_mode' : trial.suggest_categorical('seasonality_mode',['additive']),
# 'growth': trial.suggest_categorical('growth',['linear']),
# 'yearly_seasonality': trial.suggest_categorical('yearly_seasonality',[14,15]),
# 'holidays_prior_scale' : trial.suggest_categorical('holidays_prior_scale',[10])
# }
# all_params = [dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values())]
# mses = [] # Store the MSEs for each params here
# Use cross validation to evaluate all parameters
# for params in all_params:
m = Prophet(**param_grid)
m.add_regressor('month', prior_scale = prior_scale_month)
m.add_regressor('week_num', prior_scale = prior_scale_week_num)
m.add_regressor('avg_month_qty_over_df', prior_scale = prior_scale_avg_month_qty_over_df)
m.add_regressor('avg_week_nr_qty_over_df', prior_scale = prior_scale_avg_week_nr_qty_over_df)
m.fit(df_for_cv)
df_cv = cross_validation(
m, cutoffs=cutoffs, horizon="{} days".format(weeks_to_forecast*7), parallel="processes"
)
df_p = performance_metrics(df_cv, rolling_window=1)
return df_p["mse"].values[0]
# Find the best parameters
optuna_prophet = optuna.create_study(
direction="minimize", sampler=TPESampler(seed=321)
)
# * n_trials optuna hyperparameter.
#optuna_prophet.optimize(objective, n_trials=400)
optuna_prophet.optimize(objective, n_trials=1)
prophet_trial = optuna_prophet.best_trial
prophet_trial_params = prophet_trial.params
list_of_variables_outside_the_param_grid = ['prior_scale_month','prior_scale_week_num','prior_scale_avg_month_qty_over_df','prior_scale_avg_week_nr_qty_over_df']
params_outside_the_param_grid={}
param_grid = {}
for param_name in prophet_trial_params.keys():
if param_name in list_of_variables_outside_the_param_grid:
params_outside_the_param_grid.update({param_name : prophet_trial_params[param_name]})
else:
param_grid.update({param_name : prophet_trial_params[param_name]})
return param_grid, params_outside_the_param_grid
We've used Prophet before, but on less Timeseries (around 20) and with even more nr_of_trials in Optuna and never faced this issue before. Does anybody knows why is this happening? Has anybody a workaround? It always stops at the begging of the 54 trial from Optuna Search.
We've tried to run with only 1 trial on all 1000 Timeseries and it worked end - 2 - end, but still we can't figure it out why it stops on the 54th trial ?
Versions:
Optuna ---- 3.0.3.
Prophet ---- 1.1
Related
I try to come up with the derivative to the following function:
def f(x, item):
return x[item]
def df_dx(x, item):
pass
where item is as defined in the numpy doc can be list, slice, int, list of slice.
But there are so many edge cases but it feels like there should be a very easy multiplication between two tensors that should be kind of the answer. Can anyone help?
I tried something like this:
to_length = lambda i: i if type(i) == int else len(
range(i.start, (i.stop if i.stop is not None else -1), (i.step if i.step is not None else 1)))
input_shape = list(X.shape)
seg_shape = []
if type(self.item) == int or type(self.item) == slice:
seg_shape = [to_length(self.item)]
elif type(self.item) == list:
seg_shape = [to_length(i) for i in self.item]
else:
raise Exception(f"Unknown type {type(self.item)}")
v = np.zeros(seg_shape + input_shape)
if len(seg_shape) == 1:
# np.fill_diagonal(v[:, self.i:self.i + self.n], 1)
np.fill_diagonal(v[..., self.item], 1)
return v
elif len(seg_shape) == 2:
np.fill_diagonal(v[:, :, self.item[0], self.item[1]], 1)
return v
But it didn't really work.
Your defined function isn't continuous. There is a numpy gradient function. Which may do what you want.
import numpy as np
x = ( np.arange( 20 ) - 10 ) * .1
x = x*x
x
# array([1. , 0.81, 0.64, 0.49, 0.36, 0.25, 0.16, 0.09, 0.04, 0.01, 0. ,
# 0.01, 0.04, 0.09, 0.16, 0.25, 0.36, 0.49, 0.64, 0.81])
def f(x, item):
return x[item]
def df_dx(x, item):
return np.gradient( x )[ item ]
df_dx( x, None )
# array([[-0.19, -0.18, -0.16, -0.14, -0.12, -0.1 , -0.08, -0.06, -0.04,
# -0.02, 0. , 0.02, 0.04, 0.06, 0.08, 0.1 , 0.12, 0.14,
# 0.16, 0.17]])
df_dx( x, [ 2, 8, 10, 17 ] )
# array([-0.16, -0.04, 0. , 0.14])
I tried this and got an error like this:
Node: 'dnn/hiddenlayer_0/Relu'
In[0] and In[1] has different ndims: [1] vs. [1,1]
[[{{node dnn/hiddenlayer_0/Relu}}]] [Op:__inference_pruned_1112]
here is my code:
data = pd.DataFrame(
{
'age': [40.0, 20.0, 18.0, 15.0, 20.0, 30.0, 21.0, 23.0],
'click': [0, 0, 1, 1, 0, 0, 1, 0]
}
)
train, test = train_test_split(data, test_size=0.2)
age = tf.feature_column.numeric_column("age")
feature_columns = [age]
model = tf.estimator.DNNRegressor(feature_columns=feature_columns, hidden_units=[1],
model_dir='./models/dnnregressor')
serving_input_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(tf.feature_column.make_parse_example_spec([age]))
export_path = model.export_saved_model("./models/dnnregressor_serve", serving_input_fn)
age_feature = tf.train.Feature(float_list=tf.train.FloatList(value=[40]))
simple_feature = tf.train.Features(feature={
"age": age_feature
})
simple_example = tf.train.Example(features=simple_feature)
serving_model = tf.saved_model.load(export_path)
f = serving_model.signatures['serving_default']
f(inputs=tf.constant(simple_example.SerializeToString()))
that is an very simple demo, i just want to check the usage of loading saved mode to predict new examples. then i got an error said at the head of the post.
if anyone can help me!!!
change the input from tf.constant(simple_example.SerializeToString()) to
tf.constant([simple_example.SerializeToString()]), will work。
I am new in tensorflow. I use tf.feature_column to finish feature_engineering task, including crossed_column operation. After a simple LR model, I can get the feature_weight for every feature, but how do I know the feature's name? For example:
my_features = []
weight = IndicatorColumn(categorical_column=VocabularyListCategoricalColumn(key='weight', vocabulary_list=(2, 3, 0, 1), dtype=tf.int64, default_value=-1, num_oov_buckets=0))
my_features.append(weight)
vol = IndicatorColumn(categorical_column=VocabularyListCategoricalColumn(key='vol', vocabulary_list=(3, 4, 1, 2), dtype=tf.int64, default_value=-1, num_oov_buckets=0))
my_features.append(vol)
weight_X_vol = tf.feature_column.crossed_column( [feature_weight, feature_vol], has_bucket_size = 50)
my_features.append(weight_X_vol)
in feature_weight, it's one_hot feature size is 4(weight0, 1, 2, 3)
in feature_vol, it's one_hot feature size is 4(vol0, 1, 2, 3)
in feature_weight_X_vol, it's one_hot feature size is 16(weight0_vol0, weight0_vol1, ...., weight3_vol3)
so my features_size is 4 + 4 + 16 = 24
After fit a simple LR model, I can get the 24 feature_weight by model.layers[1].get_weights(), but how can I build a dict to map feature_name(key) to the feature_weight(value)? For example:
my model is :
my_model = tf.keras.Sequential([
tf.keras.layers.DenseFeatures(my_features),
tf.keras.layers.Dense(1, activatio="sigmoid")
])
feature_weight = my_model.layers[0].get_weights()
my weight is :
faeture_weight = [0.1, 0.2, 0.1, ..., 0.8, 0.4. 0.3] //lengh is 24
my feature_name is :
feature_name = [weight0, weight1, weight2, ..., vol0, vol1, ..., weight0_vol0, weight3_vol3] // lengh is 24
How can I map the key to the value? I just want to know every feature's weight for debuging.
I've trained a classification model in 0.9:
param = {
'objective': 'multi:softprob',
'num_class': 9,
'booster': 'dart',
'eta': 0.3,
'gamma': 0,
'max_depth': 6,
'alpha': 0,
'lambda': 1,
'colsample_bylevel':0.8,
'colsample_bynode': 0.8,
'colsample_bytree': 0.8,
'normalize_type': 'tree',
'rate_drop': 1.0,
'min_child_weight': 5,
'subsample': 0.5,
'num_parallel_tree': 1,
'tree_method': 'approx'
}
model = xgb.train(param, D_train, num_boost_round=1)
model.save_model('./model.bst')
Here is the sample training data:
{f1:1, f2:1, label:"1"}
Both f1 and f2 are integers.
During prediction, the results are nondeterministic with the same input. Sometimes (~1 out of 10 times) it gives equal probability for every output class.
This issue is gone when switching to XGB 1.0: Use XGB 1.0 to make predictions on a model trained in 0.9.
imported_model = xgb.Booster(model_file='./model.bst')
encoded=[[[0, 0]]
prediction_input = xgb.DMatrix(
np.array(encoded).reshape(2, -1), missing=None)
for i in range(1000):
outputs = imported_model.predict(prediction_input)
print(outputs)
Does anyone know the root cause?
I like to evaluate my object detection model with mAP (mean average precision). In https://github.com/tensorflow/models/tree/master/research/object_detection/utils/ there is object_detection_evaluation.py that I want to use.
I use following for the groundtruth boxes:
pascal_evaluator = object_detection_evaluation.PascalDetectionEvaluator(
categories, matching_iou_threshold=0.1)
groundtruth_boxes = np.array([[10, 10, 11, 11]], dtype=float)
groundtruth_class_labels = np.array([1], dtype=int)
groundtruth_is_difficult_list = np.array([False], dtype=bool)
pascal_evaluator.add_single_ground_truth_image_info(
'img2',
{
standard_fields.InputDataFields.groundtruth_boxes: groundtruth_boxes,
standard_fields.InputDataFields.groundtruth_classes: groundtruth_class_labels,
standard_fields.InputDataFields.groundtruth_difficult: groundtruth_is_difficult_list
}
)
and this for the prediction Boxes:
# Add detections
image_key = 'img2'
detected_boxes = np.array(
[ [100, 100, 220, 220], [10, 10, 11, 11]],
dtype=float)
detected_class_labels = np.array([1,1], dtype=int)
detected_scores = np.array([0.8, 0.9], dtype=float)
pascal_evaluator.add_single_detected_image_info(image_key, {
standard_fields.DetectionResultFields.detection_boxes:
detected_boxes,
standard_fields.DetectionResultFields.detection_scores:
detected_scores,
standard_fields.DetectionResultFields.detection_classes:
detected_class_labels
})
I print the results with
metrics = pascal_evaluator.evaluate()
print(metrics)
And my Question:
if I use this prediction Boxes [100, 100, 220, 220], [10, 10, 11, 11] the result is:
{'PASCAL/Precision/mAP#0.1IOU': 1.0,
'PASCAL/PerformanceByCategory/AP#0.1IOU/face': 1.0}
If I use [10, 10, 11, 11], [100, 100, 220, 220] (other Box sequence)
I get following result:
{'PASCAL/Precision/mAP#0.1IOU': 0.5,
'PASCAL/PerformanceByCategory/AP#0.1IOU/face': 0.5}
Why is that so? Or is it bug?
Cheers Michael
Although you are not so clear about it I think I found the error in your code. You mentioned you get different results for different order of bounding boxes. This seems peculiar and if true then it was surely a bug.
But, since I tested the code myself, you probably did not change the corresponding scores (detected_scores = np.array([0.8, 0.9], dtype=float)) to the bounding boxes. But this way you changes also the problem not just the order of the bounding boxes. If you apply the correct bounding boxes the mAP remains the same in both cases:
{'PascalBoxes_Precision/mAP#0.5IOU': 1.0,
'PascalBoxes_PerformanceByCategory/AP#0.5IOU/person': 1.0}