Related
I need to develop a function to save multiple models from dictionary and load the best one which gave highest accuracy on test set. Thanks in advance!
I have a model like this:
from keras.models import Sequential
from keras.layers import Dense
def create_custom_model(input_dim, output_dim, nodes, n=11, name='model'):
def create_model():
# Create model
#model = Sequential(name=name)
for i in range(n):
#nodes=[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
#for node in nodes:
model = Sequential(name=name)
model.add(Dense(n, input_dim=input_dim, activation='relu'))
model.add(Dense(output_dim, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
return create_model
models = [create_custom_model(n_features, n_classes, 8, i, 'model_{}'.format(i))
for i in range(1, 11)]
for create_model in models:
create_model().summary()
history_dict = {}
# TensorBoard Callback
cb = TensorBoard()
for create_model in models:
model = create_model()
print('Model name:', model.name)
history_callback = model.fit(X_train, Y_train,
batch_size=5,
epochs=50,
verbose=0,
validation_data=(X_test, Y_test),
callbacks=[early_stopping])
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
history_dict[model.name] = [history_callback, model]
Output:
Model name: model_1
Test loss: 0.29368123412132263
Test accuracy: 0.9066666960716248
Model name: model_2
Test loss: 0.1819317787885666
Test accuracy: 0.9466666579246521
Model name: model_3
Test loss: 0.11470139771699905
Test accuracy: 0.9599999785423279
.....
.....
How can I save all model from history_dict and load the best one which best accuracy?
I've been trying to train this CNN Model, It's a Tensorflow tutorial and I just changed the dataset ( I used fruit 360 dataset) without altering the core of the code. When it finishes training the accuracy stays constant at 0.8565 it doesn't change and when I try and test some images it almost always wrong.
What am I doing wrong?
Code output after executing
Here's the code I used
[enter image description here][1]import matplotlib.pyplot as plt
import numpy as np
import os
import PIL
import tensorflow as tf
import tarfile
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential
from tensorflow.keras import datasets, layers, models
from tensorflow import keras
import pathlib
dataset_url = "https://file.io/z5JM3sYAWXv4"
data_dir = tf.keras.utils.get_file(origin=dataset_url,
fname='tomatos',
untar=True,
extract=True)
data_dir = pathlib.Path(data_dir)
print(data_dir)
file_count = sum(len(files) for _, _, files in os.walk(r'tomatos'))
print(file_count)
batch_size = 32
img_height = 180
img_width = 180
train_ds = tf.keras.utils.image_dataset_from_directory(
data_dir,
validation_split=0.2,
subset="training",
seed=123,
image_size=(img_height, img_width),
batch_size=batch_size)
val_ds = tf.keras.utils.image_dataset_from_directory(
data_dir,
validation_split=0.2,
subset="validation",
seed=123,
image_size=(img_height, img_width),
batch_size=batch_size)
class_names = train_ds.class_names
print(class_names)
AUTOTUNE = tf.data.AUTOTUNE
train_ds = train_ds.cache().shuffle(1000).prefetch(buffer_size=AUTOTUNE)
val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)
num_classes = len(class_names)
model = Sequential([
layers.Rescaling(1./255, input_shape=(img_height, img_width, 3)),
layers.Conv2D(16, 3, padding='same', activation='relu'),
layers.MaxPooling2D(),
layers.Conv2D(32, 3, padding='same', activation='relu'),
layers.MaxPooling2D(),
layers.Conv2D(64, 3, padding='same', activation='relu'),
layers.MaxPooling2D(),
layers.Flatten(),
layers.Dense(128, activation='relu'),
layers.Dense(num_classes)
])
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
model.summary()
epochs=2
history = model.fit(
train_ds,
validation_data=val_ds,
epochs=epochs
)
data_augmentation = keras.Sequential(
[
layers.RandomFlip("horizontal",
input_shape=(img_height,
img_width,
3)),
layers.RandomRotation(0.1),
layers.RandomZoom(0.1),
]
)
model = Sequential([
data_augmentation,
layers.Rescaling(1./255),
layers.Conv2D(16, 3, padding='same', activation='relu'),
layers.MaxPooling2D(),
layers.Conv2D(32, 3, padding='same', activation='relu'),
layers.MaxPooling2D(),
layers.Conv2D(64, 3, padding='same', activation='relu'),
layers.MaxPooling2D(),
layers.Dropout(0.2),
layers.Flatten(),
layers.Dense(128, activation='relu'),
layers.Dense(num_classes)
])
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
model.summary()
epochs = 4
history = model.fit(
train_ds,
validation_data=val_ds,
epochs=epochs
)
sunflower_url = "https://puffycarrot.com/wp-content/uploads/2017/04/Green-tomatoes.jpg"
sunflower_path = tf.keras.utils.get_file('tomato2', origin=sunflower_url)
img = tf.keras.utils.load_img(
sunflower_path, target_size=(img_height, img_width)
)
img_array = tf.keras.utils.img_to_array(img)
img_array = tf.expand_dims(img_array, 0) # Create a batch
predictions = model.predict(img_array)
score = tf.nn.softmax(predictions[0])
print(
"This image most likely belongs to {} with a {:.2f} percent confidence."
.format(class_names[np.argmax(score)], 100 * np.max(score))
)
#Yaman Tarawneh, I tried replicating your above mentioned code in Google colab (using TF 2.8) and in Pycharm (using TF 2.7) and did not find the error.
Please check the output image for Pycharm :
and got the same output in Google colab :
Total params: 3,988,898
Trainable params: 3,988,898
Non-trainable params: 0
_________________________________________________________________
Epoch 1/4
78/78 [==============================] - 8s 41ms/step - loss: 0.0309 - accuracy: 0.9835 - val_loss: 5.6374e-07 - val_accuracy: 1.0000
Epoch 2/4
78/78 [==============================] - 2s 25ms/step - loss: 5.7533e-07 - accuracy: 1.0000 - val_loss: 2.7360e-07 - val_accuracy: 1.0000
Epoch 3/4
78/78 [==============================] - 2s 25ms/step - loss: 3.0400e-07 - accuracy: 1.0000 - val_loss: 1.3978e-07 - val_accuracy: 1.0000
Epoch 4/4
78/78 [==============================] - 2s 25ms/step - loss: 1.7403e-07 - accuracy: 1.0000 - val_loss: 7.2102e-08 - val_accuracy: 1.0000
This image most likely belongs to Tomato not Ripened with a 100.00 percent confidence.
For further analysis if the issue still persists, Please let us know which Python and Tensorflow version are you using.
Can anyone point to references where one can learn how to perform Quantization Aware Training (QAT) with tf.GradientTape on TensorFlow 2?
I only see this done with the tf.keras API. I do not use tf. keras, I always build customized training with tf.GradientTape provides more control over the training process. I now need to quantize a model but I only see references on how to do it using the tf. keras API.
In the official examples here, they showed QAT training with model. fit. Here is a demonstration of Quantization Aware Training using tf.GradientTape(). But for complete reference, let's do both here.
Base model training. This is directly from the official doc. For more details, please check there.
import os
import tensorflow as tf
from tensorflow import keras
import tensorflow_model_optimization as tfmot
# Load MNIST dataset
mnist = keras.datasets.mnist
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
# Normalize the input image so that each pixel value is between 0 to 1.
train_images = train_images / 255.0
test_images = test_images / 255.0
# Define the model architecture.
model = keras.Sequential([
keras.layers.InputLayer(input_shape=(28, 28)),
keras.layers.Reshape(target_shape=(28, 28, 1)),
keras.layers.Conv2D(filters=12, kernel_size=(3, 3), activation='relu'),
keras.layers.MaxPooling2D(pool_size=(2, 2)),
keras.layers.Flatten(),
keras.layers.Dense(10)
])
# Train the digit classification model
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
model.summary()
model.fit(
train_images,
train_labels,
epochs=1,
validation_split=0.1,
)
10ms/step - loss: 0.5411 - accuracy: 0.8507 - val_loss: 0.1142 - val_accuracy: 0.9705
<tensorflow.python.keras.callbacks.History at 0x7f9ee970ab90>
QAT .fit.
Now, performing QAT over the base model.
# -----------------------
# ------------- Quantization Aware Training -------------
import tensorflow_model_optimization as tfmot
quantize_model = tfmot.quantization.keras.quantize_model
# q_aware stands for for quantization aware.
q_aware_model = quantize_model(model)
# `quantize_model` requires a recompile.
q_aware_model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
q_aware_model.summary()
train_images_subset = train_images[0:1000]
train_labels_subset = train_labels[0:1000]
q_aware_model.fit(train_images_subset, train_labels_subset,
batch_size=500, epochs=1, validation_split=0.1)
356ms/step - loss: 0.1431 - accuracy: 0.9629 - val_loss: 0.1626 - val_accuracy: 0.9500
<tensorflow.python.keras.callbacks.History at 0x7f9edf0aef90>
Checking performance
_, baseline_model_accuracy = model.evaluate(
test_images, test_labels, verbose=0)
_, q_aware_model_accuracy = q_aware_model.evaluate(
test_images, test_labels, verbose=0)
print('Baseline test accuracy:', baseline_model_accuracy)
print('Quant test accuracy:', q_aware_model_accuracy)
Baseline test accuracy: 0.9660999774932861
Quant test accuracy: 0.9660000205039978
QAT tf.GradientTape().
Here is the QAT training part on the base model. Note we can also perform custom training over the base model.
batch_size = 500
train_dataset = tf.data.Dataset.from_tensor_slices((train_images_subset,
train_labels_subset))
train_dataset = train_dataset.batch(batch_size=batch_size,
drop_remainder=False)
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
optimizer = tf.keras.optimizers.Adam()
for epoch in range(1):
for x, y in train_dataset:
with tf.GradientTape() as tape:
preds = q_aware_model(x, training=True)
loss = loss_fn(y, preds)
grads = tape.gradient(loss, q_aware_model.trainable_variables)
optimizer.apply_gradients(zip(grads, q_aware_model.trainable_variables))
_, baseline_model_accuracy = model.evaluate(
test_images, test_labels, verbose=0)
_, q_aware_model_accuracy = q_aware_model.evaluate(
test_images, test_labels, verbose=0)
print('Baseline test accuracy:', baseline_model_accuracy)
print('Quant test accuracy:', q_aware_model_accuracy)
Baseline test accuracy: 0.9660999774932861
Quant test accuracy: 0.9645000100135803
I am working on predicting stock trend (up, or down).
Below is how I am handling my pre-processing.
index_ = len(df.columns) - 1
x = df.iloc[:,:index_]
x = x[['Relative_Volume', 'CurrentPrice', 'MarketCap']]
x = x.values.astype(float)
# x = x.reshape(len(x), 1, x.shape[1]).astype(float)
x = x.reshape(*x.shape, 1)
y = df.iloc[:,index_:].values.astype(float)
# x.shape = (44930, 3, 1)
# y.shape = (44930, 1)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=98 )
Then I am building my BILSTM model:
def build_nn():
model = Sequential()
model.add(Bidirectional(LSTM(128, return_sequences=True, input_shape = (x_train.shape[0], 1) , name="one")))
model.add(Dropout(0.20))
model.add(Bidirectional(LSTM(128, return_sequences=True , name="two")))
model.add(Dropout(0.20))
model.add(Bidirectional(LSTM(64, return_sequences=False , name="three")))
model.add(Dropout(0.20))
model.add(Dense(1,activation='sigmoid'))
# opt = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, decay=0.01)
opt = SGD(lr=0.01)
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
return model
filepath = "bilstmv1.h5"
chkp = ModelCheckpoint(monitor = 'val_accuracy', mode = 'auto', filepath=filepath, verbose = 1, save_best_only=True)
model = build_nn()
# model.summary()
model.fit(x_train, y_train,
epochs=3,
batch_size=256,
validation_split=0.1, callbacks=[chkp])
model.summary()
Below is the output of the loss_value:
Epoch 1/3
127/127 [==============================] - 27s 130ms/step - loss: 0.6829 - accuracy: 0.5845 - val_loss: 0.6797 - val_accuracy: 0.5803
Epoch 00001: val_accuracy improved from -inf to 0.58025, saving model to bilstmv1.h5
Epoch 2/3
127/127 [==============================] - 14s 112ms/step - loss: 0.6788 - accuracy: 0.5851 - val_loss: 0.6798 - val_accuracy: 0.5803
Epoch 00002: val_accuracy did not improve from 0.58025
Epoch 3/3
127/127 [==============================] - 14s 112ms/step - loss: 0.6800 - accuracy: 0.5822 - val_loss: 0.6798 - val_accuracy: 0.5803
Epoch 00003: val_accuracy did not improve from 0.58025
I have tried to change the optimzer, loss_function, and other modification. As you can expect, all the predictions are same since the loss function is not being changed.
You have an issue with your input shape in your first LSTM layer. Keras inputs takes (None, Your_Shape) as its input, since your input to the model can vary. You can have 1 input, 2 inputs, or infinity inputs. The only way to represent dynamic is by using None as the first input. The quickest way to do this is to change the input to (None, *input_shape), since the * will expand your input shape.
Your build function will then become:
def build_nn():
model = Sequential()
model.add(Bidirectional(LSTM(128, return_sequences=True, input_shape = (None, *x_train.shape) , name="one")))
model.add(Dropout(0.20))
model.add(Bidirectional(LSTM(128, return_sequences=True , name="two")))
model.add(Dropout(0.20))
model.add(Bidirectional(LSTM(64, return_sequences=False , name="three")))
model.add(Dropout(0.20))
model.add(Dense(1,activation='sigmoid'))
# opt = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, decay=0.01)
opt = SGD(lr=0.01)
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])
return model
Though I still advise having a look at your Optimizer as that might affect your results. You can also use -1 as an input shape which will mean auto fill, but you can only use it once.
Given 256x256 rgb input images, I'm trying to regress to predict a point on the X axis of the image (0-48000)
Initially, I tried [mobile_net -> GlobalAveragePooling2D -> several Dense layers]. I didn't realize Pooling was discarding the spatial information.
Last night, I trained on a simpler net, the loss decreased all night, but it's predicting negative values.
How can I modify this architecture to predict a 0-48000 scalar?
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(64, kernel_size=3, activation='relu', input_shape=(256,256,3)),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Conv2D(32, kernel_size=3, activation='relu'),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(1, kernel_initializer='normal'),
])
model.compile(loss='mse', optimizer='adam', metrics=['mse', 'mae', 'mape']) #
EDIT:
Inferring from my netwrok, I'm getting vastly different outputs, each run, for the SAME file. How is that possible?
Infer outputs, running multiple times on the same file:
-312864.9444580078
762.7029418945312
193352.7603149414
Here is the inference fn:
def infer(checkpoint_path):
png_file = ['3023_28338_26_m.png', '3023_28338_26_m.png'][1]
test_file = data_root + png_file
onset = png_file.strip('_m.png.').split('_')[1]
img = load_and_preprocess_from_path_label(test_file, 0)
tst = np.expand_dims(img[0], axis=0)
model = load_model_and_checkpoint(checkpoint_path)
val = model.predict(tst)[0][0] * 48000
Here is the final epoch of training.
2019-05-26 11:11:56.698907: I tensorflow/core/kernels/data/shuffle_dataset_op.cc:150] Shuffle buffer filled.
94/95 [============================>.] - ETA: 0s - loss: 0.0063 - mse: 0.0063 - mae: 0.0627 - mape: 93.2817
Epoch 00100: saving model to /media/caseybasichis/sp_data/sp_data/datasets/one_sec_onset_01/model7.ckpt
95/95 [==============================] - 47s 500ms/step - loss: 0.0063 - mse: 0.0063 - mae: 0.0626 - mape: 93.2076
Here is the latest network.
mobile_net = tf.keras.applications.ResNet50(input_shape=(256, 256, 3), include_top=False, weights='imagenet')
mobile_net.trainable=False
model = tf.keras.Sequential([
mobile_net,
tf.keras.layers.Dropout(0.25),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(512, kernel_initializer='normal', activation='relu'),
tf.keras.layers.BatchNormalization(axis=chanDim),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(1, kernel_initializer='normal', activation='linear'), # activation='sigmoid'
])
model.compile(loss='mse', optimizer='adam', metrics=['mse', 'mae', 'mape']) # mean_squared_logarithmic_error
You can simply use Sigmoid activation on the last layer and multiply the output by the scale (in a Lambda layer or preferably just scale the output out side the network)
model.add(Activation('sigmoid'))
model.add(Lambda(lambda x: 48000*x))
or
model.add(Activation('sigmoid'))
...
model.fit(x_train, y_train/48000.0)