TensorFlow 2 Quantization Aware Training (QAT) with tf.GradientTape - tensorflow

Can anyone point to references where one can learn how to perform Quantization Aware Training (QAT) with tf.GradientTape on TensorFlow 2?
I only see this done with the tf.keras API. I do not use tf. keras, I always build customized training with tf.GradientTape provides more control over the training process. I now need to quantize a model but I only see references on how to do it using the tf. keras API.

In the official examples here, they showed QAT training with model. fit. Here is a demonstration of Quantization Aware Training using tf.GradientTape(). But for complete reference, let's do both here.
Base model training. This is directly from the official doc. For more details, please check there.
import os
import tensorflow as tf
from tensorflow import keras
import tensorflow_model_optimization as tfmot
# Load MNIST dataset
mnist = keras.datasets.mnist
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
# Normalize the input image so that each pixel value is between 0 to 1.
train_images = train_images / 255.0
test_images = test_images / 255.0
# Define the model architecture.
model = keras.Sequential([
keras.layers.InputLayer(input_shape=(28, 28)),
keras.layers.Reshape(target_shape=(28, 28, 1)),
keras.layers.Conv2D(filters=12, kernel_size=(3, 3), activation='relu'),
keras.layers.MaxPooling2D(pool_size=(2, 2)),
keras.layers.Flatten(),
keras.layers.Dense(10)
])
# Train the digit classification model
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
model.summary()
model.fit(
train_images,
train_labels,
epochs=1,
validation_split=0.1,
)
10ms/step - loss: 0.5411 - accuracy: 0.8507 - val_loss: 0.1142 - val_accuracy: 0.9705
<tensorflow.python.keras.callbacks.History at 0x7f9ee970ab90>
QAT .fit.
Now, performing QAT over the base model.
# -----------------------
# ------------- Quantization Aware Training -------------
import tensorflow_model_optimization as tfmot
quantize_model = tfmot.quantization.keras.quantize_model
# q_aware stands for for quantization aware.
q_aware_model = quantize_model(model)
# `quantize_model` requires a recompile.
q_aware_model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
q_aware_model.summary()
train_images_subset = train_images[0:1000]
train_labels_subset = train_labels[0:1000]
q_aware_model.fit(train_images_subset, train_labels_subset,
batch_size=500, epochs=1, validation_split=0.1)
356ms/step - loss: 0.1431 - accuracy: 0.9629 - val_loss: 0.1626 - val_accuracy: 0.9500
<tensorflow.python.keras.callbacks.History at 0x7f9edf0aef90>
Checking performance
_, baseline_model_accuracy = model.evaluate(
test_images, test_labels, verbose=0)
_, q_aware_model_accuracy = q_aware_model.evaluate(
test_images, test_labels, verbose=0)
print('Baseline test accuracy:', baseline_model_accuracy)
print('Quant test accuracy:', q_aware_model_accuracy)
Baseline test accuracy: 0.9660999774932861
Quant test accuracy: 0.9660000205039978
QAT tf.GradientTape().
Here is the QAT training part on the base model. Note we can also perform custom training over the base model.
batch_size = 500
train_dataset = tf.data.Dataset.from_tensor_slices((train_images_subset,
train_labels_subset))
train_dataset = train_dataset.batch(batch_size=batch_size,
drop_remainder=False)
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
optimizer = tf.keras.optimizers.Adam()
for epoch in range(1):
for x, y in train_dataset:
with tf.GradientTape() as tape:
preds = q_aware_model(x, training=True)
loss = loss_fn(y, preds)
grads = tape.gradient(loss, q_aware_model.trainable_variables)
optimizer.apply_gradients(zip(grads, q_aware_model.trainable_variables))
_, baseline_model_accuracy = model.evaluate(
test_images, test_labels, verbose=0)
_, q_aware_model_accuracy = q_aware_model.evaluate(
test_images, test_labels, verbose=0)
print('Baseline test accuracy:', baseline_model_accuracy)
print('Quant test accuracy:', q_aware_model_accuracy)
Baseline test accuracy: 0.9660999774932861
Quant test accuracy: 0.9645000100135803

Related

Save multiple Keras Models and load the best one based on accuracy

I need to develop a function to save multiple models from dictionary and load the best one which gave highest accuracy on test set. Thanks in advance!
I have a model like this:
from keras.models import Sequential
from keras.layers import Dense
def create_custom_model(input_dim, output_dim, nodes, n=11, name='model'):
def create_model():
# Create model
#model = Sequential(name=name)
for i in range(n):
#nodes=[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
#for node in nodes:
model = Sequential(name=name)
model.add(Dense(n, input_dim=input_dim, activation='relu'))
model.add(Dense(output_dim, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
return create_model
models = [create_custom_model(n_features, n_classes, 8, i, 'model_{}'.format(i))
for i in range(1, 11)]
for create_model in models:
create_model().summary()
history_dict = {}
# TensorBoard Callback
cb = TensorBoard()
for create_model in models:
model = create_model()
print('Model name:', model.name)
history_callback = model.fit(X_train, Y_train,
batch_size=5,
epochs=50,
verbose=0,
validation_data=(X_test, Y_test),
callbacks=[early_stopping])
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
history_dict[model.name] = [history_callback, model]
Output:
Model name: model_1
Test loss: 0.29368123412132263
Test accuracy: 0.9066666960716248
Model name: model_2
Test loss: 0.1819317787885666
Test accuracy: 0.9466666579246521
Model name: model_3
Test loss: 0.11470139771699905
Test accuracy: 0.9599999785423279
.....
.....
How can I save all model from history_dict and load the best one which best accuracy?

How to improve the model's accuracy?

I am definitely a new beginner of tensorflow, I tried to create a simple model, but the accuracy is super low, can someone help to figure out what is wrong?
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Sequential
train_x = [[i, j] for i in range(1000) for j in range(1000)]
train_y = [[(2 * i + 3 * j) % 10] for i in range(1000) for j in range(1000)]
model = Sequential()
model.add(Dense(32, activation="relu", input_dim=2))
model.add(Dense(32, activation="relu"))
model.add(Dense(10, activation="softmax"))
model.compile(optimzier="rmsprop", loss="sparse_categorical_crossentropy", metrics=['accuracy'])
model.summary()
model.fit(train_x, train_y, epochs=10, batch_size=1000)
test_x = train_x[10:60]
test_y = train_y[10:60]
model.evaluate(test_x, test_y, batch_size=100)
Result:
loss: 2.3026 - accuracy: 0.1000
There are few ways to improve your model's accuracy
Reduce the batch size (You are using whole dataset)
Increase the number of layers, units.
Increase the number of epochs.
Use unseen data to evaluate the model. (You are using 60 elements of training data)
I suggest you to read Deep Learning with Python by Francois Chollet, section 3.5 "A multiclass classification example"

Accuracy of Auto Encoder (Image Reconstruction) model in Keras

I am trying to make a simple autoencoder model for Image reconstruction along with MSNIT dataset.
Now if I run this model, it presents me with accuracy.
'60000/60000 [==============================] - 5s 83us/sample - loss: 0.0373 - accuracy: 0.2034 - val_loss: 0.0368 - val_accuracy: 0.217
'.
but I am not sure how its calculated given that the prediction results itself are an image.
I dug deep till function "sparse_categorical_accuracy" but was not able to reach any conclusion about the formula for accuracy.
latent_dim = 64
class Autoencoder(Model):
def __init__(self, latent_dim):
super(Autoencoder, self).__init__()
self.latent_dim = latent_dim
self.encoder = tf.keras.Sequential([
layers.Flatten(),
layers.Dense(latent_dim, activation='relu'),
])
self.decoder = tf.keras.Sequential([
layers.Dense(784, activation='sigmoid'),
layers.Reshape((28, 28))
])
def call(self, x):
encoded = self.encoder(x)
decoded = self.decoder(encoded)
return decoded
autoencoder = Autoencoder(latent_dim)
autoencoder.compile(optimizer='adam', loss=losses.MeanSquaredError(),
metrics=['accuracy'])
autoencoder.fit(x_train, x_train,
epochs=10,
shuffle=True,
validation_data=(x_test, x_test))

Keras custom loss function print tensor values

I try to implement object detector like yolo. It uses complex custom loss function. So I need to print/debug its tensors. I understand, that python code only builds computing graph so standard print won't work in not eager mode.
tensorflow 1.12.0
keras 2.2.4
I tried all methods from these posts Keras custom loss function not printing value of tensor, Debugging keras tensor values and nothing works. I tried tf.Print, tf.print, callback, K.tensor_print - the same result. In console i see only standart output messages. I`m not even sure if loss function been called. The answer from this post Keras - printing intermediate tensors in loss function (tf.Print and K.print_tensor do not work...) says that loss function sometimes doesn't even called! Ok, but how to use tf.contrib.eager.defun decorator then? The example is for pure tensorflow and dont't understand how to use it in keras.
import tensorflow as tf
from keras.datasets import fashion_mnist
import matplotlib.pyplot as plt
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.layers import Flatten, Dense, Dropout
from keras.models import Sequential
from keras import optimizers
import numpy as np
from random import randrange
from keras.callbacks import LambdaCallback
import keras.backend as K
import keras
print(tf.__version__)
print(keras.__version__)
num_filters = 64
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
#reshape
x_train = x_train.reshape(60000,28,28,1)[:1000,...]
x_test = x_test.reshape(10000,28,28,1)[:100,...]
# One-hot encode the labels
y_train = tf.keras.utils.to_categorical(y_train, 10)[:1000,...]
y_test = tf.keras.utils.to_categorical(y_test, 10)[:100,]
labels = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
model = Sequential()
model.add(Conv2D(input_shape=(28,28,1), filters=num_filters,kernel_size=3,strides=(1, 1),padding="valid", activation='relu', use_bias=True))
model.add(MaxPooling2D(pool_size=2))
model.add(Dropout(0.3))
model.add(Conv2D(filters=num_filters,kernel_size=3,strides=(1, 1),padding="valid", activation='relu', use_bias=True))
model.add(MaxPooling2D(pool_size=2))
model.add(Dropout(0.3))
model.add(Flatten())
model.add(Dense(256, activation = 'relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation = 'softmax'))
#model.summary()
#loss 1
def customLoss(yTrue,yPred):
d = yPred-yTrue
d = K.print_tensor(d)
return K.mean(K.square(d), axis=-1)
#loss 2
def cat_loss(y_true, y_pred):
d = y_true - y_pred
d = tf.Print(d, [d], "Inside loss function")
return tf.reduce_mean(tf.square(d))
model.compile(loss=customLoss,
optimizer='adam')
import keras.callbacks as cbks
# 3 try to print with callback
class CustomMetrics(cbks.Callback):
def on_epoch_end(self, epoch, logs=None):
for k in logs:
if k.endswith('cat_loss'):
print(logs[k])
#checkpointer = ModelCheckpoint(filepath='model.weights.best.hdf5', verbose = 1, save_best_only=True)
model.fit(x_train,
y_train,
#verbose=1,
batch_size=16,
epochs=10,
validation_data=(x_test, y_test),
callbacks=[CustomMetrics()])
# Evaluate the model on test set
score = model.evaluate(x_test, y_test, verbose=0)
# Print test accuracy
print('\n', 'Test accuracy:', score)
rand_img = randrange(100)
result = np.argmax(model.predict(x_test[rand_img].reshape(1,28,28,1)))
plt.imshow(x_test[rand_img].reshape(28,28), cmap='gray')
plt.title(labels[result])
==========>......] - ETA: 0s - loss: 0.0243
832/1000 [=======================>......] - ETA: 0s - loss: 0.0242
Warning (from warnings module):
File "C:\Python36\lib\site-packages\keras\callbacks.py", line 122
% delta_t_median)
UserWarning: Method on_batch_end() is slow compared to the batch update (0.101474). Check your callbacks.
976/1000 [============================>.] - ETA: 0s - loss: 0.0238
992/1000 [============================>.] - ETA: 0s - loss: 0.0236
1000/1000 [==============================] - 3s 3ms/step - loss: 0.0239 - val_loss: 0.0352
Test accuracy: 0.035189545452594756```
The truth was somewhere near. Idle doesn't output tf.Print and therefore K.print_tensor() to it's shell, so when i used cmd.exe python train.py, i saw tensor output.

How to modify weights of Keras layers?

I am trying to freeze some of the weights of a layer by setting them to a specific value in Keras. How can I achieve this without moving weights to CPU ?
I checked similar questions such as modify layer weights in keras and modify layer parameters in keras
Answers suggest usage of get_weights() and 'set_weights()', however those functions moves weights between CPU and GPU.
I created a custom lambda layer and modified model.trainable_weights inside of that layer, however weights are not updated.
I used tf advanced tutorial, and just added a custom lambda layer that multiplies weights with zero.
Colab notebook with same code
from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
from tensorflow.keras.layers import Dense, Flatten, Conv2D, Lambda
from tensorflow.keras import Model
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0
# Add a channels dimension
x_train = x_train[..., tf.newaxis]
x_test = x_test[..., tf.newaxis]
def antirectifier(x):
for i,w in enumerate(model.trainable_weights):
model.trainable_weights[i] = tf.multiply(w,0)
return x
class MyModel(Model):
def __init__(self):
super(MyModel, self).__init__()
self.conv1 = Conv2D(32, 3, activation='relu')
self.flatten = Flatten()
self.d1 = Dense(128, activation='relu')
self.d2 = Dense(10, activation='softmax')
self.mask = Lambda(antirectifier)
def call(self, x):
x = self.conv1(x)
x = self.flatten(x)
x = self.d1(x)
x = self.mask(x)
return self.d2(x)
# Create an instance of the model
model = MyModel()
loss_object = tf.keras.losses.SparseCategoricalCrossentropy()
optimizer = tf.keras.optimizers.Adam()
train_loss = tf.keras.metrics.Mean(name='train_loss')
train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')
test_loss = tf.keras.metrics.Mean(name='test_loss')
test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')
#tf.function
def train_step(images, labels):
with tf.GradientTape() as tape:
predictions = model(images)
loss = loss_object(labels, predictions)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
train_loss(loss)
train_accuracy(labels, predictions)
#tf.function
def test_step(images, labels):
predictions = model(images)
t_loss = loss_object(labels, predictions)
test_loss(t_loss)
test_accuracy(labels, predictions)
EPOCHS = 5
for epoch in range(EPOCHS):
for images, labels in train_ds:
train_step(images, labels)
for test_images, test_labels in test_ds:
test_step(test_images, test_labels)
template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'
print(template.format(epoch+1,
train_loss.result(),
train_accuracy.result()*100,
test_loss.result(),
test_accuracy.result()*100))
# Reset the metrics for the next epoch
train_loss.reset_states()
train_accuracy.reset_states()
test_loss.reset_states()
test_accuracy.reset_states()
Since weights are zero, accuracy should be low. However weights are not changed.