I am trying to train a model on Google Colab, in order to play around with training on TPU. However, I am running into the following error:
AttributeError Traceback (most recent call last)
<ipython-input-82-e74efc36d872> in <module>()
----> 1 tpu_model.fit_generator(training_set, steps_per_epoch = 8000, epochs = 25)
2 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py in configure_callbacks(callbacks, model, do_validation, batch_size, epochs, steps_per_epoch, samples, verbose, count_mode, mode)
118 callback_list.model.stop_training = False
119 # pylint: disable=protected-access
--> 120 if callback_list.model._ckpt_saved_epoch is not None:
121 # The attribute `_ckpt_saved_epoch` is supposed to be None at the start of
122 # training (it should be made None at the end of successful multi-worker
AttributeError: 'KerasTPUModel' object has no attribute '_ckpt_saved_epoch'
While trying to run the following code.
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import os
import zipfile
print(tf.VERSION)
local_zip = '/home/cats_and_dogs_filtered.zip'
zip_ref = zipfile.ZipFile(local_zip, 'r')
zip_ref.extractall('/home')
zip_ref.close()
def create_model():
classifier = tf.keras.models.Sequential()
classifier.add(layers.Conv2D(32, (3, 3), input_shape=(64, 64, 3), activation='relu'))
classifier.add(layers.MaxPooling2D(pool_size=(2, 2)))
classifier.add(layers.Conv2D(32, (3, 3), activation= 'relu'))
classifier.add(layers.MaxPooling2D(pool_size=(2, 2)))
classifier.add(layers.Flatten())
classifier.add(layers.Dense(units=128, activation= 'relu'))
classifier.add(layers.Dense(units=1, activation= 'sigmoid'))
return classifier
train_datagen = ImageDataGenerator(rescale = 1./255, shear_range = 0.2, zoom_range = 0.2, horizontal_flip = True)
training_set = train_datagen.flow_from_directory('/home/cats_and_dogs_filtered/train', target_size = (64, 64), batch_size = 32, class_mode = 'binary')
model = create_model()
TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']
tpu_model = tf.contrib.tpu.keras_to_tpu_model( model, strategy=tf.contrib.tpu.TPUDistributionStrategy(tf.contrib.cluster_resolver.TPUClusterResolver(TPU_WORKER)))
tpu_model.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])
tpu_model.save_weights('./tpu_model.h5', overwrite=True)
tpu_model.fit_generator(training_set, steps_per_epoch = 8000, epochs = 25)
I am not sure what is going on. I used similar code to train it on CPU (takes a long time to train).
Related
Tried using k-cross validation from this link but with my own dataset and I got this error:
TypeError: Singleton array array(<BatchDataset element_spec=(TensorSpec(shape=(None, 180, 180, 3), dtype=tf.float32, name=None), TensorSpec(shape=(None,), dtype=tf.int32, name=None))>,
dtype=object) cannot be considered a valid collection.
Here is my code:
import numpy as np
import PIL
import tensorflow as tf
import os
from sklearn.model_selection import KFold
import numpy as np
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential
import pathlib
num_folds = 10
acc_per_fold = []
loss_per_fold = []
tf.get_logger().setLevel('ERROR')
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
dataset_path = "data"
fullPath = os.path.abspath("./" + dataset_path)
#data_dir = tf.keras.utils.get_file('photos', origin='file://'+dataset_path, extract=True)
data_dir = pathlib.Path(fullPath)
image_count = len(list(data_dir.glob('*/*.jpg')))+len(list(data_dir.glob('*/*.png')))
print(image_count)
#man = list(data_dir.glob('man/*'))
#im = PIL.Image.open(str(man[28]))
#im.show()
batch_size = 32
img_height = 180
img_width = 180
train_ds = tf.keras.utils.image_dataset_from_directory(
data_dir,
labels='inferred',
validation_split=0.2,
subset="training",
seed=123,
image_size=(img_height, img_width),
batch_size=batch_size)
val_ds = tf.keras.utils.image_dataset_from_directory(
data_dir,
labels='inferred',
validation_split=0.2,
subset="validation",
seed=123,
image_size=(img_height, img_width),
batch_size=batch_size)
class_names = train_ds.class_names
print(class_names)
normalization_layer = layers.Rescaling(1./255)
# Define the K-fold Cross Validator
kfold = KFold(n_splits=num_folds, shuffle=True, random_state=42)
fold_no = 1
for train, test in kfold.split(train_ds, val_ds):
# Define the model architecture
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(256, activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(Dense(no_classes, activation='softmax'))
# Compile the model
model.compile(loss=loss_function,
optimizer=optimizer,
metrics=['accuracy'])
# Generate a print
print('------------------------------------------------------------------------')
print(f'Training for fold {fold_no} ...')
# Fit data to model
history = model.fit(inputs[train], targets[train],
batch_size=batch_size,
epochs=no_epochs,
verbose=verbosity)
# Generate generalization metrics
scores = model.evaluate(inputs[test], targets[test], verbose=0)
print(f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores[0]}; {model.metrics_names[1]} of {scores[1]*100}%')
acc_per_fold.append(scores[1] * 100)
loss_per_fold.append(scores[0])
# Increase fold number
fold_no = fold_no + 1
# == Provide average scores ==
print('------------------------------------------------------------------------')
print('Score per fold')
for i in range(0, len(acc_per_fold)):
print('------------------------------------------------------------------------')
print(f'> Fold {i+1} - Loss: {loss_per_fold[i]} - Accuracy: {acc_per_fold[i]}%')
print('------------------------------------------------------------------------')
print('Average scores for all folds:')
print(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})')
print(f'> Loss: {np.mean(loss_per_fold)}')
print('------------------------------------------------------------------------')
## Step 1) Import Libraries
import os
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICE"] = ""
from tensorflow.python.client import device_lib;
print(device_lib.list_local_devices());
import keras
import numpy as np
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
#### Set some parameters
batch_size = 256
num_classes = 10
epochs = 3
## Step 2) Load and Prepare data|
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
print('x_train shape:', x_train.shape)
print('y_train shape:', y_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
### Prepare data
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
import matplotlib.pyplot as plt
%matplotlib inline
plt.figure()
fig_size = [20, 20]
plt.rcParams["figure.figsize"] = fig_size
idx = 0
for idx in range(1,101):
ax = plt.subplot(10, 10, idx)
img = x_train[idx,:,:,:]
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.imshow(img, cmap="Greys_r")
plt.show()
#### Preprocess class lables
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
#### Data Augmentation
datagen = ImageDataGenerator(
featurewise_center = False, # set input mean to over the dataset
samplewise_center = False, # set each sample mean to 0
featurewise_std_normalization = False, # divide inputs of the dataset
samplewise_std_normalization = False, # divide each input by its std
zca_whitening = False, # apply ZCA whitening
rotation_range = 0, # randomply rotate images in the range (degrees, 0 to 180)
width_shift_range = 0.1, # randomly shift images horiznontally (fraction of totla width)
height_shift_range = 0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip = True, # randomly flip images
vertical_flip = False) # randomly flip images
## Setp 3) Define model architecture
model = Sequential()
model.add(Conv2D(32, (3,3), padding = 'same', activation = 'relu', input_shape = x_train.shape[1:]))
model.add(Conv2D(32, (3,3), activation = 'relu',))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3,3), padding = 'same', activation = 'relu',))
model.add(Conv2D(64, (3,3), activation = 'relu',))
model.add(MaxPooling2D(pool_size = (2,2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512, activation = 'relu',))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation = 'softmax'))
## Step 4) Compile model
# model.compile(loss='categorical_crossentropy', optimizer = 'Adagrad', metrics = ['accuracy'])
model.compile(loss='categorical_crossentropy', optimizer = 'Adam', metrics = ['accuracy'])
#### Summary of model
model.summary()
## Step 5) Fit model on training data
# Compute quantities required for feature-wise normalization
# (std, mean, and principal components if ZCA whitening is applied).
datagen.fit(x_train)
# Fit the model on the batches generated by datagen.flow().
model.fit_generator(datagen.flow(x_train, y_train, batch_size = batch_size),
steps_per_epoch = x_train.shape[0] // batch_size, # // is the quotient without remainder
epochs = epochs,
validation_data = (x_test, y_test))
model.save('my_model')
And I have error
AttributeError Traceback (most recent call last)
<ipython-input-18-b71b28267016> in <module>
1 from tensorflow.keras.models import load_model
----> 2 model.save('my_model')
~\anaconda3\envs\tf\lib\site-packages\keras\engine\network.py in save(self, filepath, overwrite, include_optimizer)
1150 raise NotImplementedError
1151 from ..models import save_model
-> 1152 save_model(self, filepath, overwrite, include_optimizer)
1153
1154 #saving.allow_write_to_gcs
~\anaconda3\envs\tf\lib\site-packages\keras\engine\saving.py in save_wrapper(obj, filepath, overwrite, *args, **kwargs)
447 os.remove(tmp_filepath)
448 else:
--> 449 save_function(obj, filepath, overwrite, *args, **kwargs)
450
451 return save_wrapper
~\anaconda3\envs\tf\lib\site-packages\keras\engine\saving.py in save_model(model, filepath, overwrite, include_optimizer)
539 return
540 with H5Dict(filepath, mode='w') as h5dict:
--> 541 _serialize_model(model, h5dict, include_optimizer)
542 elif hasattr(filepath, 'write') and callable(filepath.write):
543 # write as binary stream
~\anaconda3\envs\tf\lib\site-packages\keras\engine\saving.py in _serialize_model(model, h5dict, include_optimizer)
161 layer_group[name] = val
162 if include_optimizer and model.optimizer:
--> 163 if isinstance(model.optimizer, optimizers.TFOptimizer):
164 warnings.warn(
165 'TensorFlow optimizers do not '
AttributeError: module 'keras.optimizers' has no attribute 'TFOptimizer'
I trained a sequential model and tried to save it by the code model.save(filename)
But it says AttributeError: module 'keras.optimizers' has no attribute 'TFOptimizer'
The error appears on model.save(path).
I don't know why I couldn't save the sequential model.
That error appears even when I directly import TFOptimizer from keras.optimizers
I also installed H
I had the same problem. When I changed the version of keras and tensorflow to keras2.2.5 and tensorflow1.14.0, the problem was solved.
I am building a multi-class CNN model but I am unable to compile the model due to loss shape error.
Both output layer and labels should have correct shapes; labels being (m, 1, 3) and final dense layer containing 3 perceptions with softmax activation
loss='categorical_crossentropy'
import numpy as np
import pandas as pd
from preprocess import DataLoader
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv3D, Dropout, MaxPooling3D
from tensorflow.keras.metrics import categorical_crossentropy
from tensorflow.keras import optimizers
target_width = 160
target_height = 192
target_depth = 192
num_classes = 3
batch_size = 4
data_loader = DataLoader(target_shape=(target_width, target_height, target_depth))
train, test = data_loader.Get_Data_List()
print("Train size: " + str(len(train)))
print("Test size: " + str(len(test)))
def custom_one_hot(labels):
label_dict = {"stableAD":np.array([0,0,1]),
"stableMCI":np.array([0,1,0]),
"stableNL":np.array([1,0,0])}
encoded_labels = []
for label in labels:
encoded_labels.append(label_dict[label].reshape(1,3))
return np.asarray(encoded_labels)
def additional_data_prep(train, test):
# Extract data from tuples
train_labels, train_data = zip(*train)
test_labels, test_data = zip(*test)
X_train = np.asarray(train_data)
X_test = np.asarray(test_data)
y_train = custom_one_hot(train_labels)
y_test = custom_one_hot(test_labels)
return X_train, y_train, X_test, y_test
X, y, X_test, y_test = additional_data_prep(train, test)
X = np.expand_dims(X, axis=-1).reshape((X.shape[0],target_width,target_height,target_depth,1))
X_test = np.expand_dims(X_test, axis=-1).reshape((X_test.shape[0],target_width,target_height,target_depth,1))
model = Sequential()
model.add(Conv3D(24, kernel_size=(13, 11, 11), activation='relu', input_shape=(target_width,target_height,target_depth,1), padding='same', strides=4))
model.add(MaxPooling3D(pool_size=(3, 3, 3), strides=2))
model.add(Dropout(0.1))
model.add(Conv3D(48, kernel_size=(6, 5, 5), activation='relu', padding='same'))
model.add(MaxPooling3D(pool_size=(3, 3, 3), strides=2))
model.add(Dropout(0.1))
model.add(Conv3D(24, kernel_size=(4, 3, 3), activation='relu'))
model.add(MaxPooling3D(pool_size=(3, 3, 3), strides=2))
model.add(Dropout(0.1))
model.add(Conv3D(8, kernel_size=(2, 2, 2), activation='relu'))
model.add(MaxPooling3D(pool_size=(1, 1, 1), strides=2))
model.add(Dropout(0.1))
model.add(Flatten())
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adam(learning_rate=0.0015),
metrics=['accuracy','categorical_crossentropy'])
model.fit(X, y, batch_size=batch_size, epochs=10, verbose=2, use_multiprocessing=True)
model.evaluate(X_test, y_test, verbose=2, use_multiprocessing=True)
Results in this error message:
Traceback (most recent call last):
File "train.py", line 70, in <module>
model.fit(X, y, batch_size=batch_size, epochs=10, verbose=2, use_multiprocessing=True)
File "/home/554282/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py", line 819, in fit
use_multiprocessing=use_multiprocessing)
File "/home/554282/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 235, in fit
use_multiprocessing=use_multiprocessing)
File "/home/554282/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 593, in _process_training_inputs
use_multiprocessing=use_multiprocessing)
File "/home/554282/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 646, in _process_inputs
x, y, sample_weight=sample_weights)
File "/home/554282/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py", line 2383, in _standardize_user_data
batch_size=batch_size)
File "/home/554282/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py", line 2489, in _standardize_tensors
y, self._feed_loss_fns, feed_output_shapes)
File "/home/554282/.local/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training_utils.py", line 810, in check_loss_and_target_compatibility
' while using as loss `' + loss_name + '`. '
ValueError: A target array with shape (8, 1, 3) was passed for an output of shape (None, 3) while using as loss `categorical_crossentropy`. This loss expects targets to have the same shape as the output.
The custom_one_hot function returns a [M, 1, 3] array. You should reshape that to [M, 3] since the output of the CNN is [M, 3]. M here is the batch size.
when i run these lines of code for binary classification it is running well without any problem and get a good result, but when i try to make it for many classes e.g 3 classes it give "NaN" in predict result
# Importing the Keras libraries and packages
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
# Initialising the CNN
classifier = Sequential()
# Step 1 - Convolution
classifier.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu'))
# Step 2 - Pooling
classifier.add(MaxPooling2D(pool_size = (2, 2)))
# Adding a second convolutional layer
classifier.add(Conv2D(32, (3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
classifier.add(Conv2enter code hereD(32, (3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
classifier.add(Conv2D(32, (3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
# Step 3 - Flattening
classifier.add(Flatten())
# Step 4 - Full connection
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = 3, activation = 'sigmoid'))
# Compiling the CNN
classifier.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])
# Part 2 - Fitting the CNN to the images
from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = True)
test_datagen = ImageDataGenerator(rescale = 1./255)
training_set = train_datagen.flow_from_directory('data/train',
target_size = (64, 64),
batch_size = 32,
class_mode = 'categorical')
test_set = test_datagen.flow_from_directory('data/test',
target_size = (64, 64),
batch_size = 32,
class_mode = 'categorical')
classifier.fit_generator(training_set,
steps_per_epoch = 240 ,
epochs = 25,
validation_data = test_set,
validation_steps = 30)
import numpy as np
from keras.preprocessing import image
test_image = image.load_img('2.jpeg', target_size = (64, 64))
test_image = image.img_to_array(test_image)
test_image = np.expand_dims(test_image, axis = 0)
result = classifier.predict(test_image)
training_set.class_indices
i tried these lines of code with loss function "binary" with 2 classes it worked well without any problems and get a good result that helped me with my work and the accuracy approximately '93%' .
but my project based on multi class classification, so i tried to change the loss function to 'categorical_crossentropy' and the class mod in fit_generator to 'categorical' to make it multi class, the accuracy start with 60% and grows up to 99 and suddenly drop down to 33%.
the expected result the labels of the classes
the actual result is "NaN".
thanks in advance.
For multi-class classification, usually softmax is applied on the last dense layer instead of sigmoid. Change it to softmax to see whether the issue is still there.
I am following a course on deep learning and I am doing right now the CNN networks the train set is 8000 photos 4000 cats and 4000 dogs the training set is 2000/2000 the size I am using for images is 64x64 with RGB. I am using Keras with 2 conv2d/maxpool layers of 32 filters a flatten layer and two dense layers of 128 and 1 output. My problem is that this setup is performing at 15 minutes per epoch and for 25 epochs that means 6 Hours of training at least plus sometimes on some epochs is freezing for sometimes at 7999/8000 I am running this on windows 10 and anaconda with python 3.7 and TensorFlow 1.13. Is this a good performance or I can improve it? I was expecting from the new Turing architecture better performances.
# -*- coding: utf-8 -*-
# Part 1 - Building the convolutional neural network
import tensorflow as tf
from keras import backend as K
config = tf.ConfigProto(intra_op_parallelism_threads=6,
inter_op_parallelism_threads=6,
allow_soft_placement=True,
device_count = {'CPU' : 1,
'GPU' : 1}
)
session = tf.Session(config=config)
K.set_session(session)
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense
# Initialising the CNN
classifier = Sequential()
# Step 1 - Convolution
classifier.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu'))
# Step 2 - Pooling
classifier.add(MaxPooling2D(pool_size = (2, 2)))
# Adding a second convolutional layer
classifier.add(Conv2D(32, (3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
# Step 3 - Flattening
classifier.add(Flatten())
# Step 4 - Full connection
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = 1, activation = 'sigmoid'))
# Compiling the CNN
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])
weights = classifier.get_weights()
#Part 2 - Fiting the CNN to the images
from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
horizontal_flip = True)
test_datagen = ImageDataGenerator(rescale = 1./255)
training_set = train_datagen.flow_from_directory('dataset/training_set',
target_size = (64, 64),
batch_size = 32,
class_mode = 'binary')
test_set = test_datagen.flow_from_directory('dataset/test_set',
target_size = (64, 64),
batch_size = 32,
class_mode = 'binary')
classifier.fit_generator(training_set,
steps_per_epoch = 8000,
epochs = 25,
validation_data = test_set,
validation_steps = 2000)
classifier.save("my first model")
Thank you