I run the following code in Google Colab(with GPU):
import random
random.seed(1)
import numpy as np
from numpy.random import seed
seed(1)
from tensorflow import set_random_seed
set_random_seed(2)
import pandas as pd
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.layers import Flatten, Dense, Lambda, SimpleRNN
from keras.optimizers import *
from keras.utils import np_utils
from keras.initializers import *
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, roc_auc_score, auc, precision_recall_curve
from sklearn.metrics import confusion_matrix
from keras.callbacks import EarlyStopping
from keras import backend as K
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
K.set_session(sess)
##Loading dataset train and validation files, the files are same for every run
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5)
print("***********************************************************************************************")
def make_model():
model = Sequential()
model.add(Conv2D(10,(5,5), kernel_initializer=glorot_uniform(seed=1), input_shape = (22,10,1), use_bias = True, activation = "relu", strides = 1, padding = "valid"))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Flatten())
model.add(Dense(20, kernel_initializer=glorot_uniform(seed=1), activation = "relu"))
model.add(Lambda(lambda x: tf.expand_dims(x, axis=1)))
model.add(SimpleRNN(20, kernel_initializer=glorot_uniform(seed=1), activation="relu",return_sequences=False))
model.add(Dense(1, kernel_initializer=glorot_uniform(seed=1), activation="sigmoid"))
opti = SGD(lr = 0.01)
model.compile(loss = "binary_crossentropy", optimizer = opti, metrics = ["accuracy"])
return model
model = make_model()
model.fit(x_train, y_train, validation_data = (x_validation,y_validation), epochs = 50, batch_size = 20, verbose = 2, callbacks=[es])
Despite setting all seed values, my prediction results of the model are different on subsequent runs. The training and testing of the model happens in the same Colab cell.
You are dealing with floating point numbers that are multiplied and added on different threads and can therefore happen in different order. Floating point additions and multiplications are not commutative. See What Every Computer Scientist Should Know About Floating-Point Arithmetic.
Related
I have a problem about my CNN model made using tensorflow. The goal is to predict the classes of satellite images, corresponding to the type of clouds (data extracted from the kaggle competition "Planet: Understanding the Amazon from Space"). There are 4 classes : clear, cloudy, partly cloudy and haze.
Everything works fine until I try to test the model on individual images. Then, it always predicts 2 classes and nothing else. I noticed that if I run the model again, it may predict 2 other classes among the 4. The model was trained for 10 epochs, which gave an accuracy of 0.8717.
Here is my code :
import numpy as np
import pandas as pd
import cv2
from tqdm import tqdm
import h5py
import os
os.listdir("/kaggle/input/")
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense,MaxPooling2D,Conv2D,Flatten,Dropout,Activation
from tensorflow.keras.layers import BatchNormalization
from sklearn import svm
from sklearn.model_selection import cross_val_score
import matplotlib.pyplot as plt
from tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img
from oauth2client.client import GoogleCredentials
import csv
#from keras.optimizers import RMSprop
from tensorflow.keras import Input, Model
batch_size = 128
img_width = 256
img_height = 256
train_data = ImageDataGenerator(
rescale = 1./255,
validation_split = 0.25)
train_generator = train_data.flow_from_directory(
'../input/clouds',
target_size=(img_height, img_width),
color_mode='rgb',
batch_size=batch_size,
shuffle = True,
class_mode="categorical",
subset = 'training'
)
valid_generator = train_data.flow_from_directory(
'../input/clouds',
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='categorical',
subset = 'validation'
)
num_classes = 4
model = Sequential([
Input(shape = [img_width, img_height, 3]),
Conv2D(128,4,activation = 'relu'),
MaxPooling2D(),
Conv2D(64,4,activation = 'relu'),
MaxPooling2D(),
Conv2D(32,4, activation = 'relu'),
MaxPooling2D(),
Conv2D(16,4,activation = 'relu'),
MaxPooling2D(),
Flatten(),
Dense(64, activation = 'relu'),
Dense(num_classes, activation = 'softmax')
])
model.compile(optimizer = "adam",
loss = 'categorical_crossentropy',
metrics=['accuracy'])
model.fit(train_generator, validation_data = valid_generator, epochs = 10)
img_to_predict = cv2.imread('/kaggle/input/clouds-test/clouds_test/test_3877_6013089.jpg') #an augmented image from original dataset
img_to_predict = cv2.cvtColor(img_to_predict, cv2.COLOR_BGR2RGB)
img_to_predict = np.expand_dims(cv2.resize(img_to_predict, (256,256)), axis = 0)
res = model.predict(img_to_predict)
label_map = (train_generator.class_indices)
print(label_map)
print(list(label_map)[np.argmax(res, axis = -1)[0]])
Thank you for you help.
When I use return_sequences = true, for a LSTM layer, before adding a dense layer it sometimes results in an error depending upon the environment. I believe it mainly depends on the version of tensorflow and keras. If I am using tensorflow 2.1.0 and Keras 2.3.0, I get the following error -
standardize_input_data 'with shape ' + str(data_shape)) ValueError: Error when checking target: expected dense_2 to have 3 dimensions, but got array with shape (7000, 1)
However, if I use tensorflow 2.9.1 and keras 2.9.0 I do not get any error.
Here is some minimal working sample code -
import os
import pandas as pd
from sklearn import preprocessing
from collections import deque
import random
import numpy as np
import time
import random
from keras.models import Sequential
from keras.layers import Dense, Dropout, LSTM, BatchNormalization, Input
from keras.models import load_model
import keras
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
import pickle
epochs = 10
batch_size = 64
X, y = make_classification(n_samples=10000, n_features=3, n_classes=3, n_informative=3, n_redundant=0, n_repeated=0 ,weights=[0.5,0.5,0.5])
X = X.reshape(X.shape[0], 1, 3)
y = y.reshape(-1, 1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
inputs = Input(shape=(X_train.shape[1:]))
outputs = LSTM(128, input_shape=(X_train.shape[1:]), return_sequences=True)(inputs)
outputs = Dropout(0.2)(outputs)
outputs = BatchNormalization()(outputs)
outputs = LSTM(128, input_shape=(X_train.shape[1:]), return_sequences=True)(outputs)
outputs = Dropout(0.2)(outputs)
outputs = BatchNormalization()(outputs)
outputs = Dense(32, activation="relu", kernel_initializer="glorot_uniform")(outputs)
outputs = Dropout(0.2)(outputs)
outputs = Dense(3, activation="softmax", kernel_initializer="glorot_uniform")(outputs)
model = keras.Model(inputs, outputs)
opt = keras.optimizers.Adam(lr=0.0001, decay=1e-6)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.fit(X_train, y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_test, y_test))
I've created a Keras Regressor to run a RandomizedSearch CV using a ModelCheckpoint callback, but the training overran the Colab runtime 12H limit and stopped halfway through. The models are saved in hdf5 format.
I used tensorflow_addons to add the RSquare class to monitor the R2 for train and validation sets. However, when I used keras.models.load_model, I get the following error:
As you can see from the traceback, I have passed the custom_objects parameter, but still it is not recognised.
How can I solve this?
You can see the full code example below:
import os
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.layers import Input, InputLayer
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.model_selection import RandomizedSearchCV
def build_model(n_hidden = 2, n_neurons = 64, input_shape = (X_train.shape[1],), dropout = 0):
model = Sequential()
model.add(InputLayer(input_shape = input_shape))
for i in range(n_hidden):
model.add(Dense(n_neurons, activation = 'relu'))
model.add(Dropout(dropout))
model.add(Dense(1))
model.compile(loss = 'mean_squared_error', optimizer = 'adam', metrics = [tfa.metrics.RSquare(y_shape=(1,))])
return model
keras_reg = KerasRegressor(build_model)
checkdir = os.path.join(r'/content/drive/MyDrive/COP328/Case A', 'checkpoints', datetime.datetime.now().strftime('%d-%m-%Y_%H-%M-%S'), 'imputed_log1p-{epoch:02d}-{val_r_square:.3f}.hdf5')
callbacks = [ModelCheckpoint(checkdir, save_freq='epoch', save_best_only = True, monitor = 'val_r_square', mode = 'max'),
EarlyStopping(patience = 10)]
# Here is where training got interrupted because of Colab runtime being dropped:
param_dist = {
'n_hidden' : [1,2],
'n_neurons': [8,16,32,64,128],
'dropout': [0,0.2,0.4]
}
rnd_search_cv = RandomizedSearchCV(keras_reg, param_dist, n_iter= 15, cv = 5)
rnd_search_cv.fit(X_train, y_train, epochs = 200, batch_size = 64,
validation_data = (X_valid,y_valid),
callbacks = callbacks)
# Here is where I am trying to reload one of the most promising models based on R2, and getting the error:
from keras.models import load_model
import tensorflow_addons as tfa
rnd_model = load_model(r'/content/drive/MyDrive/COP328/Case A/checkpoints/26-06-2021_17-32-29/imputed_log1p-56-1.000.hdf5', custom_objects = {'r_square': tfa.metrics.RSquare, 'val_r_square': tfa.metrics.RSquare})
This solution doesn't take into account the addons package but one possibility is to create the coefficient of determination (R^2) as a different metric without that package, and then defining it as your loss.
def coeff_determination(y_true, y_pred):
from keras import backend as K
SS_res = K.sum(K.square( y_true-y_pred ))
SS_tot = K.sum(K.square( y_true - K.mean(y_true) ) )
return ( 1 - SS_res/(SS_tot + K.epsilon()) )
Recall that minimizing MSE will maximize R^2.
I have created a project using keras and tensorflow. I used the NSL KDD dataset and coded my project in python. I also used the SGD optimizer.
I would like to fit a model then evaluate it and then check its accuracy. (So I can compare it to the results with machine learning).
Here is my complete code below, please review it.
import tensorflow as tf
from keras import backend as K
from tensorflow.python.saved_model import builder as saved_model_builder
from tensorflow.python.saved_model import tag_constants, signature_constants, signature_def_utils_impl
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import SGD
import numpy as np
sess = tf.Session()
K.set_session(sess)
K.set_learning_phase(0)
model_version = "2"
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
# Importing the dataset
dataset = pd.read_csv('KDD_Dataset.csv')
X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, 41:42].values
# Encoding categorical data X
from sklearn.preprocessing import LabelEncoder
labelencoder_X = LabelEncoder()
X[:,0] = labelencoder_X.fit_transform(X[:,0])
X[:,1] = labelencoder_X.fit_transform(X[:,1])
X[:,2] = labelencoder_X.fit_transform(X[:,2])
#
from sklearn.preprocessing import OneHotEncoder
onehotencoder_0 = OneHotEncoder(categorical_features=[0])
onehotencoder_1 = OneHotEncoder(categorical_features=[1])
onehotencoder_2 = OneHotEncoder(categorical_features=[2])
X = onehotencoder_0.fit_transform(X).toarray()
X = onehotencoder_1.fit_transform(X).toarray()
X = onehotencoder_2.fit_transform(X).toarray()
# Encoding categorical data y
from sklearn.preprocessing import LabelEncoder
labelencoder_y = LabelEncoder()
y = labelencoder_y.fit_transform(y)
max(y)
# Splitting the dataset into the Training set and Test set
#from sklearn.cross_validation import train_test_split
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size = 0.2,
random_state = 0)
# create the model
model = Sequential()
model.add(Dense(41, input_dim=8, init='uniform', activation='relu'))
model.add(Dense(20, init='uniform', activation='relu'))
model.add(Dense(1, init='uniform', activation='sigmoid'))
# compile the model
model.compile(loss='binary_crossentropy', optimizer=sgd,metrics=['accuracy'])
model.fit(X_train, y_train, validation_data=(X_test, y_test), nb_epoch=200, batch_size=5, verbose=0)
See Dense(41, input_dim=8, init='uniform', activation='relu')
The model you defined with 8 features, however your inputs have 45 features. They are not matching. You have to either make your model with 45 features to match the input, or cut the length of input feature to 8 to match your model.
Change line
model.add(Dense(41, input_dim=8, init='uniform', activation='relu'))
to
model.add(Dense(42, input_dim=42, init='uniform', activation='relu'))
and
optimizer=sgd to optimizer='sgd'
This is a keras model for sentiment analysis i need to convert it to tensorflow i couldn’t build embedding layer with tensorflow and using confusion matrix to evaluate this model? And I asked if tf-learn is the-same as tensorflow
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow import set_random_seed
set_random_seed(2)
from nltk.tokenize import word_tokenize
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer
from sklearn.preprocessing import LabelEncoder
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers.embeddings import Embedding
from keras.layers import Flatten
from keras.layers import Conv1D, MaxPooling1D
from keras.layers import Dense,Activation
from keras.layers import Dropout
from keras.callbacks import TensorBoard, ModelCheckpoint
import re
import string
import collections
import time
seed = 10
Read CSV Files
df=pd.read_csv('tweets-pos-neg.csv', usecols = ['text','airline_sentiment'])
df = df.reindex(['text','airline_sentiment'], axis=1) #reorder columns
df=df.apply(lambda x: x.astype(str).str.lower())
Normalize Text
def normalize(text):
text= re.sub(r"http\S+", r'', text)
text= re.sub(r"#\S+", r'', text)
punctuation = re.compile(r'[!"#$%&()*+,-./:;<=>?#[\]^_`{|}~|0-9]')
text = re.sub(punctuation, ' ', text)
text= re.sub(r'(.)\1\1+', r'\1', text)
return text
Cleaned Text
def prepareDataSets(df):
sentences=[]
for index, r in df.iterrows():
text= normalize(r['text'])
sentences.append([text,r['airline_sentiment']])
df_sentences=pd.DataFrame(sentences,columns=
['text','airline_sentiment'])
return df_sentences
edit_df=prepareDataSets(df)
edit_df=shuffle(edit_df)
X=edit_df.iloc[:,0]
Y=edit_df.iloc[:,1]
Split reviews to tokens
max_features = 50000
tokenizer = Tokenizer(num_words=max_features, split=' ')
tokenizer.fit_on_texts(X.values)
#convert review tokens to integers
X_seq = tokenizer.texts_to_sequences(X)
Padding Sequence to make all vectors with the same size according to MAX-length of reviews
seq_len=35
X_pad = pad_sequences(X_seq,maxlen=seq_len)
Convert target value from string to integer
le=LabelEncoder()
Y_le=le.fit_transform(Y)
Y_le_oh=to_categorical(Y_le)
Train-Test-Split
X_train, X_test, Y_train, Y_test = train_test_split(X_pad,Y_le_oh, test_size
= 0.33, random_state = 42)
X_train, X_Val, Y_train, Y_Val = train_test_split(X_train,Y_train, test_size
= 0.1, random_state = 42)
print(X_train.shape,Y_train.shape)
print(X_test.shape,Y_test.shape)
print(X_Val.shape,Y_Val.shape)
Create the model
embedding_vecor_length = 32 #no of vector columns
model_cnn = Sequential()
model_cnn.add(Embedding(max_features, embedding_vecor_length,
input_length=seq_len))
model_cnn.add(Conv1D(filters=100, kernel_size=2, padding='valid',
activation='relu', strides=1))
model_cnn.add(MaxPooling1D(2))
model_cnn.add(Flatten())
model_cnn.add(Dense(256, activation='relu'))
model_cnn.add(Dense(2, activation='softmax'))
opt=tf.keras.optimizers.Adam(lr=0.001, decay=1e-6)
model_cnn.compile(loss='binary_crossentropy', optimizer=opt, metrics=
['accuracy'])
print(model_cnn.summary())
Evaluate model
history=model_cnn.fit(X_train, Y_train, epochs=3, batch_size=32, callbacks=[tensorboard], validation_data=(X_Val, Y_Val))
scores = model_cnn.evaluate(X_test, Y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[-1]*100))
If you just need to use Tensorflow APIs to train / evaluate, you can build an Estimator using model_to_estimator function.
Here's the documentation with an example.