I am predicting classes, but there is something I don't get. In the simplified example below, I train a model to predict MNIST handwritten digits. My test set has an accuracy of 95%, when I use
model.evaluate(test_image, test_label)
However, when I use
model.predict(test_image)
and the extract the predicted labels using np.argmax(), this accuracy drops. When I run all the code again and again, this accuracy changes a lot.
I suspect now that the classes in the model are not ordered 0, 1 ... 9. Is there a way to see the class labels of a model? Or did I make another mistake?
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras.datasets.mnist import load_data
from tensorflow.keras.layers import Dense, Flatten
from tensorflow.keras.models import Sequential
import numpy as np
# Load data
(train_image, train_label), (test_image, test_label) = load_data()
# Train
model = Sequential([
Flatten(input_shape=(28,28)),
Dense(100, activation="relu"),
Dense(100, activation="relu"),
Dense(10, activation="sigmoid")
])
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics='accuracy')
history = model.fit(train_image, train_label,
batch_size=32, epochs=50,
validation_data=(test_image, test_label),
verbose = 0)
eval = model.evaluate(test_image, test_label)
print('Accuracy (auto):', eval[1]) # This is always high
# Predict and evaluate manually
predictions = model.predict(test_image)
pred = np.array([np.argmax(pred) for pred in predictions])
true = test_label
print('Accuracy (manually):', np.mean(pred == true)) # This varies a lot
Related
Im getting this error: ValueError: logits and labels must have the same shape, received ((32, 1) vs (32, 23740))
This is my full code:
import pandas as pd
from keras.models import Sequential
from keras.layers import Dense, Embedding, LSTM
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score, precision_score, recall_score, f1_score
import re
import numpy as np
from sklearn.preprocessing import StandardScaler, OneHotEncoder, LabelEncoder
from keras.preprocessing.text import Tokenizer
from keras.utils import to_categorical
from keras.utils import pad_sequences
# All data processing stuff
data = pd.read_csv('test.csv') # Load the data
# convert Sentiment type to string
data['Sentiment'] = data['Sentiment'].astype(str)
# remove special characters and convert to lowercase
data["Tweet"] = data["Tweet"].apply(lambda x: x.lower()) # Convert all tweets to lowercase
data["Tweet"] = data["Tweet"].apply(lambda x: x.replace("[^a-zA-Z0-9]", "")) # Remove special characters
data["Sentiment"] = data["Sentiment"].apply(lambda x: x.lower())
# Initialize the Tokenizer
tokenizer = Tokenizer()
# Fit the Tokenizer on the text data
tokenizer.fit_on_texts(data["Tweet"] + data["Sentiment"])
token_sequences1 = tokenizer.texts_to_matrix(data["Tweet"])
token_sequences2 = tokenizer.texts_to_matrix(data["Sentiment"])
padded_sequences1 = pad_sequences(token_sequences1)
padded_sequences2 = pad_sequences(token_sequences2)
# split the data into training and testing sets
x_train, x_test, y_train, y_test = train_test_split(
padded_sequences1, padded_sequences2, test_size=0.2, random_state=42)
# create the RNN model
model = Sequential()
# 1000 is the number of words in the vocabulary, 128 is the dimension of the embedding vector,
model.add(Embedding(1000, 128))
model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
# compile the model with a specified loss function and optimizer
# binary_crossentropy is used for binary classification problems like this one (positive or negative sentiment)
# accuracy is the metric used to evaluate the model performance (the percentage of correct predictions)
# the loss function and the optimizer can be changed to see if the model performance improves or not
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())
# train the model on the training data and put verbose=1 to see the training progress
model.fit(x_train, y_train, batch_size=32, epochs=10, verbose=1)
# evaluate the model on the testing data
y_pred = model.predict(x_test)
# calculate the evaluation metrics
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred)
recall = recall_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred)
# print the evaluation metrics
print("Accuracy: {:.2f}".format(accuracy))
print("Precision: {:.2f}".format(precision))
print("Recall: {:.2f}".format(recall))
print("F1 Score: {:.2f}".format(f1))
Im actually trying to create sentiment analyser on news headlines tweets from this data, here is a sample image of how my data looks in CSV.
Please provide a solution to this, I have tried every solution I had find
im having an odd bug that i cant seem to debug
#import all the necessary libraries and be specific so as to avoid wasting time importing everything
import sys
import matplotlib.pyplot as plt
import numpy as np
from tensorflow.keras.applications import VGG16
from tensorflow.keras import optimizers
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import sklearn.metrics as metrics
from sklearn.metrics import confusion_matrix
import seaborn as sns
model_type = 'vgg16'
# Loading the VGG Model
vgg_model = VGG16(weights='imagenet', include_top=False, input_shape=(200,200,3))
vgg_model.trainable = False
model = tf.keras.Sequential([vgg_model,
tf.keras.layers.GlobalAveragePooling2D(),
tf.keras.layers.Dropout(0.1),
tf.keras.layers.Dense(512, activation= "relu"),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Dropout(0.1),
tf.keras.layers.Dense(16, activation="softmax")
])
datagen = ImageDataGenerator(featurewise_center=True)
#retrieve the trian data using imagedatagen
train = datagen.flow_from_directory('/content/16_flowers/Train/',
class_mode='categorical', color_mode= 'rgb',batch_size=64, target_size=(200, 200))
test = datagen.flow_from_directory('/content/16_flowers/Test/',
class_mode='categorical', color_mode= 'rgb',batch_size=10, target_size=(200, 200))
base_learning_rate = 0.00005
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=base_learning_rate),loss = tf.keras.metrics.categorical_crossentropy,metrics=['accuracy'])
history = model.fit(train,epochs = 100 , validation_data = test)
#summarise_diagnostics(history)
model.save("vgg16CNN.model")
_, acc = model.evaluate(test, steps=len(test), verbose=0)
print('> %.3f' % (acc * 100.0))
plot_cm(model)
my dataset is a dataset of 16 flowers, the training data is 70 images per class and my validation/ test data is 10 images per class. when i run the code i get approx 99% training accuracy and 76% validation accuracy with 0.07 training loss and 0.7 validation loss.
but when i take the model which is produced and i use it to make predictions on the test dataset and then compare its predictions to the true classes i get around 6.25% accuracy, can anyone tell me why this might be?
below is my code to make predictions and retrun the accuracy of those predictions
datagen = ImageDataGenerator(featurewise_center=True)
test = datagen.flow_from_directory('/content/16_flowers/Test/',
class_mode='categorical', color_mode= 'rgb',batch_size=10, target_size=(200, 200))
predictions = model.predict(test)
predicted_classes = np.argmax(predictions, axis=1)
true_classes = test.classes
count = 0
for i in range(len(predicted_classes)):
print(true_classes[i],predicted_classes[i])
if true_classes[i] == predicted_classes[i]:
count +=1
print(count/160*100)
For test in flow_from_directory set shuffle=False to preserve the order of the predictions with respect to the files. Then use
classes=list(train_gen.class_indices.keys())
count=0
predictions = model.predict(test)
for i,p in enumerate(predictions):
index=np.argmax(p)
predicted_class=classes[index]
true_index=test.labels[i]
true_class= classes[true_index]
print('True class is ', true_class, . ' Predicted class is ', predicted class)
if index == true_index:
count +=1
accuracy = count * 100/len(predictions)
print('Accuracy om Test set is ', accuracy)
## MODEL IMPORTING ##
import tensorflow
import pandas as pd
import numpy as np
import os
import keras
import random
import cv2
import math
import seaborn as sns
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from tensorflow.keras.layers import Dense,GlobalAveragePooling2D,Convolution2D,BatchNormalization
from tensorflow.keras.layers import Flatten,MaxPooling2D,Dropout
from tensorflow.keras.applications import InceptionV3
from tensorflow.keras.applications.densenet import preprocess_input
from tensorflow.keras.preprocessing import image
from tensorflow.keras.preprocessing.image import ImageDataGenerator,img_to_array
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau
import warnings
warnings.filterwarnings("ignore")
WIDTH = 299
HEIGHT = 299
CLASSES = 4
base_model = InceptionV3(weights='imagenet', include_top=False)
for layer in base_model.layers:
layer.trainable = False
x = base_model.output
x = GlobalAveragePooling2D(name='avg_pool')(x)
x = Dropout(0.4)(x)
predictions = Dense(CLASSES, activation='softmax')(x)
model = Model(inputs=base_model.input, outputs=predictions)
model.summary()
model.compile(optimizer='adam', ##also tried other optimiser --> similar poor accuracy found
loss='categorical_crossentropy',
metrics=['accuracy'])
## IMAGE DATA GENERATOR ##
from keras.applications.inception_v3 import preprocess_input
train_datagen = ImageDataGenerator(
preprocessing_function=preprocess_input,
rotation_range=40,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest',
validation_split=0.2)
train_generator = train_datagen.flow_from_directory(
TRAIN_DIR,
target_size=(HEIGHT, WIDTH),
batch_size=BATCH_SIZE,
class_mode='categorical',
subset="training")
validation_generator = train_datagen.flow_from_directory(
TRAIN_DIR,
target_size=(HEIGHT, WIDTH),
batch_size=BATCH_SIZE,
class_mode='categorical',
subset="validation")
test_datagen = ImageDataGenerator(rescale=1./255)
generator_test = test_datagen.flow_from_directory(directory=TEST_DIR,
target_size=(HEIGHT, WIDTH),
batch_size=BATCH_SIZE,
class_mode='categorical',
shuffle=False)
## MODEL training ##
EPOCHS = 20
STEPS_PER_EPOCH = 320 #train_generator.n//train_generator.batch_size
VALIDATION_STEPS = 64 #validation_generator.n//validation_generator.batch_size
history = model.fit_generator(
train_generator,
epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=validation_generator,
validation_steps=VALIDATION_STEPS)
Result found:
VALIDATION ACCURACY around 0.55-0.67 fluctuating..
TRAINING ACCURACY 0.99
Questions:
What/Where is the problem in transfer learning process?
are train, validate and test data generator function parameter are chosen correctly?
Well I think you would be better off to train the entire model. So remove the code that makes the base model layers not trainable. If you look at the documentation for Inceptionv3 located here you can set pooling='max' which puts a GlobalMaxPooling2d layer as the output layer so if you do that you do not need to add your own layer as you did. Now I noticed you imported the callbacks ModelCheckpoint and ReduceLROnPlateau but you did not use them in model.fit. Using an adjustable learning rate will be beneficial to achieving a lower validation loss. ModelCheckpoint is useful to save the best model for use in predictions. See code below for implementations. save_loc is the directory where you want to store the results from ModelCheckpoint. NOTE in ModelCheckpoint I set save_weights_only=True. Reason is this is far faster than saving the entire model on each epoch for which the validation loss decreases.
checkpoint=tf.keras.callbacks.ModelCheckpoint(filepath=save_loc, monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='auto', save_freq='epoch', options=None)
lr_adjust=tf.keras.callbacks.ReduceLROnPlateau( monitor="val_loss", factor=0.5, patience=1, verbose=1, mode="auto",
min_delta=0.00001, cooldown=0, min_lr=0)
callbacks=[checkpoint, lr_adjust]
history = model.fit_generator( train_generator, epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,validation_data=validation_generator,
validation_steps=VALIDATION_STEPS, callbacks=callbacks)
model.load_weights(save_loc) # load the saved weights
# after this use the model to evaluate or predict on the test set.
# if you are satisfied with the results you can then save the entire model with
model.save(save_loc)
Be careful on the test set generator. Ensure you do the same preprocessing on the test data as you did on the training data. I noticed you only rescaled the pixels. Not sure what the preprocess function does but I would use that.
I would also remove the dropout layer initially. Monitor the training loss and validation loss on each epoch and plot the results. If the training loss continues to decrease and the validation loss trends toward increasing then you are over fitting if so then restore the dropout layer if needed. If you do evaluation or prediction on the test set, you only want to go through the test set once. So select the test batch size to that no. of test samples/test batch size is an integer and use that integer as the number of test steps. Here is a
handy function that will determine that for you where length is the number of test samples and b_max is the maximum batch size you will allow based on your memory capacity.
def get_bs(length,b_max):
batch_size=sorted([int(length/n) for n in range(1,length+1) if length % n ==0 and length/n<=b_max],reverse=True)[0]
return batch_size,int(length/batch_size)
# example of use
batch_size, step=get_bs(10000,70)
#result is batch_size= 50 steps=200
I am working on a stock prediction project and I just want to predict the gain and drop labels from the LSTM net. It is a binary classification problem.
However, my LSTM net is hard to converge even I reduce the training set a lot. Technically, it should overfit easily. But my prediction accuracy is still only 60% and loss is around 0.7 even I just feed 90 samples for training. So, I was thinking I probably made some mistakes in building the neural net. However, due to my limited ability, I cannot find the reason. Therefore, I really hope someone can take a look at my code and point out the reason! I will appreciate a lot!
My code is given below.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import collections
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers.advanced_activations import LeakyReLU
from keras.layers import Dense, LSTM, Dropout, Activation, Flatten ,BatchNormalization
from keras.utils import to_categorical, np_utils
from keras.optimizers import SGD
data = pd.read_csv("EURUSD_M5_201910210000_201910251140.csv", sep="\t")
train_cut = int(data.shape[0] * 0.8)
dataset_train = data[0:train_cut]
training_set = dataset_train["<OPEN>"].values
sc = MinMaxScaler(feature_range=(0, 1))
train_sec_scaled = sc.fit_transform(training_set.reshape(-1, 1))
X_train = []
y_train = []
step_size = 60
predic_days = 1
for i in range(step_size, len(train_sec_scaled) - predic_days):
X_train.append(train_sec_scaled[i - step_size : i, 0])
y_value = train_sec_scaled[i : i + predic_days, 0]
last_day_value = train_sec_scaled[i - 1, 0]
# 1 ==> up, 0 ==> down
if y_value[0] > last_day_value:
y_train.append([1])
else:
y_train.append([0])
X_train, y_train = np.array(X_train), np.array(y_train)
X_train = X_train.reshape(X_train.shape[0], X_train.shape[1], 1)
y_train = y_train.reshape(y_train.shape[0], y_train.shape[1])
y_train = np_utils.to_categorical(y_train, 2)
print(y_train)
print("train data generated!")
print(X_train.shape, y_train.shape)
def train():
model = Sequential()
model.add(Dropout(0.2))
model.add(LSTM(40))
model.add(BatchNormalization())
model.add(LeakyReLU(alpha=0.02))
model.add(Dropout(0.2))
model.add(Dense(30, kernel_initializer='glorot_normal'))
model.add(BatchNormalization())
model.add(LeakyReLU(alpha=0.02))
model.add(Dropout(0.2))
model.add(Dense(2, activation='softmax'))
sgd = SGD(lr=0.01, decay=1e-4, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
model.fit(X_train,
y_train,batch_size=32,epochs=10000)
model.save("trend_analysis.h5")
print("model saved!")
if __name__ == "__main__":
train()
Also, here is the dataset I used:
https://drive.google.com/open?id=1r_0Ko1F6i0F1pToTSsQF1xGt_FTtpUux
Thanks in advance!
I am trying to use hyperopt for a classification deep learning model with keras:
import numpy as np
import tensorflow as tf
import random as rn
# The below is necessary for starting Numpy generated random numbers
# in a well-defined initial state.
np.random.seed(1)
# The below is necessary for starting core Python generated random numbers
# in a well-defined state.
rn.seed(2)
# Force TensorFlow to use single thread.
# Multiple threads are a potential source of non-reproducible results.
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
from keras import backend as K
tf.set_random_seed(2)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
K.set_session(sess)
#importing libraries
import ...
from hyperas import optim
from hyperas.distributions import choice, uniform, randint
from hyperopt import Trials, STATUS_OK, tpe
def data():
return x_train_sequence, y_train, x_test_sequence, y_test
# ===============
# Model creation
# ===============
def create_model(x_train_sequence, y_train, x_test_sequence, y_test):
embedding_dim = {{choice([...])}}
lstm = {{choice([...])}}
num_epochs = {{choice([...])}}
dropout = {{uniform(0, 1)}}
batch_size = {{choice([32])}}
model = Sequential()
model.add(...)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=["binary_accuracy"])
# Fit the model and evaluate
result = model.fit(x_train_sequence, y_train,
batch_size=batch_size,
validation_data=(x_test_sequence, y_test), verbose=verbose, shuffle=True, epochs=num_epochs)
return {'loss': -validation_acc, 'status': STATUS_OK, 'model': model}
# ===============
# Apply model and find best run
# ===============
if __name__ == '__main__':
best_run, best_model = optim.minimize(model=create_model,
data=data,
algo=tpe.suggest,
rseed=np.random.seed(1),
max_evals=50,
trials=Trials())
X_train, Y_train, X_test, Y_test = data()
print("Evalutation of best performing model:")
print(best_model.evaluate(X_test, Y_test))
print("Best performing model chosen hyper-parameters:")
print(best_run)
Even though I thought I put all the necessary seeds to obtain reproducible cases. I keep getting different results even if I substitute the {{choice([...])}} with integers.
What am I missing? What should I add to seed the model properly?
Thanks so much in advance!