This question already has answers here:
Keras not training on entire dataset
(3 answers)
Closed 2 years ago.
I am facing a problem in training of a neural network in Google colab. My model is not training under the complete training dataset even after I have uploaded it in the drive and provide a proper path. Here is the code I have written
import tensorflow as tf
import tensorflow.keras as keras
from keras.models import Sequential
from keras.layers import Dense, Flatten, Activation, Dropout
from keras.optimizers import Adam
from sklearn.metrics import mean_squared_error, mean_absolute_error, max_error, r2_score
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
X=pd.read_csv('/content/drive/My Drive/ML Data/prob_232_full.dat',sep="\s+",header=None)
y=pd.read_csv('/content/drive/My Drive/ML Data/pGuess_232_full.dat',sep="\s+",header=None)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X.astype(np.float64), y.astype(np.float64), test_size = 0.25, random_state = 1)
X_train = np.array(X_train)
X_test = np.array(X_test)
# Sklearn wants the labels as one-dimensional vectors
y_train = np.array(y_train).reshape((-1,))
y_test = np.array(y_test).reshape((-1,))
ncols=X_train.shape[1]
model = Sequential()
model.add(Dense(activation="relu", input_dim=ncols, units=64, kernel_initializer="uniform"))
model.add(Dense(activation="relu", units=128, kernel_initializer="uniform"))
model.add(Dense(activation="relu", units=256, kernel_initializer="uniform"))
model.add(Dense(activation="relu", units=64, kernel_initializer="uniform"))
model.add(Dense(activation="relu", units=1, kernel_initializer="uniform"))
opt=keras.optimizers.Adam(learning_rate=0.0001)
model.compile(optimizer = opt, loss='mean_squared_error', metrics=['mean_absolute_error'])
history=model.fit(X_train, y_train, validation_data=(X_test, y_test),
batch_size = 32, epochs = 40, verbose=1)
While the size of the training set is 457500, but it shows the model is training only under 14297 training data.
Welcome to Stackoverflow.com
Dear your dataset is 457500 and you are using a batch size of 32 (in model.fit). So your total iterations for the dataset are 457500 / 32 almost equal to = 14296. The last batch contains 4 fewer examples so it is not using the last batch. So it is showing fine. It's just about understanding.
Related
When I use return_sequences = true, for a LSTM layer, before adding a dense layer it sometimes results in an error depending upon the environment. I believe it mainly depends on the version of tensorflow and keras. If I am using tensorflow 2.1.0 and Keras 2.3.0, I get the following error -
standardize_input_data 'with shape ' + str(data_shape)) ValueError: Error when checking target: expected dense_2 to have 3 dimensions, but got array with shape (7000, 1)
However, if I use tensorflow 2.9.1 and keras 2.9.0 I do not get any error.
Here is some minimal working sample code -
import os
import pandas as pd
from sklearn import preprocessing
from collections import deque
import random
import numpy as np
import time
import random
from keras.models import Sequential
from keras.layers import Dense, Dropout, LSTM, BatchNormalization, Input
from keras.models import load_model
import keras
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
import pickle
epochs = 10
batch_size = 64
X, y = make_classification(n_samples=10000, n_features=3, n_classes=3, n_informative=3, n_redundant=0, n_repeated=0 ,weights=[0.5,0.5,0.5])
X = X.reshape(X.shape[0], 1, 3)
y = y.reshape(-1, 1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
inputs = Input(shape=(X_train.shape[1:]))
outputs = LSTM(128, input_shape=(X_train.shape[1:]), return_sequences=True)(inputs)
outputs = Dropout(0.2)(outputs)
outputs = BatchNormalization()(outputs)
outputs = LSTM(128, input_shape=(X_train.shape[1:]), return_sequences=True)(outputs)
outputs = Dropout(0.2)(outputs)
outputs = BatchNormalization()(outputs)
outputs = Dense(32, activation="relu", kernel_initializer="glorot_uniform")(outputs)
outputs = Dropout(0.2)(outputs)
outputs = Dense(3, activation="softmax", kernel_initializer="glorot_uniform")(outputs)
model = keras.Model(inputs, outputs)
opt = keras.optimizers.Adam(lr=0.0001, decay=1e-6)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.fit(X_train, y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_test, y_test))
I run the following code in Google Colab(with GPU):
import random
random.seed(1)
import numpy as np
from numpy.random import seed
seed(1)
from tensorflow import set_random_seed
set_random_seed(2)
import pandas as pd
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.layers import Flatten, Dense, Lambda, SimpleRNN
from keras.optimizers import *
from keras.utils import np_utils
from keras.initializers import *
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, roc_auc_score, auc, precision_recall_curve
from sklearn.metrics import confusion_matrix
from keras.callbacks import EarlyStopping
from keras import backend as K
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
K.set_session(sess)
##Loading dataset train and validation files, the files are same for every run
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5)
print("***********************************************************************************************")
def make_model():
model = Sequential()
model.add(Conv2D(10,(5,5), kernel_initializer=glorot_uniform(seed=1), input_shape = (22,10,1), use_bias = True, activation = "relu", strides = 1, padding = "valid"))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Flatten())
model.add(Dense(20, kernel_initializer=glorot_uniform(seed=1), activation = "relu"))
model.add(Lambda(lambda x: tf.expand_dims(x, axis=1)))
model.add(SimpleRNN(20, kernel_initializer=glorot_uniform(seed=1), activation="relu",return_sequences=False))
model.add(Dense(1, kernel_initializer=glorot_uniform(seed=1), activation="sigmoid"))
opti = SGD(lr = 0.01)
model.compile(loss = "binary_crossentropy", optimizer = opti, metrics = ["accuracy"])
return model
model = make_model()
model.fit(x_train, y_train, validation_data = (x_validation,y_validation), epochs = 50, batch_size = 20, verbose = 2, callbacks=[es])
Despite setting all seed values, my prediction results of the model are different on subsequent runs. The training and testing of the model happens in the same Colab cell.
You are dealing with floating point numbers that are multiplied and added on different threads and can therefore happen in different order. Floating point additions and multiplications are not commutative. See What Every Computer Scientist Should Know About Floating-Point Arithmetic.
I tried to do human pose action recognition model,
I referred this model
I like to use LSTM for this model. So I have made some changes in train.py
My train.py code:
import pandas as pd
from enum import Enum
import numpy as np
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.layers.normalization import BatchNormalization
from keras.optimizers import Adam
from keras.models import load_model
from keras.layers import LSTM
class Actions(Enum):
sit = 0
stand = 1
walk = 2
sleep= 3
raw_data = pd.read_csv('7537real1.csv', header=0)
dataset = raw_data.values
X = dataset[0:7537, 0:36].astype(float)
Y = dataset[0:7537, 36]
encoder_Y = [0]* 4479 + [1]* 1425 + [2] * 1164 + [3] * 468
dummy_Y = np_utils.to_categorical(encoder_Y)
X_train, X_test, Y_train, Y_test = train_test_split(X, dummy_Y, test_size=0.1, random_state=9)
model = Sequential()
model.add(LSTM(4, input_shape=(36,1)))
model.add(Dense(units=4, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer=Adam(0.0001), metrics=['accuracy'])
model.fit(X_train, Y_train, batch_size=32, epochs=500, verbose=1, validation_data=(X_test, Y_test))
model.save('7537real1.h5')
My data set has 36 features and class attribute (labels:0,1,2,3)
And totally there are 7537 records in the dataset.
When I tried to build the LSTM sequential classification model I got value error.
Also I have attached dataset sample as screenshot (csv file).
How to reshape the data(array) set for this model and how to build the LSTM sequential model?
This is a keras model for sentiment analysis i need to convert it to tensorflow i couldn’t build embedding layer with tensorflow and using confusion matrix to evaluate this model? And I asked if tf-learn is the-same as tensorflow
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow import set_random_seed
set_random_seed(2)
from nltk.tokenize import word_tokenize
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer
from sklearn.preprocessing import LabelEncoder
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers.embeddings import Embedding
from keras.layers import Flatten
from keras.layers import Conv1D, MaxPooling1D
from keras.layers import Dense,Activation
from keras.layers import Dropout
from keras.callbacks import TensorBoard, ModelCheckpoint
import re
import string
import collections
import time
seed = 10
Read CSV Files
df=pd.read_csv('tweets-pos-neg.csv', usecols = ['text','airline_sentiment'])
df = df.reindex(['text','airline_sentiment'], axis=1) #reorder columns
df=df.apply(lambda x: x.astype(str).str.lower())
Normalize Text
def normalize(text):
text= re.sub(r"http\S+", r'', text)
text= re.sub(r"#\S+", r'', text)
punctuation = re.compile(r'[!"#$%&()*+,-./:;<=>?#[\]^_`{|}~|0-9]')
text = re.sub(punctuation, ' ', text)
text= re.sub(r'(.)\1\1+', r'\1', text)
return text
Cleaned Text
def prepareDataSets(df):
sentences=[]
for index, r in df.iterrows():
text= normalize(r['text'])
sentences.append([text,r['airline_sentiment']])
df_sentences=pd.DataFrame(sentences,columns=
['text','airline_sentiment'])
return df_sentences
edit_df=prepareDataSets(df)
edit_df=shuffle(edit_df)
X=edit_df.iloc[:,0]
Y=edit_df.iloc[:,1]
Split reviews to tokens
max_features = 50000
tokenizer = Tokenizer(num_words=max_features, split=' ')
tokenizer.fit_on_texts(X.values)
#convert review tokens to integers
X_seq = tokenizer.texts_to_sequences(X)
Padding Sequence to make all vectors with the same size according to MAX-length of reviews
seq_len=35
X_pad = pad_sequences(X_seq,maxlen=seq_len)
Convert target value from string to integer
le=LabelEncoder()
Y_le=le.fit_transform(Y)
Y_le_oh=to_categorical(Y_le)
Train-Test-Split
X_train, X_test, Y_train, Y_test = train_test_split(X_pad,Y_le_oh, test_size
= 0.33, random_state = 42)
X_train, X_Val, Y_train, Y_Val = train_test_split(X_train,Y_train, test_size
= 0.1, random_state = 42)
print(X_train.shape,Y_train.shape)
print(X_test.shape,Y_test.shape)
print(X_Val.shape,Y_Val.shape)
Create the model
embedding_vecor_length = 32 #no of vector columns
model_cnn = Sequential()
model_cnn.add(Embedding(max_features, embedding_vecor_length,
input_length=seq_len))
model_cnn.add(Conv1D(filters=100, kernel_size=2, padding='valid',
activation='relu', strides=1))
model_cnn.add(MaxPooling1D(2))
model_cnn.add(Flatten())
model_cnn.add(Dense(256, activation='relu'))
model_cnn.add(Dense(2, activation='softmax'))
opt=tf.keras.optimizers.Adam(lr=0.001, decay=1e-6)
model_cnn.compile(loss='binary_crossentropy', optimizer=opt, metrics=
['accuracy'])
print(model_cnn.summary())
Evaluate model
history=model_cnn.fit(X_train, Y_train, epochs=3, batch_size=32, callbacks=[tensorboard], validation_data=(X_Val, Y_Val))
scores = model_cnn.evaluate(X_test, Y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[-1]*100))
If you just need to use Tensorflow APIs to train / evaluate, you can build an Estimator using model_to_estimator function.
Here's the documentation with an example.
Running a single hidden layer MLP on MNIST, I get extremly different results for Keras and sklearn.
import numpy as np
np.random.seed(5)
import os
os.environ["CUDA_VISIBLE_DEVICES"] = '-1'
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense
from keras import regularizers
from keras.optimizers import Adam
from keras.utils import np_utils
from sklearn.neural_network import MLPClassifier
(x_train, y_train), (x_test, y_test) = mnist.load_data()
num_classes = 10
batch_data = x_train[:2000]
batch_labels = y_train[:2000]
# flat 2d images
batch_data_flat = batch_data.reshape(2000, 784)
# one-hot encoding
batch_labels_one_hot = np_utils.to_categorical(batch_labels, num_classes)
num_hidden_nodes = 100
alpha = 0.0001
batch_size = 128
beta_1 = 0.9
beta_2 = 0.999
epsilon = 1e-08
learning_rate_init = 0.001
epochs = 200
# keras
keras_model = Sequential()
keras_model.add(Dense(num_hidden_nodes, activation='relu',
kernel_regularizer=regularizers.l2(alpha),
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform'))
keras_model.add(Dense(num_classes, activation='softmax',
kernel_regularizer=regularizers.l2(alpha),
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform'))
keras_optim = Adam(lr=learning_rate_init, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon)
keras_model.compile(optimizer=keras_optim, loss='categorical_crossentropy', metrics=['accuracy'])
keras_model.fit(batch_data_flat, batch_labels_one_hot, batch_size=batch_size, epochs=epochs, verbose=0)
# sklearn
sklearn_model = MLPClassifier(hidden_layer_sizes=(num_hidden_nodes,), activation='relu', solver='adam',
alpha=alpha, batch_size=batch_size, learning_rate_init=learning_rate_init,
max_iter=epochs, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon)
sklearn_model.fit(batch_data_flat, batch_labels_one_hot)
# evaluate both on their training data
score_keras = keras_model.evaluate(batch_data_flat, batch_labels_one_hot)
score_sklearn = sklearn_model.score(batch_data_flat, batch_labels_one_hot)
print("Acc: keras %f, sklearn %f" % (score_keras[1], score_sklearn))
Outputs: Acc: keras 0.182500, sklearn 1.000000
The only difference I see is that scikit-learn computes for the Glorot initialization of the final layer sqrt(2 / (fan_in + fan_out)) vs. sqrt(6 / (fan_in + fan_out)) from Keras. But that should not cause such a difference I think. Do I forget something here?
scikit-learn 0.19.1, Keras 2.2.0 (Backend Tensorflow 1.9.0)
You should probably initialize the biases with 'zeros' and not with 'glorot_uniform'.