Email classification using word2vec - tensorflow

Question

My goal is to classify email, so each email have to correspond to a specific category like sport, clothes and so on.
My idea is to use a Recurrent Neural Network with one layer based on embedding (by word2vec). I'm using the followed schema "emojify v2" with some changing:
https://github.com/enggen/Deep-Learning-Coursera/blob/master/Sequence%20Models/Week2/Emojify/Emojify%20-%20v2.ipynb
and the word2vec pretrained model downloaded from:
http://hlt.isti.cnr.it/wordembeddings/
The problem is that this neural network does not work properly. In fact, trying to train the network on only 2 categories (20 sample). I get an accuracy of 50-60% on the training set that is of course too low.
Do you have any hint to improve my result?
What could be the problem?
a low number of samples?
a low number of nodes?
using loss='categorical_crossentropy'?
using optimizer='adam'?
the text of the email is too long?
the lenght of each email is different. This could be a problem?
I'm using keras (tensorflow).
edit1: insert code
import numpy as np
import emoji
import matplotlib.pyplot as plt
import numpy as np
# utiliti che serviranno in seguito
import csv
import numpy as np
import emoji
import pandas as pd
from sklearn.metrics import confusion_matrix
def read_glove_vecs(glove_file):
with open(glove_file, 'r') as f:
words = set()
word_to_vec_map = {}
for line in f:
line = line.strip().split()
curr_word = line[0]
words.add(curr_word)
word_to_vec_map[curr_word] = np.array(line[1:], dtype=np.float64)
i = 1
words_to_index = {}
index_to_words = {}
for w in sorted(words):
words_to_index[w] = i
index_to_words[i] = w
i = i + 1
return words_to_index, index_to_words, word_to_vec_map
def get_categories(filename):
categories_code = []
categories_name = []
with open(filename, newline='', encoding='utf-8') as f:
reader = csv.reader(f)
for row in reader:
category_code = row[1];
if (search(category_code, categories_code) == False):
categories_code.append(row[1])
categories_name.append(row[2])
Y = np.asarray(categories_code)
Z = np.asarray(categories_name)
return Y, Z
def search(category, categories):
for c in categories:
if (c == category):
return True
return False
def softmax(x):
"""Compute softmax values for each sets of scores in x."""
e_x = np.exp(x - np.max(x))
return e_x / e_x.sum()
def read_csv_for_email(filename):
name = []
text = []
category = []
category_code = []
with open(filename, newline='', encoding='utf-8') as f:
reader = csv.reader(f)
for row in reader:
name.append(row[0])
category_code.append(row[1])
category.append(row[2])
text.append(row[3])
X = np.asarray(name)
Y = np.asarray(category_code)
Z = np.asarray(category)
W = np.asarray(text)
return X, Y, Z, W
def convert_to_one_hot(Y, C):
Y = np.eye(C)[Y.reshape(-1)]
return Y
# converto in one hot vector
def my_convert_to_one_hot(Y, values):
#print(Y)
#print(values)
col = len(values)
row = len(Y)
#print(col)
#print(row)
R = np.zeros((row, col))
for res in range(row):
for i in range(col):
#print(Y[res])
#print(values[i])
if Y[res] == values[i]:
X = [0]*col;
X[i] = 1;
R[res] = np.array(X)
return R;
def print_predictions(X, pred):
print()
for i in range(X.shape[0]):
print(X[i], label_to_emoji(int(pred[i])))
def plot_confusion_matrix(y_actu, y_pred, title='Confusion matrix', cmap=plt.cm.gray_r):
df_confusion = pd.crosstab(y_actu, y_pred.reshape(y_pred.shape[0],), rownames=['Actual'], colnames=['Predicted'], margins=True)
df_conf_norm = df_confusion / df_confusion.sum(axis=1)
plt.matshow(df_confusion, cmap=cmap) # imshow
#plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(df_confusion.columns))
plt.xticks(tick_marks, df_confusion.columns, rotation=45)
plt.yticks(tick_marks, df_confusion.index)
#plt.tight_layout()
plt.ylabel(df_confusion.index.name)
plt.xlabel(df_confusion.columns.name)
def predict(X, Y, W, b, word_to_vec_map):
"""
Given X (sentences) and Y (emoji indices), predict emojis and compute the accuracy of your model over the given set.
Arguments:
X -- input data containing sentences, numpy array of shape (m, None)
Y -- labels, containing index of the label emoji, numpy array of shape (m, 1)
Returns:
pred -- numpy array of shape (m, 1) with your predictions
"""
m = X.shape[0]
pred = np.zeros((m, 1))
for j in range(m): # Loop over training examples
# Split jth test example (sentence) into list of lower case words
words = X[j].lower().split()
# Average words' vectors
avg = np.zeros((50,))
for w in words:
avg += word_to_vec_map[w]
avg = avg/len(words)
# Forward propagation
Z = np.dot(W, avg) + b
A = softmax(Z)
pred[j] = np.argmax(A)
print("Accuracy: " + str(np.mean((pred[:] == Y.reshape(Y.shape[0],1)[:]))))
return pred
import tensorflow as tf
sess = tf.Session()
from keras.layers import Input, Dense, Activation
import numpy as np
np.random.seed(0)
from keras.models import Model
from keras.layers import Dense, Input, Dropout, LSTM, Activation
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
from keras.initializers import glorot_uniform
np.random.seed(1)
# GRADED FUNCTION: sentences_to_indices
def sentences_to_indices(X, word_to_index, max_len):
"""
Converts an array of sentences (strings) into an array of indices corresponding to words in the sentences.
The output shape should be such that it can be given to `Embedding()` (described in Figure 4).
Arguments:
X -- array of sentences (strings), of shape (m, 1)
word_to_index -- a dictionary containing the each word mapped to its index
max_len -- maximum number of words in a sentence. You can assume every sentence in X is no longer than this.
Returns:
X_indices -- array of indices corresponding to words in the sentences from X, of shape (m, max_len)
"""
m = X.shape[0] # number of training examples
### START CODE HERE ###
# Initialize X_indices as a numpy matrix of zeros and the correct shape (≈ 1 line)
X_indices = np.zeros((m, max_len))
for i in range(m): # loop over training examples
# Convert the ith training sentence in lower case and split is into words. You should get a list of words.
sentence_words = X[i].lower().split()
# Initialize j to 0
j = 0
# Loop over the words of sentence_words
for w in sentence_words:
# Set the (i,j)th entry of X_indices to the index of the correct word.
if w in word_to_index:
if (j >= maxLen):
continue
X_indices[i, j] = word_to_index[w]
# Increment j to j + 1
j = j + 1
### END CODE HERE ###
return X_indices
from gensim.models import Word2Vec
model = Word2Vec.load('/home/ale/Scrivania/emoji_ita/embedding/glove_WIKI') # glove model
def getMyModels (model):
word_to_index = dict({})
index_to_word = dict({})
word_to_vec_map = dict({})
for idx, key in enumerate(model.wv.vocab):
word_to_index[key] = idx
index_to_word[idx] = key
word_to_vec_map[key] = model.wv[key]
return word_to_index, index_to_word, word_to_vec_map
code_train, category_train, category_code_train, text_train = read_csv_for_email('/home/ale/Scrivania/TestClassificazione/dataText2Categories.csv')
#FIXME
X_test, Y_test, Z_test, text_test = read_csv_for_email('C:\Users\Alessio\Desktop\3Febbraio\dataText2Categories.csv')
word_to_index, index_to_word, word_to_vec_map = getMyModels(model)
# EMBEDDING LAYER
# GRADED FUNCTION: pretrained_embedding_layer
# emb_dim define dimensionality of your GloVe word vectors (= 50)
def pretrained_embedding_layer(word_to_vec_map, word_to_index, emb_dim):
"""
Creates a Keras Embedding() layer and loads in pre-trained GloVe 50-dimensional vectors.
Arguments:
word_to_vec_map -- dictionary mapping words to their GloVe vector representation.
word_to_index -- dictionary mapping from words to their indices in the vocabulary (400,001 words)
Returns:
embedding_layer -- pretrained layer Keras instance
"""
vocab_len = len(word_to_index) + 1 # adding 1 to fit Keras embedding (requirement)
### START CODE HERE ###
# Initialize the embedding matrix as a numpy array of zeros of shape (vocab_len, dimensions of word vectors = emb_dim)
emb_matrix = np.zeros((vocab_len, emb_dim))
# Set each row "index" of the embedding matrix to be the word vector representation of the "index"th word of the vocabulary
for word, index in word_to_index.items():
emb_matrix[index, :] = word_to_vec_map[word]
# Define Keras embedding layer with the correct output/input sizes, make it trainable. Use Embedding(...). Make sure to set trainable=False.
embedding_layer = Embedding(vocab_len, emb_dim)
### END CODE HERE ###
# Build the embedding layer, it is required before setting the weights of the embedding layer. Do not modify the "None".
embedding_layer.build((None,))
# Set the weights of the embedding layer to the embedding matrix. Your layer is now pretrained.
embedding_layer.set_weights([emb_matrix])
return embedding_layer
# MODELLO EMOJIFY2
def Classifier(input_shape, word_to_vec_map, word_to_index, emb_dim, num_activation):
"""
Function creating the Emojify-v2 model's graph.
Arguments:
input_shape -- shape of the input, usually (max_len,)
word_to_vec_map -- dictionary mapping every word in a vocabulary into its 50-dimensional vector representation
word_to_index -- dictionary mapping from words to their indices in the vocabulary (400,001 words)
Returns:
model -- a model instance in Keras
"""
### START CODE HERE ###
# Define sentence_indices as the input of the graph, it should be of shape input_shape and dtype 'int32' (as it contains indices).
sentence_indices = Input(shape=input_shape, dtype=np.int32)
# Create the embedding layer pretrained with GloVe Vectors (≈1 line)
embedding_layer = pretrained_embedding_layer(word_to_vec_map, word_to_index, emb_dim)
# Propagate sentence_indices through your embedding layer, you get back the embeddings
embeddings = embedding_layer(sentence_indices)
# Propagate the embeddings through an LSTM layer with 128-dimensional hidden state
# Be careful, the returned output should be a batch of sequences.
X = LSTM(128, return_sequences=True)(embeddings)
# Add dropout with a probability of 0.5
X = Dropout(0.5)(X)
# Propagate X trough another LSTM layer with 128-dimensional hidden state
# Be careful, the returned output should be a single hidden state, not a batch of sequences.
X = LSTM(128)(X)
# Add dropout with a probability of 0.5
X = Dropout(0.5)(X)
# Propagate X through a Dense layer with softmax activation to get back a batch of 5-dimensional vectors.
X = Dense(num_activation, activation='softmax')(X)
# Add a softmax activation
X = Activation('softmax')(X)
# Create Model instance which converts sentence_indices into X.
model = Model(sentence_indices, X)
### END CODE HERE ###
return model
def max_text_lenght(filename):
max_len = 0;
max_id = 0;
with open(filename, newline='', encoding='utf-8') as f:
reader = csv.reader(f)
for row in reader:
text = row[3];
current_lenght = len(text);
if (current_lenght > max_len):
max_len = current_lenght;
max_id = row[0]
#print(max_id)
return max_len
def create_values_list():
z, y = get_categories('C:\Users\Alessio\Desktop\3Febbraio\dataText2Categories.csv')
return y;
values = create_values_list()
num_activation = len(values)
maxLen = 300
print(maxLen)
model = Classifier((maxLen,), word_to_vec_map, word_to_index, maxLen, num_activation)
model.summary()
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
Y_train_indices = sentences_to_indices(text_train, word_to_index, maxLen)
Z_train_oh = my_convert_to_one_hot(category_code_train, values)
print(Z_train_oh)
model.fit(Y_train_indices, Z_train_oh, epochs = 60, batch_size = 32, shuffle=True)
model.save('text_classification.h5')