Tensorflow why my logistic cost not change? - tensorflow

Question

I am new to tensorflow and now I want to do a classification job by self-defined layers. I have build a model which concat previous several DNN layers output to a big tensor, and then do the logistic binary classification. Like this(but without RNN):
Model Structure
This is my code:
import tensorflow as tf
import numpy as np
import pandas as pd
from load_data import load_data_from_file
TRAIN_FILE = 'train.csv'
TEST_FILE = 'test.csv'
X_train, y_train = load_data_from_file(TRAIN_FILE)
y_train = y_train[np.newaxis]
y_train = y_train.transpose()
n_x_dnn = X_train.shape[1]
n_hidden_dnn = 50
n_merge_dnn1 = 5 * n_hidden_dnn
n_dnn6 = 50
hm_epochs = 10
batch_size = 50
X = tf.placeholder(tf.float32, [None, n_x_dnn])
y = tf.placeholder(tf.float32, [None, 1])
weights = {
'dnn1': tf.Variable(tf.random_normal([n_x_dnn, n_hidden_dnn])),
'dnn2': tf.Variable(tf.random_normal([n_hidden_dnn, n_hidden_dnn])),
'dnn3': tf.Variable(tf.random_normal([n_hidden_dnn, n_hidden_dnn])),
'dnn4': tf.Variable(tf.random_normal([n_hidden_dnn, n_hidden_dnn])),
'dnn5': tf.Variable(tf.random_normal([n_hidden_dnn, n_hidden_dnn])),
'dnn_merge': tf.Variable(tf.random_normal([n_merge_dnn1, n_dnn6])),
'dnn6': tf.Variable(tf.random_normal([n_dnn6, 1]))
}
biases = {
'dnn1_b': tf.Variable(tf.random_normal([n_hidden_dnn])),
'dnn2_b': tf.Variable(tf.random_normal([n_hidden_dnn])),
'dnn3_b': tf.Variable(tf.random_normal([n_hidden_dnn])),
'dnn4_b': tf.Variable(tf.random_normal([n_hidden_dnn])),
'dnn5_b': tf.Variable(tf.random_normal([n_hidden_dnn])),
'dnn_merge_b': tf.Variable(tf.random_normal([n_dnn6])),
'dnn6_b': tf.Variable(tf.random_normal([1])),
}
def define_layers():
# dnn layer 1
dnn_layer1 = tf.add(tf.matmul(X, weights['dnn1']), biases['dnn1_b'])
dnn_layer1 = tf.nn.relu(dnn_layer1)
# dnn layer 2
dnn_layer2 = tf.add(tf.matmul(dnn_layer1, weights['dnn2']), biases['dnn2_b'])
dnn_layer2 = tf.nn.relu(dnn_layer2)
# dnn layer 3
dnn_layer3 = tf.add(tf.matmul(dnn_layer2, weights['dnn3']), biases['dnn3_b'])
dnn_layer3 = tf.nn.relu(dnn_layer3)
# dnn layer 4
dnn_layer4 = tf.add(tf.matmul(dnn_layer3, weights['dnn4']), biases['dnn4_b'])
dnn_layer4 = tf.nn.relu(dnn_layer4)
# dnn layer 5
dnn_layer5 = tf.add(tf.matmul(dnn_layer4, weights['dnn5']), biases['dnn5_b'])
dnn_layer5 = tf.nn.relu(dnn_layer5)
# merge layer
merged = tf.concat([dnn_layer1, dnn_layer2, dnn_layer3, dnn_layer4, dnn_layer5], 1)
dnn_merge = tf.add(tf.matmul(merged, weights['dnn_merge']), biases['dnn_merge_b'])
dnn_merge = tf.nn.relu(dnn_merge)
# dnn layer 6
dnn_layer6 = tf.add(tf.matmul(dnn_merge, weights['dnn6']), biases['dnn6_b'])
dnn_layer6 = tf.nn.sigmoid(dnn_layer6)
return dnn_layer6
logits = define_layers()
cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=y))
optimizer = tf.train.AdamOptimizer().minimize(cost)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(hm_epochs):
# epoch_loss = 0
i = 0
while i < len(X_train):
start = i
end = i + batch_size
batch_x = np.array(X_train[start:end])
batch_y = np.array(y_train[start:end])
_, c, l = sess.run([optimizer, cost, logits], feed_dict={X: batch_x,
y: batch_y})
# epoch_loss += c
i += batch_size
# print(c, l)
print('Epoch', epoch, 'completed out of', hm_epochs, 'loss:', c, 'logits:', l)
save_path = saver.save(sess, "model/NF_dnn_and_rnn_model.ckpt")
print("Model saved in file: %s" % save_path)
X_test, y_test = load_data_from_file(TEST_FILE)
predict = sess.run(logits, feed_dict={X: X_test})
print(predict)
And from output it shows my cost not changed:
Epoch 7 completed out of 10 loss: 1.27325 logits: [[ 1.]
Epoch 8 completed out of 10 loss: 1.27325 logits: [[ 1.]
Epoch 9 completed out of 10 loss: 1.27325 logits: [[ 1.]

Answer 1

You should remove the sigmoid activation from your last layer dnn_layer6 as sigmoid_cross_entropy_with_logits also applies sigmoid activation internally.