I have trained a model on some data using tflearn to do binary classification. The model was trained to a 97% accuracy.
I want to use model.load() in another program to predict the class of some test input data.
However, model.load() only works when I include the argument weights_only=True. When I omit that argument from model.load(), it throws an error:
NotFoundError (see above for traceback): Key is_training not found in checkpoint
When I get the model loaded and run some predictions on my small test set - the classifications seem strange.. The model predicts a perfect 1 in the 1st index every single time. To me this shouldn't be happening if the model was trained to a very high accuracy. Here's what the predictions look like (expected output on the right):
[[ 5.59889193e-22 1.00000000e+00] [0, 1]
[ 4.25160435e-22 1.00000000e+00] [0, 1]
[ 6.65333618e-23 1.00000000e+00] [0, 1]
[ 2.07748895e-21 1.00000000e+00] [0, 1]
[ 1.77639440e-21 1.00000000e+00] [0, 1]
[ 5.77486922e-18 1.00000000e+00] [1, 0]
[ 2.70562403e-19 1.00000000e+00] [1, 0]
[ 2.78288828e-18 1.00000000e+00] [1, 0]
[ 6.10306495e-17 1.00000000e+00] [1, 0]
[ 2.35787162e-19 1.00000000e+00]] [1, 0]
Note: This test data was data used to train the model so should be able to classify correctly with high accuracy.
The code for training the model:
tf.reset_default_graph()
train = pd.read_csv("/Users/darrentaggart/Library/Mobile Documents/com~apple~CloudDocs/Uni Documents/MEE4040 - Project 4/Coding Related Stuff/Neural Networks/modeltraindata_1280.csv")
test = pd.read_csv("/Users/darrentaggart/Library/Mobile Documents/com~apple~CloudDocs/Uni Documents/MEE4040 - Project 4/Coding Related Stuff/Neural Networks/modeltestdata_320.csv")
X = train.iloc[:,1:].values.astype(np.float32)
Y = np.array([np.array([int(i == l) for i in range(2)]) for l in
train.iloc[:,:1].values])
test_x = test.iloc[:,1:].values.astype(np.float32)
test_y = np.array([np.array([int(i == l) for i in range(2)]) for l in
test.iloc[:,:1].values])
X = X.reshape([-1, 16, 16, 1])
test_x = test_x.reshape([-1, 16, 16, 1])
convnet = input_data(shape=[None, 16, 16, 1], name='input')
initialization = tf.contrib.layers.variance_scaling_initializer(factor=1.0, mode='FAN_IN', uniform=False)
convnet = conv_2d(convnet, 32, 2, activation='elu',
weights_init=initialization)
convnet = max_pool_2d(convnet, 2)
convnet = tflearn.layers.normalization.batch_normalization(convnet, beta=0.0, gamma=1.0, epsilon=1e-05,
decay=0.9, stddev=0.002, trainable=True, restore=True, reuse=False, scope=None, name='BatchNormalization')
convnet = conv_2d(convnet, 64, 2, activation='elu',
weights_init=initialization)
convnet = max_pool_2d(convnet, 2)
convnet = tflearn.layers.normalization.batch_normalization(convnet, beta=0.0, gamma=1.0, epsilon=1e-05,
decay=0.9, stddev=0.002, trainable=True, restore=True, reuse=False, scope=None, name='BatchNormalization')
convnet = fully_connected(convnet, 254, activation='elu', weights_init=initialization)
convnet = dropout(convnet, 0.8)
convnet = tflearn.layers.normalization.batch_normalization(convnet, beta=0.0, gamma=1.0, epsilon=1e-05,
decay=0.9, stddev=0.002, trainable=True, restore=True, reuse=False, scope=None, name='BatchNormalization')
convnet = fully_connected(convnet, 2, activation='softmax')
adam = tflearn.optimizers.Adam(learning_rate=0.00065, beta1=0.9, beta2=0.999, epsilon=1e-08)
convnet = regression(convnet, optimizer=adam, loss='categorical_crossentropy', name='targets')
model = tflearn.DNN(convnet, tensorboard_dir='/Users/darrentaggart/Library/Mobile Documents/com~apple~CloudDocs/Uni Documents/MEE4040 - Project 4/Coding Related Stuff/Neural Networks/latest logs',
tensorboard_verbose=3)
model.fit({'input': X}, {'targets': Y}, n_epoch=100, batch_size=16,
validation_set=({'input': test_x}, {'targets': test_y}), snapshot_step=10, show_metric=True, run_id='1600 - ConvConvFC254 LR0.00065decay BN VSinit 16batchsize 100epochs')
model.save('tflearncnn.model')
Code for loading and generating predictions:
test = pd.read_csv("/Users/darrentaggart/Library/Mobile Documents/com~apple~CloudDocs/Uni Documents/MEE4040 - Project 4/Coding Related Stuff/Neural Networks/modelpredictiondata.csv")
X = test.iloc[:,1:].values.astype(np.float32)
sess=tf.InteractiveSession()
tflearn.is_training(False)
convnet = input_data(shape=[None, 16, 16, 1], name='input')
initialization = tf.contrib.layers.variance_scaling_initializer(factor=1.0, mode='FAN_IN', uniform=False)
convnet = conv_2d(convnet, 32, 2, activation='elu', weights_init=initialization)
convnet = max_pool_2d(convnet, 2)
convnet = tflearn.layers.normalization.batch_normalization(convnet, beta=0.0, gamma=1.0, epsilon=1e-05,
decay=0.9, stddev=0.002, trainable=True, restore=True, reuse=False, scope=None, name='BatchNormalization')
convnet = conv_2d(convnet, 64, 2, activation='elu', weights_init=initialization)
convnet = max_pool_2d(convnet, 2)
convnet = tflearn.layers.normalization.batch_normalization(convnet, beta=0.0, gamma=1.0, epsilon=1e-05,
decay=0.9, stddev=0.002, trainable=True, restore=True, reuse=False, scope=None, name='BatchNormalization')
convnet = fully_connected(convnet, 254, activation='elu', weights_init=initialization)
convnet = tflearn.layers.normalization.batch_normalization(convnet, beta=0.0, gamma=1.0, epsilon=1e-05,
decay=0.9, stddev=0.002, trainable=True, restore=True, reuse=False, scope=None, name='BatchNormalization')
convnet = fully_connected(convnet, 2, activation='softmax')
adam = tflearn.optimizers.Adam(learning_rate=0.00065, beta1=0.9, beta2=0.999, epsilon=1e-08)
convnet = regression(convnet, optimizer=adam, loss='categorical_crossentropy', name='targets')
model = tflearn.DNN(convnet)
if os.path.exists('{}.meta'.format('tflearncnn.model')):
model.load('tflearncnn.model', weights_only=False)
print('model loaded!')
for i in enumerate(X):
X = X.reshape([-1, 16, 16, 1])
model_out = model.predict(X)
if np.argmax(model_out) == 1: str_label='Boss'
else: str_label = 'Slot'
print(model_out)
I know it's a long shot but thought someone might be able to shed some light on the matter. Thanks.
It's been a year and a half since this question was asked, but sharing is caring after all. Using tflearn and Alexnet to binary classify an image.
The trick is to normalize after conversion to nparray. Don't forget to change the directory paths.
from __future__ import division, print_function, absolute_import
import tflearn
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.normalization import local_response_normalization
from tflearn.layers.estimator import regression
from data_utils import *
import os
from PIL import Image
from numpy import array
def res_image(f, image_shape=[224,224], grayscale=False, normalize=True):
img = load_image(f)
width, height = img.size
if width != image_shape[0] or height != image_shape[1]:
img = resize_image(img, image_shape[0], image_shape[1])
if grayscale:
img = convert_color(img, 'L')
elif img.mode == 'L':
img = convert_color(img, 'RGB')
img = pil_to_nparray(img)
if normalize: # << this here is what you need
img /= 255.
img = array(img).reshape(1, image_shape[0], image_shape[1], 3)
return img
# Building the network
network = input_data(shape=[None, 227, 227, 3])
network = conv_2d(network, 96, 11, strides=4, activation='relu')
network = max_pool_2d(network, 3, strides=2)
network = local_response_normalization(network)
network = conv_2d(network, 256, 5, activation='relu')
network = max_pool_2d(network, 3, strides=2)
network = local_response_normalization(network)
network = conv_2d(network, 384, 3, activation='relu')
network = conv_2d(network, 384, 3, activation='relu')
network = conv_2d(network, 256, 3, activation='relu')
network = max_pool_2d(network, 3, strides=2)
network = local_response_normalization(network)
network = fully_connected(network, 4096, activation='tanh')
network = dropout(network, 0.5)
network = fully_connected(network, 4096, activation='tanh')
network = dropout(network, 0.5)
network = fully_connected(network, 2, activation='softmax') # output is the number of outcomes
network = regression(network, optimizer='momentum',
loss='categorical_crossentropy',
learning_rate=0.001)
# Training
model = tflearn.DNN(network,
tensorboard_dir=R'C:\Users\b0588718\Source\Repos\AlexNet\AlexNet')
model.load('model.tfl')
f = r'C:\Users\b0588718\Source\Repos\AlexNet\AlexNet\rawdata\jpg\0\P1170047.jpg'
img = res_image(f, [227,227], grayscale=False, normalize=True)
pred = model.predict(img)
print(" %s" % pred[0])
Didn't you try model.load(<path-to-saved-model>).
Ex : model.load("./model.tflearn")
I think this will solve your problem.
Related
I use google-colab
My function is:
def createModelUsingTensorflow(nbClasses, imageSizeX, imageSizeY, imageSizeZ, args):
'''Create the Deep Neural Network Model'''
print("[+] Creating model...")
convnet = input_data(shape=[None, imageSizeX, imageSizeY, imageSizeZ], name='input')
convnet = conv_2d(convnet, 64, 2, activation='relu', weights_init="Xavier")
convnet = max_pool_2d(convnet, 2)
convnet = conv_2d(convnet, 128, 2, activation='relu', weights_init="Xavier")
convnet = max_pool_2d(convnet, 2)
convnet = conv_2d(convnet, 256, 2, activation='relu', weights_init="Xavier")
convnet = max_pool_2d(convnet, 2)
convnet = conv_2d(convnet, 512, 2, activation='relu', weights_init="Xavier")
convnet = max_pool_2d(convnet, 2)
convnet = conv_2d(convnet, 1024, 2, activation='relu', weights_init="Xavier")
convnet = max_pool_2d(convnet, 2)
convnet = conv_2d(convnet, 2048, 2, activation='relu', weights_init="Xavier")
convnet = max_pool_2d(convnet, 2)
convnet = fully_connected(convnet, 4096, activation='relu')
convnet = dropout(convnet, 0.5)
convnet = fully_connected(convnet, nbClasses, activation='softmax')
convnet = regression(convnet, optimizer='adam', loss='categorical_crossentropy', learning_rate=learningRate)
# model = tflearn.DNN(convnet, tensorboard_dir='tensorboard', tensorboard_verbose=3)
createFolder(checkpointPath)
model = tflearn.DNN(convnet, checkpoint_path='{}/model.tfl'.format(checkpointPath), max_checkpoints=1)
if args.resume and args.epochs:
try:
model.load('{}/model.tfl-{}'.format(checkpointPath, args.resume))
print(" Model retrieved and resuming training!")
except Exception as err:
print("Couldn't load the previous model", err)
raise err
else:
print(" Model created!")
return model
I get an error when I call the function
My error is:
enter image description here
because the version of the TensorFlow
What can I do ? or How do I use Glorot in my function?
thanks!!
From the image you shared, it seems you are using tflearn library instead of Tensorflow or Keras. In that case, you will have to pass tflearn.initializations.xavier() method to weights_init= in your conv_2d call.
The method signature for tflearn.initializations.xavier() is:
tflearn.initializations.xavier (uniform=True, seed=None, dtype=tf.float32)
Hope that works!
P.S: A small suggestion, always share your relevant import statements or proper library name and version for people to help you better and quicker
I am training a DCGAN model with tensorflow.keras, and I added BatchNormalization layers in both generator and discriminator.
I train gan with following steps:
1. train discriminator with real images and images from generator(using generator.predict)
2. train adversarial network(compiled with discriminator.trainable=False)
Then I found that after a few rounds the training loss returned by train_on_batch() of both generator and discriminator goes to zero. But when I use test_on_batch() the loss is still huge for generator. And the generated images are all mess.
At first I thought that it is because in the 2.'s step mentioned above when training the adversarial network, the discriminator's input only containing fake images makes the batch normalization layers get different distribution as the 1.'s step when both fake & real images were fed.
But even if I removed all batch normalization layers in discriminator, the same problem still exists. Only when all batch normalization layers were removed the problem will disappear. Also I found out that existences of Dropout layers don't make a difference. I wondering why batch normalization can cause such problem, even if in generator fed with noises with same distribution.
# Model definition
class DCGAN_128:
def __init__(self, hidden_dim):
generator = M.Sequential()
generator.add(L.Dense(128 * 8 * 8, input_shape=[hidden_dim]))
generator.add(L.Reshape([8, 8, 128]))
generator.add(L.UpSampling2D()) # [8, 8, 128]
generator.add(L.Conv2D(128, kernel_size=3, padding="same")) # [16, 16, 128]
generator.add(L.LayerNormalization()) # 4
generator.add(L.ReLU())
generator.add(L.UpSampling2D()) # [32, 32, 128]
generator.add(L.Conv2D(64, kernel_size=5, padding="same")) # [32, 32, 64]
generator.add(L.LayerNormalization()) # 8
generator.add(L.ReLU())
generator.add(L.UpSampling2D()) # [64, 64, 128]
generator.add(L.Conv2D(32, kernel_size=7, padding="same")) # [64, 64, 32]
generator.add(L.LayerNormalization()) # 12
generator.add(L.ReLU())
generator.add(L.UpSampling2D()) # [128, 128, 32]
generator.add(L.Conv2D(3, kernel_size=3, padding="same", activation=A.sigmoid)) # [128, 128, 3]
discriminator = M.Sequential()
discriminator.add(L.Conv2D(32, kernel_size=5, strides=2, padding="same", input_shape=[128, 128, 3]))
discriminator.add(L.LeakyReLU())
# discriminator.add(L.Dropout(0.25)) # [64, 64, 32]
discriminator.add(L.Conv2D(64, kernel_size=3, strides=2, padding="same"))
# discriminator.add(L.BatchNormalization(epsilon=1e-5)) # 4
discriminator.add(L.LeakyReLU())
# discriminator.add(L.Dropout(0.25)) # [32, 32, 64]
discriminator.add(L.Conv2D(128, kernel_size=3, strides=2, padding="same"))
discriminator.add(L.LayerNormalization()) # 8
discriminator.add(L.LeakyReLU()) # [16, 16, 128]
discriminator.add(L.Dropout(0.25))
discriminator.add(L.Conv2D(256, kernel_size=3, strides=2, padding="same"))
discriminator.add(L.LayerNormalization()) # 12
discriminator.add(L.LeakyReLU()) # [8, 8, 256]
discriminator.add(L.Dropout(0.25))
discriminator.add(L.Conv2D(512, kernel_size=3, strides=2, padding="same"))
discriminator.add(L.LeakyReLU()) # [4, 4, 512]
discriminator.add(L.Flatten())
discriminator.add(L.Dense(1, activation=A.sigmoid))
self.model_gen = generator
self.model_dis = discriminator
self.adv_input = L.Input([hidden_dim])
self.adv_output = discriminator(generator(self.adv_input))
self.model_adversarial = M.Model(self.adv_input, self.adv_output)
# Training
dcgan = hidden_dim = 100
DCGAN_128(hidden_dim)
data_loader = AnimeFacesLoader([128, 128])
batch_size = 32
n_rounds = 40000
dis_model = dcgan.model_dis
gen_model = dcgan.model_gen
adv_model = dcgan.model_adversarial
gen_model.summary()
adv_model.summary()
dis_model.compile(Opt.Adam(0.0002), Lo.binary_crossentropy)
dis_model.trainable = False
adv_model.compile(Opt.Adam(0.0002), Lo.binary_crossentropy)
layer_outputs = [layer.output for layer in dis_model.layers]
visual_model = tf.keras.Model(dis_model.input, layer_outputs)
for rounds in range(n_rounds):
# Get output images
if rounds % 100 == 0 and rounds > 0:
noise = np.random.uniform(-1, 1, [16, hidden_dim])
tiled_images = np.zeros([4*128, 4*128, 3]).astype(np.uint8)
generated_imgs = gen_model.predict(noise)
generated_imgs *= 256
generated_imgs = generated_imgs.astype(np.uint8)
for i in range(16):
tiled_images[int(i / 4)*128: int(i / 4)*128 + 128,
int(i % 4)*128: int(i % 4)*128 + 128, :] = generated_imgs[i, :, :, :]
Image.fromarray(tiled_images).save("Output/DCGAN/" + "rounds_{0}.jpg".format(rounds))
'''
layer_visualization = visual_model.predict(generated_imgs[:1])
for i in range(len(layer_visualization)):
plt.imshow(layer_visualization[i][0, :, :, 0])
plt.show()
'''
# train discriminator on real & fake images
real_imgs = data_loader.get_batch(batch_size)
real_ys = np.ones([batch_size, 1])
noise = np.random.uniform(-1, 1, [batch_size, hidden_dim])
fake_ys = np.zeros([batch_size, 1])
fake_imgs = gen_model.predict(noise)
imgs = np.concatenate([real_imgs, fake_imgs], axis=0)
ys = np.concatenate([real_ys, fake_ys], axis=0)
loss_dis = dis_model.train_on_batch(imgs, ys)
print("Round {}, Loss dis:{:.4f}".format(rounds, loss_dis))
loss_dis_test = dis_model.test_on_batch(imgs, ys)
print(loss_dis_test)
noise = np.random.uniform(-1, 1, [batch_size, hidden_dim])
fake_ys = np.ones([batch_size, 1])
loss_gen = adv_model.train_on_batch(noise, fake_ys)
print("Round {}, Loss gen:{:.4f}".format(rounds, loss_gen))
loss_gen_test = adv_model.test_on_batch(noise, fake_ys)
print(loss_gen_test)
this question has been asked several times already, but I don't seem to be able to adapt previous solutions to my code. I would therefore appreciate any advice on how to solve this. I have tried using pdb and set a trace point right before the problem, which didn't give me much information.
I am adapting this tutorial to my problem:
https://www.oreilly.com/ideas/visualizing-convolutional-neural-networks
Data Shape:
x_train.shape: (1161, 68, 68, 1)
x_test.shape: (216, 68, 68, 1)
y_test.shape: (216,)
y_train.shape: (1161,)
Where the error occurs:
#Train the Model
steps = int(x_train.shape[0]/batchSize)
for i in range(numEpochs):
print(i)
accHist = []
accHist2 = []
#x_train, y_train = imf.shuffle(x_train, y_train)
for j in range(steps):
print(j)
#Calculate our current step
step = i * steps + j
#Feed forward batch of train images into graph and log accuracy
acc = sess.run([accuracy], feed_dict={X: x_train[(j*batchSize):((j+1)*batchSize),:,:,:], Y_: np.array(y_train[(j*batchSize):((j+1)*batchSize)]).reshape(1,30), keepRate1: 1, keepRate2: 1})
print(accHist)
accHist.append(acc)
#Back propigate using adam optimizer to update weights and biases.
sess.run(train_step, feed_dict={X: x_train[(j*batchSize):((j+1)*batchSize),:,:,:], Y_: np.array(y_train[(j*batchSize):((j+1)*batchSize)]).reshape(1,30), keepRate1: 0.2, keepRate2: 0.5})
print("success")
print('Epoch number {} Training Accuracy: {}'.format(i+1, np.mean(accHist)))
#Feed forward all test images into graph and log accuracy
for k in range(int(x_test.shape[0]/batchSize)):
acc = sess.run(accuracy, feed_dict={X: x_test[(k*batchSize):((k+1)*batchSize),:,:,:], Y_: np.array(y_test[(k*batchSize):((k+1)*batchSize)]).reshape(1,30), keepRate1: 1, keepRate2: 1})
accHist2.append(acc)
print("Test Set Accuracy: {}".format(np.mean(accHist2)))
I am getting the following error message:
InvalidArgumentError: logits and labels must be same size: logits_size=[30,30] labels_size=[1,30]
[[Node: cross_entropy_7/SoftmaxCrossEntropyWithLogits = SoftmaxCrossEntropyWithLogits[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"](cross_entropy_7/Reshape, cross_entropy_7/Reshape_1)]]
Following the tutorial, I thought the logins were set here:
#FULLY CONNECTED 3 & SOFTMAX OUTPUT
with tf.name_scope('softmax') as scope:
fc2w = tf.Variable(tf.truncated_normal([512, classes], dtype=tf.float32,
stddev=1e-1), name='weights3_2')
fc2b = tf.Variable(tf.constant(1.0, shape=[classes], dtype=tf.float32),
trainable=True, name='biases3_2')
Ylogits = tf.nn.bias_add(tf.matmul(fc1_drop, fc2w), fc2b)
Y = tf.nn.softmax(Ylogits)
print(Ylogits.shape) here gives me: (?, 30). Classes is set at 30 so this seems to make sense.
This seems to be the functions that doesn't work, so I printed the shapes:
with tf.name_scope('cross_entropy'):
print(Ylogits.shape)
print(Y.shape)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Y_)
loss = tf.reduce_mean(cross_entropy)
Which gave me:
(?, 30)
(?, 30)
When executing the line for back propagation above though this does not seem to work. Can anyone help?
In response to comment (this basically is the tutorial code from the link mentioned above):
Place Holders:
classes = 30
X = tf.placeholder(tf.float32, name="X-placeholder", shape=(None, 68, 68, 1))
Y_ = tf.placeholder(tf.float32, [None, classes], name="Y_-placeholder")
keepRate1 = tf.placeholder(tf.float32, name="keepRate1-placeholder")
keepRate2 = tf.placeholder(tf.float32, name="keepRate2-placeholder")
Model:
# CONVOLUTION 1 - 1
with tf.name_scope('conv1_1'):
filter1_1 = tf.Variable(tf.truncated_normal([3, 3, 1, 32], dtype=tf.float32,
stddev=1e-1), name='weights1_1')
stride = [1,1,1,1]
conv = tf.nn.conv2d(X, filter1_1, stride, padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[32], dtype=tf.float32),
trainable=True, name='biases1_1')
out = tf.nn.bias_add(conv, biases)
conv1_1 = tf.nn.relu(out)
# CONVOLUTION 1 - 2
with tf.name_scope('conv1_2'):
filter1_2 = tf.Variable(tf.truncated_normal([3, 3, 32, 32], dtype=tf.float32,
stddev=1e-1), name='weights1_2')
conv = tf.nn.conv2d(conv1_1, filter1_2, [1,1,1,1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[32], dtype=tf.float32),
trainable=True, name='biases1_2')
out = tf.nn.bias_add(conv, biases)
conv1_2 = tf.nn.relu(out)
# POOL 1
with tf.name_scope('pool1'):
pool1_1 = tf.nn.max_pool(conv1_2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1_1')
pool1_1_drop = tf.nn.dropout(pool1_1, keepRate1)
# CONVOLUTION 2 - 1
with tf.name_scope('conv2_1'):
filter2_1 = tf.Variable(tf.truncated_normal([3, 3, 32, 64], dtype=tf.float32,
stddev=1e-1), name='weights2_1')
conv = tf.nn.conv2d(pool1_1_drop, filter2_1, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),
trainable=True, name='biases2_1')
out = tf.nn.bias_add(conv, biases)
conv2_1 = tf.nn.relu(out)
# CONVOLUTION 2 - 2
with tf.name_scope('conv2_2'):
filter2_2 = tf.Variable(tf.truncated_normal([3, 3, 64, 64], dtype=tf.float32,
stddev=1e-1), name='weights2_2')
conv = tf.nn.conv2d(conv2_1, filter2_2, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),
trainable=True, name='biases2_2')
out = tf.nn.bias_add(conv, biases)
conv2_2 = tf.nn.relu(out)
# POOL 2
with tf.name_scope('pool2'):
pool2_1 = tf.nn.max_pool(conv2_2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2_1')
pool2_1_drop = tf.nn.dropout(pool2_1, keepRate1)
#FULLY CONNECTED 1
with tf.name_scope('fc1') as scope:
shape = int(np.prod(pool2_1_drop.get_shape()[1:]))
fc1w = tf.Variable(tf.truncated_normal([shape, 512], dtype=tf.float32,
stddev=1e-1), name='weights3_1')
fc1b = tf.Variable(tf.constant(1.0, shape=[512], dtype=tf.float32),
trainable=True, name='biases3_1')
pool2_flat = tf.reshape(pool2_1_drop, [-1, shape])
out = tf.nn.bias_add(tf.matmul(pool2_flat, fc1w), fc1b)
fc1 = tf.nn.relu(out)
fc1_drop = tf.nn.dropout(fc1, keepRate2)
#FULLY CONNECTED 3 & SOFTMAX OUTPUT
with tf.name_scope('softmax') as scope:
fc2w = tf.Variable(tf.truncated_normal([512, classes], dtype=tf.float32,
stddev=1e-1), name='weights3_2')
fc2b = tf.Variable(tf.constant(1.0, shape=[classes], dtype=tf.float32),
trainable=True, name='biases3_2')
Ylogits = tf.nn.bias_add(tf.matmul(fc1_drop, fc2w), fc2b)
Y = tf.nn.softmax(Ylogits)
numEpochs = 400
batchSize = 30
alpha = 1e-5
with tf.name_scope('cross_entropy'):
print(Ylogits.shape)
print(Y.shape)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Y_)
loss = tf.reduce_mean(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(Y, 1), tf.argmax(Y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
with tf.name_scope('train'):
train_step = tf.train.AdamOptimizer(learning_rate=alpha).minimize(loss)
#Create Session and insert variables
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
The tensor shape (?, 30) denotes that the batch size is not set, so you can feed any batch size data to your graph, the problem is that then you can run into these kinds of problems, and need to keep track of the tensor shapes in your head.
The thing you need to fix is: either you have 30 images in one batch, but only 1 label in one batch, which needs to be fixed, because you cannot compute loss for 30 images with only one label, you either need to decrease number of images to 1 or increase label batch size to 30, it could also be that somewhere you are reshaping the tensors incorrectly.
I would look at where you read your data in, and then batch it, that is most likely where the problem will be, or at places where you are reshaping them.
Post your entire code, it would be more helpful.
I am trying to use Tensorflow to classify some object representations. I used the same architecture as in the Tensorflow Cifar-10 example, with the last layer defined as:
with tf.variable_scope('sigmoid_linear') as scope:
weights = _variable_with_weight_decay('weights', [192, num_classes],
stddev=1 / 192.0, wd=0.0)
biases = _variable_on_cpu('biases', [num_classes],
initializer)
sigmoid_linear = tf.add(tf.matmul(local4, weights), biases, name=scope.name)
_activation_summary(sigmoid_linear)
return sigmoid_linear
In my case, num_classes is 2, and the amount of channels in the representation fed to the neural network is 8. Furthermore, I'm currently debugging with only 5 examples. The output of the last layer has a shape of[40,2]. I expect the first dimension is due to 5 examples * 8 channels and the second due to the number of classes.
In order to use compare the logits and the labels using e.g. tensorflow.nn.SparseSoftmaxCrossEntropyWithLogits I need them to have a common shape. How can I interpret the current content of the logits in the current shape, and how can I reduce the first dimension of the logits to be the same as num_classes?
Edit: the shape of the input to the inference function has a shape of [5,101,1008,8]. The inference function is defined as:
def inference(representations):
"""Build the model.
Args:
STFT spectra: spectra returned from distorted_inputs() or inputs().
Returns:
Logits.
"""
# conv1
with tf.variable_scope('conv1') as scope:
kernel = _variable_with_weight_decay('weights',
shape=[5, 5, nChannels, 64],
stddev=5e-2,
wd=0.0)
conv = tf.nn.conv2d(representations, kernel, [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu('biases', [64], initializer,
)
pre_activation = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(pre_activation, name=scope.name)
_activation_summary(conv1)
# pool1
pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
padding='SAME', name='pool1')
# norm1
norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75,
name='norm1')
# conv2
with tf.variable_scope('conv2') as scope:
kernel = _variable_with_weight_decay('weights',
shape=[5, 5, 64, 64],
stddev=5e-2,
wd=0.0)
conv = tf.nn.conv2d(norm1, kernel, [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu('biases', [64], initializer)
pre_activation = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(pre_activation, name=scope.name)
_activation_summary(conv2)
# norm2
norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75,
name='norm2')
# pool2
pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1], padding='SAME', name='pool2')
# local3
with tf.variable_scope('local3') as scope:
# Move everything into depth so we can perform a single matrix multiply.
reshape = tf.reshape(pool2, [batch_size, -1])
dim = reshape.get_shape()[1].value
weights = _variable_with_weight_decay('weights', shape=[dim, 384],
stddev=0.04, wd=0.004)
biases = _variable_on_cpu('biases', [384], initializer)
local3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name)
_activation_summary(local3)
# local4
with tf.variable_scope('local4') as scope:
weights = _variable_with_weight_decay('weights', shape=[384, 192],
stddev=0.04, wd=0.004)
biases = _variable_on_cpu('biases', [192], initializer)
local4 = tf.nn.relu(tf.matmul(local3, weights) + biases, name=scope.name)
_activation_summary(local4)
with tf.variable_scope('sigmoid_linear') as scope:
weights = _variable_with_weight_decay('weights', [192, num_classes],
stddev=1 / 192.0, wd=0.0)
biases = _variable_on_cpu('biases', [num_classes],
initializer)
sigmoid_linear = tf.add(tf.matmul(local4, weights), biases, name=scope.name)
_activation_summary(sigmoid_linear)
return sigmoid_linear
After more debugging I could find the problem. The posted code with the layers, originally from the Tensorflow tutorial, works well (of course it does). I printed all shapes, after each layer, and found out that the number 40 was not due to 5 examples * 8 channels, but that I had previously set batch_size = 40, and thus also higher than the amount of training examples. The mismatch began after the reshaping in the local layer 3. The question can now be closed.
I am trying to generate an unrecognizable image which can fool Vggnet. I used the following vgg model for tensorflow. I add some modification for calculating the gradient. In the ending part, you can see my modification for calculating the gradient respect to the given image (is it correct? I am trying to generate an image to whom the vggnet assign high probability at class 1). With this gradient, I update the random image for fooling the vggnet. But this is not so successful. I can't generate an image with high probability. The maximum probability I got is around 0.001. How can I make it keep increasing?
.
Vggnet model
#
# Davi Frossard, 2016 #
# VGG16 implementation in TensorFlow #
# Details: #
# http://www.cs.toronto.edu/~frossard/post/vgg16/ #
# #
# Model from https://gist.github.com/ksimonyan/211839e770f7b538e2d8#file-readme-md #
# Weights from Caffe converted using https://github.com/ethereon/caffe-tensorflow #########################################################################################
import tensorflow as tf
import numpy as np
from scipy.misc import imread, imresize
from imagenet_classes import class_names
class vgg16:
def __init__(self, imgs, weights=None, sess=None):
self.imgs = imgs
self.convlayers()
self.fc_layers()
self.probs = tf.nn.softmax(self.fc3l, name= 'prob')
if weights is not None and sess is not None:
self.load_weights(weights, sess)
def convlayers(self):
self.parameters = []
# zero-mean input
with tf.name_scope('preprocess') as scope:
mean = tf.constant([123.68, 116.779, 103.939], dtype=tf.float32, shape=[1, 1, 1, 3], name='img_mean')
images = self.imgs-mean
# conv1_1
with tf.name_scope('conv1_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 3, 64], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv1_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv1_2
with tf.name_scope('conv1_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 64], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.conv1_1, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[64], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv1_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# pool1
self.pool1 = tf.nn.max_pool(self.conv1_2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1')
# conv2_1
with tf.name_scope('conv2_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.pool1, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv2_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv2_2
with tf.name_scope('conv2_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 128], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.conv2_1, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv2_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# pool2
self.pool2 = tf.nn.max_pool(self.conv2_2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2')
# conv3_1
with tf.name_scope('conv3_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 256], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.pool2, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv3_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv3_2
with tf.name_scope('conv3_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.conv3_1, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv3_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv3_3
with tf.name_scope('conv3_3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.conv3_2, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[256], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv3_3 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# pool3
self.pool3 = tf.nn.max_pool(self.conv3_3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool3')
# conv4_1
with tf.name_scope('conv4_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 512], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.pool3, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv4_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv4_2
with tf.name_scope('conv4_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.conv4_1, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv4_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv4_3
with tf.name_scope('conv4_3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.conv4_2, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv4_3 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# pool4
self.pool4 = tf.nn.max_pool(self.conv4_3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool4')
# conv5_1
with tf.name_scope('conv5_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.pool4, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv5_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv5_2
with tf.name_scope('conv5_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.conv5_1, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv5_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv5_3
with tf.name_scope('conv5_3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512], dtype=tf.float32,
stddev=1e-1), name='weights')
conv = tf.nn.conv2d(self.conv5_2, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[512], dtype=tf.float32),
trainable=True, name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv5_3 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# pool5
self.pool5 = tf.nn.max_pool(self.conv5_3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool4')
def fc_layers(self):
# fc1
with tf.name_scope('fc1') as scope:
shape = int(np.prod(self.pool5.get_shape()[1:]))
fc1w = tf.Variable(tf.truncated_normal([shape, 4096],
dtype=tf.float32,
stddev=1e-1), name='weights')
fc1b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32),
trainable=True, name='biases')
pool5_flat = tf.reshape(self.pool5, [-1, shape])
fc1l = tf.nn.bias_add(tf.matmul(pool5_flat, fc1w), fc1b)
self.fc1 = tf.nn.relu(fc1l)
self.parameters += [fc1w, fc1b]
# fc2
with tf.name_scope('fc2') as scope:
fc2w = tf.Variable(tf.truncated_normal([4096, 4096],
dtype=tf.float32,
stddev=1e-1), name='weights')
fc2b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32),
trainable=True, name='biases')
fc2l = tf.nn.bias_add(tf.matmul(self.fc1, fc2w), fc2b)
self.fc2 = tf.nn.relu(fc2l)
self.parameters += [fc2w, fc2b]
# fc3
with tf.name_scope('fc3') as scope:
fc3w = tf.Variable(tf.truncated_normal([4096, 1000],
dtype=tf.float32,
stddev=1e-1), name='weights')
fc3b = tf.Variable(tf.constant(1.0, shape=[1000], dtype=tf.float32),
trainable=True, name='biases')
self.fc3l = tf.nn.bias_add(tf.matmul(self.fc2, fc3w), fc3b)
self.parameters += [fc3w, fc3b]
###################### Modified part######################
with tf.name_scope('grad') as scope:
temp = np.zeros(1000)
temp[0] = 1
vec = tf.constant(temp, dtype='float32', name = 'goal')
loss = tf.reduce_mean(tf.square(tf.sub(tf.nn.softmax(self.fc3l), vec)))
self.grad = tf.gradients(loss, self.imgs)[-1]
##############################################################
def load_weights(self, weight_file, sess):
weights = np.load(weight_file)
keys = sorted(weights.keys())
for i, k in enumerate(keys):
print i, k, np.shape(weights[k])
sess.run(self.parameters[i].assign(weights[k]))
Create session
#
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
sess = tf.Session()
imgs = tf.placeholder(tf.float32, [None, 224, 224, 3])
vgg = vgg16(imgs, 'vgg16_weights.npz', sess)
Generate new image for fooling
#
imarray = np.random.rand(224,224,3) * 255
imarray = imarray.astype('float32')
feed_dict = {vgg.imgs: [imarray]}
prob_list = []
prob_list.append(sess.run(vgg.probs, feed_dict={vgg.imgs: [imarray]})[0][0])
lamda = 0.1
#mean = np.array([123.68, 116.779, 103.939])
print 'start'
for i in range(1000):
rst = sess.run(vgg.grad, feed_dict)
imarray -= lamda * (rst[0]*255)
feed_dict = {vgg.imgs: [imarray]}
prob_list.append(sess.run(vgg.probs, feed_dict={vgg.imgs: [imarray]})[0][0])
I'm surprised that the shapes of the gradient and the image match.
You are taking the derivative of the loss with respect to the parameters, is should be with respect to the image placeholder. Excuse me, if I'm missing something obvious, I can't run the code right now.
The computation of the loss is based on fc3l, the final output is probs. I don't see where probs is computed in the VGG code. Maybe there are layers in between. You could plot the first component of fc3l instead, see if that goes up.
You should probably base the loss on probs.