I am trying to implement convolutional layers for text classification from this blog post with some modification to suit my needs.
In the blog, there is only one convolution layer while I'd like mine to have two convolutional layers followed by ReLU and max-pooling.
The code so far is:
vocab_size = 2000
embedding_size = 100
filter_height = 5
filter_width = embedding_size
no_of_channels = 1
no_of_filters = 256
sequence_length = 50
filter_size = 3
no_of_classes = 26
input_x = tf.placeholder(tf.int32, [None, sequence_length], name="input_x")
input_y = tf.placeholder(tf.float32, [None, no_of_classes], name="input_y")
# Defining the embedding layer:
with tf.device('/cpu:0'), tf.name_scope("embedding"):
W = tf.Variable(tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0), name="W")
embedded_chars = tf.nn.embedding_lookup(W, input_x)
embedded_chars_expanded = tf.expand_dims(embedded_chars, -1)
# Convolution block:
with tf.name_scope("convolution-block"):
filter_shape = [filter_height, embedding_size, no_of_channels, no_of_filters]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[no_of_filters]), name="b")
conv1 = tf.nn.conv2d(embedded_chars_expanded,
W,
strides = [1,1,1,1],
padding = "VALID",
name = "conv1")
conv2 = tf.nn.conv2d(conv1,
W,
strides = [1,1,1,1],
padding = "VALID",
name = "conv2")
Here, W is the filter matrix.
However, this gives the error:
ValueError: Dimensions must be equal, but are 256 and 1 for 'convolution-block_16/conv2' (op: 'Conv2D') with input shapes: [?,46,1,256], [5,100,1,256].
I realise I have erred in the dimensions of the layer, but I am unable to fix it or put in the correct dimensions.
If anybody could provide any guidance/help, it'd be really helpful.
Thank you.
Can't quite understand what you code to do, but change as follows will fix your problem.
with tf.name_scope("convolution-block"):
filter_shape = [filter_height, embedding_size, no_of_channels, no_of_channels #change the output channel as input#]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[no_of_filters]), name="b")
conv1 = tf.nn.conv2d(embedded_chars_expanded,
W,
strides = [1,1,1,1],
padding = "SAME", ##Change the padding scheme
name = "conv1")
conv2 = tf.nn.conv2d(conv1,
W,
strides = [1,1,1,1],
padding = "VALID",
name = "conv2")
Related
I have a network. the one before last layer is a dense layer. I want the last layer to return both the max value from the layer before, and the index of that max value.
so if the output of the dense layer is [0,4,5,120,1], the last layer should return [120, 3].
the loss I need the network to work with is calculated only based on the max value, not the index. therefore, I wrote a loss function for the second output, the index, that always returns zero - but if there is a better solution I would like to hear it, in addition to how to fix this error.
the code is:
def ignor_loss(preds, trues):
return 0
# build deep q network
inputs = Input((240,256,3))
pool0 = MaxPooling2D()(inputs)
conv1 = Conv2D(30,3,activation='relu')(pool0)
pool1 = MaxPooling2D()(conv1)
conv2 = Conv2D(40,3,activation='relu')(pool1)
pool2 = MaxPooling2D()(conv2)
conv3 = Conv2D(50,3,activation='relu')(pool2)
pool3 = MaxPooling2D()(conv3)
conv4 = Conv2D(60,3,activation='relu')(pool3)
pool4 = MaxPooling2D()(conv4)
conv5 = Conv2D(80,3,activation='relu')(pool4)
flat = Flatten()(conv5)
dense1 = Dense(70)(flat)
dense2 = Dense(40)(dense1)
values = Dense(env.action_space.n)(dense2)
max_, ind = Lambda(lambda x : [K.max(x),K.argmax(x)])(values)
m = Model(inputs, [max_, ind])
m.compile('adam', ['mse',ignor_loss])
and the error is:
TypeError: unsupported operand type(s) for -: 'int' and 'NoneType'
any ideas?
EDIT:
here is my updated code:
# build deep q network
inputs = Input((240,256,3))
pool0 = MaxPooling2D()(inputs)
conv1 = Conv2D(30,3,activation='relu')(pool0)
pool1 = MaxPooling2D()(conv1)
conv2 = Conv2D(40,3,activation='relu')(pool1)
pool2 = MaxPooling2D()(conv2)
conv3 = Conv2D(50,3,activation='relu')(pool2)
pool3 = MaxPooling2D()(conv3)
conv4 = Conv2D(60,3,activation='relu')(pool3)
pool4 = MaxPooling2D()(conv4)
conv5 = Conv2D(80,3,activation='relu')(pool4)
flat = Flatten()(conv5)
dense1 = Dense(70)(flat)
dense2 = Dense(40)(dense1)
q_values = Dense(env.action_space.n)(dense2)
max_q = Lambda(lambda x : K.max(x), name='max')(q_values)
ind = Lambda(lambda x : K.argmax(x), name='ind')(q_values)
m = Model(inputs, [max_q,ind])
m.compile('adam', {'max':'mse','ind':'mse'}, loss_weights=[1., 0.0])
I still get the same error:
unsupported operand type(s) for -: 'int' and 'NoneType'
I need to know why this error heppens? any ideas?
EDIT 2:
now I added the keepdims=True to the max function and K.expand_dims to the argmax func, like this:
q_values = Dense(env.action_space.n)(dense2)
max_q = Lambda(lambda x : K.max(x, keepdims=True), name='max')(q_values)
ind = Lambda(lambda x : K.expand_dims(K.argmax(x)), name='ind')(q_values)
m = Model(inputs, [max_q,ind])
m.compile('adam', {'max':'mse','ind':'mse'}, loss_weights=[1., 0.0])
but I get a different error:
TypeError: Expected int64, got 0.0 of type 'float' instead.
I think that this is a cleaner solution
1 step: fit the model on the max
X = np.random.uniform(0,1, (2,240,256,3))
y = np.random.uniform(0,1, 2)
inputs = Input((240,256,3))
pool0 = MaxPooling2D()(inputs)
conv1 = Conv2D(30,3,activation='relu')(pool0)
pool1 = MaxPooling2D()(conv1)
conv2 = Conv2D(40,3,activation='relu')(pool1)
pool2 = MaxPooling2D()(conv2)
conv3 = Conv2D(50,3,activation='relu')(pool2)
pool3 = MaxPooling2D()(conv3)
conv4 = Conv2D(60,3,activation='relu')(pool3)
pool4 = MaxPooling2D()(conv4)
conv5 = Conv2D(80,3,activation='relu')(pool4)
flat = Flatten()(conv5)
dense1 = Dense(70)(flat)
dense2 = Dense(40)(dense1)
values = Dense(10)(dense2) # in my case env.action_space.n is 10
max_ = Lambda(lambda x: tf.reduce_max(x, axis=1, keepdims=True))(values)
m = Model(inputs, max_)
m.compile('adam', 'mse')
m.fit(X,y, epochs=3)
2 step: make inference with the fitted model returning max and argmax (this simply require to build a new model)
ind = Lambda(lambda x: tf.expand_dims(tf.argmax(x, axis=1),-1))(values)
final_model = Model(inputs, [max_, ind])
final_model.predict(X) this return max and argmax
EDIT: here a compact model which operate all the operation. if u have two outputs u need to pass to keras two targets. for this reason, the second target is generated by me as an array of 0 (it has no impact)
def ignor_loss(trues, preds):
return 0.
X = np.random.uniform(0,1, (2,240,256,3))
y = np.random.uniform(0,1, 2)
inputs = Input((240,256,3))
pool0 = MaxPooling2D()(inputs)
conv1 = Conv2D(30,3,activation='relu')(pool0)
pool1 = MaxPooling2D()(conv1)
conv2 = Conv2D(40,3,activation='relu')(pool1)
pool2 = MaxPooling2D()(conv2)
conv3 = Conv2D(50,3,activation='relu')(pool2)
pool3 = MaxPooling2D()(conv3)
conv4 = Conv2D(60,3,activation='relu')(pool3)
pool4 = MaxPooling2D()(conv4)
conv5 = Conv2D(80,3,activation='relu')(pool4)
flat = Flatten()(conv5)
dense1 = Dense(70)(flat)
dense2 = Dense(40)(dense1)
values = Dense(10)(dense2) # in my case env.action_space.n is 10
max_ = Lambda(lambda x: tf.reduce_max(x, axis=1, keepdims=True), name='max')(values)
ind = Lambda(lambda x: tf.expand_dims(tf.argmax(x, axis=1),-1), name='ind')(values)
m = Model(inputs, [max_,ind])
m.compile('adam', loss={'max':'mse', 'ind':ignor_loss},
loss_weights={'max':1., 'ind':0.})
m.fit(X, {'max':y, 'ind':np.zeros_like(y)}, epochs=3)
m.predict(X)
I have some neural network (tensorflow)
n_steps = 10
n_inputs = 3
n_outputs = 1
n_neurons = 100
n_layers = 3
X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])
y = tf.placeholder(tf.float32, [None, n_steps, n_outputs])
layers = []
for i in range(n_layers):
layers.append(tf.contrib.rnn.BasicRNNCell(num_units=n_neurons, activation=tf.nn.relu))
multi_layer_cell = tf.contrib.rnn.MultiRNNCell(layers)
rnn_outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X, dtype=tf.float32)
Like this (below) is correct? It is working but i'm not sure ;)
training = tf.placeholder_with_default(True,shape=())
X_dropout = tf.layers.dropout(X,dropout_rate,training=training)
rnn_outputs, states = tf.nn.dynamic_rnn(multi_layer_cell, X_dropout, dtype=tf.float32)
How to add into this neural network tensorflow dropout?
Thanks for any sugestions!
Your code just does dropout for input X, and you should use tf.contrib.rnn.DropoutWrapper(link).
layers = []
for i in range(n_layers):
layers.append(tf.contrib.rnn.DropoutWrapper(tf.contrib.rnn.BasicRNNCell(num_units=n_neurons
, activation=tf.nn.relu)
,output_keep_prob=1-dropout_rate))
I am a beginner with tensorflow and I was tinkering around with convnet for image recognition.However after I save my model I am getting an error while restoring it.
this is my tensor graph components ->
Y_train = to_categorical(y_train,num_classes=4)
Y_test = to_categorical(y_test,num_classes=4)
X = tf.placeholder(tf.float32, shape=(None, 64,64,3))
Y = tf.placeholder(tf.float32, shape=(None, 4))
w1 = tf.get_variable("w1", [4,4,3,8], initializer = tf.contrib.layers.xavier_initializer(seed = 0))
w2 = tf.get_variable("w2", [2,2,8,16], initializer = tf.contrib.layers.xavier_initializer(seed = 0))
Z1 = tf.nn.conv2d(X,w1, strides = [1,1,1,1], padding = 'SAME')
A1 = tf.nn.relu(Z1)
P1 = tf.nn.max_pool(A1, ksize = [1,8,8,1], strides = [1,8,8,1], padding = 'SAME')
Z2 = tf.nn.conv2d(P1,w2, strides = [1,1,1,1], padding = 'SAME')
A2 = tf.nn.relu(Z2)
P2 = tf.nn.max_pool(A2, ksize = [1,4,4,1], strides = [1,4,4,1], padding = 'SAME')
P2 = tf.contrib.layers.flatten(P2)
Z3 = tf.contrib.layers.fully_connected(P2,4,activation_fn=None)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = Z3, labels = Y))
optimizer = tf.train.AdamOptimizer(0.004).minimize(cost)
this is a basic convnet which i have successfully trained and tested.
However the problem I am facing is that after all the epochs have been completed for training, How do I save this model such that I can use it again in some file say predict.py where in I can just import it and make predictions
So i read some blogs on save and restore and did that but then I was getting the below mention error
Attempting to use uninitialized value fully_connected/biases [[Node:
fully_connected/biases/read = IdentityT=DT_FLOAT,
_device="/job:localhost/replica:0/task:0/device:CPU:0"]]
so given the convnet what should I do to use that model ? can someone provide working code for both the model file and predict file.
I am training a CNN in tensorflow but the program seems to have stuck at the first tf.nn.conv2d step. I have imported the cifar 10 dataset from keras and running my code in floydhub.
I haven't started the session yet, this is just the computation graph.
Link to the complete notebook
This is the part where it gets stuck:
# forward propagation
# convolution layer 1
c1 = tf.nn.conv2d(x_train, w1, strides = [1,1,1,1], padding = 'VALID')
# activation function for c1: relu
r1 = tf.nn.relu(c1)
# maxpooling
p1 = tf.nn.max_pool(r1, ksize = [1,2,2,1], strides = [1,2,2,1], padding = 'SAME')
# convolution layer 2
c2 = tf.nn.conv2d(p1, w2, strides = [1,1,1,1], padding='VALID')
# activation function for c2: relu
r2 = tf.nn.relu(c2)
# maxpooling
p2 = tf.nn.max_pool(r2, ksize = [1,2,2,1], strides = [1,2,2,1], padding = 'SAME')
# flattening the previous max pool layer
l1 = tf.contrib.layers.flatten(p2)
# fully connected layer
final = tf.contrib.layers.fully_connected(l1, 10, activation_fn = None)
EDIT:-
This is how I imported the dataset
# importing 50000 images of size 32x32x3
i = 0
# this list holds the images
img_base = []
for img in glob.glob("train\\*.png"):
img_base.append(cv2.imread(img))
# x_train is a nx32x32x3 matrix of n no. of images
x_train = np.array(img_base[0:40000]).astype(np.float32)
placeholders for training images and labels
# creating placeholders
x = tf.placeholder(tf.float32, [None, 32, 32, 3])
y = tf.placeholder(tf.float32, [None, 10])
weight initialisation`
tf.reset_default_graph()
# creating weights
w1 = tf.get_variable('w1', [4,4,3,10], initializer=tf.contrib.layers.xavier_initializer())
w2 = tf.get_variable('w2', [4,4,10,15], initializer=tf.contrib.layers.xavier_initializer())
my input variables
IMG_SIZE_PX=50
SLICE_COUNT=20
n_classes=2
x=tf.placeholder('float')
y=tf.placeholder('float')
keep_rate=0.8
keep_prob=tf.placeholder(tf.float32)
my convolution 3d function
def conv3d(x, W):
return tf.nn.conv3d(x, W, strides=[1,1,1,1,1], padding='SAME')
my maxpooling 3d function
def maxpool3d(x):
return tf.nn.max_pool3d(x, ksize=[1,2,2,2,1], strides=[1,2,2,2,1],
padding='SAME')
this is my network
def convolutional_neural_network(x):
my network weights
weights = {'W_conv1':tf.Variable(tf.random_normal([3,3,3,1,32])),
'W_conv2':tf.Variable(tf.random_normal([3,3,3,32,64])),
'W_fc':tf.Variable(tf.random_normal([ 54080 ,1024])),#here 54080
is the input tensor value
'out':tf.Variable(tf.random_normal([1024, n_classes]))}
my network biases
biases = {'b_conv1':tf.Variable(tf.random_normal([32])),
'b_conv2':tf.Variable(tf.random_normal([64])),
'b_fc':tf.Variable(tf.random_normal([1024])),
'out':tf.Variable(tf.random_normal([n_classes]))}
here is my input x
x = tf.reshape(x, shape=[-1, IMG_SIZE_PX, IMG_SIZE_PX, SLICE_COUNT, 1])
my 2 hidden layers(convolution+maxpooling)
conv1 = tf.nn.relu(conv3d(x, weights['W_conv1']) + biases['b_conv1'])
conv1 = maxpool3d(conv1)
conv2 = tf.nn.relu(conv3d(conv1, weights['W_conv2']) + biases['b_conv2'])
conv2 = maxpool3d(conv2)
my fully connected layer
fc = tf.reshape(conv2,[-1, 54080 ])
fc = tf.nn.relu(tf.matmul(fc, weights['W_fc'])+biases['b_fc'])
fc = tf.nn.dropout(fc, keep_rate)
my output layer
output = tf.matmul(fc, weights['out'])+biases['out']
return output
my input numpy arrays
much_data = np.load('D:/muchdata-50-50-20.npy')
train_data = much_data[-10:]
validation_data = much_data[-2:]
finally training my network
def train_neural_network(x):
outl = convolutional_neural_network(x)#don't know this is my output
layer
model=Model(input=x, output=outl)
model.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=
['accuracy'])
train_neural_network(x)#train the net
my error is thiskeras meta data is missing
any help can be appreciated