I am trying to set custom layer weights in a keras sequential model. However, in doing so I get the error "Layer weights shape (6,6) not compatible with provided weight shape (6, ).
I see there are other questions on here regarding that error. But my issue is that I reshape my input array to (6, ) in my code. Also, I am using sypder. After running the code it shows the size of weights1 to be (6,) in the variable explorer window. Also, as another check, I typed weights1.shape in the console, and got (6, ).
So it appears the shape of my weights array is in fact (6, ), but I am still getting that error.
Here are the parts of my code I think are relevant, if I miss something just let me know:
population_size = 100
output11 = 1
activation = 'elu'
out_activation='sigmoid'
epochs = 500
opt = SGD(lr=.01, momentum =.9)
weights = [[]for _ in range(population_size)]
for i in range(population_size):
for j in range(6):
weights[i].append(random.uniform(0, 1000))
for i in range(len(array)):
if i == 0:
inc.append(0)
elif array[i][4] > array[i-1][4]:
inc.append(1)
elif array[i][4] <= array[i-1][4]:
inc.append(0)
inc = np.array(inc)
inc = np.reshape(inc, (-1,1))
array = np.hstack((array, inc))
input_train = array[0:1259,0:6]
output_train = array[0:1259, 6]
input_validate = array[1259:1887, 0:6]
output_validate = array[1259:1887, 6]
input_test = array[1887: , 0:6]
output_test = array[1887: , 6]
weights1 = np.array(weights[0])
weights1.shape = (6,)
out_test2d =np.reshape(output_test,(len(output_test),1))
out_train2d = np.reshape(output_test, (len(output_test),1))
weights1 = np.array(weights[0])
weights1.shape=(6,)
model = Sequential();
model.add(Dense(6, use_bias = False, name = "hiddenL1", input_shape = (None,6)) )
model.add(Dense(output11, use_bias=False, name="output", activation=out_activation))
print(model.summary())
opt = tf.keras.optimizers.Adam(learning_rate=0.01, epsilon=None, decay=0.05, beta_1 = 1,beta_2 = .5,amsgrad=False)
model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['accuracy'])
model.layers[0].set_weights([weights1])
Related
There is a 2-class classification problem, and my loss function is custom. The labels are categorical, and the final activation function is Softmax. During the training, the loss is printed, but the val_loss is nan(inf). Using model.evaluate(X_train,Y_train) at the end of training, the train loss is the same as the vaidation loss, and both are nan.
This is my custom loss function.
def custom_loss(y_true, y_pred):
import tensorflow as tf
bce = tf.keras.losses.BinaryCrossentropy(
from_logits=False,
label_smoothing=0.0,
axis=-1,
reduction="none",
name="binary_crossentropy",
)
intra = tf.constant(1, dtype=tf.float64)
inter = tf.constant(0.01, dtype=tf.float64)
zeros = tf.gather_nd(y_pred,tf.where(tf.argmin(y_true, axis = 1)))
ones = tf.gather_nd(y_pred,tf.where(tf.argmax(y_true, axis = 1)))
centroid_zero = tf.reduce_mean(zeros,0)
centroid_one = tf.reduce_mean(ones,0)
loss_zero_intra = tf.math.squared_difference(zeros,centroid_zero)
loss_one_intra = tf.math.squared_difference(ones,centroid_zero)
loss_zero_intra = tf.cast(loss_zero_intra, tf.float64)
loss_one_intra = tf.cast(loss_one_intra, tf.float64)
loss_intra = tf.zeros_like(y_pred, tf.float64)
loss_intra = tf.tensor_scatter_nd_update(loss_intra,tf.where(tf.argmin(y_true, axis = 1)),loss_zero_intra)
loss_intra = tf.tensor_scatter_nd_update(loss_intra,tf.where(tf.argmax(y_true, axis = 1)),loss_one_intra)
loss_inter_value = tf.math.sqrt(tf.math.squared_difference(centroid_zero[0],centroid_one[0]) +
tf.math.squared_difference(centroid_zero[1],centroid_one[1]))
loss_inter = tf.fill(tf.shape(y_pred),loss_inter_value)
binary_cross_entropy= tf.tile(tf.expand_dims(bce(y_true,y_pred),axis=1),
tf.constant([1,2],tf.int32))
loss_intra = tf.cast(loss_intra, tf.float64)
loss_inter = tf.cast(loss_inter, tf.float64)
binary_cross_entropy= tf.cast(binary_cross_entropy, tf.float64)
loss = tf.math.multiply(intra, loss_intra) - tf.math.multiply(inter, loss_inter) + binary_cross_entropy
return loss
And Also you can see my model code here:
def create_model(kernelLength = 32, nb_classes = 2, Chans = 19, Samples = 512,
dropoutRate = 0.5 , F1 = 8, D = 2, F2 = 16, norm_rate = 0.25,
dropoutType = 'Dropout', optimizer_type = 'Adam', lr=0.0005, **kwargs):
K.clear_session()
gc.collect()
if dropoutType == 'SpatialDropout2D':
dropoutType = SpatialDropout2D
elif dropoutType == 'Dropout':
dropoutType = Dropout
else:
raise ValueError('dropoutType must be one of SpatialDropout2D '
'or Dropout, passed as a string.')
input1 = Input(shape = (1, Chans, Samples))
block1 = Conv2D(F1, (1, kernelLength), padding = 'same',
input_shape = (1, Chans, Samples),
use_bias = False)(input1)
block1 = BatchNormalization(axis = 1)(block1)
block1 = DepthwiseConv2D((Chans, 1), use_bias = False,
depth_multiplier = D,
depthwise_constraint = max_norm(1.))(block1)
block1 = BatchNormalization(axis = 1)(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling2D((1, 4))(block1)
block1 = dropoutType(dropoutRate)(block1)
block2 = SeparableConv2D(F2, (1, 16),
use_bias = False, padding = 'same')(block1)
block2 = BatchNormalization(axis = 1)(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D((1, 8))(block2)
block2 = dropoutType(dropoutRate)(block2)
flatten = Flatten(name = 'flatten')(block2)
dense = Dense(nb_classes, name = 'dense',
kernel_constraint = max_norm(norm_rate))(flatten)
softmax = Activation('softmax', name = 'softmax')(dense)
model = Model(inputs=input1, outputs=softmax)
if optimizer_type == 'Adam':
optimizer = Adam(learning_rate = lr)
if optimizer_type == 'Adamax':
optimizer = Adamax(learning_rate = lr)
if optimizer_type == 'AdamW':
optimizer = AdamW(learning_rate = lr)
model.compile(loss=custom_loss, optimizer=optimizer, metrics = ['accuracy'])
return model
The custom_loss function returns three distinct terms. One of them is the binary_cross_entropy. The model works fine with this term, which works the same as binary_cross_entropy in Keras. Therefore, there is no problem with the data. The train and validation accuracy improves throughout training, and the train loss decreases. The number of validation samples is the same as the number of train samples.
After the training was accomplished, by using the model.evaluation(X,Y) function, the loss was shown as "nan," however calculating the loss using the custom loss function, resulting in a "number" not a "nan".
Increasing the batch size, scaling the data, and using clipnorm or clipvalue within the optimizer all had no effect. Furthermore, no nan appears in the model predictions (y_pred).I suspect that the problem is caused by the below extreme value inside the model prediction: An example of model prediction with extremes:
Can anyone suggest a solution to this problem?
Thanks in advance.
My model:
model = Sequential()
model.add( LSTM(25, batch_input_shape = (None, None, 19), return_sequences = True ) )
model.add(Dense(4, activation = 'tanh'))
model.compile(loss='mean_squared_error', optimizer ='adam', metrics = ['accuracy'])
some example of input data shape:
input_list[0].shape = (7,19)
input_list[1].shape = (8,19)
input_list[2].shape = (17,19)
some example of output data shape:
output_list[0].shape = (7,4)
output_list[1].shape = (8,4)
output_list[2].shape = (17,4)
input_list.shape = (233,)
output_list.shape = (233,)
error while:
d_loss = model.fit(input_list,output_list,validation_split=0.33,nb_epoch=100,verbose=1,shuffle=True, batch_size = 1)
error: ValueError: Error when checking input: expected lstm_22_input to have 3 dimensions, but got array with shape (233, 1)
Just increase the dimensions, by np.expand_dims(x, axis= 0). It will become three dimensional.
I am relatively new to TF and am wondering how to get tensor slice dynamically, from a unknown shape of tensor?
I want to get the weights from the last layer (output_layer) and do softmax and then only look at those indices on the 2-nd dimensions (from the out_reshape). The numpy-type of striding didnt work, so I am using tf.gather instead (after changing the axis so that the desired axis is on the first axis).
And this works:
out_reshape = tf.gather(out_reshape, [1,2,3,4])
This outputs a tensor with [4, 3, ?] (as we expected). But I want to change the indices based on the data fed to T (instead of [1,2,3,4] as shown above).
this gives an unpredicted result (as shown in the code below):
out_reshape = tf.gather(out_reshape, T)
and
out_reshape.shape
this gives TensorShape(None)), but I was expecting to get [?, 3, ?], where the first value is the same length as T (the data fed into the T is an 1-d ndarray, such as [100, 200, 300, 400]).
What is going on here? Why its output shape collapses to None?
The entire code is something like this:
graph = tf.Graph()
tf.reset_default_graph()
with graph.as_default():
y=tf.placeholder(tf.float32, shape =(31, 3, None), name = 'Y_observed') # (samples x) ind x seqlen
T=tf.placeholder(tf.int32, shape =(None), name = "T_observed")
x = tf.placeholder(tf.float32, shape = (None, None, 4) , name = 'X_observed')
model = Conv1D(filters = 16,
kernel_size = filter_width,
padding = "same",
activation='relu')(x)
model = Conv1D(filters = 16,
kernel_size = filter_width,
padding = "same",
activation='relu')(model)
model = Conv1D(filters = n_output_channels,
kernel_size = 1,
padding = "same",
activation='relu')(model)
model_output = tf.identity(model, name='last_layer')
output_layer = tf.get_default_graph().get_tensor_by_name('last_layer:0')
out = output_layer[:, 512:, :]
out_norm = tf.nn.softmax( out, axis=1 )
out_reshape = tf.transpose(out_norm, (1, 2, 0)) # this gives a [?,3,?] tensor
out_reshape = tf.gather(out_reshape, T) # --> Problematic part !
...
updates = tf.train.AdamOptimizer(1e-4).minimize....
...
I am a beginner with tensorflow and I was tinkering around with convnet for image recognition.However after I save my model I am getting an error while restoring it.
this is my tensor graph components ->
Y_train = to_categorical(y_train,num_classes=4)
Y_test = to_categorical(y_test,num_classes=4)
X = tf.placeholder(tf.float32, shape=(None, 64,64,3))
Y = tf.placeholder(tf.float32, shape=(None, 4))
w1 = tf.get_variable("w1", [4,4,3,8], initializer = tf.contrib.layers.xavier_initializer(seed = 0))
w2 = tf.get_variable("w2", [2,2,8,16], initializer = tf.contrib.layers.xavier_initializer(seed = 0))
Z1 = tf.nn.conv2d(X,w1, strides = [1,1,1,1], padding = 'SAME')
A1 = tf.nn.relu(Z1)
P1 = tf.nn.max_pool(A1, ksize = [1,8,8,1], strides = [1,8,8,1], padding = 'SAME')
Z2 = tf.nn.conv2d(P1,w2, strides = [1,1,1,1], padding = 'SAME')
A2 = tf.nn.relu(Z2)
P2 = tf.nn.max_pool(A2, ksize = [1,4,4,1], strides = [1,4,4,1], padding = 'SAME')
P2 = tf.contrib.layers.flatten(P2)
Z3 = tf.contrib.layers.fully_connected(P2,4,activation_fn=None)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = Z3, labels = Y))
optimizer = tf.train.AdamOptimizer(0.004).minimize(cost)
this is a basic convnet which i have successfully trained and tested.
However the problem I am facing is that after all the epochs have been completed for training, How do I save this model such that I can use it again in some file say predict.py where in I can just import it and make predictions
So i read some blogs on save and restore and did that but then I was getting the below mention error
Attempting to use uninitialized value fully_connected/biases [[Node:
fully_connected/biases/read = IdentityT=DT_FLOAT,
_device="/job:localhost/replica:0/task:0/device:CPU:0"]]
so given the convnet what should I do to use that model ? can someone provide working code for both the model file and predict file.
I am trying to write a simple program using TensorFlow to predict the next number in a sequence.
I am not experienced in TensorFlow so instead of starting from scratch I started with this guide: http://monik.in/a-noobs-guide-to-implementing-rnn-lstm-using-tensorflow/
However, in contrast to the implementation in the link above I do not want to treat the problem as a classification problem - where I only have n possible outcomes - but instead just calculate a single value for a sequence.
I tried modifying the code to fit my problem:
import numpy as np
import random
from random import shuffle
import tensorflow as tf
NUM_EXAMPLES = 10000
train_input = ['{0:020b}'.format(i) for i in range(2**20)]
shuffle(train_input)
train_input = [map(int,i) for i in train_input]
ti = []
for i in train_input:
temp_list = []
for j in i:
temp_list.append([j])
ti.append(np.array(temp_list))
train_input = ti
train_output = []
for i in train_input:
count = 0
for j in i:
if j[0] == 1:
count+=1
#temp_list = ([0]*21)
#temp_list[count]=1
#train_output.append(temp_list)
train_output.append(count)
test_input = train_input[NUM_EXAMPLES:]
test_output = train_output[NUM_EXAMPLES:]
train_input = train_input[:NUM_EXAMPLES]
train_output = train_output[:NUM_EXAMPLES]
print "test and training data loaded"
target = tf.placeholder(tf.float32, [None, 1])
data = tf.placeholder(tf.float32, [None, 20,1]) #Number of examples, number of input, dimension of each input
#target = tf.placeholder(tf.float32, [None, 1])
#print('target shape: ', target.get_shape())
#print('shape[0]', target.get_shape()[1])
#print('int(shape) ', int(target.get_shape()[1]))
num_hidden = 24
cell = tf.nn.rnn_cell.LSTMCell(num_hidden)
val, _ = tf.nn.dynamic_rnn(cell, data, dtype=tf.float32)
val = tf.transpose(val, [1, 0, 2])
print('val shape, ', val.get_shape())
last = tf.gather(val, int(val.get_shape()[0]) - 1)
weight = tf.Variable(tf.truncated_normal([num_hidden, int(target.get_shape()[1])]))
bias = tf.Variable(tf.constant(0.1, shape=[target.get_shape()[1]]))
#prediction = tf.nn.softmax(tf.matmul(last, weight) + bias)
prediction = tf.matmul(last, weight) + bias
cross_entropy = -tf.reduce_sum(target - prediction)
optimizer = tf.train.AdamOptimizer()
minimize = optimizer.minimize(cross_entropy)
mistakes = tf.not_equal(tf.argmax(target, 1), tf.argmax(prediction, 1))
error = tf.reduce_mean(tf.cast(mistakes, tf.float32))
init_op = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init_op)
batch_size = 100
no_of_batches = int(len(train_input)) / batch_size
epoch = 500
for i in range(epoch):
ptr = 0
for j in range(no_of_batches):
inp, out = train_input[ptr:ptr+batch_size], train_output[ptr:ptr+batch_size]
ptr+=batch_size
sess.run(minimize,{data: inp, target: out})
print "Epoch ",str(i)
incorrect = sess.run(error,{data: test_input, target: test_output})
#print sess.run(prediction,{data: [[[1],[0],[0],[1],[1],[0],[1],[1],[1],[0],[1],[0],[0],[1],[1],[0],[1],[1],[1],[0]]]})
#print('Epoch {:2d} error {:3.1f}%'.format(i + 1, 100 * incorrect))
sess.close()
It is still work in progress, since the input is bogus as well as the cross entropy calculation.
However, my main problem is that the code doesn't compile at all.
I get this error:
ValueError: Cannot feed value of shape (100,) for Tensor
u'Placeholder:0', which has shape '(?, 1)'
The number 100 comes from the "batch_size" and the (?, 1) comes from the fact that my prediction is a one dimensional number. However, I do not have any idea where the problem is in my code?
Can anyone help me get the dimensions to match?
This error means your targets placeholder is being fed something with the wrong shape. To fix it, I think you should reshape something like test_output.reshape([-1, 1])
To fix the placeholders shape, change your code to
for i in range(epoch):
ptr = 0
for j in range(no_of_batches):
inp = train_input[ptr:ptr+batch_size]
out = train_output[ptr:ptr+batch_size]
ptr+=batch_size
out = np.reshape(out, (100,1)) #reshape
sess.run(minimize,{data: inp, target: out})
print ("Epoch ",str(i))
test_output = np.reshape(test_output, (1038576,1)) #reshape
incorrect = sess.run(error,{data: test_input, target: test_output})