The code of my encoder model is given below, I have made it using functional API(TF 2.0)
embed_obj = EndTokenLayer()
def encoder_model(inp):
input_1 = embed_obj(inp)
h = Masking([(lambda x: x*0)(x) for x in range(128)])(input_1)
lstm1 , state_h, state_c = LSTM(512, return_sequences=True, return_state=True)(h)
model = Model(inputs=input_1, outputs=[lstm1, state_h, state_c])
return model
And when I'm calling my model:
for x,y in train.take(1):
k = x
model = encoder_model(k)
I'm getting the following error:
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-98-46e9c9596137> in <module>()
2 for x,y in train.take(1):
3 k = x
----> 4 model = encoder_model(k)
7 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in op(self)
1111 def op(self):
1112 raise AttributeError(
-> 1113 "Tensor.op is meaningless when eager execution is enabled.")
1114
1115 #property
AttributeError: Tensor.op is meaningless when eager execution is enabled.
In TF2, static graph (preventing eager execution with dynamic graph) can be constructed by using a decorator.
Try #tf.function decorator
#tf.function
def encoder_model(inp):
input_1 = embed_obj(inp)
h = Masking([(lambda x: x*0)(x) for x in range(128)])(input_1)
lstm1 , state_h, state_c = LSTM(512, return_sequences=True, return_state=True)(h)
model = Model(inputs=input_1, outputs=[lstm1, state_h, state_c])
return model
Then call the function
for x,y in train.take(1):
k = x
model = encoder_model(k)
Related
I get this error when I try :
model = LSTMPolicy(sequence_length, n_x)
model.predict(X_train[:batch_size])
ValueError: Exception encountered when calling layer 'lstm_policy_26'
(type LSTMPolicy). in user code:
....
ValueError: If a RNN is stateful, it needs to know its batch size.
Specify the batch size of your input tensors:
If using a Sequential model, specify the batch size by passing a batch_input_shape argument to your first layer.
If using the functional API, specify the batch size by passing a batch_shape argument to your Input layer.
Call arguments received by layer 'lstm_policy_26' (type LSTMPolicy):
• inputs=tf.Tensor(shape=(None, 20, 79), dtype=float32)
But I already added it and still get the same error.
class LSTMPolicy(Model):
def __init__(self, sequence_length, n_x):
super(LSTMPolicy, self).__init__()
self.sequence_length = sequence_length
self.n_x = n_x
self.build_model()
def build_model(self):
self.lstm1 = LSTM(units, return_sequences=True, return_state=True, stateful=True, name='lstm1', batch_input_shape=(batch_size, self.sequence_length, self.n_x))
self.dropout1 = Dropout(0.3)
self.lstm2 = LSTM(units, return_sequences=True, return_state=True, stateful=True, name='lstm2')
self.dropout2 = Dropout(0.3)
self.lstm3 = LSTM(units, return_state=True, stateful=True, name='lstm3')
self.dropout3 = Dropout(0.3)
self.dense2 = Dense(self.n_x, activation="softmax")
def call(self, inputs):
x, final_memory_state1, final_carry_state1 = self.lstm1(inputs)
x = self.dropout1(x)
x, final_memory_state2, final_carry_state2 = self.lstm2(x)
x = self.dropout2(x)
x, final_memory_state3, final_carry_state3 = self.lstm3(x)
x = self.dense1(x)
x = self.dropout3(x)
output = self.dense2(x)
return output, (final_memory_state1,
final_carry_state1,
final_memory_state2,
final_carry_state2,
final_memory_state3,
final_carry_state3)
def sampleAction(self, observation, init_state):
self.lstm1.states[0] = init_state[0]
self.lstm1.states[1] = init_state[1]
self.lstm2.states[0] = init_state[2]
self.lstm2.states[1] = init_state[3]
self.lstm3.states[0] = init_state[4]
self.lstm3.states[1] = init_state[5]
return self.call(observation)
I am trying to predict uncertainty in a regression problem using Dropout during testing as per Yarin Gal's article. I created a class using Keras's backend function as provided by this stack overflow question's answer. The class takes a NN model as input and randomly drops neurons during testing to give a stochastic estimate rather than deterministic output for a time-series forecasting.
I create a simple encoder-decoder model as shown below for the forecasting with 0.1 dropout during training:
input_sequence = Input(shape=(lookback, train_x.shape[2]))
encoder = LSTM(128, return_sequences=False)(input_sequence)
r_vec = RepeatVector(forward_pred)(encoder)
decoder = LSTM(128, return_sequences=True, dropout=0.1)(r_vec) #maybe use dropout=0.1
output = TimeDistributed(Dense(train_y.shape[2], activation='linear'))(decoder)
# optimiser = optimizers.Adam(clipnorm=1)
enc_dec_model = Model(input_sequence, output)
enc_dec_model.compile(loss="mean_squared_error",
optimizer="adam",
metrics=['mean_squared_error'])
enc_dec_model.summary()
After that, I define and call the DropoutPrediction class.
# Define the class:
class KerasDropoutPrediction(object):
def __init__(self ,model):
self.f = K.function(
[model.layers[0].input,
K.learning_phase()],
[model.layers[-1].output])
def predict(self ,x, n_iter=10):
result = []
for _ in range(n_iter):
result.append(self.f([x , 1]))
result = np.array(result).reshape(n_iter ,x.shape[0] ,x.shape[1]).T
return result
# Call the object:
kdp = KerasDropoutPrediction(enc_dec_model)
y_pred_do = kdp.predict(x_test,n_iter=100)
y_pred_do_mean = y_pred_do.mean(axis=1)
However, in the line
kdp = KerasDropoutPrediction(enc_dec_model), when I call the LSTM model,
I got the following error message which says the input has to be a Keras Tensor. Can anyone help me with this error?
Error Message:
ValueError: Found unexpected instance while processing input tensors for keras functional model. Expecting KerasTensor which is from tf.keras.Input() or output from keras layer call(). Got: 0
To activate Dropout at inference time, you simply have to specify training=True (TF>2.0) in the layer of interest (in the last LSTM layer in your case)
with training=False
inp = Input(shape=(10, 1))
x = LSTM(1, dropout=0.3)(inp, training=False)
m = Model(inp,x)
# m.compile(...)
# m.fit(...)
X = np.random.uniform(0,1, (1,10,1))
output = []
for i in range(0,100):
output.append(m.predict(X)) # always the same
with training=True
inp = Input(shape=(10, 1))
x = LSTM(1, dropout=0.3)(inp, training=True)
m = Model(inp,x)
# m.compile(...)
# m.fit(...)
X = np.random.uniform(0,1, (1,10,1))
output = []
for i in range(0,100):
output.append(m.predict(X)) # always different
In your example, this becomes:
input_sequence = Input(shape=(lookback, train_x.shape[2]))
encoder = LSTM(128, return_sequences=False)(input_sequence)
r_vec = RepeatVector(forward_pred)(encoder)
decoder = LSTM(128, return_sequences=True, dropout=0.1)(r_vec, training=True)
output = TimeDistributed(Dense(train_y.shape[2], activation='linear'))(decoder)
enc_dec_model = Model(input_sequence, output)
enc_dec_model.compile(
loss="mean_squared_error",
optimizer="adam",
metrics=['mean_squared_error']
)
enc_dec_model.fit(train_x, train_y, epochs=10, batch_size=32)
and the KerasDropoutPrediction:
class KerasDropoutPrediction(object):
def __init__(self, model):
self.model = model
def predict(self, X, n_iter=10):
result = []
for _ in range(n_iter):
result.append(self.model.predict(X))
result = np.array(result)
return result
kdp = KerasDropoutPrediction(enc_dec_model)
y_pred_do = kdp.predict(test_x, n_iter=100)
y_pred_do_mean = y_pred_do.mean(axis=0)
I'm trying to build a simple MNIST Model and this is what I've built -
training_loader = DataLoader(training_dataset, 128, shuffle = True)
validation_loader = DataLoader(validation_dataset, 128)
class mnistmodel(nn.Module):
def __init__(self):
super().__init__()
self.linear1 = nn.Linear(784, 10)
self.linear2 = nn.Linear(10, 5)
self.linear3 = nn.Linear(5, 10)
def forward(self, xb):
xb.reshape(-1, 784)
predicted = F.relu(self.linear1(xb))
predicted.reshape(-1, 10)
predicted = F.relu(self.linear2(predicted))
predicted.reshape(-1, 5)
predicted = self.linear3(predicted)
return predicted
def training_step(self, batch):
images, labels = batch
predicted = self(images)
loss = F.cross_entropy(predicted, labels)
return loss
def validation_step(self, batch):
images, labels = batch
predicted = self(images)
loss = F.cross_entropy(predicted, labels)
_, preds = torch.max(predicted, dim=1)
accuracy = torch.tensor(torch.sum(preds == labels).item() / len(preds))
return {'validation_loss': loss, 'validation_accuracy': accuracy}
def validation_epoch_end(self, outputs):
batch_losses = [x['validation_loss'] for x in outputs]
epoch_loss = torch.stack(batch_losses).mean()
batch_accs = [x['validation_acc'] for x in outputs]
epoch_acc = torch.stack(batch_accs).mean()
return {'validation_loss': epoch_loss.item(), 'validation_accuracy': epoch_acc.item()}
def epoch_end(self, epoch, result):
print(f"Epoch [{epoch}], val_loss: {result['validation_loss']}, val_acc: {result['validation_acc']}")
model = mnistmodel()
def fit_mnist(epochs, lr, model, training_loader, validation_loader, optimizer_function=torch.optim.SGD):
optimizer = optimizer_function(model.parameters(), lr)
history = []
for epoch in range(epochs):
for batch in training_loader:
loss = model.training_step(batch)
loss.backward()
optimizer.step()
optimizer.zero_grad()
result = evaluate(model, validation_loader)
model.epoch_end(epoch, result)
history.append(result)
return history
history1 = fit_mnist(5, 0.001, model, training_loader, validation_loader)
I get the following error -
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-162-48e4fe0cc2d9> in <module>()
----> 1 history1 = fit_mnist(5, 0.001, model, training_loader, validation_loader)
6 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
1751 if has_torch_function_variadic(input, weight):
1752 return handle_torch_function(linear, (input, weight), input, weight, bias=bias)
-> 1753 return torch._C._nn.linear(input, weight, bias)
1754
1755
RuntimeError: mat1 and mat2 shapes cannot be multiplied (3584x28 and 784x10)
I'm new to pytorch but as far as I understand the shapes seem to be fine, what is going wrong here?
I've checked manually and the dimensions seem to be fine. I even tried multiple things removing adding reshape commands but this error persists
I've found the mistake I need to reassign the reshape which I missed and spent the rest of the time debugging.
I would like to wrap numpy codes using tf.py_fucn in a customized lambda layer using keras.
Notice: only for simplicity I'm showing a simple np.power function.
Here is what I've done
def my_func(x):
return np.power(x, 2)
def my_lambda_func(x):
return tf.py_function(my_func, [x], tf.float32)
def model():
inp = Input(shape=(2,))
x = Dense(128)(inp)
x = Dense(128)(x)
z = Lambda(my_lambda_func)(x)
output = Dense(1)(z)
model = Model(inputs=inp, outputs=output)
return model
model = model ()
model.compile(optimizer='adam', loss='mse')
Then I get this error
ValueError Traceback (most recent call last)
in ()
21
22
---> 23 model = model ()
24 model.compile(optimizer='adam', loss='mse')
3 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/core.py in build(self, input_shape)
1179 last_dim = tensor_shape.dimension_value(input_shape[-1])
1180 if last_dim is None:
-> 1181 raise ValueError('The last dimension of the inputs to Dense '
1182 'should be defined. Found None.')
1183 self.input_spec = InputSpec(min_ndim=2, axes={-1: last_dim})
ValueError: The last dimension of the inputs to Dense should be defined. Found None.
You may need to specify the output shape of the Lambda layer z, the tf.py_function would give None as output shape which does not sit well with the Dense layer that follows it. You should try:
z = Lambda(my_lambda_func)(x)
z.set_shape(x.shape)
Trying to add Densenet121 functional block to the model.
I need Keras model to be written in this format, not using
model=Sequential()
model.add()
method
What's wrong the function, build_img_encod
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-62-69dd207148e0> in <module>()
----> 1 x = build_img_encod()
3 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/input_spec.py in assert_input_compatibility(input_spec, inputs, layer_name)
164 spec.min_ndim is not None or
165 spec.max_ndim is not None):
--> 166 if x.shape.ndims is None:
167 raise ValueError('Input ' + str(input_index) + ' of layer ' +
168 layer_name + ' is incompatible with the layer: '
AttributeError: 'Functional' object has no attribute 'shape'
def build_img_encod( ):
base_model = DenseNet121(input_shape=(150,150,3),
include_top=False,
weights='imagenet')
for layer in base_model.layers:
layer.trainable = False
flatten = Flatten(name="flatten")(base_model)
img_dense_encoder = Dense(1024, activation='relu',name="img_dense_encoder", kernel_regularizer=regularizers.l2(0.0001))(flatten)
model = keras.models.Model(inputs=base_model, outputs = img_dense_encoder)
return model
The reason why you get that error is that you need to provide the input_shape of the base_model, instead of the base_model per say.
Replace this line: model = keras.models.Model(inputs=base_model, outputs = img_dense_encoder)
with: model = keras.models.Model(inputs=base_model.input, outputs = img_dense_encoder)
def build_img_encod( ):
dense = DenseNet121(input_shape=(150,150,3),
include_top=False,
weights='imagenet')
for layer in dense.layers:
layer.trainable = False
img_input = Input(shape=(150,150,3))
base_model = dense(img_input)
flatten = Flatten(name="flatten")(base_model)
img_dense_encoder = Dense(1024, activation='relu',name="img_dense_encoder", kernel_regularizer=regularizers.l2(0.0001))(flatten)
model = keras.models.Model(inputs=img_input, outputs = img_dense_encoder)
return model
This worked..