Since I load my data (images) from the structured folders, I utilize the flow_from_directory function of the ImageDataGenerator class, which is provided by Keras. I've no issues while feeding this data to a CNN model. But when it comes to an LSTM model, getting the following error: ValueError: Error when checking input: expected lstm_1_input to have 3 dimensions, but got array with shape (64, 28, 28, 1). How can I reduce the dimension of the input data while reading it via ImageDataGenerator objects to be able to use an LSTM model instead of a CNN?
p.s. The shape of the input images is (28, 28) and they are grayscale.
train_valid_datagen = ImageDataGenerator(validation_split=0.2)
train_gen = train_valid_datagen.flow_from_directory(
directory=TRAIN_IMAGES_PATH,
target_size=(28, 28),
color_mode='grayscale',
batch_size=64,
class_mode='categorical',
shuffle=True,
subset='training'
)
Update: The LSTM model code:
inp = Input(shape=(28, 28, 1))
inp = Lambda(lambda x: squeeze(x, axis=-1))(inp) # from 4D to 3D
x = LSTM(num_units, dropout=dropout, recurrent_dropout=recurrent_dropout, activation=activation_fn, return_sequences=True)(inp)
x = BatchNormalization()(x)
x = Dense(128, activation=activation_fn)(x)
output = Dense(nb_classes, activation='softmax', kernel_regularizer=l2(0.001))(x)
model = Model(inputs=inp, outputs=output)
you start feeding your network with 4D data like your images in order to have the compatibility with ImageDataGenerator and then you have to reshape them in 3D format for LSTM.
These are the possibilities:
with only one channel you can simply squeeze the last dimension
inp = Input(shape=(28, 28, 1))
x = Lambda(lambda x: tf.squeeze(x, axis=-1))(inp) # from 4D to 3D
x = LSTM(32)(x)
if you have multiple channels (this is the case of RGB images or if would like to apply a RNN after a Conv2D) a solution can be this
inp = Input(shape=(28, 28, 1))
x = Conv2D(32, 3, padding='same', activation='relu')(inp)
x = Reshape((28,28*32))(x) # from 4D to 3D
x = LSTM(32)(x)
the fit can be computed as always with model.fit_generator
UPDATE: model review
inp = Input(shape=(28, 28, 1))
x = Lambda(lambda x: squeeze(x, axis=-1))(inp) # from 4D to 3D
x = LSTM(32, dropout=dropout, recurrent_dropout=recurrent_dropout, activation=activation_fn, return_sequences=False)(x)
x = BatchNormalization()(x)
x = Dense(128, activation=activation_fn)(x)
output = Dense(nb_classes, activation='softmax', kernel_regularizer=l2(0.001))(x)
model = Model(inputs=inp, outputs=output)
model.summary()
pay attention when you define inp variable (don't overwrite it)
set return_seq = False in LSTM in order to have 2D output
Related
I'm using keras to implement a functional CNN model where I have images with the size of 64x64x1. with 6 convolutional layer like this :
num_classes = 5
def get_model():
##creating CNN functional api for learning
input_ = keras.layers.Input(shape=[64, 64,1])
##first layer of convolutional layer
Conv1 = keras.layers.Conv2D(32, kernel_size=5,activation=tf.nn.relu)(input_layer)
#second convolutional layer
Conv12 = keras.layers.Conv2D(32, kernel_size=5,activation=tf.nn.relu)(Conv1)
#third convolutional layer
Conv13 = keras.layers.Conv2D(32, kernel_size=5,activation=tf.nn.relu)(Conv12)
##i add max pooling with a stride of 2
Max1 = keras.layers.MaxPool2D(2, strides=2)(Conv13)
##i add a second layer of convlutional layer
Conv2 = keras.layers.Conv2D(64, kernel_size=5,activation=tf.nn.relu)(Max1)
##adding second convolutional layer
Conv21 = keras.layers.Conv2D(64, kernel_size=5,activation=tf.nn.relu)(Conv2)
##adding third convolutional layer
Conv23 = keras.layers.Conv2D(64, kernel_size=5,activation=tf.nn.relu)(Conv21)
##i add another layer of max pooling
Max2 = keras.layers.MaxPool2D(2, strides=2)(Conv23)
##here i execute data flatting, i will change this to use attention layer Att.
Flat = keras.layers.Flatten()(Max2)
#i add another dense architecture
Dense1= keras.layers.Dense(2048,activation=tf.nn.relu)(Flat)
Dense2= keras.layers.Dense(700,activation=tf.nn.relu)(Dense1)
#i add now the output layer with softmax
# Output layer, class prediction.
output = keras.layers.Dense(num_classes,activation=tf.nn.softmax)(Dense2)
model = Model(inputs=input_, outputs=output)
##end of creating CNN using functional api
##defining loss function and training data and epoche. I modify the optimizer to rmsprop
optimize_rmsprop = keras.optimizers.RMSprop(learning_rate=0.001, epsilon=1e-08, decay=0.0)
model.compile(loss="sparse_categorical_crossentropy", optimizer=optimizer_rmsprop,metrics=["accuracy"])
###i return the model
return model
to get better performance i want to add this attention layer to the above CNN :
#already imported
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import Model
from keras.layers import Dropout
# Variable-length int sequences. 64*64*1
query_input = tf.keras.Input(shape=(4096,), dtype='int32')
value_input = tf.keras.Input(shape=(4096,), dtype='int32')
# Embedding lookup.
token_embedding = tf.keras.layers.Embedding(input_dim=1000, output_dim=64)
# Query embeddings of shape [batch_size, Tq, dimension].
query_embeddings = token_embedding(query_input)
# Value embeddings of shape [batch_size, Tv, dimension].
value_embeddings = token_embedding(value_input)
# CNN layer.
cnn_layer = tf.keras.layers.Conv1D(
filters=100,
kernel_size=4,
# Use 'same' padding so outputs have the same shape as inputs.
padding='same')
# Query encoding of shape [batch_size, Tq, filters].
query_seq_encoding = cnn_layer(query_embeddings)
# Value encoding of shape [batch_size, Tv, filters].
value_seq_encoding = cnn_layer(value_embeddings)
# Query-value attention of shape [batch_size, Tq, filters].
query_value_attention_seq = tf.keras.layers.Attention()(
[query_seq_encoding, value_seq_encoding])
# Reduce over the sequence axis to produce encodings of shape
# [batch_size, filters].
query_encoding = tf.keras.layers.GlobalAveragePooling1D()(
query_seq_encoding)
query_value_attention = tf.keras.layers.GlobalAveragePooling1D()(
query_value_attention_seq)
# Concatenate query and document encodings to produce a DNN input layer.
input_layer = tf.keras.layers.Concatenate()(
[query_encoding, query_value_attention])
but the problem is i don't know how to link the attention layer with my cnn model because when i connect the first convolutional network with the attention layer like this :
Conv1 = keras.layers.Conv2D(32, kernel_size=5)(input_layer)
I get this error :
ValueError: Input 0 of layer "conv2d" is incompatible with the layer: expected min_ndim=4, found ndim=2. Full shape received: (None, 200)
can someone show me how to add an attention layer to the CNN model.
Updated Answer...
keras.backend.clear_session()
class AttentionLayer(tf.keras.layers.Layer):
def __init__(self,
output_dims):
super(AttentionLayer, self).__init__()
self.output_dims = output_dims
self.embeddings = tf.keras.layers.Embedding(input_dim=4096, output_dim = output_dims)
self.conv = tf.keras.layers.Conv1D(2048, 4 , padding='same')
self.attn_layer = tf.keras.layers.Attention()
self.global_pooling_1 = tf.keras.layers.GlobalAveragePooling1D()
self.global_pooling_2 = tf.keras.layers.GlobalAveragePooling1D()
def call (self, query_input, value_input):
batch_size = tf.shape(query_input)[0]
query_input = tf.reshape(query_input, (batch_size, 4096))
value_input = tf.reshape(value_input, (batch_size, 4096))
# Query embeddings of shape [batch_size, Tq, dimension].
query_embeddings = self.embeddings(query_input)
# Value embeddings of shape [batch_size, Tv, dimension].
value_embeddings = self.embeddings(value_input)
# Query encoding of shape [batch_size, Tq, filters].
query_seq_encoding = self.conv(query_embeddings)
# Value encoding of shape [batch_size, Tv, filters].
value_seq_encoding = self.conv(value_embeddings)
# Query-value attention of shape [batch_size, Tq, filters].
query_value_attention_seq = self.attn_layer(
[query_seq_encoding, value_seq_encoding])
# Reduce over the sequence axis to produce encodings of shape
# [batch_size, filters].
query_encoding = self.global_pooling_1(
query_seq_encoding)
query_value_attention = self.global_pooling_2(
query_value_attention_seq)
# Concatenate query and document encodings to produce a DNN input layer.
input_layer = tf.keras.layers.Concatenate()(
[query_encoding, query_value_attention])
input_layer = tf.reshape(input_layer , (batch_size, 64,64 ,1))
return input_layer
keras.backend.clear_session()
num_classes = 5
class Model(tf.keras.Model):
def __init__(self):
super(Model, self).__init__()
self.attn_layer = AttentionLayer(64)
##first layer of convolutional layer
self.Conv1 = keras.layers.Conv2D(32, kernel_size=5,activation=tf.nn.relu)
#second convolutional layer
self.Conv12 = keras.layers.Conv2D(32, kernel_size=5,activation=tf.nn.relu)
#third convolutional layer
self.Conv13 = keras.layers.Conv2D(32, kernel_size=5,activation=tf.nn.relu)
##i add max pooling with a stride of 2
self.Max1 = keras.layers.MaxPool2D(2, strides=2)
##i add a second layer of convlutional layer
self.Conv2 = keras.layers.Conv2D(64, kernel_size=5,activation=tf.nn.relu)
##adding second convolutional layer
self.Conv21 = keras.layers.Conv2D(64, kernel_size=5,activation=tf.nn.relu)
##adding third convolutional layer
self.Conv23 = keras.layers.Conv2D(64, kernel_size=5,activation=tf.nn.relu)
##i add another layer of max pooling
self.Max2 = keras.layers.MaxPool2D(2, strides=2)
##here i execute data flatting, i will change this to use attention layer Att.
self.Flat = keras.layers.Flatten()
#i add another dense architecture
self.Dense1= keras.layers.Dense(2048,activation=tf.nn.relu)
self.Dense2= keras.layers.Dense(700,activation=tf.nn.relu)
#i add now the output layer with softmax
# Output layer, class prediction.
self.outputs = keras.layers.Dense(num_classes,activation=tf.nn.softmax)
def call(self, x):
x = self.attn_layer(x , x)
x = self.Conv1(x)
x = self.Conv12(x)
x = self.Conv13(x)
x = self.Max1(x)
x = self.Conv2(x)
x = self.Conv21(x)
x = self.Conv23(x)
x = self.Max2(x)
x = self.Flat(x)
x = self.Dense1(x)
x = self.Dense2(x)
return self.outputs(x)
model = Model()
x = np.random.randint(0 ,2 , size = (8, 64, 64,1))
y = np.random.randint(0,5, size=(8,1))
print(model(x).shape)
optimize_rmsprop = keras.optimizers.RMSprop(learning_rate=0.001, epsilon=1e-08, decay=0.0)
model.compile(loss="sparse_categorical_crossentropy", optimizer=optimize_rmsprop,metrics=["accuracy"])
model.fit(x, y, epochs=1)
Output:
I am trying to make a spatio temporal graph convolutional network where a gcn layer is sandwiched between two tomporal cnn layer. The code is following:
inputs = Input(shape=(train_x.shape[1],train_x.shape[2],train_x.shape[3]), batch_size=None)
# temporal convolution
y = tf.keras.layers.Conv1D(128, 9, activation='relu')(inputs)
#graph convolution
y = tf.keras.layers.Conv2D(32, (1,1), activation='relu')(y)
n, v, t, kc = y.shape
y = tf.reshape(y,(n, 1, kc//1, t, v))
y = tf.einsum('nkctv,kvw->nwtc', y, AD_tensor)
#temporal convolution
y = tf.keras.layers.Conv1D(16, 9, activation='relu')(y)
concat = Flatten()(y)
fc = Dense(units=80, activation='relu')(concat)
fc1 = Dense(units=40, activation='relu')(fc)
fc2 = Dense(units=40, activation='relu')(fc1)
fc3 = Dense(units=80, activation='relu')(fc2)
out = Dense(1, activation = 'sigmoid')(fc3)
model = Model(inputs, out)
model.compile(loss='mse', optimizer= Adam(lr=0.0001))
model.fit(train_x, train_y, validation_data = (valid_x,valid_y), epochs=300, batch_size=2)
When I run this code it shows me this type error:
TypeError: Failed to convert object of type <class 'tuple'> to Tensor.
Contents: (None, 1, 32, 72, 25). Consider casting elements to a supported type.
To use Tensorflow operations with Keras layers, you should wrap them in a Lambda layer as such. The Lambda layer takes a function as its argument.
y = tf.keras.layers.Lambda(lambda x: tf.reshape(n, v, t, kc))(y)
However, for reshaping, Keras already provides a layer for this operation, so you could do
y = tf.keras.layers.Reshape(shape=(v, t, kc))(y)
The layer version of reshaping already takes into account the batch dimension, so you only need to specify the other dimensions.
For the einsum operation, you can use
y = tf.keras.layers.Lambda(lambda x: tf.einsum('nkctv,kvw->nwtc', x[0], x[1]))([y, AD_tensor])
I want to construct a variational autoencoder in Keras (2.2.4, with TensorFlow backend), here is my code:
dims = [1000, 256, 64, 32]
x_inputs = Input(shape=(dims[0],), name='inputs')
h = x_inputs
# internal layers in encoder
for i in range(n_stacks-1):
h = Dense(dims[i + 1], activation='relu', kernel_initializer='glorot_uniform', name='encoder_%d' % i)(h)
# hidden layer
z_mean = Dense(dims[-1], kernel_initializer='glorot_uniform', name='z_mean')(h)
z_log_var = Dense(dims[-1], kernel_initializer='glorot_uniform', name='z_log_var')(h)
z = Lambda(sampling, output_shape=(dims[-1],), name='z')([z_mean, z_log_var])
encoder = Model(inputs=x_inputs, outputs=z, name='encoder')
encoder_z_mean = Model(inputs=x_inputs, outputs=z_mean, name='encoder_z_mean')
# internal layers in decoder
latent_inputs = Input(shape=(dims[-1],), name='latent_inputs')
h = latent_inputs
for i in range(n_stacks-1, 0, -1):
h = Dense(dims[i], activation='relu', kernel_initializer='glorot_uniform', name='decoder_%d' % i)(h)
# output
outputs = Dense(dims[0], activation='relu', kernel_initializer='glorot_uniform' name='mean')
decoder = Model(inputs=latent_inputs, outputs=outputs, name='decoder')
ae_output = decoder(encoder_z_mean(x_inputs))
ae = Model(inputs=x_inputs, outputs=ae_output, name='ae')
ae.summary()
vae_output = decoder(encoder(x_inputs))
vae = Model(inputs=x_inputs, outputs=vae_output, name='vae')
vae.summary()
The problem is I can print the summary of the "ae" and "vae" models, but when I train the ae model, it says
tensorflow.python.framework.errors_impl.InvalidArgumentError: You must feed a value for placeholder tensor 'latent_inputs' with dtype float and shape [?,32]
In the model "decoder" is supposed to connect to the output of "encoder_z_mean" layer in the ae model. But when I print the summary of the "ae" model, "decoder" is actually connected to "encoder_z_mean[1][0]". Should it be "encoder_z_mean[0][0]"?
A few corrections:
x_inputs is already the input of the encoders, don't call it again with encoder_z_mean(x_inputs) or with encoder(x_inputs)
Besides creating a second node (the 1 that you are worried with, and that is not a problem), it may be the source of the error because it's not an extra input, but the same input
A healthy usage of this would need the creation of a new Input(...) tensor to be called
The last Dense layer is not being called on a tensor. You probably want (h) there.
Do it this way:
# output - called h in the last layer
outputs = Dense(dims[0], activation='relu', kernel_initializer='glorot_uniform' name='mean')(h)
#unchanged
decoder = Model(inputs=latent_inputs, outputs=outputs, name='decoder')
#adjusted inputs
ae_output = decoder(encoder_z_mean.output)
ae = Model(encoder_z_mean.input, ae_output, name='ae')
ae.summary()
vae_output = decoder(encoder.output)
vae = Model(encoder.input, vae_output, name='vae')
vae.summary()
It's possible that the [1][0] still occurs with the decoder, but this is not a problem at all. It means that the decoder itself has its own input node (number 0), and you created an extra input node (number 1) when you called it with the output of another model. This is harmless. The node 1 will be used while node 0 will be ignored.
I have time series data input 72 value by separate last 6 value for test prediction. I want to use CONV1D with LSTM.
This is my code.
df = pd.read_csv('D://data.csv',
engine='python')
df['DATE_'] = pd.to_datetime(df['DATE_']) + MonthEnd(1)
df = df.set_index('DATE_')
df.head()
split_date = pd.Timestamp('03-01-2015')
train = df.loc[:split_date, ['COLUMN3DATA']]
test = df.loc[split_date:, ['COLUMN3DATA']]
sc = MinMaxScaler()
train_sc = sc.fit_transform(train)
test_sc = sc.transform(test)
X_train = train_sc[:-1]
y_train = train_sc[1:]
X_test = test_sc[:-1]
y_test = test_sc[1:]
################### Convolution #######################
X_train_t = X_train[None,:]
print(X_train_t.shape)
X_test_t = X_test[:, None]
K.clear_session()
model = Sequential()
model.add(Conv1D(6, 3, activation='relu', input_shape=(12,1)))
model.add(LSTM(6, input_shape=(1,3), return_sequences=True))
model.add(LSTM(3))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam' )
model.summary()
model.fit(X_train_t, y_train, epochs=400, batch_size=10, verbose=1)
y_pred = model.predict(X_test_t)
When I run it show error like this
ValueError: Error when checking input: expected conv1d_1_input to have shape (None, 12, 1) but got array with shape (1, 64, 1)
How to use conv1D with lstm
The problem is between your input data and your input shape.
You said in the model that your input shape is (12,1) (= batch_shape=(None,12,1))
But your data X_train_t has shape (1,64,1).
Either you fix the input shape of the model, or you fix your data if this is not the expected shape.
For variable lengths/timesteps, you can use input_shape=(None,1).
You don't need an input_shape in the second layer.
my input variables
IMG_SIZE_PX=50
SLICE_COUNT=20
n_classes=2
x=tf.placeholder('float')
y=tf.placeholder('float')
keep_rate=0.8
keep_prob=tf.placeholder(tf.float32)
my convolution 3d function
def conv3d(x, W):
return tf.nn.conv3d(x, W, strides=[1,1,1,1,1], padding='SAME')
my maxpooling 3d function
def maxpool3d(x):
return tf.nn.max_pool3d(x, ksize=[1,2,2,2,1], strides=[1,2,2,2,1],
padding='SAME')
this is my network
def convolutional_neural_network(x):
my network weights
weights = {'W_conv1':tf.Variable(tf.random_normal([3,3,3,1,32])),
'W_conv2':tf.Variable(tf.random_normal([3,3,3,32,64])),
'W_fc':tf.Variable(tf.random_normal([ 54080 ,1024])),#here 54080
is the input tensor value
'out':tf.Variable(tf.random_normal([1024, n_classes]))}
my network biases
biases = {'b_conv1':tf.Variable(tf.random_normal([32])),
'b_conv2':tf.Variable(tf.random_normal([64])),
'b_fc':tf.Variable(tf.random_normal([1024])),
'out':tf.Variable(tf.random_normal([n_classes]))}
here is my input x
x = tf.reshape(x, shape=[-1, IMG_SIZE_PX, IMG_SIZE_PX, SLICE_COUNT, 1])
my 2 hidden layers(convolution+maxpooling)
conv1 = tf.nn.relu(conv3d(x, weights['W_conv1']) + biases['b_conv1'])
conv1 = maxpool3d(conv1)
conv2 = tf.nn.relu(conv3d(conv1, weights['W_conv2']) + biases['b_conv2'])
conv2 = maxpool3d(conv2)
my fully connected layer
fc = tf.reshape(conv2,[-1, 54080 ])
fc = tf.nn.relu(tf.matmul(fc, weights['W_fc'])+biases['b_fc'])
fc = tf.nn.dropout(fc, keep_rate)
my output layer
output = tf.matmul(fc, weights['out'])+biases['out']
return output
my input numpy arrays
much_data = np.load('D:/muchdata-50-50-20.npy')
train_data = much_data[-10:]
validation_data = much_data[-2:]
finally training my network
def train_neural_network(x):
outl = convolutional_neural_network(x)#don't know this is my output
layer
model=Model(input=x, output=outl)
model.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=
['accuracy'])
train_neural_network(x)#train the net
my error is thiskeras meta data is missing
any help can be appreciated