System information
OS Platform and Distribution :CentOS Linux release 7.7.1908
-TensorFlow version:2.3.0
I try to convert the tensorflow offical image caption model to TFLite model
I try to convert the tf.keras.Model 's encoder and decoder model as following:
import tensorflow as tf
embedding_dim = 256
units = 512
top_k = 5000
vocab_size = top_k + 1
features_shape = 2048
attention_features_shape = 64
class BahdanauAttention(tf.keras.Model):
def __init__(self, utils):
super(BahdanauAttention, self).__init__()
self.W1 = tf.keras.layers.Dense(utils)
self.W2 = tf.keras.layers.Dense(utils)
self.V = tf.keras.layers.Dense(1)
def call(self, features, hidden):
# features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)
# hidden shape == (batch_size, hidden_size)
# hidden_with_time_axis shape == (batch_size, 1, hidden_size)
hidden_with_time_axis_shape = tf.expand_dims(hidden, 1)
# score shape == (batch_size, 64, hidden_size)
score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis_shape))
# attention_weights shape == (batch_size, 64, 1)
# you get 1 at the last axis because you are applying score to self.V
attention_weights = tf.nn.softmax(self.V(score), axis=1)
# context_vector shape after sum == (batch_size, hidden_size)
context_vector = attention_weights * features
context_vector = tf.reduce_sum(context_vector, axis=1)
return context_vector, attention_weights
class CNN_Encoder(tf.keras.Model):
#由于您已经提取了特征并使用pickle进行了转储
#该编码器通过完全连接的层传递这些特征
def __init__(self, embedding):
super(CNN_Encoder, self).__init__()
# shape after fc == (batch_size, 64, embedding_dim)
self.fc = tf.keras.layers.Dense(embedding_dim)
# #tf.function(input_signature=[tf.TensorSpec(shape=(1, 64, features_shape),dtype=tf.float32)])
#tf.function
def call(self, x):
x = self.fc(x)
x = tf.nn.relu(x)
return x
class RNN_Decoder(tf.keras.Model):
def __init__(self, embedding_dim, units, vocab_size):
super(RNN_Decoder, self).__init__()
self.units = units
self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)
self.gru = tf.keras.layers.GRU(self.units,
return_sequences=True,
return_state=True,
recurrent_initializer='glorot_uniform',
unroll = True)
self.fc1 = tf.keras.layers.Dense(self.units)
self.fc2 = tf.keras.layers.Dense(vocab_size)
self.attention = BahdanauAttention(self.units)
#tf.function(input_signature=[tf.TensorSpec(shape=[1, 1], dtype=tf.int32, name='x'),
tf.TensorSpec(shape=[1, 64, 256], dtype=tf.float32, name='feature'),
tf.TensorSpec(shape=[1, 512], dtype=tf.float32, name='hidden')])
#tf.function
def call(self, x , features, hidden):
#将注意力定义为一个单独的模型
context_vector, attention_weights = self.attention(features, hidden)
#x shape after passing through embedding == (batch_size, 1, embedding_dim)
x = self.embedding(x)
#x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)
x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)
#将concated后的的向量传递给GRU
output, state = self.gru(x)
#shape == (batch_size, max_length, hidden_size)
x = self.fc1(output)
#x shape == (batch_size, max_length, hidden_size)
x = tf.reshape(x, (-1, x.shape[2]))
# output shape == (batch_size * max_length, vocab)
x = self.fc2(x)
return x, state, attention_weights
def reset_states(self, batch_size):
return tf.zeros((batch_size, self.units))
encoder = CNN_Encoder(embedding_dim)
decoder = RNN_Decoder(embedding_dim, units, vocab_size)`
encoder._set_inputs(tf.TensorSpec(shape=(1, 64, features_shape),dtype=tf.float32))
decoder._set_inputs([tf.TensorSpec(shape=[1, 1], dtype=tf.int32, name='x'),
tf.TensorSpec(shape=[1, 64, 256], dtype=tf.float32, name='feature'),
tf.TensorSpec(shape=[1, 512], dtype=tf.float32, name='hidden')])
encoder_converter = tf.lite.TFLiteConverter.from_keras_model(encoder)
decoder_converter = tf.lite.TFLiteConverter.from_keras_model(decoder)
encoder_model = encoder_converter.convert()
decoder_model = decoder_converter.convert()
open("encoder_model.tflite", "wb").write(encoder_model)
open("decoder_model.tflite", "wb").write(decoder_model)
The error messge is
ValueError: Structure of Python function inputs does not match input_signature:
inputs: (
[<tf.Tensor 'x:0' shape=(1, 1) dtype=int32>, <tf.Tensor 'feature:0' shape=(1, 64, 256) dtype=float32>, <tf.Tensor 'hidden:0' shape=(1, 512) dtype=float32>])
input_signature: (
TensorSpec(shape=(1, 1), dtype=tf.int32, name='x'),
TensorSpec(shape=(1, 64, 256), dtype=tf.float32, name='feature'),
TensorSpec(shape=(1, 512), dtype=tf.float32, name='hidden'))
I think the function input is the same as the input signature.How can I fix the problem?
i also had the same issue. found a simple solution to this. the call method of tf.keras.Model class accepts only single input, so you need to pack all your inputs inside a list/tuple
https://github.com/tensorflow/tensorflow/issues/32488#issuecomment-560248754
Related
I want to use TensorFlow Keras(v2.2) model fit in mnist with multiple outputs and losses, but it failed.
My costume model will return a list [logits, embedding]. logits is 2D tensor [batch , 10] and embedding is also 2D tensor [batch, 64].
class MyModel(tf.keras.Model):
def __init__(self):
super(MyModel, self).__init__()
self.reshape = tf.keras.layers.Reshape((28, 28, 1))
self.conv2D1 = tf.keras.layers.Conv2D(filters=8, kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')
self.maxPool1 = tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="same")
self.conv2D2 = tf.keras.layers.Conv2D(filters=8, kernel_size=(3,3), strides=(1, 1), padding='same', activation='relu')
self.maxPool2 = tf.keras.layers.MaxPooling2D(pool_size=2)
self.flatten = tf.keras.layers.Flatten(data_format="channels_last")
self.dropout = tf.keras.layers.Dropout(tf.compat.v1.placeholder_with_default(0.25, shape=[], name="dropout"))
self.dense1 = tf.keras.layers.Dense(64, activation=None)
self.dense2 = tf.keras.layers.Dense(10, activation=None)
def call(self, inputs, training):
x = self.reshape(inputs)
x = self.conv2D1(x)
x = self.maxPool1(x)
if training:
x = self.dropout(x)
x = self.conv2D2(x)
x = self.maxPool2(x)
if training:
x = self.dropout(x)
x = self.flatten(x)
x = self.dense1(x)
embedding = tf.math.l2_normalize(x, axis=1)
logits = self.dense2(embedding)
return [logits, embedding]
loss_0 is normal cross_entropy
def loss_0(y_true, y_pred):
loss_0 = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y_true, logits=y_pred[0]))
loss_1 is triplet_semihard_loss
def loss_1(y_true, y_pred):
loss_1 = tfa.losses.triplet_semihard_loss(y_true=y_true, y_pred=y_pred[1], distance_metric="L2")
return loss_1
When I use model fit, I can only get logits tensor in each loss. I can't get embedding tensor. y_pred[0] and y_pred[1] is not work. Any suggestion?
model = MyModel()
model.compile(optimizer=tf.keras.optimizers.Adam(lr=1e-3), loss=[loss_0, loss_1], loss_weights=[0.1, 0.1])
history = model.fit(train_dataset, epochs=5)
I want build subclass of tf.keras.Model and want to see the model structure with summary function. But it not works. The following is my code:
import tensorflow as tf
class MyModel(tf.keras.Model):
def __init__(self):
super(MyModel, self).__init__()
self.conv1 = tf.keras.layers.Conv2D(32, 3, activation='relu')
self.flatten = tf.keras.layers.Flatten()
self.d1 = tf.keras.layers.Dense(128, activation='relu')
self.d2 = tf.keras.layers.Dense(10, activation='softmax')
def call(self, x):
x = self.conv1(x)
x = self.flatten(x)
x = self.d1(x)
return self.d2(x)
model = MyModel()
model.summary()
The error:
ValueError: This model has not yet been built. Build the model first
by calling build() or calling fit() with some data, or specify an
input_shape argument in the first layer(s) for automatic build.
You need to call each layer once to infer shapes and then call build() method of the tf.keras.Model with model's input shape as argument:
import tensorflow as tf
import numpy as np
class MyModel(tf.keras.Model):
def __init__(self):
super(MyModel, self).__init__()
self.conv1 = tf.keras.layers.Conv2D(32, 3, activation='relu')
self.flatten = tf.keras.layers.Flatten()
self.d1 = tf.keras.layers.Dense(128, activation='relu')
self.d2 = tf.keras.layers.Dense(10, activation='softmax')
x = np.random.normal(size=(1, 32, 32, 3))
x = tf.convert_to_tensor(x)
_ = self.call(x)
def call(self, x):
x = self.conv1(x)
x = self.flatten(x)
x = self.d1(x)
return self.d2(x)
model = MyModel()
model.build((32, 32, 3))
model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d (Conv2D) multiple 896
_________________________________________________________________
flatten (Flatten) multiple 0
_________________________________________________________________
dense (Dense) multiple 3686528
_________________________________________________________________
dense_1 (Dense) multiple 1290
=================================================================
Total params: 3,688,714
Trainable params: 3,688,714
Non-trainable params: 0
_________________________________________________________________
A better solution is listed here. You need to provide a model method to infer the model explicitly.
import tensorflow as tf
from tensorflow.keras.layers import Input
class MyModel(tf.keras.Model):
def __init__(self):
super().__init__()
self.dense = tf.keras.layers.Dense(1)
def call(self, inputs, **kwargs):
return self.dense(inputs)
def model(self):
x = Input(shape=(1))
return Model(inputs=[x], outputs=self.call(x))
MyModel().model().summary()
Editing #Vlad's answer to avoid this error ValueError: Input 0 of layer conv2d_10 is incompatible with the layer: : expected min_ndim=4, found ndim=3. Full shape received: (32, 32, 3)
Change this line from:
model.build((32, 32, 3 ))
To:
model.build((None, 32, 32, 3 ))
Final Code:
class MyModel(tf.keras.Model):
def __init__(self):
super(MyModel, self).__init__()
self.conv1 = tf.keras.layers.Conv2D(32, 3, activation='relu')
self.flatten = tf.keras.layers.Flatten()
self.d1 = tf.keras.layers.Dense(128, activation='relu')
self.d2 = tf.keras.layers.Dense(10, activation='softmax')
x = np.random.normal(size=(1, 32, 32, 3))
x = tf.convert_to_tensor(x)
_ = self.call(x)
def call(self, x):
x = self.conv1(x)
x = self.flatten(x)
x = self.d1(x)
return self.d2(x)
model = MyModel()
model.build((None, 32, 32, 3 ))
model.summary()
Based on How can I use TensorFlow's sampled softmax loss function in a Keras model?, I created this code:
class SampledSoftmax(tensorflow.keras.layers.Layer):
def __init__(self, **kwargs):
super(SampledSoftmax, self).__init__(**kwargs)
def call(self, inputs):
def f1(inputs):
return tf.nn.sampled_softmax_loss(
inputs[0]._keras_history[0].weights[0],
inputs[0]._keras_history[0].bias,
tf.reshape(tf.argmax(inputs[1], 1), [-1, 1]),
inputs[0],
8192,
817496)
def f2(inputs):
logits = tf.matmul(inputs[0], tf.transpose(inputs[0]._keras_history[0].weights[0]))
logits = tf.nn.bias_add(logits, inputs[0]._keras_history[0].bias)
return tf.nn.softmax_cross_entropy_with_logits_v2(
labels=inputs[1],
logits=logits)
return tf.cond(K.learning_phase(), true_fn=f1(inputs), false_fn=f2(inputs))
and when used with the following model:
#model
input_layer = Input(shape=(None,), dtype='int32')
target_input = Input(shape=(None,vocab_size), dtype='int8')
embedding_layer = Embedding(vocab_size,
EMBEDDING_DIM,
trainable=True,
mask_zero=True) (input_layer)
common = LSTM(LSTM_UNITS, return_sequences=True,dropout=0.2, recurrent_dropout=0.2)(embedding_layer)
common = (Dense(PROJ_UNITS, activation='linear'))(common)
out = (Dense(vocab_size, name='output_layer'))(common)
out = (SampledSoftmax())([out, target_input])
model = Model(inputs=[input_layer,target_input], outputs=out)
it failed with this error:
ValueError: Shape must be rank 2 but is rank 3 for 'sampled_softmax/sampled_softmax_loss/MatMul' (op: 'MatMul') with input shapes: [?,?,817496], [?,817496].
I made some progress based on google search:
class MyLayer(tensorflow.keras.layers.Dense):
def __init__(self, num_sampled, num_classes, mode, **kwargs):
self.num_sampled = num_sampled
self.num_classes = num_classes
self.mode = mode
super(MyLayer, self).__init__(num_classes, **kwargs)
self.input_spec = [InputSpec(ndim=2)]
def build(self, input_shape):
#self.input_spec = [InputSpec(shape=input_shape)]
super(MyLayer, self).build(input_shape) # Be sure to call this somewhere!
def call(self, inputs_and_labels):
inputs, labels = inputs_and_labels
if self.mode == "train":
loss = tf.nn.sampled_softmax_loss(
weights=self.kernel,
biases=self.bias,
labels=tf.reshape(tf.argmax(labels, 1), [-1, 1]),
inputs=inputs,
num_sampled=self.num_sampled,
num_classes=self.num_classes,
num_true=1)
elif self.mode == "eval":
logits = tf.matmul(inputs, tf.transpose(self.kernel))
logits = tf.nn.bias_add(logits, self.bias)
loss = tf.nn.softmax_cross_entropy_with_logits(
labels=labels,
logits=logits)
return loss
def compute_output_shape(self, input_shape):
dense_shape, classes_shape = input_shape
return (dense_shape[0], )
and the error now:
The error now:
ValueError: Layer my_layer expects 1 inputs, but it received 2 input tensors. Inputs received: [<tf.Tensor 'dense/BiasAdd:0' shape=(?, ?, 512) dtype=float32>, <tf.Tensor 'input_2:0' shape=(?, ?, 817496) dtype=int8>]
I tried to use self.input_spec but it does not work until now.
I am looking at the examples/image_orc.py example in Keras, when I run it I see something like
_______________
max2 (MaxPooling2D) (None, 32, 16, 16) 0 conv2[0][0]
____________________________________________________________________________________________________
reshape (Reshape) (None, 32, 256) 0 max2[0][0]
____________________________________________________________________________________________________
dense1 (Dense) (None, 32, 32) 8224 reshape[0][0]
_____________________________________________________________________________________
The Dense layer outputs a tensor 32x32. I am trying to replicate this in pur TensorFlow where tf.matmul would be used, but how can I output 32x32 using matmul?
Addition:
I am not trying to replicate the Keras example exactly,
w = 128; h = 64
# junk image, only one
dataset = np.zeros((1,w,h,1))
import tensorflow as tf
pool_size = 1
num_filters = 16
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
inputs = tf.placeholder(tf.float32, [None, w, h, 1])
W_conv1 = weight_variable([3, 3, 1, num_filters])
b_conv1 = bias_variable([num_filters])
h_conv1 = tf.nn.relu(conv2d(inputs, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)
W_conv2 = weight_variable([3, 3, num_filters, num_filters])
b_conv2 = bias_variable([num_filters])
h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)
h_pool2_flat = tf.reshape(h_pool2, [-1, 32, 256])
W_fc1 = weight_variable([256, 32])
b_fc1 = bias_variable([32])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
print inputs.shape
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
output = sess.run(h_pool2_flat, feed_dict={inputs: dataset})
print 'output',output.shape
And I get
ValueError: Shape must be rank 2 but is rank 3 for 'MatMul_5' (op: 'MatMul') with input shapes: [?,32,256], [256,32].
A smaller example
import numpy as np
import tensorflow as tf
dataset = np.zeros((3,2,4))
inputs = tf.placeholder(tf.float32, [None, 2, 4])
print inputs
W = tf.zeros((4,5))
print W
W2 = tf.matmul(inputs, W)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
output = sess.run(W2, feed_dict={inputs: dataset})
print 'output',output.shape
This also gives similar error
ValueError: Shape must be rank 2 but is rank 3 for 'MatMul_12' (op: 'MatMul') with input shapes: [?,2,4], [4,5].
Any ideas?
Thanks,
That 32 is there because it was in the previous layer. It keeps unchanged.
The tf.matmul multiplies considering the two last dimensions, as stated here. (See the examples taking more than two dimensions)
I see you've got a Dense(32) there, with input size = 256.
This means that the weights matrix is (256,32). In keras, the multiplication as seen here is inputs x kernel.
So, if you have the input tensor shaped as (?, any, 256), and the weights matrix shaped as (256,32), all you need is:
output = tf.matmul(input,weights)
This will output a shape (?, any, 32) - any is there untouched because it just was there before.
You may also want to sum the biases, which will follow the same principle. You need a bias vector of shape (32,).
Below code creates some tensors, which are implicit. I wonder how I can view values of these tensors:
<tf.Variable 'rnn/basic_lstm_cell/kernel:0' shape=(43, 160) dtype=float32_ref>
<tf.Variable 'rnn/basic_lstm_cell/bias:0' shape=(160,) dtype=float32_ref>
<tf.Variable 'rnn/basic_lstm_cell/kernel/Adagrad:0' shape=(43, 160) dtype=float32_ref>
<tf.Variable 'rnn/basic_lstm_cell/bias/Adagrad:0' shape=(160,) dtype=float32_ref>
<tf.Variable 'softmax/W/Adagrad:0' shape=(40, 10) dtype=float32_ref>
<tf.Variable 'softmax/b/Adagrad:0' shape=(10,) dtype=float32_ref>
Here is code itself.
import tensorflow as tf
import numpy as np
VECTOR_SIZE = 3
SEQUENCE_LENGTH = 5
BATCH_SIZE = 7
STATE_SIZE = 40
NUM_CLASSES = 10
LEARNING_RATE = 0.1
x = tf.placeholder(tf.float32, [BATCH_SIZE, SEQUENCE_LENGTH, VECTOR_SIZE],
name='input_placeholder')
y = tf.placeholder(tf.int32, [BATCH_SIZE, SEQUENCE_LENGTH],
name='labels_placeholder')
init_state = tf.zeros([BATCH_SIZE, STATE_SIZE])
rnn_inputs = tf.unstack(x, axis = 1)
y_as_list = tf.unstack(y, axis=1)
cell = tf.contrib.rnn.BasicLSTMCell(STATE_SIZE, state_is_tuple = True)
rnn_outputs, final_state = tf.contrib.rnn.static_rnn(cell, rnn_inputs,
initial_state=(init_state,init_state))
with tf.variable_scope('softmax'):
W = tf.get_variable('W', [STATE_SIZE, NUM_CLASSES])
b = tf.get_variable('b', [NUM_CLASSES], initializer=tf.constant_initializer(0.0))
logits = [tf.matmul(rnn_output, W) + b for rnn_output in rnn_outputs]
predictions = [tf.nn.softmax(logit) for logit in logits]
losses = [tf.nn.sparse_softmax_cross_entropy_with_logits(labels=label, logits=logit) for \
logit, label in zip(logits, y_as_list)]
total_loss = tf.reduce_mean(losses)
train_step = tf.train.AdagradOptimizer(LEARNING_RATE).minimize(total_loss)
X = np.ones([BATCH_SIZE, SEQUENCE_LENGTH, VECTOR_SIZE])
Y = np.ones([BATCH_SIZE, SEQUENCE_LENGTH])
saver = tf.train.Saver()
sess = tf.Session()
sess.run(tf.global_variables_initializer())
batch_total_loss = sess.run([total_loss, train_step],
feed_dict = {x:X,y:Y})
save_path = saver.save(sess, "/tmp/model.ckpt")
for el in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES):
print(el)
Use sess.run():
for el in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES):
print(el) # this will print the tensor's name, shape, data type
print(sess.run(el)) # this will print the tensor's current value