I am currently working on a project where I build a Network in Keras like so:
inputstuff = Input(shape=(32,), name='main_input')
encoded = Dense(16, activation='relu',init='he_normal', activity_regularizer=regularizers.l1(0.01))(inputstuff)
[other layers...]
decoded = Dense(32, activation='relu',init='he_normal', name='main_output')(decoded)
autoencoder = Model(input=inputstuff, output=decoded)
Next to save this file as a tensorflow model, I do the following:
sessK=K.get_session()
saver = tf.train.Saver()
model_path = "/tmp/KerasGlobalAEModel.ckpt"
save_path = saver.save(sessK, model_path)
Next I want to load the file from another program:
model_path = "/tmp/KerasGlobalAEModel.ckpt"
tf.train.NewCheckpointReader(model_path)
with tf.Session() as sess:
# Initialize variables
init = tf.initialize_all_variables()
sess.run(init)
saver = tf.train.import_meta_graph('/tmp/KerasGlobalAEModel.ckpt.meta')
# Restore model weights from previously saved model
saver.restore(sess, model_path)
test = [some data]
graph = tf.get_default_graph()
MyAnswers = sess.run(Y*, feed_dict={X*: test})
If this was the same file I would be able to use:
X* = model.input
Y* = model.output
However, this and all other things I have tried failed.
Here is a list of what DOES NOT work.
Y = graph.get_tensor_by_name("main_output")
Y = model.output
Y = autoencoder.output
I realize I am a noobie at this, and this might be a dumb question but I would appreciate any guidance.
Thank you,
Related
I'm trying to save and restore parameters of a tensorflow model. The code does save the parameters to the given path, but when I try to restore the last checkpoint, perform an operation, and then save again, it doesn't return the last checkpoint. what should I do?
import tensorflow as tf
import os
v1 = tf.Variable(1.32, name="v1")
v2 = tf.Variable(1.33, name="v2")
saver = tf.train.Saver()
init = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init)
print (v2.eval(sess))
saver.save(sess, "/tmp/model")
print("Model restored.")
for i in range(10):
ckpt = tf.train.get_checkpoint_state(os.path.dirname('/tmp/'))
print(ckpt)
sess = tf.Session()
if(ckpt and ckpt.model_checkpoint_path):
saver.restore(sess, ckpt.model_checkpoint_path)
x = sess.run("v2:0")
x = x + 1
print(x)
saver.save(sess, "/tmp/model", global_step = i)
sess.close()
Currently I have resnet_v2_50.ckpt from tensorflow's open sourced pretrained model. I am trying to serve this model in Go cause my backend for my web app is going to be in Go. If I were to create my own model and train it, and then save it. I have no trouble with serving it in Go but I am trying to use pre-trained model to save time on my end.
Here's a simple example of how I save my model
mnist = input_data.read_data_sets(DATA_DIR, one_hot=True)
# Recall that each image is 28x28
x = tf.placeholder(tf.float32, [None, 784], name='imageinput')
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
y = tf.add(tf.matmul(x, W), b)
labels = tf.placeholder(tf.float32, [None, 10])
cross_entropy_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=y))
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy_loss)
with tf.Session() as sess:
with tf.device("/cpu:0"):
sess.run(tf.global_variables_initializer())
for i in range(1000):
batch_x, batch_label = mnist.train.next_batch(100)
loss, _ = sess.run([cross_entropy_loss, train_step], feed_dict={x: batch_x, labels: batch_label})
print '%d: %f' % (i + 1, loss)
infer = tf.argmax(y, axis=1, name='infer')
truth = tf.argmax(labels, axis=1)
correct_prediction = tf.equal(infer, truth)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print sess.run(accuracy, feed_dict={x: mnist.test.images, labels: mnist.test.labels})
print 'Time to save the graph!'
builder = tf.saved_model.builder.SavedModelBuilder('mnist_model')
builder.add_meta_graph_and_variables(sess, ['serve'])
builder.save()
I can load it using tensorflow in Go
model, err := tf.LoadSavedModel("./tf_mnist_py/mnist_model", []string{"serve"}, nil)
if err != nil {
fmt.Printf("Error loading saved model: %s\n", err.Error())
return
}
defer model.Session.Close()
But now when it comes to pretrained model, I am dealing with ckpt files. One solution I have is to load it up in Python and then save it as protobuf.
from tensorflow.python.tools import inspect_checkpoint as ckpt
ckpt.print_tensors_in_checkpoint_file('./resnet50/resnet_v2_50.ckpt',
tensor_name='',
all_tensors=False,
all_tensor_names=False)
tf.reset_default_graph()
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, './resnet50/resnet_v2_50.ckpt')
print 'Model is restored'
print 'Time to save the graph!'
builder = tf.saved_model.builder.SavedModelBuilder('resnet_50_model')
builder.add_meta_graph_and_variables(sess, ['serve'])
builder.save()
However, this gives me an error saying that ValueError: No variables to save. I can fix it by declaring a variable
v1 = tf.get_variable('total_loss/ExponentialMovingAverage', shape=[])
But here comes my question, does that mean I must declare EVERY variable in the ResNet50 and have tensorflow load the values from ckpt file into those variables and then I perform save? Is there a shortcut to do this?
You should import the variables as well from the .meta file if you have it available with the pretrained model like
saver = tf.train.import_meta_graph('./resnet50/resnet_v2_50.meta')
Detailed tutorial here.
If you don't have the .meta available, but you do have the network generating code, like with resnet_v2_50 in tensorflow/models/blob/master/research/slim/nets/resnet_v2.py then you should import that file and run the resnet_v2_50 function, which will define all the variables for you. Then restore the checkpoint.
When freezing a graph and then running it elsewhere (mobile device), the output is of low quality compared to the inference on the server on my semantic segmentation model. It is basically a messy version of what would run on the server. It is executing successfully, but it appears as though something was not initialized prior to freezing, even though the method to load the model between the export script and inference scripts is nearly identical.
The exported model can be run on the same images over and over and produce the same results for a given set of images, as expected.
However, each time the model is frozen, using the exact same script and checkpoint, it creates a different output for a given set of images.
def main():
args = get_arguments()
if args.dataset == 'cityscapes':
num_classes = cityscapes_class
else:
num_classes = ADE20k_class
shape = [320, 320]
x = tf.placeholder(dtype=tf.float32, shape=(shape[0], shape[1], 3), name="input")
img_tf = preprocess(x)
model = model_config[args.model]
net = model({'data': img_tf}, num_classes=num_classes, filter_scale=args.filter_scale)
raw_output = net.layers['conv6_cls']
raw_output_up = tf.image.resize_bilinear(raw_output, size=shape, align_corners=True)
raw_output_maxed = tf.argmax(raw_output_up, axis=3, name="output")
# Init tf Session
config = tf.ConfigProto()
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
model_path = model_paths[args.model]
ckpt = tf.train.get_checkpoint_state(model_path)
if ckpt and ckpt.model_checkpoint_path:
input_checkpoint = ckpt.model_checkpoint_path
loader = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=True)
load(loader, sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found at %s.' % model_path)
exit()
print("Loaded Model")
# We retrieve the protobuf graph definition
graph = tf.get_default_graph()
input_graph_def = graph.as_graph_def()
# We use a built-in TF helper to export variables to constants
output_graph_def = graph_util.convert_variables_to_constants(
sess, # The session is used to retrieve the weights
input_graph_def, # The graph_def is used to retrieve the nodes
output_node_names.split(",") # The output node names are used to select the usefull nodes
)
# Finally we serialize and dump the output graph to the filesystem
with tf.gfile.GFile("model/output_graph.pb", "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." % len(output_graph_def.node))
In my training file(train.py), I write:
def deep_part(self):
with tf.variable_scope("deep-part"):
y_deep = tf.reshape(self.embeddings, shape=[-1, self.field_size * self.factor_size]) # None * (F*K)
# self.deep_layers = 2
for i in range(0,len(self.deep_layers)):
y_deep = tf.contrib.layers.fully_connected(y_deep, self.deep_layers[i], \
activation_fn=self.deep_layers_activation, scope = 'fc%d' % i)
return y_deep
now in predict file(predict.py), I restore the checkpoint, but I dont know how to reload the "deep-part" network's weights and biases.Because I think the "fully_conncted" function might hide the weights and biases.
I wrote a lengthy explanation here. A short summary:
By saver.save(sess, '/tmp/my_model') Tensorflow produces multiple files:
checkpoint
my_model.data-00000-of-00001
my_model.index
my_model.meta
The checkpoint file checkpoint is just a pointer to the latest version of our model-weights and it is simply a plain text file containing
$ !cat /tmp/model/checkpoint
model_checkpoint_path: "/tmp/my_model"
all_model_checkpoint_paths: "/tmp/my_model"
The others are binary files containing the graph (.meta) and weights (.data*).
You can help yourself by running
import tensorflow as tf
import numpy as np
data = np.arange(9 * 1).reshape(1, 9).astype(np.float32)
plhdr = tf.placeholder(tf.float32, shape=[1, 9], name='input')
print plhdr.name
activation = tf.layers.dense(plhdr, 10, name='fc')
print activation.name
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
expected = sess.run(activation, {plhdr: data})
print expected
saver = tf.train.Saver(tf.global_variables())
saver.save(sess, '/tmp/my_model')
tf.reset_default_graph()
with tf.Session() as sess:
# load the computation graph (the fully connected + placeholder)
loader = tf.train.import_meta_graph('/tmp/my_model.meta')
sess.run(tf.global_variables_initializer())
plhdr = tf.get_default_graph().get_tensor_by_name('input:0')
activation = tf.get_default_graph().get_tensor_by_name('fc/BiasAdd:0')
actual = sess.run(activation, {plhdr: data})
assert np.allclose(actual, expected) is False
# now load the weights
loader = loader.restore(sess, '/tmp/my_model')
actual = sess.run(activation, {plhdr: data})
assert np.allclose(actual, expected) is True
I attempt to read an RNN network through import_graph_def() and do inference.
But I cannot use tf.trainable_variables() to get any variables.
In the following code, tf.trainable_variables() returns [] (a list with nothing)
Also, when I use saver = tf.train.Saver(), tensorflow reports "no variables to save"
def eval_on_test(graph_path):
batch_size = 80
train_begin = 0
train_end = 3000
with tf.Graph().as_default() as graph:
with open(graph_path, 'rb') as f:
tf_graph = tf.GraphDef()
print("Loading graph_def from {}".format(graph_path))
tf_graph.ParseFromString(f.read())
return_elements = tf.import_graph_def(tf_graph, name="", return_elements=['input_x:0', 'output_y:0', 'pred:0', 'loss:0'])
X = return_elements[0]
Y = return_elements[1]
pred = return_elements[2]
loss = return_elements[3]
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
print("graph loaded, start testing")
with tf.Session(config=tf_config) as sess:
init_op = sess.graph.get_operation_by_name('init')
sess.run(init_op)
print(tf.trainable_variables())
batch_index,train_x,train_y=get_train_data(batch_size,time_step,train_begin,train_end)
for batch in range(len(batch_index)-1):
loss_ = sess.run(loss, feed_dict={X:train_x[batch_index[batch]:batch_index[batch+1]],Y:train_y[batch_index[batch]:batch_index[batch+1]]})
print(batch, loss_)
Any help would be appreciated.
import_graph_def will only restore graph but will not restore collections such as GLOBAL_VARIABLES, that's why Saver can't find any variables in graph, to solve this, you can try tf.train.import_meta_graph which will also restore all collections.