I am trying to convert a Linear Classifier based on this example that works for exporting a DNN Classifier:
print("\n====== classifier model_dir, latest_checkpoint ===========")
print(classifier.model_dir)
print(classifier.latest_checkpoint())
debug = False
with tf.Session() as sess:
# First let's load meta graph and restore weights
latest_checkpoint_path = classifier.latest_checkpoint()
saver = tf.train.import_meta_graph(latest_checkpoint_path + '.meta')
saver.restore(sess, latest_checkpoint_path)
# Get the input and output tensors needed for toco.
# These were determined based on the debugging info printed / saved below.
input_tensor = sess.graph.get_tensor_by_name("dnn/input_from_feature_columns/input_layer/concat:0")
input_tensor.set_shape([1, 10])
out_tensor = sess.graph.get_tensor_by_name("dnn/logits/BiasAdd:0")
out_tensor.set_shape([1, 5])
# Pass the output node name we are interested in.
# Based on the debugging info printed / saved below, pulled out the
# name of the node for the logits (before the softmax is applied).
frozen_graph_def = tf.graph_util.convert_variables_to_constants(
sess, sess.graph_def, output_node_names=["dnn/logits/BiasAdd"])
if debug is True:
print("\nORIGINAL GRAPH DEF Ops ===========================================")
ops = sess.graph.get_operations()
for op in ops:
if "BiasAdd" in op.name or "input_layer" in op.name:
print([op.name, op.values()])
# save original graphdef to text file
with open("estimator_graph.pbtxt", "w") as fp:
fp.write(str(sess.graph_def))
print("\nFROZEN GRAPH DEF Nodes ===========================================")
for node in frozen_graph_def.node:
print(node.name)
# save frozen graph def to text file
with open("estimator_frozen_graph.pbtxt", "w") as fp:
fp.write(str(frozen_graph_def))
tflite_model = tf.contrib.lite.toco_convert(frozen_graph_def, [input_tensor], [out_tensor])
open("estimator_model.tflite", "wb").write(tflite_model)
but I don't know which tensor to use in this section:
input_tensor = sess.graph.get_tensor_by_name("dnn/input_from_feature_columns/input_layer/concat:0")
input_tensor.set_shape([1, 10])
out_tensor = sess.graph.get_tensor_by_name("dnn/logits/BiasAdd:0")
out_tensor.set_shape([1, 3])
I have tried as input tensor:
linear/linear_model/linear_model/weighted_sum:0
shape: 1,5
(because I couldn't find a tensor that works with 1,10)
and as output tensor with: linear/head/predictions/probabilities:0
shape 1,5
but when I tried to use it in android device the shape of the output tensor is no longer 1,5 but 1,10
And I don't know how to interpret this result, maybe the problem is that I Don't know which tensor to choose as input to the toco_convert function
Related
I am trying to get to run a bit of sample code from github in order to learn Working with Tensorflow 2 and the YOLO Framework. My Laptop has a M1000M Graphics Card and I installed the CUDA Platform from NVIDIA from here.
So the Code in question is this bit:
tf.compat.v1.disable_eager_execution()
_MODEL_SIZE = (416, 416)
_CLASS_NAMES_FILE = './data/labels/coco.names'
_MAX_OUTPUT_SIZE = 20
def main(type, iou_threshold, confidence_threshold, input_names):
class_names = load_class_names(_CLASS_NAMES_FILE)
n_classes = len(class_names)
model = Yolo_v3(n_classes=n_classes, model_size=_MODEL_SIZE,
max_output_size=_MAX_OUTPUT_SIZE,
iou_threshold=iou_threshold,
confidence_threshold=confidence_threshold)
if type == 'images':
batch_size = len(input_names)
batch = load_images(input_names, model_size=_MODEL_SIZE)
inputs = tf.compat.v1.placeholder(tf.float32, [batch_size, *_MODEL_SIZE, 3])
detections = model(inputs, training=False)
saver = tf.compat.v1.train.Saver(tf.compat.v1.global_variables(scope='yolo_v3_model'))
with tf.compat.v1.Session() as sess:
saver.restore(sess, './weights/model.ckpt')
detection_result = sess.run(detections, feed_dict={inputs: batch})
draw_boxes(input_names, detection_result, class_names, _MODEL_SIZE)
print('Detections have been saved successfully.')
While executing this (also wondering why starting the detection.py doesnt use GPU in the first place), I get the Error Message:
File "C:\SDKs etc\Python 3.8\lib\site-packages\tensorflow\python\client\session.py", line 1451, in _call_tf_sessionrun
return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
tensorflow.python.framework.errors_impl.UnimplementedError: The Conv2D op currently only supports the NHWC tensor format on the CPU. The op was given the format: NCHW
[[{{node yolo_v3_model/conv2d/Conv2D}}]]
Full Log see here.
If I am understanding this correctly, the format of inputs = tf.compat.v1.placeholder(tf.float32, [batch_size, *_MODEL_SIZE, 3]) is already NHWC (Model Size is a tuple of 2 Numbers) and I don't know how I need to change things in Code to get this running on CPU.
If I am understanding this correctly, the format of inputs =
tf.compat.v1.placeholder(tf.float32, [batch_size, *_MODEL_SIZE, 3]) is
already NHWC (Model Size is a tuple of 2 Numbers) and I don't know how
I need to change things in Code to get this running on CPU.
Yes you are. But look here:
def __init__(self, n_classes, model_size, max_output_size, iou_threshold,
confidence_threshold, data_format=None):
"""Creates the model.
Args:
n_classes: Number of class labels.
model_size: The input size of the model.
max_output_size: Max number of boxes to be selected for each class.
iou_threshold: Threshold for the IOU.
confidence_threshold: Threshold for the confidence score.
data_format: The input format.
Returns:
None.
"""
if not data_format:
if tf.test.is_built_with_cuda():
data_format = 'channels_first'
else:
data_format = 'channels_last'
And later:
def __call__(self, inputs, training):
"""Add operations to detect boxes for a batch of input images.
Args:
inputs: A Tensor representing a batch of input images.
training: A boolean, whether to use in training or inference mode.
Returns:
A list containing class-to-boxes dictionaries
for each sample in the batch.
"""
with tf.compat.v1.variable_scope('yolo_v3_model'):
if self.data_format == 'channels_first':
inputs = tf.transpose(inputs, [0, 3, 1, 2])
Solution:
check tf.test.is_built_with_cuda() work as expected
if not - set order manually when create model:
model = Yolo_v3(n_classes=n_classes, model_size=_MODEL_SIZE,
max_output_size=_MAX_OUTPUT_SIZE,
iou_threshold=iou_threshold,
confidence_threshold=confidence_threshold,
data_format = 'channels_last')
I am using Tensorflow and Python to define and train a NN in the following manner:
# clear old tensor values that cause a problem when rerunning
tf.reset_default_graph()
#network dims
hiddenneurons=12
input_dim=3
# Set up the model
def neural_net_model(X_data,input_dim):
W_1 = tf.Variable(tf.random_uniform([input_dim,hiddenneurons]),name='W_1')
b_1 = tf.Variable(tf.zeros([hiddenneurons]),name='b_1')
layer_1 = tf.add(tf.matmul(X_data,W_1), b_1)
layer_1 = tf.nn.tanh(layer_1)
# layer 1 multiplying and adding bias then activation function
W_O = tf.Variable(tf.random_uniform([hiddenneurons,1]),name='W_O')
b_O = tf.Variable(tf.zeros([1]),name='b_O')
output = tf.add(tf.matmul(layer_1,W_O), b_O)
# O/p layer multiplying and adding bias then activation function
return output
xs = tf.placeholder("float")
ys = tf.placeholder("float")
output = neural_net_model(xs,3)
cost = tf.reduce_mean(tf.square(output-ys))
train = tf.train.GradientDescentOptimizer(0.001).minimize(cost)
c_t = []
c_test = []
with tf.Session() as sess:
# Initiate session and initialize all vaiables
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
for i in range(10):
for j in range(X_train.shape[0]):
sess.run([cost,train],feed_dict={xs:X_train.values[j,:].reshape(1,3), ys:y_train.values[j]})
# Run cost and train with each sample
c_t.append(sess.run(cost, feed_dict={xs:X_train,ys:y_train}))
c_test.append(sess.run(cost, feed_dict={xs:X_test,ys:y_test}))
print('Epoch :',i,'Cost :',c_t[i])
plt.scatter(i,c_t[i])
pred = sess.run(output, feed_dict={xs:X_test})
print(xs)
# predict output of test data after training
print('Cost :',sess.run(cost, feed_dict={xs:X_test,ys:y_test}))
# save model
saver.save(sess,'save folder')
print('Model saved')
At this point, if I output the trainable variables with tf.trainable_variables(), I get four variables of the expected shape for W_1:0, W_O:0, b_1:0, b_O:0.
I want to be able have a different file which restores the model, then uses the same weights and biases as saved, to allow me to run a variety of testing data through.
I am having trouble restoring the model and persuading it to reuse the past weights and biases. My restore code looks like this:
# clear old tensor values
tf.reset_default_graph()
newsaver = tf.train.import_meta_graph(modelloc)
def neural_net_model(X_data,input_dim):
W_1 = tf.Variable(tf.random_uniform([input_dim,hiddenneurons]))
b_1 = tf.Variable(tf.zeros([hiddenneurons]))
layer_1 = tf.add(tf.matmul(X_data,W_1), b_1)
layer_1 = tf.nn.tanh(layer_1)
W_O = tf.Variable(tf.random_uniform([hiddenneurons,1]))
b_O = tf.Variable(tf.zeros([1]))
output = tf.add(tf.matmul(layer_1,W_O), b_O)
xs = tf.placeholder("float")
ys = tf.placeholder("float")
output = neural_net_model(xs,3)
with tf.Session() as sessr:
sessr.run(tf.global_variables_initializer())
newsaver.restore(sessr, tf.train.latest_checkpoint('folder where .ckpt files are'))
pred = sessr.run(output, feed_dict={xs:X_test})
At this point, if I type tf.trainable_variables(), I get the details of the four tensors W_1:0, W_O:0, b_O:0, b_1:0, plus four new ones Variable_0, Variable_1:0, Variable_2:0, Variable_3_0. This means that the data is tested on these new variables and does not give the desired result. I don't seem to be able to use the restored weights and biases W_1,W_O, b_1,b_O.
I KNOW that I am reinitialising the variables when I don't need to, and this is the problem, and I have read this post here in detail. I have also read this, and this and many others. If I remove the repitition of the model definition, 'output', or 'neural_net_model' becomes undefined and the code doesn't work. If I try to specify W_1 etc. in any other way, the code doesn't work.
In my training file(train.py), I write:
def deep_part(self):
with tf.variable_scope("deep-part"):
y_deep = tf.reshape(self.embeddings, shape=[-1, self.field_size * self.factor_size]) # None * (F*K)
# self.deep_layers = 2
for i in range(0,len(self.deep_layers)):
y_deep = tf.contrib.layers.fully_connected(y_deep, self.deep_layers[i], \
activation_fn=self.deep_layers_activation, scope = 'fc%d' % i)
return y_deep
now in predict file(predict.py), I restore the checkpoint, but I dont know how to reload the "deep-part" network's weights and biases.Because I think the "fully_conncted" function might hide the weights and biases.
I wrote a lengthy explanation here. A short summary:
By saver.save(sess, '/tmp/my_model') Tensorflow produces multiple files:
checkpoint
my_model.data-00000-of-00001
my_model.index
my_model.meta
The checkpoint file checkpoint is just a pointer to the latest version of our model-weights and it is simply a plain text file containing
$ !cat /tmp/model/checkpoint
model_checkpoint_path: "/tmp/my_model"
all_model_checkpoint_paths: "/tmp/my_model"
The others are binary files containing the graph (.meta) and weights (.data*).
You can help yourself by running
import tensorflow as tf
import numpy as np
data = np.arange(9 * 1).reshape(1, 9).astype(np.float32)
plhdr = tf.placeholder(tf.float32, shape=[1, 9], name='input')
print plhdr.name
activation = tf.layers.dense(plhdr, 10, name='fc')
print activation.name
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
expected = sess.run(activation, {plhdr: data})
print expected
saver = tf.train.Saver(tf.global_variables())
saver.save(sess, '/tmp/my_model')
tf.reset_default_graph()
with tf.Session() as sess:
# load the computation graph (the fully connected + placeholder)
loader = tf.train.import_meta_graph('/tmp/my_model.meta')
sess.run(tf.global_variables_initializer())
plhdr = tf.get_default_graph().get_tensor_by_name('input:0')
activation = tf.get_default_graph().get_tensor_by_name('fc/BiasAdd:0')
actual = sess.run(activation, {plhdr: data})
assert np.allclose(actual, expected) is False
# now load the weights
loader = loader.restore(sess, '/tmp/my_model')
actual = sess.run(activation, {plhdr: data})
assert np.allclose(actual, expected) is True
My question is about context and the TensorFlow default sessions and graph.
The problem:
Tensorflow is unable to feed a placeholder in the following scenario:
Function Test defines a graph.
Function Test_Once defines a session.
When Function Test calls Test_Once -> Feeding fails.
When I change the code so function Test declares the graph + the session -> all is working.
Here is the code:
def test_once(g, saver, summary_writer, logits, images, summary_op):
"""Run a session once for a givven test image.
Args:
saver: Saver.
summary_writer: Summary writer.
logits:
summary_op: Summary op.
"""
with tf.Session(graph=g) as sess:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
# Restores from checkpoint
saver.restore(sess, ckpt.model_checkpoint_path)
# extract global_step from it.
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
else:
print('No checkpoint file found')
return
images.astype(np.float32)
predictions = sess.run(logits, feed_dict={'InputPlaceHolder/TestInput:0':images})
summary = tf.Summary()
summary.ParseFromString(sess.run(summary_op))
summary_writer.add_summary(summary, global_step)
return (predictions)
def test():
"""Test LCPR with a test image"""
with tf.Graph().as_default() as g:
# Get image for testing
images, labels = lcpr.test_input()
# Build a Graph that computes the logits predictions from the
# inference model.
with tf.name_scope('InputPlaceHolder'):
test_image_placeholder = tf.placeholder(tf.float32, (None,None,None,3), 'TestInput')
# Display the training images in the visualizer.
# The 'max_outputs' default is 3. Not stated. (Max number of batch elements to generate images for.)
#tf.summary.image('input_images', test_image_placeholder)
with tf.name_scope('Inference'):
logits = lcpr.inference(test_image_placeholder)
# Restore the moving average version of the learned variables for eval.
variable_averages = tf.train.ExponentialMovingAverage(
lcpr.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
# Build the summary operation based on the TF collection of Summaries.
writer = tf.summary.FileWriter("/tmp/lcpr/test")
writer.add_graph(g)
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.test_dir, g)
#Sadly, this will not work:
predictions = test_once(g, saver, summary_writer, logits, images, summary_op)
'''Alternative working option :
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
# Restores from checkpoint
saver.restore(sess, ckpt.model_checkpoint_path)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/cifar10_train/model.ckpt-0,
# extract global_step from it.
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
else:
print('No checkpoint file found')
return
x = sess.run(logits, feed_dict={'InputPlaceHolder/TestInput:0':images})
print(x)
'''
The above code yeilds an error that the placeholder is not fed:
InvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'InputPlaceHolder/TestInput' with dtype float
And it's not that TensorFlow does not recognize the placeholder. If I change the name from 'InputPlaceHolder/TestInput:0' to 'InputPlaceHolder/TestInput:1' I receive a message calming that 'InputPlaceHolder/TestInput' exists but has only 1 output. This makes sense, and I guess the session runs on my default graph.
Things only work for me if I stay within the same def:
If I change the code by running the commented part (starting ' with tf.Session() as sess:) directly from within the first function all works.
I wonder what am I missing?
My guess that is context related, maybe not assigning the session to the graph?
Solved. Stupid mistake
test_once calls sess.run twice. On the second time, indeed no placeholder is fed.... : summary.ParseFromString(sess.run(summary_op))
Tensorboard provides embedding visualization of tensorflow variables by using tf.train.Saver(). The following is a working example (from this answer)
import os
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from tensorflow.contrib.tensorboard.plugins import projector
LOG_DIR = '/tmp/emb_logs/'
metadata = os.path.join(LOG_DIR, 'metadata.tsv')
mnist = input_data.read_data_sets('MNIST_data')
#Variables
images = tf.Variable(mnist.test.images, name='images')
weights=tf.Variable(tf.random_normal([3,3,1,16]))
biases=tf.Variable(tf.zeros([16]))
#Tensor from the variables
x = tf.reshape(images, shape=[-1, 28, 28, 1])
conv_layer=tf.nn.conv2d(x, weights, [1,1,1,1], padding="SAME")
conv_layer=tf.add(conv_layer, biases)
y = tf.reshape(conv_layer, shape=[-1, 28*28*16])
with open(metadata, 'wb') as metadata_file:
for row in mnist.test.labels:
metadata_file.write('%d
' % row)
with tf.Session() as sess:
saver = tf.train.Saver([images])
sess.run(images.initializer)
saver.save(sess, os.path.join(LOG_DIR, 'images.ckpt'))
config = projector.ProjectorConfig()
# One can add multiple embeddings.
embedding = config.embeddings.add()
embedding.tensor_name = images.name
# Link this tensor to its metadata file (e.g. labels).
embedding.metadata_path = metadata
# Saves a config file that TensorBoard will read during startup.
projector.visualize_embeddings(tf.summary.FileWriter(LOG_DIR), config)
How can I visualize embeddings from a tensorflow tensor, like y in the code above?
Simply replacing
saver = tf.train.Saver([images])
with
saver = tf.train.Saver([y])
doesn't work, because of the following error:
474 var = ops.convert_to_tensor(var, as_ref=True)
475 if not BaseSaverBuilder._IsVariable(var):
476 raise TypeError("Variable to save is not a Variable: %s" % var)
477 name = var.op.name
478 if name in names_to_saveables:
TypeError: Variable to save is not a Variable: Tensor("Reshape_11:0", shape=(10000, 12544), dtype=float32)
Does anyone know of an alternative to generate tensorboard embedding visualizations of a tf.tensor?
You could create a new variable you can assign the value of y to.
y_var = tf.Variable(tf.shape(y))
saver = tf.train.Saver([y_var])
assign_op = y_var.assign(y)
...
# Training
sess.run((loss, assign_op))
saver.save(...)
tf.summary.tensor_summary will save a summary for an arbitrary Tensor.