I wrote following code and executed, then a PB file called test.pb generated.
import tensorflow as tf
import numpy as np
x = tf.placeholder(tf.float32, shape=[1,2,3,4], name="x")
mu, sigma = tf.nn.moments(x, [0,1,2])
in_data = np.array([i for i in range(24)]).reshape([1,2,3,4])
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
print sess.run([mu, sigma], feed_dict={x: in_data})
tf.train.write_graph(sess.graph_def, "./", "test.pb")
In test.pb, there's a node:
node {
name: "moments/normalize/divisor"
op: "Reciprocal"
input: "moments/sufficient_statistics/Const"
input: "^moments/sufficient_statistics/mean_ss"
input: "^moments/sufficient_statistics/var_ss"
attr {
key: "T"
value {
type: DT_FLOAT
}
}
}
My question is that what the punctuation ^ mean in ^moments/sufficient_statistics/mean_ss and ^moments/sufficient_statistics/var_ss?
Related
I have an issue with Tensorflow model that is converted from Pytorch -> Onnx -> Tensorflow. The issue is the converted Tensorflow model expects the input in Pytorch format that is (batch size, number channels, height, width) but not in Tensorflow format (batch size, height, width, number channel). Therefore, I cannot use the model to process further with Vitis AI.
So I would like to ask is there is any ways to convert this Pytorch input format to Tensorflow format by using tools from Onnx, Tensorflow 1, or others?
My code is as below:
Pytorch -> Onnx
from hardnet import hardnet
import torch
import onnx
ckpt = torch.load('../hardnet.pth')
model_state_dict = ckpt['model_state_dict']
optimizer_state_dict = ckpt['optimizer_state_dict']
model = hardnet(11)
model.load_state_dict(model_state_dict)
model.eval()
dummy_input = torch.randn(1, 3, 1080, 1920)
input_names = ['input0']
output_names = ['output0']
output_file = 'hardnet.onnx'
torch.onnx.export(model, dummy_input, output_file, verbose=True,
input_names=input_names, output_names=output_names,
opset_version=11, keep_initializers_as_inputs=True)
onnx_model = onnx.load(output_file)
onnx.checker.check_model(onnx_model)
print('Passed Onnx')
Onnx -> Tensorflow 1 (using Tensorflow 1.15)
import cv2
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import onnx
from onnx_tf.backend import prepare
output_file = 'hardnet.onnx'
onnx_model = onnx.load(output_file)
output = prepare(onnx_model)
output.export_graph('hardnet.pb')
tf.compat.v1.disable_eager_execution()
def load_pb(path_to_pb: str):
"""From: https://stackoverflow.com/questions/51278213/what-is-the-use-of-a-pb-file-in-tensorflow-and-how-does-it-work
"""
with tf.gfile.GFile(path_to_pb, "rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
with tf.Graph().as_default() as graph:
tf.import_graph_def(graph_def, name='')
return graph
graph = load_pb('hardnet.pb')
input = graph.get_tensor_by_name('input0:0')
output = graph.get_tensor_by_name('output0:0')
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
img = cv2.imread('train_0.jpg', cv2.IMREAD_COLOR)
img = cv2.resize(img, (1920, 1080))
img = img/255
img = img - mean
img = img/std
img = np.expand_dims(img, -1)
# To Pytorch format.
img = np.transpose(img, (3, 2, 0, 1))
img = img
with tf.Session(graph=graph) as sess:
pred = sess.run(output, {input: img})
You could wrap your Pytorch model into another one that would do the transpose you want to have in TensorFlow. See the following example:
Let's say you have the following toy NN:
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.rnn = nn.LSTM(10, 20, 2)
def forward(self, x):
h0 = torch.zeros(2, 3, 20)
c0 = torch.zeros(2, 3, 20)
return self.rnn(x, (h0, c0))
the exemplary pytorch/tensorflow input shape would be :
>> pytorch_input = torch.randn(5, 3, 10)
>> tf_input = torch.transpose(pytorch_input, 1, 2)
>> print("PyTorch input shape: ", pytorch_input.shape)
>> print("TensorFlow input shape: ", tf_input.shape)
PyTorch input shape: torch.Size([5, 3, 10])
TensorFlow input shape: torch.Size([5, 10, 3])
Now, the wrapper which will first transpose input and then pass transposed input to some model:
class NetTensorFlowWrapper(nn.Module):
def __init__(self, main_module: nn.Module):
super(NetTensorFlowWrapper, self).__init__()
self.main_module = main_module
def forward(self, x):
x = torch.transpose(x, 1, 2)
return self.main_module(x)
Then, this is possible:
net = Net()
net_wrapper = NetTensorFlowWrapper(net)
net(pytorch_input)
net_wrapper(tf_input)
and then, when you finally save your models like you did previously via torch.onnx.export and read their graph via onnx package (not torch.onnx) you will have...
for Net- input 5x3x10 and no transpose layer
graph torch-jit-export (
%input0[FLOAT, 5x3x10]
{
%76 = Shape(%input0)
%77 = Constant[value = <Scalar Tensor []>]()
for NetTensorFlowWrapper- input 5x10x3 and transpose layer
graph torch-jit-export (
%input0[FLOAT, 5x10x3]
{
%9 = Transpose[perm = [0, 2, 1]](%input0)
%77 = Shape(%9)
%78 = Constant[value = <Scalar Tensor []>]()
...
I have a tensorflow graph (stored in a protobuffer file) with placeholder operations as inputs. I want to wrap this graph as a keras layer or model.
Here is an example:
with tf.Graph().as_default() as gf:
x = tf.placeholder(tf.float32, shape=(None, 123), name='x')
c = tf.constant(100, dtype=tf.float32, name='C')
y = tf.multiply(x, c, name='y')
with tf.gfile.GFile("test_graph/y.pb", "wb") as f:
raw = gf.as_graph_def().SerializeToString()
f.write(raw)
Load back as a tensorflow graph:
persisted_sess = tf.Session()
with persisted_sess.as_default():
with gfile.FastGFile("./test_graph/y.pb",'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
persisted_sess.graph.as_default()
tf.import_graph_def(graph_def, name='')
for i, op in enumerate(persisted_sess.graph.get_operations()):
tensor = persisted_sess.graph.get_tensor_by_name(op.name + ':0')
print(i, '\t', op.name, op.type, tensor)
x_tensor = persisted_sess.graph.get_tensor_by_name('x:0')
y_tensor = persisted_sess.graph.get_tensor_by_name('y:0')
We can see the x and y operations and tensors:
0 x Placeholder Tensor("x:0", shape=(?, 123), dtype=float32)
1 C Const Tensor("C:0", shape=(), dtype=float32)
2 y Mul Tensor("y:0", shape=(?, 123), dtype=float32)
Then I try to wrap it into a keras model using different method:
Method 1:
output_layer = Lambda(lambda x: y_tensor, name='output_y')(x_tensor)
model = Model(inputs=[x_tensor], outputs=[output_layer]) # ERROR!
This already produce error InvalidArgumentError: You must feed a value for placeholder tensor 'x' with dtype float and shape [?,123] [[{{node x}}]]
Method 2:
input_x = Input(name='x', shape=(123,), dtype='float32')
output_layer = Lambda(lambda x: y_tensor, name='output_y')(input_x)
model = Model(inputs=[input_x], outputs=[output_layer]) # OK
model.predict({'x': np.ones((3, 123), dtype=np.float32)}) # ERROR!
This causes the same error at the predict call.
The closest info I can find relating to my question is this, but it doesn't address the handling of placeholders. What would be the correct way to do this?
I figured out the way. We need to use InputLayer instead of Input.
First the codes that create the demo tensorflow graph PB:
def dump_model(): # just to hide all vars during creation demo
import numpy as np
import sys
import tensorflow as tf
with tf.Graph().as_default() as gf:
x = tf.placeholder(tf.float32, shape=(None, 123), name='x')
b = tf.placeholder(tf.float32, shape=(None, 123), name='b')
c = tf.constant(100, dtype=tf.float32, name='C')
y = tf.multiply(x, c, name='y')
z = tf.add(y, x, name='z')
print(x, b, c, y, z)
with tf.gfile.GFile("test_graph/y.pb", "wb") as f:
raw = gf.as_graph_def().SerializeToString()
print(type(raw), len(raw))
f.write(raw)
dump_model()
Then import the graph and find the input/output tensors:
import numpy as np
import sys
import tensorflow as tf
persisted_sess = tf.Session()
with tf.Session().as_default() as session:
with tf.gfile.FastGFile("./test_graph/y.pb",'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
persisted_sess.graph.as_default()
tf.import_graph_def(graph_def, name='')
print(persisted_sess.graph.get_name_scope())
for i, op in enumerate(persisted_sess.graph.get_operations()):
tensor = persisted_sess.graph.get_tensor_by_name(op.name + ':0')
print(i, '\t', op.name, op.type, tensor)
x_tensor = persisted_sess.graph.get_tensor_by_name('x:0')
b_tensor = persisted_sess.graph.get_tensor_by_name('b:0')
y_tensor = persisted_sess.graph.get_tensor_by_name('y:0')
z_tensor = persisted_sess.graph.get_tensor_by_name('z:0')
Then we can create the keras model and make inference:
from tensorflow.keras.layers import Lambda, InputLayer
from tensorflow.keras import Model
from tensorflow.python.util import nest
from tensorflow.python.keras.utils import layer_utils
input_x = InputLayer(name='x', input_tensor=x_tensor)
input_x.is_placeholder = True # this is the critical bits
input_b = InputLayer(name='b2', input_tensor=b_tensor) # note the keras name can be different than the tf name
input_b.is_placeholder = True
output_y = Lambda(lambda x: y_tensor, name='output_y')(input_x.output)
output_z = Lambda(lambda x_b: z_tensor, name='output_z')([input_x.output, input_b.output])
base_model_inputs = nest.flatten([layer_utils.get_source_inputs(input_x.output),
layer_utils.get_source_inputs(input_b.output)])
base_model = Model(base_model_inputs, [output_y, output_z])
y_out, z_out = base_model.predict({'x': np.ones((3, 123), dtype=np.float32),
'b2': np.full((3, 123), 100.0, dtype=np.float32)})
y_out.shape, z_out.shape
And we can even create a new model from the base model:
from tensorflow.keras.layers import Add
derived_output = Add(name='derived')([output_y, output_z])
derived_model = Model(base_model.inputs, [derived_output])
derived_out = derived_model.predict({'x': np.ones((3, 123), dtype=np.float32),
'b2': np.full((3, 123), 100.0, dtype=np.float32)})
derived_out.shape
I had an issue with colocate information in the GraphDef.
Here are the high-level steps I follow
Train an Estimator using tf.train.ExponentialMovingAverage and use the EMA predictions for the PREDICT mode
Export to SavedModel
Reload the GraphDef from the SavedModel and remove unnecessary nodes with extract_sub_graph
Freeze the resulting graph (make variables into constants using checkpoint data) with freeze_graph_with_def_protos
At step 4. I get an error ValueError: Node 'layer/kernel/ExponentialMovingAverage' expects to be colocated with unknown node 'layer/kernel'
Here is the code I use to train the model
# train.py
import logging
from pathlib import Path
import sys
import tensorflow as tf
def ema_getter(ema):
def _ema_getter(getter, name, *args, **kwargs):
var = getter(name, *args, **kwargs)
ema_var = ema.average(var)
return ema_var if ema_var else var
return _ema_getter
def model_fn(features, labels, mode, params):
# pylint: disable=unused-argument
"""Dummy model_fn"""
if isinstance(features, dict): # For serving
features = features['feature']
predictions = tf.layers.dense(features, 1, name="layer")
predictions = tf.identity(predictions, name="predictions")
ema = tf.train.ExponentialMovingAverage(1.0)
variables = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name)
ema_op = ema.apply(variables)
with tf.variable_scope(tf.get_variable_scope(), reuse=True, custom_getter=ema_getter(ema)):
predictions_ema = tf.layers.dense(features, 1, name="layer")
predictions_ema = tf.identity(predictions_ema, name="predictions_ema")
if mode == tf.estimator.ModeKeys.PREDICT:
preds = {
"predictions_ema": predictions_ema
}
return tf.estimator.EstimatorSpec(mode, predictions=preds)
else:
loss = tf.nn.l2_loss(predictions - labels)
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(
mode, loss=loss)
elif mode == tf.estimator.ModeKeys.TRAIN:
train_op = tf.train.AdamOptimizer(learning_rate=0.5).minimize(
loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(
mode, loss=loss, train_op=tf.group([train_op, ema_op]))
else:
raise NotImplementedError()
def train_generator_fn():
for number in range(100):
yield [number, number], [2 * number]
def train_input_fn():
shapes, types = (2, 1), (tf.float32, tf.float32)
dataset = tf.data.Dataset.from_generator(
train_generator_fn, output_types=types, output_shapes=shapes)
dataset = dataset.batch(20).repeat(200)
return dataset
def serving_input_receiver_fn():
"""Serving input_fn that builds features from placeholders
Returns
-------
tf.estimator.export.ServingInputReceiver
"""
number = tf.placeholder(dtype=tf.float32, shape=[None, 1], name='number')
receiver_tensors = {'number': number}
features = tf.tile(number, multiples=[1, 2])
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
if __name__ == '__main__':
# Logging
Path('model').mkdir(exist_ok=True)
tf.logging.set_verbosity(logging.INFO)
handlers = [
logging.FileHandler('model/train.log'),
logging.StreamHandler(sys.stdout)
]
logging.getLogger('tensorflow').handlers = handlers
# Train estimator
estimator = tf.estimator.Estimator(model_fn, 'model', params={})
estimator.train(train_input_fn)
# Export
estimator.export_saved_model('saved_model', serving_input_receiver_fn)
and the code I use to optimize the graph
# optimize.py
from pathlib import Path
import tensorflow as tf
from tensorflow.python.tools.freeze_graph import freeze_graph_with_def_protos
from tensorflow.python.framework.graph_util import extract_sub_graph
from tensorflow.core.framework.graph_pb2 import GraphDef
def optimize_and_export(export_dir: str, output: str):
with tf.Session() as sess:
g = tf.saved_model.loader.load(sess, ["serve"], export_dir)
inference_graph = extract_sub_graph(g.graph_def, ["predictions_ema"])
g = freeze_graph_with_def_protos(
inference_graph,
None,
None,
"predictions_ema",
None,
None,
None,
None,
None,
input_saved_model_dir=export_dir,
saved_model_tags=["serve"],
)
tf.io.write_graph(g, logdir=str(Path(output).parent), name=Path(output).name, as_text=False)
if __name__ == '__main__':
export_dir = str(sorted(Path('saved_model').glob('*'))[0])
print(f"Reloading from {export_dir}")
optimize_and_export(export_dir, 'saved_model/final')
My understanding is that tf.train.ExponentialMovingAverage adds colocation information between the nodes (the orignal and the EMA version).
NB: colocation seems to mean "these variables should be located on the same device"
This colocate information is present in the graph protobuf (the SavedModel export).
When extracting the subgraph, only the EMA versions of the variables are kept, but the colocation information is preserved, which causes issues when creating the Graph, which tries to find the original colocated variables (not present anymore).
I found a way around by modifying the protobuf manually and removing all colocation information with
def optimize_and_export(export_dir: str, output: str):
with tf.Session() as sess:
g = tf.saved_model.loader.load(sess, ["serve"], export_dir)
inference_graph = extract_sub_graph(g.graph_def, ["predictions_ema"])
# Remove colocate information from GraphDef
for node in inference_graph.node:
if "_class" in node.attr:
del node.attr["_class"]
tf.logging.warning(f"Removing _class attr of {node.name}")
g = freeze_graph_with_def_protos(
inference_graph,
None,
None,
"predictions_ema",
None,
None,
None,
None,
None,
input_saved_model_dir=export_dir,
saved_model_tags=["serve"],
)
tf.io.write_graph(g, logdir=str(Path(output).parent), name=Path(output).name, as_text=False)
I am using the TensorFlow Object Detection API. I trained a model and extracted a graph, but during inference I get messages that seem like prints of the graph layers and training parameters. Here's a taste:
Optimizing fused batch norm node name: "SecondStageFeatureExtractor/InceptionV2/Mixed_5c/Branch_3/Conv2d_0b_1x1/BatchNorm/FusedBatchNorm"
op: "FusedBatchNorm"
input: "SecondStageFeatureExtractor/InceptionV2/Mixed_5c/Branch_3/Conv2d_0b_1x1/Conv2D"
input: "SecondStageFeatureExtractor/InceptionV2/Mixed_5c/Branch_3/Conv2d_0b_1x1/BatchNorm/gamma"
input: "SecondStageFeatureExtractor/InceptionV2/Mixed_5c/Branch_3/Conv2d_0b_1x1/BatchNorm/beta"
input:
"SecondStageFeatureExtractor/InceptionV2/Mixed_5c/Branch_3/Conv2d_0b_1x1/BatchNorm/moving_mean"
input:
"SecondStageFeatureExtractor/InceptionV2/Mixed_5c/Branch_3/Conv2d_0b_1x1/BatchNorm/moving_variance"
device: "/job:localhost/replica:0/task:0/device:GPU:0"
attr { key:
"T" value {
type: DT_FLOAT } }
attr { key: "data_format" value {
s: "NHWC" } }
And many more.
How can I suppress these messages?
I already tried os.environ['TF_CPP_MIN_LOG_LEVEL'] and tf.logging.set_verbosity(tf.logging.ERROR). They do not suppress the object detection API messages.
My inference code is similar to the one presented here. Here is my code:
detection_graph = tf.Graph()
with detection_graph.as_default():
od_graph_def = tf.GraphDef()
with tf.gfile.GFile(PATH_TO_FROZEN_GRAPH, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
images = glob.glob(TEST_IMAGE_PATHS)
fig, ax = plt.subplots(1)
with detection_graph.as_default():
with tf.Session() as sess:
# Get handles to input and output tensors
ops = tf.get_default_graph().get_operations()
all_tensor_names = {output.name for op in ops for output in op.outputs}
tensor_dict = {}
for key in ['num_detections', 'detection_boxes', 'detection_scores', 'detection_classes']:
tensor_name = key + ':0'
if tensor_name in all_tensor_names:
tensor_dict[key] = tf.get_default_graph().get_tensor_by_name(tensor_name)
image_tensor = tf.get_default_graph().get_tensor_by_name('image_tensor:0')
# Run inference
for image_filename in images:
image = nd.imread(image_filename)
image = imresize(image, 50)
output_dict = sess.run(tensor_dict, feed_dict={image_tensor: np.expand_dims(image, 0)})
# all outputs are float32 numpy arrays, so convert types as appropriate
boxes = output_dict['detection_boxes'][0]
plt.imshow(image)
shape = image.shape
for i in range(int(output_dict['num_detections'][0])):
box_y = boxes[i][0]*shape[0]
box_x = boxes[i][1]*shape[1]
box_h = (boxes[i][2] - boxes[i][0])*shape[0]
box_w = (boxes[i][3] - boxes[i][1])*shape[1]
ax.add_patch(patches.Rectangle((box_x, box_y), box_w, box_h, linewidth=1, edgecolor='r', facecolor='none'))
fig.show()
plt.waitforbuttonpress()
ax.clear()
I am new to tensorflow and are reading mnist_export.py in tensorflow serving example.
There is something here I cannot understand:
sess = tf.InteractiveSession()
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
x = tf.identity(tf_example['x'], name='x') # use tf.identity() to assign name
Above, serialized_tf_example is a Tensor.
I have read the api document tf.parse_example but it seems that serialized is serialized Example protos like:
serialized = [
features
{ feature { key: "ft" value { float_list { value: [1.0, 2.0] } } } },
features
{ feature []},
features
{ feature { key: "ft" value { float_list { value: [3.0] } } }
]
So how to understand tf_example = tf.parse_example(serialized_tf_example, feature_configs) here as serialized_tf_example is a Tensor, not Example proto?
Here serialized_tf_example is serialized string of a tf.train.Example. See tf.parse_example for the usage. Reading data chapter gives some example link.
tf_example.SerializeToString() converts tf.train.Example to string and tf.parse_example parses the serialized string to a dict.
The below mentioned code provides the simple example of using parse_example
import tensorflow as tf
sess = tf.InteractiveSession()
serialized_tf_example = tf.placeholder(tf.string, shape=[1], name='serialized_tf_example')
feature_configs = {'x': tf.FixedLenFeature(shape=[1], dtype=tf.float32)}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
feature_dict = {'x': tf.train.Feature(float_list=tf.train.FloatList(value=[25]))}
example = tf.train.Example(features=tf.train.Features(feature=feature_dict))
f = example.SerializeToString()
sess.run(tf_example,feed_dict={serialized_tf_example:[f]})