Related
I have an exported Frozen Graph .pb file, and converted it to tflite using
graph_def_file = "model.pb"
input_arrays = ["Placeholder"]
input_shape = {"Placeholder": [1024, 2048, 3]}
output_arrays = ["final_output"]
converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph(
graph_def_file, input_arrays, output_arrays, input_shape)
tflite_model = converter.convert()
with open("my_model.tflite", "wb") as f:
f.write(tflite_model)
And I could not restore my model with
interpreter = tf.lite.Interpreter(model_path="my_model.tflite")
interpreter.resize_tensor_input(0, [1024, 2048, 3], strict=True)
interpreter.allocate_tensors()
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-28-3cfaee7dc51c> in <module>
1 interpreter = tf.lite.Interpreter(model_path=model_path)
2 interpreter.resize_tensor_input(0, [1024, 2048, 3], strict=True)
----> 3 interpreter.allocate_tensors()
~/.local/lib/python3.9/site-packages/tensorflow/lite/python/interpreter.py in allocate_tensors(self)
512 def allocate_tensors(self):
513 self._ensure_safe()
--> 514 return self._interpreter.AllocateTensors()
515
516 def _safe_to_run(self):
RuntimeError: tensorflow/lite/kernels/conv.cc:349 input->dims->data[3] != filter->dims->data[3] (65 != 64)Node number 11 (CONV_2D) failed to prepare.Failed to apply the default TensorFlow Lite delegate indexed at 0.
However, I can successfully reload my .pb file frozen graph by using
with tf.gfile.GFile('my_model.pb', "rb") as pb:
graph_def = tf.GraphDef()
graph_def.ParseFromString(pb.read())
with tf.Graph().as_default() as graph:
tf.import_graph_def(
graph_def,
name="",
)
and also successfully generated result by
node_in = graph.get_tensor_by_name('Placeholder:0')
node_out = graph.get_tensor_by_name('final_output:0')
with tf.Session(graph=graph) as sess: # Session()
# sess.run(tf.global_variables_initializer())
feed_dict = {node_in: input_img}
pred = sess.run(node_out, feed_dict)
print(pred)
sess.close()
I've checked the node 11 of the .tflite file in Netron, but everything seemed fine.
Node 11
What could be the problem?
I had an issue with colocate information in the GraphDef.
Here are the high-level steps I follow
Train an Estimator using tf.train.ExponentialMovingAverage and use the EMA predictions for the PREDICT mode
Export to SavedModel
Reload the GraphDef from the SavedModel and remove unnecessary nodes with extract_sub_graph
Freeze the resulting graph (make variables into constants using checkpoint data) with freeze_graph_with_def_protos
At step 4. I get an error ValueError: Node 'layer/kernel/ExponentialMovingAverage' expects to be colocated with unknown node 'layer/kernel'
Here is the code I use to train the model
# train.py
import logging
from pathlib import Path
import sys
import tensorflow as tf
def ema_getter(ema):
def _ema_getter(getter, name, *args, **kwargs):
var = getter(name, *args, **kwargs)
ema_var = ema.average(var)
return ema_var if ema_var else var
return _ema_getter
def model_fn(features, labels, mode, params):
# pylint: disable=unused-argument
"""Dummy model_fn"""
if isinstance(features, dict): # For serving
features = features['feature']
predictions = tf.layers.dense(features, 1, name="layer")
predictions = tf.identity(predictions, name="predictions")
ema = tf.train.ExponentialMovingAverage(1.0)
variables = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name)
ema_op = ema.apply(variables)
with tf.variable_scope(tf.get_variable_scope(), reuse=True, custom_getter=ema_getter(ema)):
predictions_ema = tf.layers.dense(features, 1, name="layer")
predictions_ema = tf.identity(predictions_ema, name="predictions_ema")
if mode == tf.estimator.ModeKeys.PREDICT:
preds = {
"predictions_ema": predictions_ema
}
return tf.estimator.EstimatorSpec(mode, predictions=preds)
else:
loss = tf.nn.l2_loss(predictions - labels)
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(
mode, loss=loss)
elif mode == tf.estimator.ModeKeys.TRAIN:
train_op = tf.train.AdamOptimizer(learning_rate=0.5).minimize(
loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(
mode, loss=loss, train_op=tf.group([train_op, ema_op]))
else:
raise NotImplementedError()
def train_generator_fn():
for number in range(100):
yield [number, number], [2 * number]
def train_input_fn():
shapes, types = (2, 1), (tf.float32, tf.float32)
dataset = tf.data.Dataset.from_generator(
train_generator_fn, output_types=types, output_shapes=shapes)
dataset = dataset.batch(20).repeat(200)
return dataset
def serving_input_receiver_fn():
"""Serving input_fn that builds features from placeholders
Returns
-------
tf.estimator.export.ServingInputReceiver
"""
number = tf.placeholder(dtype=tf.float32, shape=[None, 1], name='number')
receiver_tensors = {'number': number}
features = tf.tile(number, multiples=[1, 2])
return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)
if __name__ == '__main__':
# Logging
Path('model').mkdir(exist_ok=True)
tf.logging.set_verbosity(logging.INFO)
handlers = [
logging.FileHandler('model/train.log'),
logging.StreamHandler(sys.stdout)
]
logging.getLogger('tensorflow').handlers = handlers
# Train estimator
estimator = tf.estimator.Estimator(model_fn, 'model', params={})
estimator.train(train_input_fn)
# Export
estimator.export_saved_model('saved_model', serving_input_receiver_fn)
and the code I use to optimize the graph
# optimize.py
from pathlib import Path
import tensorflow as tf
from tensorflow.python.tools.freeze_graph import freeze_graph_with_def_protos
from tensorflow.python.framework.graph_util import extract_sub_graph
from tensorflow.core.framework.graph_pb2 import GraphDef
def optimize_and_export(export_dir: str, output: str):
with tf.Session() as sess:
g = tf.saved_model.loader.load(sess, ["serve"], export_dir)
inference_graph = extract_sub_graph(g.graph_def, ["predictions_ema"])
g = freeze_graph_with_def_protos(
inference_graph,
None,
None,
"predictions_ema",
None,
None,
None,
None,
None,
input_saved_model_dir=export_dir,
saved_model_tags=["serve"],
)
tf.io.write_graph(g, logdir=str(Path(output).parent), name=Path(output).name, as_text=False)
if __name__ == '__main__':
export_dir = str(sorted(Path('saved_model').glob('*'))[0])
print(f"Reloading from {export_dir}")
optimize_and_export(export_dir, 'saved_model/final')
My understanding is that tf.train.ExponentialMovingAverage adds colocation information between the nodes (the orignal and the EMA version).
NB: colocation seems to mean "these variables should be located on the same device"
This colocate information is present in the graph protobuf (the SavedModel export).
When extracting the subgraph, only the EMA versions of the variables are kept, but the colocation information is preserved, which causes issues when creating the Graph, which tries to find the original colocated variables (not present anymore).
I found a way around by modifying the protobuf manually and removing all colocation information with
def optimize_and_export(export_dir: str, output: str):
with tf.Session() as sess:
g = tf.saved_model.loader.load(sess, ["serve"], export_dir)
inference_graph = extract_sub_graph(g.graph_def, ["predictions_ema"])
# Remove colocate information from GraphDef
for node in inference_graph.node:
if "_class" in node.attr:
del node.attr["_class"]
tf.logging.warning(f"Removing _class attr of {node.name}")
g = freeze_graph_with_def_protos(
inference_graph,
None,
None,
"predictions_ema",
None,
None,
None,
None,
None,
input_saved_model_dir=export_dir,
saved_model_tags=["serve"],
)
tf.io.write_graph(g, logdir=str(Path(output).parent), name=Path(output).name, as_text=False)
(I'm on TF 1.7 right now, in case that matters.)
I'm trying to initialize and then save a model and associated metagraph in one script (init.py) so that I can load the model and resume training from a second script (train.py). The model is initialized with placeholders for training example and label, to be replaced with real tensors during training. Yet when I try create some real tensors in train.py (from a Dataset), I get a stack trace to the effect that my iterator hasn't been initialized. The trace points to the import_meta_graph() call and happens the same whether I use a oneshot iterator (which shouldn't require initialization) or an initializable iterator which I am actually initializing.
Am I missing something, conceptually, for how two graphs get spliced together?
I want to believe that this is a common use case for saving and restoring metagraphs, but I can't find any examples of it on the internet. How do others feed their real data into a restored model?
Caused by op 'IteratorGetNext_1', defined at:
File "src/tictactoe/train.py", line 47, in <module>
meta_graph, input_map={'example': example, 'label': label})
File "/home/mason/dev/rust/seraphim/lib/python3.5/site-packages/tensorflow/python/training/saver.py", line 1927, in import_meta_graph
**kwargs)
File "/home/mason/dev/rust/seraphim/lib/python3.5/site-packages/tensorflow/python/framework/meta_graph.py", line 741, in import_scoped_meta_graph
producer_op_list=producer_op_list)
File "/home/mason/dev/rust/seraphim/lib/python3.5/site-packages/tensorflow/python/util/deprecation.py", line 432, in new_func
return func(*args, **kwargs)
File "/home/mason/dev/rust/seraphim/lib/python3.5/site-packages/tensorflow/python/framework/importer.py", line 577, in import_graph_def
op_def=op_def)
File "/home/mason/dev/rust/seraphim/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 3290, in create_op
op_def=op_def)
File "/home/mason/dev/rust/seraphim/lib/python3.5/site-packages/tensorflow/python/framework/ops.py", line 1654, in __init__
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
FailedPreconditionError (see above for traceback): GetNext() failed because theiterator has not been initialized. Ensure that you have run the initializer operation for this iterator before getting the next element.
[[Node: IteratorGetNext_1 = IteratorGetNext[output_shapes=[[?,19], [?,9]], output_types=[DT_UINT8, DT_FLOAT], _device="/job:localhost/replica:0/task:0/device:CPU:0"](Iterator_1)]]
Complete code of both scripts here:
# init.py
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from pathlib import Path
import argparse
import os
import tensorflow as tf
parser = argparse.ArgumentParser(description='Initialize a TicTacToe expert model.')
parser.add_argument('name', metavar='foo-model', help='Model prefix')
args = parser.parse_args()
model_dir = "src/tictactoe/saved_models/" + args.name + "/" + args.name
with tf.Session() as sess:
example = tf.placeholder(tf.uint8, shape=[1, 9 * 2 + 1], name ='example')
label = tf.placeholder(tf.float32, shape=[1, 9], name='label')
dense = tf.layers.dense(tf.cast(example, tf.float32), units=64, activation=tf.nn.relu)
logits = tf.layers.dense(dense, units=9, activation=tf.nn.relu)
softmax = tf.nn.softmax(logits, name='softmax')
tf.add_to_collection('softmax', softmax)
sess = tf.Session()
init = tf.group(
tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init)
loss = tf.losses.mean_squared_error(labels=label, predictions=softmax)
optimizer = tf.train.GradientDescentOptimizer(.01)
train = optimizer.minimize(loss, name='train')
tf.add_to_collection('train', train)
saver = tf.train.Saver()
saved = saver.save(sess, model_dir, global_step=0)
print("Model saved in path: %s" % saved)
Here's the training script.
# train.py
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from pathlib import Path
import argparse
import glob
import os
import tensorflow as tf
parser = argparse.ArgumentParser(description='Initialize a TicTacToe expert model.')
parser.add_argument('name', metavar='foo-model', help='Model prefix')
args = parser.parse_args()
model_dir = "src/tictactoe/saved_models/" + args.name
saver_prefix = "src/tictactoe/saved_models/" + args.name + "/" + args.name
latest_checkpoint = tf.train.latest_checkpoint(model_dir)
meta_graph = ".".join([latest_checkpoint, "meta"])
num_epochs = 100
minibatch_size = 128
dataset_dir = "src/tictactoe/gamedata"
def make_dataset(minibatch_size, dataset_dir):
files = glob.glob("{}/*.tfrecord".format(dataset_dir))
print(files)
dataset = tf.data.TFRecordDataset(files)
dataset = dataset.map(parse)
dataset = dataset.shuffle(buffer_size=100000)
dataset = dataset.batch(minibatch_size)
return dataset
def parse(bytes):
features = {"game": tf.FixedLenFeature((), tf.string),
"choice": tf.FixedLenSequenceFeature((), tf.float32, allow_missing=True)}
parsed_features = tf.parse_single_example(bytes, features)
game = tf.decode_raw(parsed_features["game"], tf.uint8)
choice = parsed_features["choice"]
return tf.reshape(game, [19]), tf.reshape(choice, [9])
with tf.Session() as sess:
dataset = make_dataset(minibatch_size, dataset_dir)
iterator = dataset.make_initializable_iterator()
sess.run(iterator.initializer)
example, label = iterator.get_next()
saver = tf.train.import_meta_graph(
meta_graph, input_map={'example': example, 'label': label})
print("{}".format(meta_graph))
saver.restore(sess, latest_checkpoint)
print("{}".format(latest_checkpoint))
train_op = tf.get_collection('train_op')[0]
for i in range(num_epochs):
sess.run(iterator.initializer)
while True:
try:
sess.run(train_op)
except tf.errors.OutOfRangeError:
break
print(saver.save(sess, saver_prefix, global_step=step))
I believe I've found the issue. The issue is that my Saver in train.py is saving the real input tensors that I've mapped in. When I try to restore, those real input tensors are restored from the disk, but not initialized.
So: after running input.py one time, the following train.py script successfully trains. But when I run again, the extra input tensors that it's mapped into the graph are restored but not initialized. It's a bit strange because I am mapping them out again when I restore, so I wouldn't think it'd be necessary to initialize them. I found that tf.report_uninitialized_variables() was crucial for debugging the issue.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from pathlib import Path
import argparse
import glob
import os
import tensorflow as tf
parser = argparse.ArgumentParser(description='Initialize a TicTacToe expert model.')
parser.add_argument('name', metavar='foo-model', help='Model prefix')
args = parser.parse_args()
model_dir = "src/tictactoe/saved_models/" + args.name
saver_prefix = "src/tictactoe/saved_models/" + args.name + "/" + args.name
latest_checkpoint = tf.train.latest_checkpoint(model_dir)
meta_graph = ".".join([latest_checkpoint, "meta"])
num_epochs = 100
minibatch_size = 128
dataset_dir = "src/tictactoe/gamedata"
def make_dataset(minibatch_size, dataset_dir):
files = glob.glob("{}/*.tfrecord".format(dataset_dir))
print(files)
dataset = tf.data.TFRecordDataset(files)
dataset = dataset.map(parse)
dataset = dataset.shuffle(buffer_size=100000)
dataset = dataset.batch(minibatch_size)
return dataset
def parse(bytes):
features = {"game": tf.FixedLenFeature((), tf.string),
"choice": tf.FixedLenSequenceFeature((), tf.float32, allow_missing=True)}
parsed_features = tf.parse_single_example(bytes, features)
game = tf.decode_raw(parsed_features["game"], tf.uint8)
choice = parsed_features["choice"]
return tf.reshape(game, [19]), tf.reshape(choice, [9])
with tf.Session() as sess:
dataset = make_dataset(minibatch_size, dataset_dir)
iterator = dataset.make_initializable_iterator()
example, label = iterator.get_next()
# print("before iterator", sess.run(tf.report_uninitialized_variables()))
saver = tf.train.import_meta_graph(meta_graph, input_map={'example': example, 'label': label})
print("{}".format(meta_graph))
saver.restore(sess, latest_checkpoint)
print("{}".format(latest_checkpoint))
train_op = tf.get_collection('train_op')[0]
init = tf.get_collection('init')[0]
for i in range(num_epochs):
sess.run(iterator.initializer)
while True:
try:
sess.run(train_op)
except tf.errors.OutOfRangeError:
break
print(saver.save(sess, saver_prefix))
I'm deploying a text matching tensorflow program on docker referring to the official website, the installation steps and the test steps are all OK, including the server running status as well as OK, but client has a problem.
Let me explain it in detail.
This is my model graph with four inputs:
This is my model server:
import os import shutil
import tensorflow as tf from preprocess import Word2Vec, TestQA, WebQA from ABCNN import ABCNN from utils import build_path import numpy as np Max_len = 40 d0 = 300 ''' Loads the saved bcnn model, injects additional layers for the input transformation and export the model into protobuf format '''
# Command line arguments tf.app.flags.DEFINE_string('checkpoint_dir', './models/',
"Directory where to read training checkpoints.") tf.app.flags.DEFINE_string('output_dir', './models-export',
"Directory where to export the model.") tf.app.flags.DEFINE_integer('model_version', 1,
"Version number of the model.") FLAGS = tf.app.flags.FLAGS
def test(w, l2_reg, epoch, max_len, model_type, num_layers, data_type, classifier, num_classes):
model_path = build_path("./models/", data_type, model_type, num_layers)
model = ABCNN(s=max_len, w=w, l2_reg=l2_reg, model_type=model_type, num_classes=num_classes, num_layers=num_layers)
with tf.Session() as sess:
saver = tf.train.Saver()
print(model_path + "-" + str(12))
saver.restore(sess, model_path + "-" + str(12))
x1 = tf.placeholder(tf.float32, shape=[None, d0, max_len])
x2 = tf.placeholder(tf.float32, shape=[None, d0, max_len])
y = tf.placeholder(tf.int32, shape=[None])
features = tf.placeholder(tf.float32, shape=[None, 4]) #num_features = 4
export_path = os.path.join(
tf.compat.as_bytes(FLAGS.output_dir),
tf.compat.as_bytes(str(FLAGS.model_version)))
if os.path.exists(export_path):
shutil.rmtree(export_path)
# create model builder
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
# create tensors info
inputs = {
"x1": tf.saved_model.utils.build_tensor_info(x1),
"x2": tf.saved_model.utils.build_tensor_info(x2),
"label": tf.saved_model.utils.build_tensor_info(y),
"features": tf.saved_model.utils.build_tensor_info(features)
}
output = {"predict_score": tf.saved_model.utils.build_tensor_info(model.prediction)}
# build prediction signature
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs=inputs,
outputs=output,
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME
)
)
# save the model
legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict_score': prediction_signature
},
legacy_init_op=legacy_init_op)
builder.save()
print("Successfully exported BCNN model version '{}' into '{}'".format(
FLAGS.model_version, FLAGS.output_dir))
def main(_):
# default parameters
params = {
"ws": 4,
"l2_reg": 0.0004,
"epoch": 20,
"max_len": 40,
"model_type": "BCNN",
"num_layers": 2,
"num_classes": 2,
"data_type": "WebQA",
"classifier": "LR",
# "word2vec": Word2Vec()
}
test(w=int(params["ws"]), l2_reg=float(params["l2_reg"]), epoch=int(params["epoch"]),
max_len=int(params["max_len"]), model_type=params["model_type"],
num_layers=int(params["num_layers"]), data_type=params["data_type"],
classifier=params["classifier"], num_classes=params["num_classes"])
if __name__ == '__main__':
tf.app.run()
And this is my client:
import sys sys.path.insert(0, \"./\")
# from tensorflow_serving_client.protos import predict_pb2, prediction_service_pb2_grpc from tensorflow_serving.apis import predict_pb2 from tensorflow_serving.apis import prediction_service_pb2 from grpc.beta import implementations
# import grpc import tensorflow as tf import numpy as np from tensorflow.python.framework import dtypes import time from preprocess import Word2Vec, MSRP, WikiQA,WebQA
im_name = "dir/files" if __name__ == '__main__':
test_data = WebQA(word2vec=Word2Vec(), max_len=40)
test_data.open_file(mode="test")
s1s, s2s, labels, features = test_data.only_batch()
s1s = s1s[0]
s2s = s2s[0]
labels = labels[0]
features = features[0]
print(s1s.shape)
print(s2s.shape)
print(labels.shape)
print(features.shape)
start_time = time.time()
channel = implementations.insecure_channel("localhost", 9000)
stub = prediction_service_pb2.beta_create_PredictionService_stub(channel)
request = predict_pb2.PredictRequest()
request.model_spec.name = "bcnn"
request.model_spec.signature_name = "predict_score"
request.inputs["x1"].CopyFrom(tf.contrib.util.make_tensor_proto(s1s, dtype=dtypes.float32))
request.inputs["x2"].CopyFrom(tf.contrib.util.make_tensor_proto(s2s, dtype=dtypes.float32))
request.inputs["label"].CopyFrom(tf.contrib.util.make_tensor_proto(labels, dtype=dtypes.int32))
request.inputs["features"].CopyFrom(tf.contrib.util.make_tensor_proto(features, dtype=dtypes.float32))
response = stub.Predict(request, 10.0)
results = {}
for key in response.outputs:
tensor_proto = response.outputs[key]
nd_array = tf.contrib.util.make_ndarray(tensor_proto)
results[key] = nd_array
print("cost %ss to predict: " % (time.time() - start_time))
print(results["predict_score"])
And this is the error:
root#15bb1c2766e3:/ABCNN-master# python3 client.py
(1, 300, 40)
(1, 300, 40)
(1,)
(1, 4)
WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/contrib/learn/python/learn/datasets/base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.
Instructions for updating:
Use the retry module or similar alternatives.
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/grpc/beta/_client_adaptations.py", line 193, in _blocking_unary_unary
credentials=_credentials(protocol_options))
File "/usr/local/lib/python3.5/dist-packages/grpc/_channel.py", line 487, in __call__
return _end_unary_response_blocking(state, call, False, deadline)
File "/usr/local/lib/python3.5/dist-packages/grpc/_channel.py", line 437, in _end_unary_response_blocking
raise _Rendezvous(state, None, None, deadline)
grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with (StatusCode.INVALID_ARGUMENT, You must feed a value for placeholder tensor 'x2' with dtype float and shape [?,300,40]
[[Node: x2 = Placeholder[dtype=DT_FLOAT, shape=[?,300,40], _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]])>
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "client.py", line 50, in <module>
response = stub.Predict(request, 10.0)
File "/usr/local/lib/python3.5/dist-packages/grpc/beta/_client_adaptations.py", line 309, in __call__
self._request_serializer, self._response_deserializer)
File "/usr/local/lib/python3.5/dist-packages/grpc/beta/_client_adaptations.py", line 195, in _blocking_unary_unary
raise _abortion_error(rpc_error_call)
grpc.framework.interfaces.face.face.AbortionError: AbortionError(code=StatusCode.INVALID_ARGUMENT, details="You must feed a value for placeholder tensor 'x2' with dtype float and shape [?,300,40]
[[Node: x2 = Placeholder[dtype=DT_FLOAT, shape=[?,300,40], _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]")
ns.py", line 309, in __call__er# on3.5/dist-packages/grpc/beta/_client_adaptatio
I have printed the shape of features: (1,4), why does it still say I have fed the wrong tensor 'features'? I can not get it.
Thanks for your any suggestion in advance.
I have read many similar questions and just cannot get this to work properly.
I have my model being trained well and checkpoint files are being made every epoch. I want to have it so the program can continue from epoch x once reloaded and also for it to print that is on that epoch with every iteration. I could simply save the data outside of the checkpoint file, however I was also wanting to do this to give me confidence everything else is also being stored properly.
Unfortunately the value in the epoch/global_step variable is always still 0 when I restart.
import tensorflow as tf
import numpy as np
import tensorflow as tf
import numpy as np
# more imports
def extract_number(f): # used to get latest checkpint file
s = re.findall("epoch(\d+).ckpt",f)
return (int(s[0]) if s else -1,f)
def restore(init_op, sess, saver): # called to restore or just initialise model
list = glob(os.path.join("./params/e*"))
if list:
file = max(list,key=extract_number)
saver.restore(sess, file[:-5])
sess.run(init_op)
return
with tf.Graph().as_default() as g:
# build models
total_batch = data.train.num_examples / batch_size
epochLimit = 51
saver = tf.train.Saver()
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
saver = tf.train.Saver()
init_op = tf.global_variables_initializer()
restore(init_op, sess, saver)
epoch = global_step.eval()
while epoch < epochLimit:
total_batch = data.train.num_examples / batch_size
for i in range(int(total_batch)):
sys.stdout.flush()
voxels = newData.eval()
batch_z = np.random.uniform(-1, 1, [batch_size, z_size]).astype(np.float32)
sess.run(opt_G, feed_dict={z:batch_z, train:True})
sess.run(opt_D, feed_dict={input:voxels, z:batch_z, train:True})
with open("out/loss.csv", 'a') as f:
batch_loss_G = sess.run(loss_G, feed_dict={z:batch_z, train:False})
batch_loss_D = sess.run(loss_D, feed_dict={input:voxels, z:batch_z, train:False})
msgOut = "Epoch: [{0}], i: [{1}], G_Loss[{2:.8f}], D_Loss[{3:.8f}]".format(epoch, i, batch_loss_G, batch_loss_D)
print(msgOut)
epoch=epoch+1
sess.run(global_step.assign(epoch))
saver.save(sess, "params/epoch{0}.ckpt".format(epoch))
batch_z = np.random.uniform(-1, 1, [batch_size, z_size]).astype(np.float32)
voxels = sess.run(x_, feed_dict={z:batch_z})
v = voxels[0].reshape([32, 32, 32]) > 0
util.save_binvox(v, "out/epoch{0}.vox".format(epoch), 32)
I also update the global step variable using assign at the bottom. Any ideas? Any help would be greatly appreciated.
When you call sess.run(init_op) after restoring this resets all variables to their initial values. Comment that line out and things should work.
My original code was wrong for several reasons because I was trying so many things. The first responder Alexandre Passos gives a valid point, but I believe what changed the game was also the use of scopes (maybe?).
Below is the working updated code if it helps anyone:
import tensorflow as tf
import numpy as np
# more imports
def extract_number(f): # used to get latest checkpint file
s = re.findall("epoch(\d+).ckpt",f)
return (int(s[0]) if s else -1,f)
def restore(sess, saver): # called to restore or just initialise model
list = glob(os.path.join("./params/e*"))
if list:
file = max(list,key=extract_number)
saver.restore(sess, file[:-5])
return saver, True, sess
saver = tf.train.Saver()
init_op = tf.global_variables_initializer()
sess.run(init_op)
return saver, False , sess
batch_size = 100
learning_rate = 0.0001
beta1 = 0.5
z_size = 100
save_interval = 1
data = dataset.read()
total_batch = data.train.num_examples / batch_size
def fill_queue():
for i in range(int(total_batch*epochLimit)):
sess.run(enqueue_op, feed_dict={batch: data.train.next_batch(batch_size)}) # runnig in seperate thread to feed a FIFOqueue
with tf.variable_scope("glob"):
global_step = tf.get_variable(name='global_step', initializer=0,trainable=False)
# build models
epochLimit = 51
saver = tf.train.Saver()
with tf.Session() as sess:
saver,rstr,sess = restore(sess, saver)
with tf.variable_scope("glob", reuse=True):
epocht = tf.get_variable(name='global_step', trainable=False, dtype=tf.int32)
epoch = epocht.eval()
while epoch < epochLimit:
total_batch = data.train.num_examples / batch_size
for i in range(int(total_batch)):
sys.stdout.flush()
voxels = newData.eval()
batch_z = np.random.uniform(-1, 1, [batch_size, z_size]).astype(np.float32)
sess.run(opt_G, feed_dict={z:batch_z, train:True})
sess.run(opt_D, feed_dict={input:voxels, z:batch_z, train:True})
with open("out/loss.csv", 'a') as f:
batch_loss_G = sess.run(loss_G, feed_dict={z:batch_z, train:False})
batch_loss_D = sess.run(loss_D, feed_dict={input:voxels, z:batch_z, train:False})
msgOut = "Epoch: [{0}], i: [{1}], G_Loss[{2:.8f}], D_Loss[{3:.8f}]".format(epoch, i, batch_loss_G, batch_loss_D)
print(msgOut)
epoch=epoch+1
sess.run(global_step.assign(epoch))
saver.save(sess, "params/epoch{0}.ckpt".format(epoch))
batch_z = np.random.uniform(-1, 1, [batch_size, z_size]).astype(np.float32)
voxels = sess.run(x_, feed_dict={z:batch_z})
v = voxels[0].reshape([32, 32, 32]) > 0
util.save_binvox(v, "out/epoch{0}.vox".format(epoch), 32)