Error while sequentially exporting two different inference graphs in tensorflow object detection API - tensorflow

Question

I am using Tensorflow object detection API and I have trained two separate models( FRCNN Inception V2 and SSD Mobilenet V2). In my code flow, when both of the models have been trained, I need to export inference graphs. The following is the code for the same:
# Dependencies
import tensorflow as tf
import glob
import os
import re
from google.protobuf import text_format
from object_detection import exporter
from object_detection.protos import pipeline_pb2
class ExportInferenceGraph():
def __init__(self):
# input parameters for exporter
self.pipeline_config_path = None
self.trained_checkpoint_prefix = None
self.output_directory = None
#############################################################################
'''
code used form export_inference_graph.py file from Tensorflow github
'''
#############################################################################
flags = tf.app.flags
flags.DEFINE_string('input_type', 'image_tensor', 'Type of input node. Can be '
'one of [`image_tensor`, `encoded_image_string_tensor`, '
'`tf_example`]')
flags.DEFINE_string('input_shape', None,
'If input_type is `image_tensor`, this can explicitly set '
'the shape of this input tensor to a fixed size. The '
'dimensions are to be provided as a comma-separated list '
'of integers. A value of -1 can be used for unknown '
'dimensions. If not specified, for an `image_tensor, the '
'default shape will be partially specified as '
'`[None, None, None, 3]`.')
flags.DEFINE_string('config_override', '',
'pipeline_pb2.TrainEvalPipelineConfig '
'text proto to override pipeline_config_path.')
flags.DEFINE_boolean('write_inference_graph', False,
'If true, writes inference graph to disk.')
self.FLAGS = flags.FLAGS
#############################################################################
# method to get latest checkpoint files
def get_latest_checkpoints(self, trainingDir):
# getting list of all meta files
metaFiles = glob.glob(trainingDir + '/*.meta')
tempList = []
# sorting based on num_steps
for _file in metaFiles:
tempList.append(int(re.findall(r'[0-9]+', os.path.basename(_file))[0]))
tempList.sort(reverse = True)
# returning path of latest checkpoint file
return trainingDir + '/model.ckpt-' + str(tempList[0])
# parsing flags and exporting graphs
def export(self, pipeline_config_path, trained_checkpoint_dir, output_directory):
# path to store exported inference graphs
if not os.path.exists(output_directory):
os.makedirs(output_directory)
# getting latest checkpoint file
self.trained_checkpoint_prefix = self.get_latest_checkpoints(trained_checkpoint_dir)
print(self.trained_checkpoint_prefix)
self.pipeline_config_path = pipeline_config_path
self.output_directory = output_directory
#############################################################################
'''
code used form export_inference_graph.py file from Tensorflow
'''
#############################################################################
pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
with tf.gfile.GFile(self.pipeline_config_path, 'r') as f:
text_format.Merge(f.read(), pipeline_config)
text_format.Merge(self.FLAGS.config_override, pipeline_config)
if self.FLAGS.input_shape:
input_shape = [
int(dim) if dim != '-1' else None
for dim in self.FLAGS.input_shape.split(',')
]
else:
input_shape = None
exporter.export_inference_graph(
self.FLAGS.input_type, pipeline_config, self.trained_checkpoint_prefix,
self.output_directory, input_shape = input_shape,
write_inference_graph = self.FLAGS.write_inference_graph)
#############################################################################
This is the code snippet which shows how I call the methods for two different models:
export = ExportInferenceGraph()
export.export(pipeline_config_path = TRAIN_CONFIG_FILES[0],
trained_checkpoint_dir = MODEL_TRAIN_DIRS[0],
output_directory = INFERENCE_SUB_DIRS[0])
export.export(pipeline_config_path = TRAIN_CONFIG_FILES[1],
trained_checkpoint_dir = MODEL_TRAIN_DIRS[1],
output_directory = INFERENCE_SUB_DIRS[1])
But I am getting the following error:
2020-01-22 17:42:47.520891: W tensorflow/core/framework/op_kernel.cc:1502] OP_REQUIRES failed at save_restore_v2_ops.cc:184 : Not
found: Key BoxPredictor_0/BoxEncodingPredictor/biases not found in checkpoint
Traceback (most recent call last):
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\client\session.py", line 1356, in _do_call
return fn(*args)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\client\session.py", line 1341, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\client\session.py", line 1429, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.NotFoundError: Key BoxPredictor_0/BoxEncodingPredictor/biases not found in checkpoint
[[{{node save_1/RestoreV2}}]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\training\saver.py", line 1286, in restore
{self.saver_def.filename_tensor_name: save_path})
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\client\session.py", line 950, in run
run_metadata_ptr)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\client\session.py", line 1173, in _run
feed_dict_tensor, options, run_metadata)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\client\session.py", line 1350, in _do_run
run_metadata)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\client\session.py", line 1370, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.NotFoundError: Key BoxPredictor_0/BoxEncodingPredictor/biases not found in checkpoint
[[node save_1/RestoreV2 (defined at ../TensorFlow/models/research\object_detection\exporter.py:323) ]]
Original stack trace for 'save_1/RestoreV2':
File "ocr_train.py", line 99, in <module>
output_directory = INFERENCE_SUB_DIRS[1])
File "D:\shubham\wsp\OCR\export_inference_graph.py", line 109, in export
write_inference_graph = self.FLAGS.write_inference_graph)
File "../TensorFlow/models/research\object_detection\exporter.py", line 489, in export_inference_graph
write_inference_graph=write_inference_graph)
File "../TensorFlow/models/research\object_detection\exporter.py", line 418, in _export_inference_graph
trained_checkpoint_prefix=checkpoint_to_use)
File "../TensorFlow/models/research\object_detection\exporter.py", line 323, in write_graph_and_checkpoint
tf.import_graph_def(inference_graph_def, name='')
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\util\deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\importer.py", line 443, in import_graph_def
_ProcessNewOps(graph)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\importer.py", line 236, in _ProcessNewOps
for new_op in graph._add_new_tf_operations(compute_devices=False): # pylint: disable=protected-access
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\ops.py", line 3751, in _add_new_tf_operations
for c_op in c_api_util.new_tf_operations(self)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\ops.py", line 3751, in <listcomp>
for c_op in c_api_util.new_tf_operations(self)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\ops.py", line 3641, in _create_op_from_tf_operation
ret = Operation(c_op, self)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\ops.py", line 2005, in __init__
self._traceback = tf_stack.extract_stack()
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\training\saver.py", line 1296, in restore
names_to_keys = object_graph_key_mapping(save_path)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\training\saver.py", line 1614, in object_graph_key_mapping
object_graph_string = reader.get_tensor(trackable.OBJECT_GRAPH_PROTO_KEY)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\pywrap_tensorflow_internal.py", line 678, in get_tensor
return CheckpointReader_GetTensor(self, compat.as_bytes(tensor_str))
tensorflow.python.framework.errors_impl.NotFoundError: Key _CHECKPOINTABLE_OBJECT_GRAPH not found in checkpoint
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "ocr_train.py", line 99, in <module>
output_directory = INFERENCE_SUB_DIRS[1])
File "D:\shubham\wsp\OCR\export_inference_graph.py", line 109, in export
write_inference_graph = self.FLAGS.write_inference_graph)
File "../TensorFlow/models/research\object_detection\exporter.py", line 489, in export_inference_graph
write_inference_graph=write_inference_graph)
File "../TensorFlow/models/research\object_detection\exporter.py", line 418, in _export_inference_graph
trained_checkpoint_prefix=checkpoint_to_use)
File "../TensorFlow/models/research\object_detection\exporter.py", line 327, in write_graph_and_checkpoint
saver.restore(sess, trained_checkpoint_prefix)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\training\saver.py", line 1302, in restore
err, "a Variable name or other graph key that is missing")
tensorflow.python.framework.errors_impl.NotFoundError: Restoring from checkpoint failed. This is most likely due to a Variable name or other graph key that is missing from the checkpoint. Please ensure that you have not altered the graph expected based on the
checkpoint. Original error:
Key BoxPredictor_0/BoxEncodingPredictor/biases not found in checkpoint
[[node save_1/RestoreV2 (defined at ../TensorFlow/models/research\object_detection\exporter.py:323) ]]
Original stack trace for 'save_1/RestoreV2':
File "ocr_train.py", line 99, in <module>
output_directory = INFERENCE_SUB_DIRS[1])
File "D:\shubham\wsp\OCR\export_inference_graph.py", line 109, in export
write_inference_graph = self.FLAGS.write_inference_graph)
File "../TensorFlow/models/research\object_detection\exporter.py", line 489, in export_inference_graph
write_inference_graph=write_inference_graph)
File "../TensorFlow/models/research\object_detection\exporter.py", line 418, in _export_inference_graph
trained_checkpoint_prefix=checkpoint_to_use)
File "../TensorFlow/models/research\object_detection\exporter.py", line 323, in write_graph_and_checkpoint
tf.import_graph_def(inference_graph_def, name='')
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\util\deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\importer.py", line 443, in import_graph_def
_ProcessNewOps(graph)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\importer.py", line 236, in _ProcessNewOps
for new_op in graph._add_new_tf_operations(compute_devices=False): # pylint: disable=protected-access
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\ops.py", line 3751, in _add_new_tf_operations
for c_op in c_api_util.new_tf_operations(self)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\ops.py", line 3751, in <listcomp>
for c_op in c_api_util.new_tf_operations(self)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\ops.py", line 3641, in _create_op_from_tf_operation
ret = Operation(c_op, self)
File "C:\Users\Lenovo\Anaconda3\envs\wsp\lib\site-packages\tensorflow\python\framework\ops.py", line 2005, in __init__
self._traceback = tf_stack.extract_stack()
NOTE: I could see the first graph exported successfully but it throws the error for the second graph. I interchanged calling of the export methods, it still failed at second export.
I am still new to Tensorflow and need some help here. I guess it is using the same graph which it created for the first model.

Answer 1

I dig deep into the Tensorflow directory and reached to method _export_inference_graph. The path is TensorFlow/models/research/object_detection/exporter.py. Adding this line at the end of the function solved my problem.
tf.reset_default_graph()