How do you run a validation loop during `slim.learning.train()` - tensorflow

Question

I'm looking at this answer for running evaluation metrics during training:
How to use evaluation_loop with train_loop in tf-slim
and it seems like overriding train_step_fn=train_step_fn is the reasonable approach. But I want to run a validation loop, not evaluation. My graph is something like this:
with tf.Graph().as_default():
train_dataset = slim.dataset.Dataset(data_sources= "train_*.tfrecord")
train_images, _, train_labels = load_batch(train_dataset,
batch_size=mini_batch_size,
is_training=True)
val_dataset = slim.dataset.Dataset(data_sources= "validation_*.tfrecord")
val_images, _, val_labels = load_batch(val_dataset,
batch_size=mini_batch_size,
is_training=False)
with slim.arg_scope(vgg.vgg_arg_scope(weight_decay=0.0005)):
net, end_points = vgg.vgg_16(train_images,
num_classes=10,
is_training=is_training)
predictions = tf.nn.softmax(net)
labels = train_labels
...
init_fn = slim.assign_from_checkpoint_fn(
checkpoint_path,
slim.get_variables_to_restore(exclude=['vgg_16/fc8']),
ignore_missing_vars=True
)
final_loss = slim.learning.train(train_op, TRAIN_LOG,
train_step_fn=train_step_fn,
init_fn=init_fn,
global_step=global_step,
number_of_steps=steps,
save_summaries_secs=60,
save_interval_secs=600,
session_config=sess_config,
)
I want to add something like this to do a validation loop with a mini-batch against the current weights for the network
def validate_on_checkpoint(sess, *args, **kwargs ):
loss,mean,stddev = sess.run([val_loss, val_rms_mean, val_rms_stddev],
feed_dict={images: val_images,
labels: val_labels,
is_training: is_training })
validation_writer = tf.train.SummaryWriter(LOG_DIR + '/validation')
validation_writer.add_summary(loss, global_step)
validation_writer.add_summary(mean, global_step)
validation_writer.add_summary(stddev, global_step)
def train_step_fn(sess, *args, **kwargs):
total_loss, should_stop = train_step(sess, *args, **kwargs)
if train_step_fn.step % FLAGS.validation_every_n_step == 0:
validate_on_checkpoint(sess, *args, **kwargs )
train_step_fn.step += 1
return [total_loss, should_stop]
but I got an error=Graph is finalized and cannot be modified.
Conceptually I'm not sure how I should add this. The training loop needs to needs gradients, dropouts, and weight updates for the net, but the validation loop skips all of that. I keep getting variations on Graph is finalized and cannot be modified. if I try to modify the Graph or XXX is not defined if I use an if is_training: else: approach

Answer 1

I figured out one way to make this work from a few other stackoverflow answers. Here are the basics:
1) get inputs and labels for both train and validation datasets
x_train, y_train = produce_batch(320)
x_validation, y_validation = produce_batch(320)
2) use reuse=True to reuse model weights between the train and validation loop. Here's one way:
with tf.variable_scope("model") as scope:
# Make the model, reuse weights for validation batches
predictions, nodes = regression_model(inputs, is_training=True)
scope.reuse_variables()
val_predictions, _ = regression_model(val_inputs, is_training=False)
3) define your losses, put your validation losses in a different collection so it does not get added to train losses in tf.losses.get_losses()
loss = tf.losses.mean_squared_error(labels=targets, predictions=predictions)
total_loss = tf.losses.get_total_loss()
val_loss = tf.losses.mean_squared_error(labels=val_targets, predictions=val_predictions,
loss_collection="validation"
)
4) define a train_step_fn() to trigger the validation loop, as necessary
VALIDATION_INTERVAL = 1000 . # validate every 1000 steps
# slim.learning.train(train_step_fn=train_step_fn)
def train_step_fn(sess, train_op, global_step, train_step_kwargs):
"""
slim.learning.train_step():
train_step_kwargs = {summary_writer:, should_log:, should_stop:}
"""
train_step_fn.step += 1 # or use global_step.eval(session=sess)
# calc training losses
total_loss, should_stop = slim.learning.train_step(sess, train_op, global_step, train_step_kwargs)
# validate on interval
if train_step_fn.step % VALIDATION_INTERVAL == 0:
validiate_loss, validation_delta = sess.run([val_loss, summary_validation_delta])
print(">> global step {}: train={} validation={} delta={}".format(train_step_fn.step,
total_loss, validiate_loss, validiate_loss-total_loss))
return [total_loss, should_stop]
train_step_fn.step = 0
5) add train_step_fn() to your training loop
# Run the training inside a session.
final_loss = slim.learning.train(
train_op,
train_step_fn=train_step_fn,
...
)
see the complete results in this Colaboratory notebook