Related
I am learning Sagemaker and I have this entry point:
import os
import tensorflow as tf
from tensorflow.python.estimator.model_fn import ModeKeys as Modes
INPUT_TENSOR_NAME = 'inputs'
SIGNATURE_NAME = 'predictions'
LEARNING_RATE = 0.001
def model_fn(features, labels, mode, params):
# Input Layer
input_layer = tf.reshape(features[INPUT_TENSOR_NAME], [-1, 28, 28, 1])
# Convolutional Layer #1
conv1 = tf.layers.conv2d(
inputs=input_layer,
filters=32,
kernel_size=[5, 5],
padding='same',
activation=tf.nn.relu)
# Pooling Layer #1
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)
# Convolutional Layer #2 and Pooling Layer #2
conv2 = tf.layers.conv2d(
inputs=pool1,
filters=64,
kernel_size=[5, 5],
padding='same',
activation=tf.nn.relu)
pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)
# Dense Layer
pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])
dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)
dropout = tf.layers.dropout(
inputs=dense, rate=0.4, training=(mode == Modes.TRAIN))
# Logits Layer
logits = tf.layers.dense(inputs=dropout, units=10)
# Define operations
if mode in (Modes.PREDICT, Modes.EVAL):
predicted_indices = tf.argmax(input=logits, axis=1)
probabilities = tf.nn.softmax(logits, name='softmax_tensor')
if mode in (Modes.TRAIN, Modes.EVAL):
global_step = tf.train.get_or_create_global_step()
label_indices = tf.cast(labels, tf.int32)
loss = tf.losses.softmax_cross_entropy(
onehot_labels=tf.one_hot(label_indices, depth=10), logits=logits)
tf.summary.scalar('OptimizeLoss', loss)
if mode == Modes.PREDICT:
predictions = {
'classes': predicted_indices,
'probabilities': probabilities
}
export_outputs = {
SIGNATURE_NAME: tf.estimator.export.PredictOutput(predictions)
}
return tf.estimator.EstimatorSpec(
mode, predictions=predictions, export_outputs=export_outputs)
if mode == Modes.TRAIN:
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(loss, global_step=global_step)
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
if mode == Modes.EVAL:
eval_metric_ops = {
'accuracy': tf.metrics.accuracy(label_indices, predicted_indices)
}
return tf.estimator.EstimatorSpec(
mode, loss=loss, eval_metric_ops=eval_metric_ops)
def serving_input_fn(params):
inputs = {INPUT_TENSOR_NAME: tf.placeholder(tf.float32, [None, 784])}
return tf.estimator.export.ServingInputReceiver(inputs, inputs)
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'image_raw': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64),
})
image = tf.decode_raw(features['image_raw'], tf.uint8)
image.set_shape([784])
image = tf.cast(image, tf.float32) * (1. / 255)
label = tf.cast(features['label'], tf.int32)
return image, label
def train_input_fn(training_dir, params):
return _input_fn(training_dir, 'train.tfrecords', batch_size=100)
def eval_input_fn(training_dir, params):
return _input_fn(training_dir, 'test.tfrecords', batch_size=100)
def _input_fn(training_dir, training_filename, batch_size=100):
test_file = os.path.join(training_dir, training_filename)
filename_queue = tf.train.string_input_producer([test_file])
image, label = read_and_decode(filename_queue)
images, labels = tf.train.batch(
[image, label], batch_size=batch_size,
capacity=1000 + 3 * batch_size)
return {INPUT_TENSOR_NAME: images}, labels
def neo_preprocess(payload, content_type):
import logging
import numpy as np
import io
logging.info('Invoking user-defined pre-processing function')
if content_type != 'application/x-image' and content_type != 'application/vnd+python.numpy+binary':
raise RuntimeError('Content type must be application/x-image or application/vnd+python.numpy+binary')
f = io.BytesIO(payload)
image = np.load(f)*255
return image
### NOTE: this function cannot use MXNet
def neo_postprocess(result):
import logging
import numpy as np
import json
logging.info('Invoking user-defined post-processing function')
# Softmax (assumes batch size 1)
result = np.squeeze(result)
result_exp = np.exp(result - np.max(result))
result = result_exp / np.sum(result_exp)
response_body = json.dumps(result.tolist())
content_type = 'application/json'
return response_body, content_type
And I am training it
estimator = TensorFlow(entry_point='cnn_fashion_mnist.py',
role=role,
input_mode='Pipe',
training_steps=1,
evaluation_steps=1,
train_instance_count=1,
output_path=output_path,
train_instance_type='ml.c5.2xlarge',
base_job_name='mnist')
so far it is trying correctly and it tells me that everything when well, but when I check the output there is nothing there or if I try to deploy it I get the error saying it couldn't find the model because there is nothing in the bucker, any ideas or extra configurations? Thank you
Looks like you are using one of the older Tensorflow versions.
We would recommend switching to a newer more straight-forward way of running Tensorflow in SageMaker (script mode) by switching to a more recent Tensorflow version.
You can read more about it in our documentation:
https://sagemaker.readthedocs.io/en/stable/using_tf.html
Here is an example that might help:
https://github.com/awslabs/amazon-sagemaker-examples/blob/master/sagemaker-python-sdk/tensorflow_script_mode_training_and_serving/tensorflow_script_mode_training_and_serving.ipynb
Are you sure that your entry point has code that is really executed? You need a "main" / top--level code outside of functions. This code is executed as soon as you start the training. At least in my running examples.
import os
import tensorflow as tf
from tensorflow.python.estimator.model_fn import ModeKeys as Modes
INPUT_TENSOR_NAME = 'inputs'
SIGNATURE_NAME = 'predictions'
LEARNING_RATE = 0.001
ADD CODE FOR CREATION OF ESTIMATOR + TRAIN +....
ADD CODE THAT SAVES YOUR MODEL(e.g. joblib.dump(xxx, path)
In addition for executing the training, your "estimator = TensorFlow(..." should be followed by "estimater.fit(...)"-like call.
Have you double-checked in the protocolls for your training request in the aws console which part of your code was executed?
I have the following example code to train and evaluate a cnn mnist model using tensorflow's estimator api:
def model_fn(features, labels, mode):
images = tf.reshape(features, [-1, 28, 28, 1])
model = Model()
logits = model(images)
predicted_logit = tf.argmax(input=logits, axis=1, output_type=tf.int32)
if mode == tf.estimator.ModeKeys.PREDICT:
probabilities = tf.nn.softmax(logits)
predictions = {
'predicted_logit': predicted_logit,
'probabilities': probabilities
}
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
else:
...
def mnist_train_and_eval(_):
train_data, train_labels, eval_data, eval_labels, val_data, val_labels = get_mnist_data()
# Create a input function to train
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x= train_data,
y=train_labels,
batch_size=_BATCH_SIZE,
num_epochs=1,
shuffle=True)
# Create a input function to eval
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x= eval_data,
y=eval_labels,
batch_size=_BATCH_SIZE,
num_epochs=1,
shuffle=False)
# Create a estimator with model_fn
image_classifier = tf.estimator.Estimator(model_fn=model_fn, model_dir=_MODEL_DIR)
# Finally, train and evaluate the model after each epoch
for _ in range(_NUM_EPOCHS):
image_classifier.train(input_fn=train_input_fn)
metrics = image_classifier.evaluate(input_fn=eval_input_fn)
How can I use the estimator.export_savedmodel to save the trained model for later inference? How should I write the serving_input_receiver_fn?
Thank you very much for your help!
You create a function with a dictionary of input features. Placeholder should match the shape of your image, with first dimension for batch_size.
def serving_input_receiver_fn():
x = tf.placeholder(tf.float32, [None, Shape])
inputs = {'x': x}
return tf.estimator.export.ServingInputReceiver(features=inputs, receiver_tensors=inputs)
Or you can use TensorServingInputReceiver which doesn't required dict mapping
inputs = tf.placeholder(tf.float32, [None, 32*32*3])
tf.estimator.export.TensorServingInputReceiver(inputs, inputs)
This function returns new instance of ServingInputReceiver, which is passed to export_savedmodel or tf.estimator.FinalExporter
...
image_classifier.export_savedmodel(saved_dir, serving_input_receiver_fn)
I tried to write a good structured Neural network model with Tensorflow. But I met a problem about feed the data from tfrecord into the graph. The code is as below, it hangs on at the following function, how can I make it work?
images, labels = network.load_tfrecord_data(1)
this function can not get the features (images) and labels from my datafile, .tfrecords?
Any idea will be appreciated?
from __future__ import division
from __future__ import print_function
import datetime
import numpy as np
import tensorflow as tf
layers = tf.contrib.layers
losses = tf.contrib.losses
metrics = tf.contrib.metrics
LABELS = 10
WIDTH = 28
HEIGHT = 28
HIDDEN = 100
def read_and_decode_single_example(filename):
filename_queue = tf.train.string_input_producer([filename], num_epochs=None)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image': tf.FixedLenFeature([50176], tf.int64)
})
label = features['label']
image = features['image']
image = tf.reshape(image, [-1, 224, 224, 1])
label = tf.one_hot(label - 1, 11, dtype=tf.int64)
return label, image
class Network:
def __init__(self, logdir, experiment, threads):
# Construct the graph
with tf.name_scope("inputs"):
self.images = tf.placeholder(tf.float32, [None, WIDTH, HEIGHT, 1], name="images")
self.labels = tf.placeholder(tf.int64, [None], name="labels")
# self.keep_prob = keep_prob
self.keep_prob = tf.placeholder(tf.float32, name="keep_prob")
flattened_images = layers.flatten(self.images)
hidden_layer = layers.fully_connected(flattened_images, num_outputs=HIDDEN, activation_fn=tf.nn.relu, scope="hidden_layer")
output_layer = layers.fully_connected(hidden_layer, num_outputs=LABELS, activation_fn=None, scope="output_layer")
loss = losses.sparse_softmax_cross_entropy(labels=self.labels, logits=output_layer, scope="loss")
self.training = layers.optimize_loss(loss, None, None, tf.train.AdamOptimizer(), summaries=['loss', 'gradients', 'gradient_norm'], name='training')
with tf.name_scope("accuracy"):
predictions = tf.argmax(output_layer, 1, name="predictions")
accuracy = metrics.accuracy(predictions, self.labels)
tf.summary.scalar("training/accuracy", accuracy)
self.accuracy = metrics.accuracy(predictions, self.labels)
with tf.name_scope("confusion_matrix"):
confusion_matrix = metrics.confusion_matrix(predictions, self.labels, weights=tf.not_equal(predictions, self.labels), dtype=tf.float32)
confusion_image = tf.reshape(confusion_matrix, [1, LABELS, LABELS, 1])
# Summaries
self.summaries = {'training': tf.summary.merge_all() }
for dataset in ["dev", "test"]:
self.summaries[dataset] = tf.summary.scalar(dataset + "/loss", loss)
self.summaries[dataset] = tf.summary.scalar(dataset + "/accuracy", accuracy)
self.summaries[dataset] = tf.summary.image(dataset + "/confusion_matrix", confusion_image)
# Create the session
self.session = tf.Session(config=tf.ConfigProto(inter_op_parallelism_threads=threads,
intra_op_parallelism_threads=threads))
self.session.run(tf.global_variables_initializer())
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")
self.summary_writer = tf.summary.FileWriter("{}/{}-{}".format(logdir, timestamp, experiment), graph=self.session.graph, flush_secs=10)
self.steps = 0
def train(self, images, labels, keep_prob):
self.steps += 1
feed_dict = {self.images: self.session.run(images), self.labels: self.session.run(labels), self.keep_prob: keep_prob}
if self.steps == 1:
metadata = tf.RunMetadata()
self.session.run(self.training, feed_dict, options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE), run_metadata=metadata)
self.summary_writer.add_run_metadata(metadata, 'step1')
elif self.steps % 100 == 0:
_, summary = self.session.run([self.training, self.summaries['training']], feed_dict)
self.summary_writer.add_summary(summary, self.steps)
else:
self.session.run(self.training, feed_dict)
def evaluate(self, dataset, images, labels):
feed_dict ={self.images: images, self.labels: labels, self.keep_prob: 1}
summary = self.summaries[dataset].eval({self.images: images, self.labels: labels, self.keep_prob: 1}, self.session)
self.summary_writer.add_summary(summary, self.steps)
def load_tfrecord_data(self, training):
training = training
if training:
label, image = read_and_decode_single_example("mhad_Op_train.tfrecords")
# print(self.session.run(image))
else:
label, image = read_and_decode_single_example("mhad_Op_test.tfrecords")
# image = tf.cast(image, tf.float32) / 255.
images_batch, labels_batch = tf.train.shuffle_batch(
[image, label], batch_size=50, num_threads=2,
capacity=80,
min_after_dequeue=30)
return images_batch, labels_batch
if __name__ == '__main__':
# Fix random seed
np.random.seed(42)
tf.set_random_seed(42)
# Parse arguments
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', default=256, type=int, help='Batch size.')
parser.add_argument('--epochs', default=50, type=int, help='Number of epochs.')
parser.add_argument('--logdir', default="logs", type=str, help='Logdir name.')
parser.add_argument('--exp', default="mnist-final-confusion_matrix_customized_loss", type=str, help='Experiment name.')
parser.add_argument('--threads', default=1, type=int, help='Maximum number of threads to use.')
args = parser.parse_args()
# Load the data
keep_prob = 1
# Construct the network
network = Network(logdir=args.logdir, experiment=args.exp, threads=args.threads)
# Train
for i in range(args.epochs):
images, labels = network.load_tfrecord_data(1)
network.train(images, labels, keep_prob)
print('current epoch', i)
You need to start the queue before using images, labels in your model.
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
images, labels = network.load_tfrecord_data(1)
...
coord.request_stop()
coord.join(threads)
Check this tutorial for a full example
I am trying to make use of tensorflow protobuffer feeding pipeline. The easiest way seemed to use tf.estimator.Estimator with tf.contrib.data.TFRecordDataset. However, I came across the issue that it creates a new Graph in spite of being launched within with g.as_default(). In following code I see that both model tensors and tensors returned by the TFRecordDataset are the same before I feed them to Estimator, but become different within the Estimator. Any ideas how to put them on the same graph?
# coding: utf-8
import sys
import tensorflow as tf
from keras.applications.inception_v3 import InceptionV3
import numpy as np
final_activation='linear'
g = tf.Graph()
with g.as_default():
model = InceptionV3(weights='imagenet',
include_top=True,
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
def model_fn(mode, features, labels, params):
optimizer = params["optimizer"]
opt_params= params.get("opt_params", {})
predictions = model(features)
if (mode == tf.estimator.ModeKeys.TRAIN or
mode == tf.estimator.ModeKeys.EVAL):
loss = tf.contrib.keras.backend.categorical_crossentropy(predictions, labels)
#loss = tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logyhat)
else:
loss = None
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = getattr(tf.train, optimizer)
train_op = optimizer(opt_params).minimize(loss)
else:
train_op = None
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op)
def parser(record):
keys_to_features = {
'height': tf.FixedLenFeature([], tf.int64),
'width': tf.FixedLenFeature([], tf.int64),
'image_raw': tf.FixedLenFeature([], tf.string),
'label': tf.FixedLenFeature([], tf.int64)
}
features = tf.parse_single_example(
record,
features=keys_to_features)
# Convert from a scalar string tensor to a uint8 tensor
image = tf.decode_raw(features['image_raw'], tf.float32)
height = tf.cast(features['height'], tf.int32)
width = tf.cast(features['width'], tf.int32)
image_shape = tf.stack([height, width, 3])
image = tf.reshape(image, image_shape)
label = tf.cast(features["label"], tf.int32)
return image, label
def get_dataset_inp_fn(filenames, epochs=20):
def dataset_input_fn():
dataset = tf.contrib.data.TFRecordDataset(filenames)
# Use `Dataset.map()` to build a pair of a feature dictionary and a label
# tensor for each example.
dataset = dataset.map(parser)
dataset = dataset.shuffle(buffer_size=10000)
dataset = dataset.batch(32)
dataset = dataset.repeat(epochs)
iterator = dataset.make_one_shot_iterator()
features, labels = iterator.get_next()
return features, labels
return dataset_input_fn
inpfun = get_dataset_inp_fn(["mydataset.tfrecords"], epochs=20)
x,y = inpfun()
print("X", x.graph)
print("DEFAULT", g)
print("MODEL", model.input.graph)
# everything is on the same graph
if not x.graph is tf.get_default_graph():
raise ValueError()
with tf.Session(graph=g) as sess:
est = tf.estimator.Estimator(
model_fn,
model_dir=None,
config=None,
params={"optimizer": "AdamOptimizer",
"opt_params":{}}
)
est.train(inpfun)
I'm trying to display my embeddings in tensorboard. When I open embeddings tab of tensorboard I get: "Computing PCA..." and tensorboard hangs infinitely.
Before that it does load my tensor of shape 200x128. It does find the metadata file too.
I tried that on TF versions 0.12 and 1.1 with the same result.
features = np.zeros(shape=(num_batches*batch_size, 128), dtype=float)
embedding_var = tf.Variable(features, name='feature_embedding')
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = 'feature_embedding'
metadata_path = os.path.join(self.log_dir, 'metadata.tsv')
embedding.metadata_path = metadata_path
with tf.Session(config=self.config) as sess:
tf.global_variables_initializer().run()
restorer = tf.train.Saver()
restorer.restore(sess, self.pretrained_model_path)
with open(metadata_path, 'w') as f:
for step in range(num_batches):
batch_images, batch_labels = data.next()
for label in batch_labels:
f.write('%s\n' % label)
feed_dict = {model.images: batch_images}
features[step*batch_size : (step+1)*batch_size, :] = \
sess.run(model.features, feed_dict)
sess.run(embedding_var.initializer)
projector.visualize_embeddings(tf.summary.FileWriter(self.log_dir), config)
I don't know what was wrong in the code above, but I rewrote it in a different way (below), and it works. The difference is when and how the embedding_var is initialized.
I also made a gist to copy-paste code from out of this.
# a numpy array for embeddings and a list for labels
features = np.zeros(shape=(num_batches*self.batch_size, 128), dtype=float)
labels = []
# compute embeddings batch by batch
with tf.Session(config=self.config) as sess:
tf.global_variables_initializer().run()
restorer = tf.train.Saver()
restorer.restore(sess, self.pretrained_model)
for step in range(num_batches):
batch_images, batch_labels = data.next()
labels += batch_labels
feed_dict = {model.images: batch_images}
features[step*self.batch_size : (step+1)*self.batch_size, :] = \
sess.run(model.features, feed_dict)
# write labels
metadata_path = os.path.join(self.log_dir, 'metadata.tsv')
with open(metadata_path, 'w') as f:
for label in labels:
f.write('%s\n' % label)
# write embeddings
with tf.Session(config=self.config) as sess:
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = 'feature_embedding'
embedding.metadata_path = metadata_path
embedding_var = tf.Variable(features, name='feature_embedding')
sess.run(embedding_var.initializer)
projector.visualize_embeddings(tf.summary.FileWriter(self.log_dir), config)
saver = tf.train.Saver({"feature_embedding": embedding_var})
saver.save(sess, os.path.join(self.log_dir, 'model_features'))
It's a bug. It's fixed in tensorflow 1.13