Related
I try to print the actual value of the tensor which is loaded from my custom tfds dataset.
I can't figure out how to do it. I am using Tensorflow2 so the session is no longer encouraged.
I tried using .numpy() tf.print. tf.executing.eagerly() but nothing works. It either prints
just the tensor object showing me the shape or in case of .numpy() it throws the error in the title.
I need the value of tensors and I need to bring it back to numpy in order to debugg the code.
This is how I create the dataset:
class dt(tfds.core.GeneratorBasedBuilder):
''' Dataset builder'''
# DOuble check
VERSION = tfds.core.Version('1.0.0')
RELEASE_NOTES = {
'1.0.0': 'Initial release.',
}
def _info(self) ->tfds.core.DatasetInfo:
'''Dataset metadata'''
return tfds.core.DatasetInfo(
builder=self,
features=tfds.features.FeaturesDict({
"id": tf.int64,
"image": tfds.features.Image(shape=(IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), encoding_format='png'),
"mask": tfds.features.Image(shape=(IMG_HEIGHT, IMG_WIDTH, 1), encoding_format='png'),
"label": tfds.features.ClassLabel(names=CLASSES),
}),
supervised_keys=('image', 'mask')
)
def _split_generators(self, dl_manager: tfds.download.DownloadManager):
'''Splitgenerator for train and test splits'''
path = DATASETS_ROOT
return {
"train": self._generate_examples(
images_path=os.path.join(path, "train/rgb"),
masks_path=os.path.join(path, "train/masks")
),
"test": self._generate_examples(
images_path=os.path.join(path, "test/rgb"),
masks_path=os.path.join(path, "test/masks")
)
}
def _generate_examples(self, images_path, masks_path):
'''Generator of examples for each split'''
for i, (image, mask) in enumerate(zip(glob.glob(images_path + "/*.png"), glob.glob(masks_path + "/*.png"))):
yield i, {
"id": i,
"image": image,
"mask": mask,
"label": CLASSES[3],
}
This is how I try to extract the numpy array
def custom_load_X_Y(training=True):
if training:
dt, dt_info = tfds.load("dt", split="train", shuffle_files=True, as_supervised=True, with_info=True)
print(f'EAGERLY {tf.executing_eagerly()}')
print(f'MOde type {type(dt)}')
tf.print(f"aaaaa {dt.numpy()} aaaaaa")
Console output:
Console output
You can print the image dataset by using tfds.show_examples()
import tensorflow_datasets as tfds
ds, ds_info = tfds.load('cifar10', split='train', with_info=True)
fig = tfds.show_examples(ds, ds_info)
output:
For more details please refer to this gist.Thank You.
Ask a Question
I success convert mxnet model to onnx but it failed when inference .The model 's shape is (1,1,100,100)
convert code
sym = 'single-symbol.json'
params = '/single-0090.params'
input_shape = (1, 1, 100, 100)
onnx_file = './model.onnx'
converted_model_path = onnx_mxnet.export_model(sym, params, [input_shape], np.float32, onnx_file,verbose=True)
model= onnx.load_model(converted_model_path)
checker.check_graph(model.graph)
checker.check_model(model)
output
INFO:root:Input shape of the model [(1, 1, 100, 100)]
INFO:root:Exported ONNX file ./model.onnx saved to disk
inference code
sess = ort.InferenceSession("./model.onnx")
output
onnxruntime.capi.onnxruntime_pybind11_state.RuntimeException:
[ONNXRuntimeError] : 6 : RUNTIME_EXCEPTION :
Exception during initialization:
/onnxruntime/core/providers/cpu/nn/pool_attributes.h:77
onnxruntime::PoolAttributes::PoolAttributes(const OpNodeProtoHelper<onnxruntime::ProtoHelperNodeContext> &,
const std::string &, int) pads[dim] < kernel_shape[dim] &&
pads[dim + kernel_shape.size()] < kernel_shape[dim] was false.
Pad should be smaller than kernel.
Question
mxnet pooling node json
{
"op": "Pooling",
"name": "pool1_fwd",
"attrs": {
"count_include_pad": "True",
"global_pool": "False",
"kernel": "(4, 4)",
"layout": "NCHW",
"pad": "(4, 4)",
"pool_type": "avg",
"pooling_convention": "valid",
"stride": "(4, 4)"
},
"inputs": [[46, 0, 0]]
}
I change the "pad": "(4, 4)" to "pad": "(3, 3)" smaller than "kernel": "(4, 4), then try convert again.
sess = ort.InferenceSession("./model.onnx")
output = sess.run(None, {"data": data.astype(np.float32)})
it worked,but the output value is not right.
how to fix it ?
BTW:convert the mxnet model to ncnn all is right(not change anything,pad=(4,4),kernel=(4,4))
Further information
python:3.8
onnx:1.10.2
mxnet:1.8.0
I fix it,recode model by pytorch and copy weights,use nn.ZeroPad2d(4) before avgpooling:
self.pad = nn.ZeroPad2d(4)
self.pool = nn.AvgPool2d(kernel_size=(4,4),stride=(4,4))
X = self.pool(self.pad(self.conv(X)))
i am founding an error in plant disease detection using resnet50 deep learning model every time it raises an error message in decode_predictions
error
expects a batch of predictions (i.e. a 2D array of shape (samples, 1000)). Found array with shape: (1, 38)"
enter code here
model = ResNet50(weights='imagenet',include_top=False,classes=38)
try:
model = load_model('/content/drive/My
Drive/color/checkpoints/ResNet50_model_weights.h5')
print("model loaded")
except:
print("model not loaded")
img_path = '/content/drive/My Drive/color/test/0/appleblackrot188.jpg'
img = image.load_img(img_path, target_size=(300, 300))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
preds = model.predict(x)
print('Predicted:', decode_predictions(preds,top=3)[0])
You can try using the preprocesing function:
import tensorflow as tf
# Using the keras wrapper on tensorflow (it must be the same using just keras).
IMAGE = [] # From image source, i did it from the camera.
toPred = tf.keras.applications.resnet50.preprocess_input(np.expand_dims(tf.keras.preprocessing.image.img_to_array(IMAGE), axis=0))
Maybe that can help :)
decode_predictions works only for ImageNet (no. of classes = 1000). For these 38 classes of plants, you have to write your own decode predictions based on the ground truth label you've assigned for each plant.
First, you need an index JSON file and create a new decode_predictions function.
For example
This HAM10000 that has 7 classes and you need to split to each folder like this
then make an index JSON file like this
{
"0" : [
"AKIEC",
"akiec"
],
"1" : [
"BCC",
"bcc"
],
"2" : [
"BKL",
"bkl"
],
"3" : [
"DF",
"df"
],
"4" : [
"MEL",
"mel"
],
"5" : [
"NV",
"nv"
],
"6" : [
"VASC",
"vasc"
]
}
Then try this code
def decode_predictions(preds, top=4, class_list_path='/content/SKIN_DATA/HAM10000_index.json'):
if len(preds.shape) != 2 or preds.shape[1] != 7: # your classes number
raise ValueError('`decode_predictions` expects '
'a batch of predictions '
'(i.e. a 2D array of shape (samples, 1000)). '
'Found array with shape: ' + str(preds.shape))
index_list = json.load(open(class_list_path))
results = []
for pred in preds:
top_indices = pred.argsort()[-top:][::-1]
result = [tuple(index_list[str(i)]) + (pred[i],) for i in top_indices]
result.sort(key=lambda x: x[2], reverse=True)
results.append(result)
return results
I am trying to use parse_single_example to process the TFRecordDataset file.
But It looks it can not handle the categorical_column in the Example proto.
For example:
In the example, there are 3 features. "a", "b", and "month".
mouth is an int64 type feature, here I'd like to use parse this field and expand to a categorical signal directly.
I am not sure if parse_single_example support it or my method is totally wrong?
def _parse_function(example_proto):
print(example_proto)
features = {"a": tf.VarLenFeature(tf.float32),
"b": tf.VarLenFeature(tf.float32),
"month": tf.feature_column.categorical_column_with_identity(key='month', num_buckets=12),
"label": tf.FixedLenFeature((), tf.float32, default_value=0.0)}
parsed_features = tf.parse_single_example(example_proto, features)
return tf.concat([tf.sparse_tensor_to_dense(parsed_features["a"]),
tf.sparse_tensor_to_dense(parsed_features["b"]),
parsed_features["month"]], 0),\
tf.cast(parsed_features["close_pct"] + tf.convert_to_tensor(1.0, tf.float32), tf.int64)
https://www.tensorflow.org/api_docs/python/tf/io/parse_single_example
https://www.tensorflow.org/api_docs/python/tf/feature_column/categorical_column_with_identity
Use tf.one_hot after parse the example.
return tf.concat([tf.sparse_tensor_to_dense(parsed_features["a"]),
tf.sparse_tensor_to_dense(parsed_features["b"]),
tf.one_hot(parsed_features["month"], categorical_size, axis=-1)], 0),\
tf.cast(parsed_features["close_pct"] + tf.convert_to_tensor(1.0, tf.float32), tf.int64)
I'm running a wide_deep.py script for linear regression in tensorflow.
I have cloned the models directory also as a part of process.
But i'm getting a error like AttributeError: 'list' object has no attribute 'model_dir'.
If I hard code this particular variable, I m getting other errors as AttributeError: 'list' object has no attribute 'data_dir' and so on .
Code:
"""Example code for TensorFlow Wide & Deep Tutorial using tf.estimator API."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import shutil
from absl import app as absl_app
from absl import flags
import tensorflow as tf # pylint: disable=g-bad-import-order
from official.utils.flags import core as flags_core
from official.utils.logs import hooks_helper
from official.utils.misc import model_helpers
_CSV_COLUMNS = [
'age', 'workclass', 'fnlwgt', 'education', 'education_num',
'marital_status', 'occupation', 'relationship', 'race', 'gender',
'capital_gain', 'capital_loss', 'hours_per_week', 'native_country',
'income_bracket'
]
_CSV_COLUMN_DEFAULTS = [[0], [''], [0], [''], [0], [''], [''], [''], [''], [''],
[0], [0], [0], [''], ['']]
_NUM_EXAMPLES = {
'train': 32561,
'validation': 16281,
}
LOSS_PREFIX = {'wide': 'linear/', 'deep': 'dnn/'}
def define_wide_deep_flags():
"""Add supervised learning flags, as well as wide-deep model type."""
flags_core.define_base()
flags.adopt_module_key_flags(flags_core)
flags.DEFINE_enum(
name="model_type", short_name="mt", default="wide_deep",
enum_values=['wide', 'deep', 'wide_deep'],
help="Select model topology.")
flags_core.set_defaults(data_dir='/tmp/census_data',
model_dir='/tmp/census_model',
train_epochs=40,
epochs_between_evals=2,
batch_size=40)
def build_model_columns():
"""Builds a set of wide and deep feature columns."""
# Continuous columns
age = tf.feature_column.numeric_column('age')
education_num = tf.feature_column.numeric_column('education_num')
capital_gain = tf.feature_column.numeric_column('capital_gain')
capital_loss = tf.feature_column.numeric_column('capital_loss')
hours_per_week = tf.feature_column.numeric_column('hours_per_week')
education = tf.feature_column.categorical_column_with_vocabulary_list(
'education', [
'Bachelors', 'HS-grad', '11th', 'Masters', '9th', 'Some-college',
'Assoc-acdm', 'Assoc-voc', '7th-8th', 'Doctorate', 'Prof-school',
'5th-6th', '10th', '1st-4th', 'Preschool', '12th'])
marital_status = tf.feature_column.categorical_column_with_vocabulary_list(
'marital_status', [
'Married-civ-spouse', 'Divorced', 'Married-spouse-absent',
'Never-married', 'Separated', 'Married-AF-spouse', 'Widowed'])
relationship = tf.feature_column.categorical_column_with_vocabulary_list(
'relationship', [
'Husband', 'Not-in-family', 'Wife', 'Own-child', 'Unmarried',
'Other-relative'])
workclass = tf.feature_column.categorical_column_with_vocabulary_list(
'workclass', [
'Self-emp-not-inc', 'Private', 'State-gov', 'Federal-gov',
'Local-gov', '?', 'Self-emp-inc', 'Without-pay', 'Never-worked'])
# To show an example of hashing:
occupation = tf.feature_column.categorical_column_with_hash_bucket(
'occupation', hash_bucket_size=1000)
# Transformations.
age_buckets = tf.feature_column.bucketized_column(
age, boundaries=[18, 25, 30, 35, 40, 45, 50, 55, 60, 65])
# Wide columns and deep columns.
base_columns = [
education, marital_status, relationship, workclass, occupation,
age_buckets,
]
crossed_columns = [
tf.feature_column.crossed_column(
['education', 'occupation'], hash_bucket_size=1000),
tf.feature_column.crossed_column(
[age_buckets, 'education', 'occupation'], hash_bucket_size=1000),
]
wide_columns = base_columns + crossed_columns
deep_columns = [
age,
education_num,
capital_gain,
capital_loss,
hours_per_week,
tf.feature_column.indicator_column(workclass),
tf.feature_column.indicator_column(education),
tf.feature_column.indicator_column(marital_status),
tf.feature_column.indicator_column(relationship),
# To show an example of embedding
tf.feature_column.embedding_column(occupation, dimension=8),
]
return wide_columns, deep_columns
def build_estimator(model_dir, model_type):
"""Build an estimator appropriate for the given model type."""
wide_columns, deep_columns = build_model_columns()
hidden_units = [100, 75, 50, 25]
# Create a tf.estimator.RunConfig to ensure the model is run on CPU, which
# trains faster than GPU for this model.
run_config = tf.estimator.RunConfig().replace(
session_config=tf.ConfigProto(device_count={'GPU': 0}))
if model_type == 'wide':
return tf.estimator.LinearClassifier(
model_dir=model_dir,
feature_columns=wide_columns,
config=run_config)
elif model_type == 'deep':
return tf.estimator.DNNClassifier(
model_dir=model_dir,
feature_columns=deep_columns,
hidden_units=hidden_units,
config=run_config)
else:
return tf.estimator.DNNLinearCombinedClassifier(
model_dir=model_dir,
linear_feature_columns=wide_columns,
dnn_feature_columns=deep_columns,
dnn_hidden_units=hidden_units,
config=run_config)
def input_fn(data_file, num_epochs, shuffle, batch_size):
"""Generate an input function for the Estimator."""
assert tf.gfile.Exists(data_file), (
'%s not found. Please make sure you have run data_download.py and '
'set the --data_dir argument to the correct path.' % data_file)
def parse_csv(value):
print('Parsing', data_file)
columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS)
features = dict(zip(_CSV_COLUMNS, columns))
labels = features.pop('income_bracket')
return features, tf.equal(labels, '>50K')
# Extract lines from input files using the Dataset API.
dataset = tf.data.TextLineDataset(data_file)
if shuffle:
dataset = dataset.shuffle(buffer_size=_NUM_EXAMPLES['train'])
dataset = dataset.map(parse_csv, num_parallel_calls=5)
# We call repeat after shuffling, rather than before, to prevent separate
# epochs from blending together.
dataset = dataset.repeat(num_epochs)
dataset = dataset.batch(batch_size)
return dataset
def export_model(model, model_type, export_dir):
"""Export to SavedModel format.
Args:
model: Estimator object
model_type: string indicating model type. "wide", "deep" or "wide_deep"
export_dir: directory to export the model.
"""
wide_columns, deep_columns = build_model_columns()
if model_type == 'wide':
columns = wide_columns
elif model_type == 'deep':
columns = deep_columns
else:
columns = wide_columns + deep_columns
feature_spec = tf.feature_column.make_parse_example_spec(columns)
example_input_fn = (
tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec))
model.export_savedmodel(export_dir, example_input_fn)
def run_wide_deep(flags_obj):
"""Run Wide-Deep training and eval loop.
Args:
flags_obj: An object containing parsed flag values.
"""
# Clean up the model directory if present
shutil.rmtree(flags_obj.model_dir, ignore_errors=True)
model = build_estimator(flags_obj.model_dir, flags_obj.model_type)
train_file = os.path.join(flags_obj.data_dir, 'adult.data')
test_file = os.path.join(flags_obj.data_dir, 'adult.test')
# Train and evaluate the model every `flags.epochs_between_evals` epochs.
def train_input_fn():
return input_fn(
train_file, flags_obj.epochs_between_evals, True, flags_obj.batch_size)
def eval_input_fn():
return input_fn(test_file, 1, False, flags_obj.batch_size)
loss_prefix = LOSS_PREFIX.get(flags_obj.model_type, '')
train_hooks = hooks_helper.get_train_hooks(
flags_obj.hooks, batch_size=flags_obj.batch_size,
tensors_to_log={'average_loss': loss_prefix + 'head/truediv',
'loss': loss_prefix + 'head/weighted_loss/Sum'})
# Train and evaluate the model every `flags.epochs_between_evals` epochs.
for n in range(flags_obj.train_epochs // flags_obj.epochs_between_evals):
model.train(input_fn=train_input_fn, hooks=train_hooks)
results = model.evaluate(input_fn=eval_input_fn)
# Display evaluation metrics
print('Results at epoch', (n + 1) * flags_obj.epochs_between_evals)
print('-' * 60)
for key in sorted(results):
print('%s: %s' % (key, results[key]))
if model_helpers.past_stop_threshold(
flags_obj.stop_threshold, results['accuracy']):
break
# Export the model
if flags_obj.export_dir is not None:
export_model(model, flags_obj.model_type, flags_obj.export_dir)
def main(_):
run_wide_deep(flags.FLAGS)
if __name__ == '__main__':
tf.logging.set_verbosity(tf.logging.INFO)
define_wide_deep_flags()
absl_app.run(main)
Hunter, I tried to run without hardcoding but still faced issues with attributes , so I tried to hard code to avoid this .
But, The issue is resolved now.
I cloned the directory again and instead of copying the wide_deep.py to another directory and run from there(which I was doing before), I ran directly from the same directory and now it is working fine.