Using tf.app.run() to call main function from a class - tensorflow

The question is simple, I'm trying to use tf.app.run() to call the main function in a class. But, the following code gives me an error. Any help is appreciated.
import tensorflow as tf
import sys
# Where to find data
tf.app.flags.DEFINE_string('f1', '', 'feature 1')
tf.app.flags.DEFINE_string('f2', '', 'feature 2')
FLAGS = tf.app.flags.FLAGS
class Test(object):
def __init__(self):
pass
def main(self, args):
print(FLAGS.__flag.iteritems())
def main(args):
test = Test()
test.main(args)
if __name__ == '__main__':
tf.app.run(main)
And here is the error:
Traceback (most recent call last):
File "test.py", line 21, in <module>
tf.app.run(main)
File "/Users/yaserkeneshloo/anaconda/envs/env27/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 126, in run
_sys.exit(main(argv))
File "test.py", line 18, in main
test.main(args)
File "test.py", line 14, in main
print(FLAGS.__flag.iteritems())
File "/Users/yaserkeneshloo/anaconda/envs/env27/lib/python2.7/site-packages/tensorflow/python/platform/flags.py", line 85, in __getattr__
return wrapped.__getattr__(name)
File "/Users/yaserkeneshloo/anaconda/envs/env27/lib/python2.7/site-packages/absl/flags/_flagvalues.py", line 470, in __getattr__
raise AttributeError(name)
AttributeError: _Test__flag

The issue is that when you try to access a property of FLAGS beginning with double underscore from inside a class, it prefixes the property with the class name. So instead of FLAGS.__flags it tries to find FLAGS._Test__flags which does not exist.
This is true for both setting or getting a value. So if you set the value from outside of the class, you have to prefix it with _Test (because you named your class Test.) If you set the flags inside the class, you don't need to prefix, because it does the prefixing automatically also on assigning a value.
So basically you have no problem at all with your code because your own flags don't start with double underscore, except you cannot use the internal __flags property to print all flags. You can access them individually though.
See the code below for detailed examples. (Also, the default value in the DEFINE line is the second parameter, not the third.)
# Where to find data
tf.app.flags.DEFINE_string('_Test__f1', 'feature 1', 'feature 1')
tf.app.flags.DEFINE_string('__f2', 'feature 2', 'feature 2')
FLAGS = tf.app.flags.FLAGS
print( FLAGS.__f2 ) # prints "feature 2"
class Test(object):
def __init__(self):
pass
def main(self, args):
print( FLAGS.__f1 ) # prints "feature 1"
FLAGS.__f1 = 'foobar' # assignment works the same way
print( FLAGS.__f1 ) # prints "foobar"
print( FLAGS.__f2 ) # AttributeError: _Test__f2
def main(args):
test = Test()
test.main(args)
if __name__ == '__main__':
tf.app.run(main)

The problem is that FLAGS doesn't have an attribute named __flag. If you want to print the string corresponding to the f1 flag, call print(FLAGS.f1).

This will do the trick:
import tensorflow as tf
import sys
# Where to find data
tf.app.flags.DEFINE_string('f1', '', 'feature 1')
tf.app.flags.DEFINE_string('f2', '', 'feature 2')
FLAGS = tf.app.flags.FLAGS
class Test(object):
def __init__(self):
pass
def main(self, args):
flags = getattr(FLAGS,"__flags")
print([(k,v) for (k,v) in flags.iteritems()])
def main(args):
test = Test()
test.main(args)
if __name__ == '__main__':
tf.app.run(main)

Related

keras.models.load_model() gives ValueError

I have saved the trained model and the weights as below.
model, history, score = fit_model(model, train_batches, val_batches, callbacks=[callback])
model.save('./model')
model.save_weights('./weights')
Then I tried to get the saved model as the following way
if __name__ == '__main__':
model = keras.models.load_model('./model', compile= False,custom_objects={"F1Score": tfa.metrics.F1Score})
test_batches, nb_samples = test_gen(dataset_test_path, 32, img_width, img_height)
predict, loss, acc = predict_model(model,test_batches, nb_samples)
print(predict)
print(acc)
print(loss)
But it gives me an error. What should I do to overcome this?
Traceback (most recent call last):
File "test_pro.py", line 34, in <module>
model = keras.models.load_model('./model',compile= False,custom_objects={"F1Score": tfa.metrics.F1Score})
File "/home/dcs2016csc007/.local/lib/python3.8/site-packages/tensorflow/python/keras/saving/save.py", line 212, in load_model
return saved_model_load.load(filepath, compile, options)
File "/home/dcs2016csc007/.local/lib/python3.8/site-packages/tensorflow/python/keras/saving/saved_model/load.py", line 138, in load
keras_loader.load_layers()
File "/home/dcs2016csc007/.local/lib/python3.8/site-packages/tensorflow/python/keras/saving/saved_model/load.py", line 379, in load_layers
self.loaded_nodes[node_metadata.node_id] = self._load_layer(
File "/home/dcs2016csc007/.local/lib/python3.8/site-packages/tensorflow/python/keras/saving/saved_model/load.py", line 407, in _load_layer
obj, setter = revive_custom_object(identifier, metadata)
File "/home/dcs2016csc007/.local/lib/python3.8/site-packages/tensorflow/python/keras/saving/saved_model/load.py", line 921, in revive_custom_object
raise ValueError('Unable to restore custom object of type {} currently. '
ValueError: Unable to restore custom object of type _tf_keras_metric currently. Please make sure that the layer implements `get_config`and `from_config` when saving. In addition, please use the `custom_objects` arg when calling `load_model()`.
Looking at the source code for Keras, the error is raised when trying to load a model with a custom object:
def revive_custom_object(identifier, metadata):
"""Revives object from SavedModel."""
if ops.executing_eagerly_outside_functions():
model_class = training_lib.Model
else:
model_class = training_lib_v1.Model
revived_classes = {
constants.INPUT_LAYER_IDENTIFIER: (
RevivedInputLayer, input_layer.InputLayer),
constants.LAYER_IDENTIFIER: (RevivedLayer, base_layer.Layer),
constants.MODEL_IDENTIFIER: (RevivedNetwork, model_class),
constants.NETWORK_IDENTIFIER: (RevivedNetwork, functional_lib.Functional),
constants.SEQUENTIAL_IDENTIFIER: (RevivedNetwork, models_lib.Sequential),
}
parent_classes = revived_classes.get(identifier, None)
if parent_classes is not None:
parent_classes = revived_classes[identifier]
revived_cls = type(
compat.as_str(metadata['class_name']), parent_classes, {})
return revived_cls._init_from_metadata(metadata) # pylint: disable=protected-access
else:
raise ValueError('Unable to restore custom object of type {} currently. '
'Please make sure that the layer implements `get_config`'
'and `from_config` when saving. In addition, please use '
'the `custom_objects` arg when calling `load_model()`.'
.format(identifier))
The method will only work fine with the custom objects of the types defined in revived_classes. As you can see, it currently only works with input layer, layer, model, network, and sequential custom objects.
In your code, you pass an tfa.metrics.F1Score class in the custom_objects argument, which is of type METRIC_IDENTIFIER, therefore, not supported (probably because it doesn't implement the get_config and from_config functions as the error output says):
keras.models.load_model('./model', compile=False, custom_objects={"F1Score": tfa.metrics.F1Score})
It's been a while since I last worked with Keras but maybe you can try and follow what was proposed in this other related answer and wrap the call to tfa.metrics.F1Score in a method. Something like this (adjust it to your needs):
def f1(y_true, y_pred):
metric = tfa.metrics.F1Score(num_classes=3, threshold=0.5)
metric.update_state(y_true, y_pred)
return metric.result()
keras.models.load_model('./model', compile=False, custom_objects={'f1': f1})

Instance of has no member

Maybe I'm getting something basic confused but I cant seem to work out how to fix this issue.
The compiler gives me
Traceback (most recent call last):
File "python", line 153, in <module>
File "python", line 90, in finish
File "python", line 25, in registered
AttributeError: 'Marathon' object has no attribute 'runnerList'
Which all seem to be the same issue. Surely the instance does have members. I'm not sure why it thinks it doesn't.
class Marathon:
# Creator
# -------
runnersList = []
timesList = []
def __init__(self):
"""Set up this marathon without any runners."""
# Inspectors
# ----------
# These are called anytime.
def registered(self, runner):
"""Return True if runner has registered, otherwise False."""
for item in self.runnerList:
if item == runner:
return True
else:
return False

How to use tf.train.Saver in SessionRunHook?

I have trained many sub-models, each sub-models is a part of the last model. And then I want to use those pretrained sub models to initial the last model's parameters. I try to use SessionRunHook to load other ckpt file's model parameters to initial the last model's.
I tried the follow code but failed. Hope some advices. Thanks!
The error info is:
Traceback (most recent call last):
File "train_high_api_local.py", line 282, in <module>
tf.app.run()
File "/Users/zhouliaoming/anaconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 124, in run
_sys.exit(main(argv))
File "train_high_api_local.py", line 266, in main
clf_.train(input_fn=lambda: read_file([tables[0]], epochs_per_eval), steps=None, hooks=[hook_test]) # input yield: x, y
File "/Users/zhouliaoming/anaconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/estimator/estimator.py", line 314, in train
.......
File "/Users/zhouliaoming/anaconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/training/monitored_session.py", line 674, in create_session
hook.after_create_session(self.tf_sess, self.coord)
File "train_high_api_local.py", line 102, in after_create_session
saver = tf.train.Saver([ti]) # TODO: ERROR INFO: Graph is finalized and cannot be modified.
.......
File "/Users/zhouliaoming/anaconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3135, in create_op
self._check_not_finalized()
File "/Users/zhouliaoming/anaconda3/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2788, in _check_not_finalized
raise RuntimeError("Graph is finalized and cannot be modified.")
RuntimeError: Graph is finalized and cannot be modified.
and the code detail is:
class SetTensor(session_run_hook.SessionRunHook):
""" like tf.train.LoggingTensorHook """
def after_create_session(self, session, coord):
""" Called when new TensorFlow session is created: graph is finalized and ops can no longer be added. """
graph = tf.get_default_graph()
ti = graph.get_tensor_by_name("h_1_15/bias:0")
with session.as_default():
with tf.name_scope("rewrite"):
saver = tf.train.Saver([ti]) # TODO: ERROR INFO: Graph is finalized and cannot be modified.
saver.restore(session, "/Users/zhouliaoming/data/credit_dnn/model_retrain/rm_gene_v2_sall/model.ckpt-2102")
pass
def main(unused_argv):
""" train """
norm_all_func = lambda x: tf.cond(x>1, lambda: tf.log(x), lambda: tf.identity(x))
feature_columns=[[tf.feature_column.numeric_column(COLUMNS[i], shape=fi, normalizer_fn=lambda x: tf.py_func(weight_norm2, [x], tf.float32) )] for i, fi in enumerate(FEA_DIM)] # normlized: running OK!
## use self-defined model
param = {"learning_rate": 0.0001, "feature_columns": feature_columns, "isanalysis": FLAGS.isanalysis, "isall": False}
clf_ = tf.estimator.Estimator(model_fn=model_fn_wide2deep, params=param, model_dir=ckpt_dir)
hook_test = SetTensor(["h_1_15/bias", "h_1_15/kernel"])
epochs_per_eval = 1
for n in range(int(FLAGS.num_epochs/epochs_per_eval)):
# train num_epochs
clf_.train(input_fn=lambda: read_file([tables[0]], epochs_per_eval), steps=None, hooks=[hook_test]) # input yield: x, y
SessionRunHook is not meant for this use case. As the error says, you cannot change the graph once sess.run() has been invoked.
You can assign variables using saver.restore() in your "normal code". You don't have to be inside any hooks.
Also, if you want to restore many variables and can match them to their names and shapes in a checkpoint, you might want to take a look at https://gist.github.com/iganichev/d2d8a0b1abc6b15d4a07de83171163d4. It shows some example code to restore a subset of variables.
You can do this:
class SaveAtEnd(tf.train.SessionRunHook):
def begin(self):
self._saver = # create your saver
def end(self, session):
self._saver.save(session, ...)

How to write custom PySpark serializer

I want to write a custom pyspark serializer. There is very little documentation online apart from details here.
My logic is as follows:
If I receive my special object, then I use custom logic to serialize/deserialize.
Otherwise, use cPickle for the same.
The custom serializer looks like the following:
from pyspark.serializers import FramedSerializer
class CustomSerializer(FramedSerializer):
import cPickle as pickle
import CustomClass
def dumps(self, obj):
if isinstance(obj, CustomClass):
bytes_str = obj.serialize()
bytes_str = '\1' + bytes_str
elif isinstance(obj, CustomClass.Location):
bytes_str = obj.serialize()
bytes_str = '\2' + bytes_str
else:
bytes_str = pickle.dumps(obj)
bytes_str = '\0' + bytes_str
return bytes_str
def loads(self, bytes_str):
c = bytes_str[0]
if c=='\1':
obj = CustomClass()
obj.parse_from_string(bytes_str[1:])
elif c=='\2':
obj = CustomClass.Location()
obj.parse_from_string(bytes_str[1:])
else:
obj = pickle.loads(bytes_str[1:])
return obj
While initiating SparkContext, I make sure I specify the custom serializer:
serializer = CustomSerializer()
sc = SparkContext(appName='MyApp', serializer=serializer)
However, I still get error:
Caused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):
File "/mnt/yarn/usercache/user/appcache/application_1507044435666_0035/container_1507044435666_0035_01_000002/pyspark.zip/pyspark/worker.py", line 174, in main
process()
File "/mnt/yarn/usercache/user/appcache/application_1507044435666_0035/container_1507044435666_0035_01_000002/pyspark.zip/pyspark/worker.py", line 169, in process
serializer.dump_stream(func(split_index, iterator), outfile)
File "/mnt/yarn/usercache/user/appcache/application_1507044435666_0035/container_1507044435666_0035_01_000002/pyspark.zip/pyspark/serializers.py", line 272, in dump_stream
bytes = self.serializer.dumps(vs)
File "<ipython-input-2-536808351108>", line 14, in dumps
PicklingError: Can't pickle <class 'CustomClass.Location'>: attribute lookup Location failed
What am I missing?
Thanks.

Guess number, no such attribute, python 3

I'm learning OOP in python and was trying to run this small game in OOP style, but for some reason system doesn't find object's attributes.
Here's the problem:
Traceback (most recent call last):
File "HelloUsername.py", line 47, in <module>
newGameGTN = GuessTheNumber()
File "HelloUsername.py", line 6, in __init__
self.start_game()
File "HelloUsername.py", line 32, in start_game
player = player_choice()
NameError: name 'player_choice' is not defined
On this code in python 3:
from random import randint
class GuessTheNumber(object):
"""docstring for GuessTheNumber"""
def __init__(self):
self.start_game()
self.player_choice()
self.compare_numbers()
def player_choice(self):
choice = int(input("Choose your number: "))
if choice in range(101):
return(choice)
else:
print("Please enter a number 0-100")
player_choice()
def compare_numbers(self, computer, player):
if player == computer:
return(0)
elif player > computer:
return(1)
elif player < computer:
return(-1)
def start_game(self):
computer = randint(0, 100)
turn = 0
for turn in range(3):
player = player_choice()
x = compare_numbers(computer, player)
print(computer)
if x == -1:
print("too small")
elif x == 1:
print("too big")
elif x == 0:
print("you win")
break
turn += 1
print("game over")
newGameGTN = GuessTheNumber()
newGameGTN.start_game()
NameError is not the same as AttributeError (which you mention in the question's summary). A NameError exception means that the name referenced in your code does not exist. A name can be a local variable, or a variable in an enclosing scope.
All methods in a class need to be called on an instance of that class. (staticmethods and classmethods not withstanding) Instead of name = player_choice() you need to write name = self.player_choice(). Likewise for all other occurrences where you call a method defined in the class.