I am working on tensorflow federated (tff). The problem arises when I call the iterative process and pass on the instance of the model created.
I have declared a keras model which is used at both the server and at the client. I basically want to modify the client model's weights and biases in order to perform mathematical functions on them. These mathematical functions are performed on the models' weights and biases by using function named 'processed'.
The processed function can return weights and layers both in tf or numpy format (the funtion 'processed' just uses tf.convert_to_tensor command to convert the numpy returned weights/biases from the get/set_weights command used)
def create_keras_model():
return tf.keras.models.Sequential([
tf.keras.layers.Conv2D(filters=32, kernel_size=[5, 5],name='conv2d_1',activation=tf.nn.relu, use_bias=True, bias_initializer =tf.initializers.lecun_normal(seed=137), input_shape=(28 ,28 ,1 )),
tf.keras.layers.MaxPool2D(pool_size=[2, 2], strides=2),
tf.keras.layers.Conv2D(filters=32, kernel_size=[5 5],name='conv2d_2',activation=tf.nn.relu, use_bias = True, bias_initializer=tf.initializers.lecun_normal(seed=137)),
tf.keras.layers.MaxPool2D(pool_size=[2 2], strides=2),
tf.keras.layers.Reshape(target_shape=(4 * 3 * 3)),
tf.keras.layers.Dense(units= 50, activation=tf.nn.relu, use_bias=True, bias_initializer=tf.initializers.lecun_normal(seed=137), name='dense_1'),
tf.keras.layers.Dense(units=10 , use_bias=True, bias_initializer=tf.initializers.lecun_normal(seed=137), activation=tf.nn.softmax, name='dense_2' ),
])
I create an instant of the above Keras model named as
net_1 = create_keras_model()
Now I am accessing the weights and biases by using the following function (proccessed) and assigning them new weights and biases after some mathmatical operations
def processed(net_1,L_norm):
a=net_1.get_layer('conv2d_1').get_weights()[0]
b=net_1.get_layer('conv2d_1').get_weights()[1]
c= net_1.get_layer('conv2d_2').get_weights()[0]
d=net_1.get_layer('conv2d_2').get_weights()[1]
e= net_1.get_layer('dense_1').get_weights()[0]
f= net_1.get_layer('dense_1').get_weights()[1]
g=net_1.get_layer('dense_2').get_weights()[0]
h= net_1.get_layer('dense_2').get_weights()[1]
L1,B1,L2,B2,L3,B3,L4,B4 = processing_work(a,b,c,d,e,f,g,h,L_norm)
L1=tf.convert_to_tensor(L1)
B1=tf.convert_to_tensor(B1)
L2=tf.convert_to_tensor(L2)
B2=tf.convert_to_tensor(B2)
L3=tf.convert_to_tensor(L3)
B3=tf.convert_to_tensor(B3)
L4=tf.convert_to_tensor(L4)
B4=tf.convert_to_tensor(B4)
net_1.get_layer('conv2d_1').set_weights([L1,B1])
net_1.get_layer('conv2d_2').set_weights([L2,B2])
net_1.get_layer('dense_1').set_weights([L3,B3])
net_1.get_layer('dense_2').set_weights([L4,B4])
return net_1
Following that, I use the following function
def model_fn_of():
# We _must_ create a new model here, and _not_ capture it from an external
# scope. TFF will call this within different graph contexts.
local_model = processed(create_keras_model(),L_norm)
return tff.learning.from_keras_model(
local_model,
input_spec=preprocessed_example_dataset.element_spec,
loss=tf.keras.losses.SparseCategoricalCrossentropy(),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])
Then I call the iterator of tff.
iterative_process = tff.learning.algorithms.build_weighted_fed_avg(
model_fn_of,
client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.02),
server_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=1.00))
However I get the following error:
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-32-651e903cc722> in <module>
2 model_fn_of,
3 client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.02),
----> 4 server_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=1.00))
8 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/ops.py in __getattr__(self, name)
511 from tensorflow.python.ops.numpy_ops import np_config
512 np_config.enable_numpy_behavior()""".format(type(self).__name__, name))
--> 513 self.__getattribute__(name)
514
515 #staticmethod
AttributeError: 'Tensor' object has no attribute 'numpy'
Can someone please help me with this. I dont know what is wrong and from which part of the code the error is even popping out. Please note that the dtype of weights/biases of model layers before and after using the 'processed' function are same.
Thanks in Advance
Related
for learning i tried to to use tensorflow probability to fit a simple 1 dimensional function (x,y) but i have realized i do something wrong when i call a model inside another model and i really need to understand what.
here is my model
mlp1 = tf.keras.Sequential(
[tf.keras.layers.Input(shape=(1), dtype=tf.float32),
Dense(20, activation="relu", name="layer1"),
Dense(20, activation="relu", name="layer2"),
Dense(1, activation="softplus",name="layer3"),
]
)
class subq0(tf.keras.Model):
def __init__(self):
super().__init__()
self.mlp1=mlp1()
def call(self, inputs):
sigma= self.mlp1(inputs)
return sigma
polynom = subq0()
optimizer = tf.optimizers.SGD(learning_rate=0.0001,momentum=0.9)
polynom.compile(optimizer=optimizer, loss= "mse" )
polynom.build(input_shape=(1,))
polynom.summary()
polynomial.fit(X_train, y_train , epochs= 100, verbose=0, batch_size=64 ,validation_data=(X_val,y_val) )
the error i get is
ValueError: You cannot build your model by calling build if your
layers do not support float type inputs. Instead, in order to
instantiate and build your model, call your model on real tensor data
(of the correct dtype).
The actual error from call is:
'tensorflow.python.framework.ops.EagerTensor' object is not callable.
I am implementing a custom layer in Keras. If I print the shape of the input passed to the call method, I get None as the first element. Why is that? Shouldn't the first element be the batch size?
def call(self, x):
print(x.shape) # (None, ...)
When I call model.fit, I am passing the batch size
batch_size = 50
model.fit(x_train, y_train, ..., batch_size=batch_size)
So, when is the method call actually called? And what is the recommended way of getting the batch size in the method call?
None means it is a dynamic shape. It can take any value depending on the batch size you choose.
When you define a model by default it is defined to support any batch size you can choose. This is what the None means. In TensorFlow 1.* the input to your model is an instance of tf.placeholder().
If you don't use the keras.InputLayer() with specified batch size you get the first dimension None by default:
import tensorflow as tf
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Dense(units=2, input_shape=(2, )))
print(model.inputs[0].get_shape().as_list()) # [None, 2]
print(model.inputs[0].op.type == 'Placeholder') # True
When you do use keras.InputLayer() with specified batch size you can define the input placeholder with fixed batch size:
import tensorflow as tf
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.InputLayer((2,), batch_size=50))
model.add(tf.keras.layers.Dense(units=2, input_shape=(2, )))
print(model.inputs[0].get_shape().as_list()) # [50, 2]
print(model.inputs[0].op.type == 'Placeholder') # True
When you specify the batch size to the model.fit() method these input placeholders have already been defined and you cannot modify their shape. The batch size for model.fit() is used only to split the data you provided to batches.
If you define your input layer with batch size 2 and then you pass different value of a batch size to the model.fit() method you will get ValueError:
import tensorflow as tf
import numpy as np
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.InputLayer((2,), batch_size=2)) # <--batch_size==2
model.add(tf.keras.layers.Dense(units=2, input_shape=(2, )))
model.compile(optimizer=tf.keras.optimizers.Adam(),
loss='categorical_crossentropy')
x_train = np.random.normal(size=(10, 2))
y_train = np.array([[0, 1] for _ in range(10)])
model.fit(x_train, y_train, batch_size=3) # <--batch_size==3
This will raise:
ValueError: Thebatch_sizeargument value 3 is incompatible with the specified batch size of your Input Layer: 2
I faced the same issue and found that using tf.shape(your_variable) instead of your_variable.shape solved the problem. As tf.shape(your_variable) is dynamically evaluated later when the fit function is called.
reference
https://github.com/tensorflow/tensorflow/issues/36991#issuecomment-590448880
To get the value in integers consider output_shape variable.
Minimum working example
import tensorflow as tf
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.InputLayer((2,), batch_size=50))
model.add(tf.keras.layers.Dense(units=2, input_shape=(2, )))
print(model.output_shape)
print(type(model.output_shape[0]), type(model.output_shape[1]))
Output:
(50, 2)
<class 'int'> <class 'int'>
I've created a model based on VGG16:
base_model = VGG16(weights='imagenet', include_top=False,
input_tensor=next_batch["image"],
input_shape=INPUT_SHAPE)
x = base_model.output
# model customization follows, not relevant
Note that in the snippet I specify input_tensor which comes from tf.data.Dataset.
I've trained the model for some epochs and then saved it using model.save("model.h5")
Now, when I try to load the model using load_model("model.h5") and continue training it, I get:
Traceback (most recent call last):
tensorflow.python.framework.errors_impl.InvalidArgumentError: You must
feed a value for placeholder tensor 'input_1' with dtype float and
shape [?,512,512,3]
[[{{node input_1}} = Placeholderdtype=DT_FLOAT, shape=[?,512,512,3],
_device="/job:localhost/replica:0/task:0/device:GPU:0"]]
So the question is, how to specify input_tensor on a loaded model?
After reading through the code responsible for model loading, I came up with the following solution:
class InputLayerFix:
#staticmethod
def from_config(config):
return InputLayer(input_tensor=next_batch["image"],
dtype=config["dtype"],
name=config["name"],
batch_input_shape=config["batch_input_shape"])
custom_objects = {"InputLayer": InputLayerFix}
model.load_model("model.h5", custom_objects=custom_objects)
This code relies on the fact that the deserialization routine first checks custom_objects for classes to instantiate (https://github.com/tensorflow/tensorflow/blob/r1.12/tensorflow/python/keras/utils/generic_utils.py#L155)
I have created a custom layer (called GraphGather) in Keras, yet the output tensor prints as :
Tensor("graph_gather/Tanh:0", shape=(?, ?), dtype=float32)
For some reason the shape is being returned as (?,?), which is causing the next dense layer to raise the following error:
ValueError: The last dimension of the inputs to Dense should be defined. Found None.
The GraphGather layer code is as follows:
class GraphGather(tf.keras.layers.Layer):
def __init__(self, batch_size, num_mols_in_batch, activation_fn=None, **kwargs):
self.batch_size = batch_size
self.num_mols_in_batch = num_mols_in_batch
self.activation_fn = activation_fn
super(GraphGather, self).__init__(**kwargs)
def build(self, input_shape):
super(GraphGather, self).build(input_shape)
def call(self, x, **kwargs):
# some operations (most of def call omitted)
out_tensor = result_of_operations() # this line is pseudo code
if self.activation_fn is not None:
out_tensor = self.activation_fn(out_tensor)
out_tensor = out_tensor
return out_tensor
def compute_output_shape(self, input_shape):
return (self.num_mols_in_batch, 2 * input_shape[0][-1])}
I have also tried hardcoding compute_output_shape to be:
python
def compute_output_shape(self, input_shape):
return (64, 150)
```
Yet the output tensor when printed is still
Tensor("graph_gather/Tanh:0", shape=(?, ?), dtype=float32)
which causes the ValueError written above.
System information
Have written custom code
**OS Platform and Distribution*: Linux Ubuntu 16.04
TensorFlow version (use command below): 1.5.0
Python version: 3.5.5
I had the same problem. My workaround was to add the following lines to the call method:
input_shape = tf.shape(x)
and then:
return tf.reshape(out_tensor, self.compute_output_shape(input_shape))
I haven't run into any problems with it yet.
If Johnny's answer doesn't work, I found another way to get around this is to follow advice here https://github.com/tensorflow/tensorflow/issues/38296#issuecomment-623698709
which is to call the set_shape method on the output of your layer.
E.g.
l=GraphGather(...)
y=l(x)
y.set_shape( l.compute_output_shape(x.shape) )
This only works if you are using the functional API.
(1) I'm trying to fine-tune a VGG-16 network using TFSlim by loading pretrained weights into all layers except thefc8 layer. I achieved this by using the TF-SLIm function as follows:
import tensorflow as tf
import tensorflow.contrib.slim as slim
import tensorflow.contrib.slim.nets as nets
vgg = nets.vgg
# Specify where the Model, trained on ImageNet, was saved.
model_path = 'path/to/vgg_16.ckpt'
# Specify where the new model will live:
log_dir = 'path/to/log/'
images = tf.placeholder(tf.float32, [None, 224, 224, 3])
predictions = vgg.vgg_16(images)
variables_to_restore = slim.get_variables_to_restore(exclude=['fc8'])
restorer = tf.train.Saver(variables_to_restore)
init = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init)
restorer.restore(sess,model_path)
print "model restored"
This works fine as long as I do not change the num_classes for the VGG16 model. What I would like to do is to change the num_classes from 1000 to 200. I was under the impression that if I did this modification by defining a new vgg16-modified class that replaces the fc8 to produce 200 outputs, (along with a variables_to_restore = slim.get_variables_to_restore(exclude=['fc8']) that everything will be fine and dandy. However, tensorflow complains of a dimensions mismatch:
InvalidArgumentError (see above for traceback): Assign requires shapes of both tensors to match. lhs shape= [1,1,4096,200] rhs shape= [1,1,4096,1000]
So, how does one really go about doing this ? The documentation for TFSlim is really patchy and there are several versions scattered on Github - so not getting much help there.
You can try using slim's way of restoring — slim.assign_from_checkpoint.
There is related documentation in the slim sources:
https://github.com/tensorflow/tensorflow/blob/129665119ea60640f7ed921f36db9b5c23455224/tensorflow/contrib/slim/python/slim/learning.py
Corresponding part:
*************************************************
* Fine-Tuning Part of a model from a checkpoint *
*************************************************
Rather than initializing all of the weights of a given model, we sometimes
only want to restore some of the weights from a checkpoint. To do this, one
need only filter those variables to initialize as follows:
...
# Create the train_op
train_op = slim.learning.create_train_op(total_loss, optimizer)
checkpoint_path = '/path/to/old_model_checkpoint'
# Specify the variables to restore via a list of inclusion or exclusion
# patterns:
variables_to_restore = slim.get_variables_to_restore(
include=["conv"], exclude=["fc8", "fc9])
# or
variables_to_restore = slim.get_variables_to_restore(exclude=["conv"])
init_assign_op, init_feed_dict = slim.assign_from_checkpoint(
checkpoint_path, variables_to_restore)
# Create an initial assignment function.
def InitAssignFn(sess):
sess.run(init_assign_op, init_feed_dict)
# Run training.
slim.learning.train(train_op, my_log_dir, init_fn=InitAssignFn)
Update
I tried the following:
import tensorflow as tf
import tensorflow.contrib.slim as slim
import tensorflow.contrib.slim.nets as nets
images = tf.placeholder(tf.float32, [None, 224, 224, 3])
predictions = nets.vgg.vgg_16(images)
print [v.name for v in slim.get_variables_to_restore(exclude=['fc8']) ]
And got this output (shortened):
[u'vgg_16/conv1/conv1_1/weights:0',
u'vgg_16/conv1/conv1_1/biases:0',
…
u'vgg_16/fc6/weights:0',
u'vgg_16/fc6/biases:0',
u'vgg_16/fc7/weights:0',
u'vgg_16/fc7/biases:0',
u'vgg_16/fc8/weights:0',
u'vgg_16/fc8/biases:0']
So it looks like you should prefix scope with vgg_16:
print [v.name for v in slim.get_variables_to_restore(exclude=['vgg_16/fc8']) ]
gives (shortened):
[u'vgg_16/conv1/conv1_1/weights:0',
u'vgg_16/conv1/conv1_1/biases:0',
…
u'vgg_16/fc6/weights:0',
u'vgg_16/fc6/biases:0',
u'vgg_16/fc7/weights:0',
u'vgg_16/fc7/biases:0']
Update 2
Complete example that executes without errors (at my system).
import tensorflow as tf
import tensorflow.contrib.slim as slim
import tensorflow.contrib.slim.nets as nets
s = tf.Session(config=tf.ConfigProto(gpu_options={'allow_growth':True}))
images = tf.placeholder(tf.float32, [None, 224, 224, 3])
predictions = nets.vgg.vgg_16(images, 200)
variables_to_restore = slim.get_variables_to_restore(exclude=['vgg_16/fc8'])
init_assign_op, init_feed_dict = slim.assign_from_checkpoint('./vgg16.ckpt', variables_to_restore)
s.run(init_assign_op, init_feed_dict)
In the example above vgg16.ckpt is a checkpoint saved by tf.train.Saver for 1000 classes VGG16 model.
Using this checkpoint with all variables of 200 classes model (including fc8) gives the following error:
init_assign_op, init_feed_dict = slim.assign_from_checkpoint('./vgg16.ckpt', slim.get_variables_to_restore())
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
1 init_assign_op, init_feed_dict = slim.assign_from_checkpoint(
----> 2 './vgg16.ckpt', slim.get_variables_to_restore())
/usr/local/lib/python2.7/dist-packages/tensorflow/contrib/framework/python/ops/variables.pyc in assign_from_checkpoint(model_path, var_list)
527 assign_ops.append(var.assign(placeholder_value))
528
--> 529 feed_dict[placeholder_value] = var_value.reshape(var.get_shape())
530
531 assign_op = control_flow_ops.group(*assign_ops)
ValueError: total size of new array must be unchanged