I have a customer Tensorflow op. that wrote in C++ and was build successfully to call in Tensorflow code as
from libs.customer_op import customer_op
output = customer_op(x, filter=w, rates=[1, 1, rate, rate], padding="SAME", strides=[1, 1, stride, stride])
Now, I am using Keras with Tensorflow backend. Is it possible to call my above function in Keras. Do we need do some extra register step?
Update: Thanks Matias Valdenegro for your suggestion. I have tried it. This is my full code in tensorflow and what I have done in Keras.
-Tensorflow code
def my_conv(input,num_o,kernel_size, stride):
num_x = input.shape[3].value
offset = slim.conv2d(input, 18, [kernel_size, kernel_size], stride=stride, activation_fn=None, scope='offset', normalizer_fn=None)
w = tf.get_variable('weights', shape=[num_o, num_x, kernel_size, kernel_size],
initializer=tf.contrib.layers.xavier_initializer())
output = customer_conv(x, filter=w, offset=offset,padding="SAME")
-Keras code:
def my_conv(input, num_o, kernel_size, stride):
num_x = input.shape[3].value
offset = KL.Conv2D(18, (kernel_size, kernel_size), strides=(stride,stride))(input)
w = KI.TruncatedNormal(mean=0.0, stddev=0.05, seed=None)
output = Lambda(lambda x: deform_conv_op(x, filter=w, offset=offset, padding="SAME"))(input)
return output
So, this is the place that I will call the function
class CustomerCNN():
def __init__(self, mode):
self.mode = mode
def build(self, mode):
# Inputs
input_image = KL.Input(
shape=config.IMAGE_SHAPE.tolist(), name="input_image")
f1 = Lambda(lambda x: my_conv(x, 256, 3, 1))(input_image)
For above solution, I still remain the issue:
How to initial weight with shape as shape=[num_o, num_x, kernel_size, kernel_size] in Keras
How to call my customer conv my_conv in the class CustomerCNN? Do we need one more Lambda function as I did
You can just call it with a lambda layer:
output = Lambda(lambda x: customer_op(x, filter=w, rates=[1, 1, rate, rate],
padding="SAME", strides=[1, 1, stride, stride]))(input)
Related
I am trying to create this custom ANN using tensorflow. Here is image of the toy network and code.
import tensorflow as tf
import numpy as np
in = np.array([1, 2, 3, 4], , dtype="float32")
y_true = np.array([10, 11], , dtype="float32")
# w is vector of weights
# y_pred = np.array([in[0]*w[0]+in[1]*w[0]], [in[2]*w[1]+in[3]*w[1]] )
# y_pred1 = 1 / (1 + tf.math.exp(-y_pred)) # sigmoid activation function
def loss_fun(y_true, y_pred1):
loss1 = tf.reduce_sum(tf.pow(y_pred1 - y_true, 2))
# model.compile(loss=loss_fun, optimizer='adam', metrics=['accuracy'])
The output of this network goes to another ANN to the right and I know that stuff, but don't know how can I create the connections, update the w, y_pred, and compile the model. Any help?
Something like this ought to work
import tensorflow as tf
import numpy as np
def y_pred(x, w):
return [x[0]*w[0]+x[1]*w[0], x[2]*w[1]+x[3]*w[1]]
def loss_fun(y_true, y_pred):
return tf.reduce_sum(tf.pow(y_pred - y_true, 2))
x = np.array([1, 2, 3, 4], dtype="float32")
y_true = np.array([10, 11], dtype="float32")
w = tf.Variable(initial_value=np.random.normal(size=(2)), name='weights', dtype=tf.float32)
xt = tf.convert_to_tensor(x)
yt = tf.convert_to_tensor(y_true)
sgd_opt = tf.optimizers.SGD()
training_steps = 100
display_steps = 10
for step in range(training_steps):
with tf.GradientTape() as tape:
tape.watch(w)
yp = y_pred(xt, w)
loss = loss_fun(yt, yp)
dl_dw = tape.gradient(loss, w)
sgd_opt.apply_gradients(zip([dl_dw], [w]))
if step % display_steps == 0:
print(loss, w)
I tried Convolution Neural Network with Tensorflow.
However, Shape causes an error.
First is part of the main function.
while True:
with mss.mss() as sct:
Game_Scr = np.array(sct.grab(Game_Scr_pos))[:,:,:3]
cv2.imshow('Game_Src', Game_Scr)
cv2.waitKey(0)
Game_Scr = cv2.resize(Game_Scr, dsize=(960, 540), interpolation=cv2.INTER_AREA)
print(Game_Scr.shape)
print(Convolution(Game_Scr))
Second is my called function.
def Convolution(img):
kernel = tf.Variable(tf.truncated_normal(shape=[4], stddev=0.1))
sess = tf.Session()
with tf.Session() as sess:
img = img.astype('float32')
Bias1 = tf.Variable(tf.truncated_normal(shape=[4],stddev=0.1))
conv2d = tf.nn.conv2d(img, kernel, strides=[1, 1, 1, 1], padding='SAME')# + Bias1
conv2d = sess.run(conv2d)
return conv2d
ValueError: Shape must be rank 4 but is rank 3 for 'Conv2D' (op: 'Conv2D') with input shapes: [540,960,3], [4].
I tried changing the shape many times, but I get the same error.
Try replacing
img = img.astype('float32')
with
img = tf.expand_dims(img.astype('float32'), 0)
The dimention of tf.nn.conv2d input shoul be 4, (batch_size, image_hight, image_with, image_channels). You where missing the batch_size, tf.expand_dims just add that dimention (with a batch_size of 1 since you only have one image).
As per the official documentation here, input tensor should be of shape [batch, in_height, in_width, in_channels] and a filter / kernel tensor should be of shape [filter_height, filter_width, in_channels, out_channels].
Try by changing your Convolution function to something like this:
def Convolution(img):
kernel = tf.Variable(tf.truncated_normal(shape=[200, 200, 3, 3], stddev=0.1))
sess = tf.Session()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
img = img.astype('float32')
conv2d = tf.nn.conv2d(np.expand_dims(img, 0), kernel, strides=[1, 1, 1, 1], padding='SAME')# + Bias1
conv2d = sess.run(conv2d)
return conv2d
I'm trying to convert my tensorflow code to tensorflow eager. The problem is the forward pass predicts only the same actions for different input values in eager mode. The normal tensorflow code with graph works fine. I've only changed the network. The agent is the same I'm used with normal tensorflow. What could be the problem with the network? The forward pass is in the function get_probs()
Another issue is the eager network is very slow. I think the graph execution is 2-3 times faster.
Example probs for one episode in eager
...
[0.31471518 0.33622807 0.34905672]
[0.31472355 0.3363353 0.34894115]
[0.31482834 0.33600125 0.34917045]
[0.31461707 0.33643782 0.34894508]
[0.31466153 0.33620775 0.34913075]
[0.31461093 0.33637658 0.3490125 ]
[0.31452385 0.33623937 0.34923682]
[0.31438416 0.33645296 0.3491629 ]
[0.31471425 0.3363982 0.34888753]
[0.314866 0.33610862 0.34902537]
[0.31489033 0.33622313 0.34888652]
...
Example probs for one episode with tensorflow graph
...
[0.25704077 0.46056205 0.28239718]
[0.20610097 0.49288744 0.30101162]
[0.24638997 0.5338215 0.2197885 ]
[0.22581507 0.51206875 0.2621162 ]
[0.19064051 0.5398092 0.26955026]
[0.24399564 0.4424694 0.313535 ]
[0.25321653 0.48051655 0.26626688]
[0.2241595 0.43447506 0.3413655 ]
[0.20665398 0.5128011 0.28054494]
[0.2943201 0.39530927 0.3103706 ]
...
Network
import tensorflow as tf
from keras.layers import *
import numpy as np
tf.enable_eager_execution()
print(tf.executing_eagerly())
class PGEagerAtariNetwork:
def __init__(self, state_space, action_space, lr):
self.state_space = state_space
self.action_space = action_space
self.model = tf.keras.Sequential()
self.model.add(InputLayer(input_shape=(84, 84, 4)))
# Conv
self.model.add(Conv2D(filters=32, kernel_size=[8, 8], strides=[4, 4], activation='relu', name='conv1'))
self.model.add(Conv2D(filters=64, kernel_size=[4, 4], strides=[2, 2], activation='relu', name='conv2'))
self.model.add(Conv2D(filters=128, kernel_size=[4, 4], strides=[2, 2], activation='relu', name='conv3'))
# Flatten
self.model.add(Flatten(name='flatten'))
# Fully connected
self.model.add(Dense(units=512, activation='relu', name='fc1'))
# Logits
self.model.add(Dense(units=self.action_space, activation=None, name='logits'))
self.model.summary()
# Optimizer
self.optimizer = tf.train.AdamOptimizer(learning_rate=lr)
def get_probs(self, s):
s = s[np.newaxis, :]
logits = self.model(s)
probs = tf.nn.softmax(logits).numpy().squeeze()
return probs
def update_policy(self, s, r, a):
with tf.GradientTape() as tape:
loss = self.calc_loss(s, r, a)
grads = tape.gradient(loss, self.model.trainable_variables)
self.optimizer.apply_gradients(zip(grads, self.model.trainable_variables),
global_step=tf.train.get_or_create_global_step())
def calc_loss(self, s, r, a):
logits = self.model(s)
policy_loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=a, logits=logits)
policy_loss = tf.reduce_mean(policy_loss * tf.stop_gradient(r))
loss = tf.reduce_mean(policy_loss)
return loss
I defined a model function which named "drrn_model". While I was training my model, I use model by:
shared_model = tf.make_template('shared_model', drrn_model)
train_output = shared_model(train_input, is_training=True)
It begin training step by step, and I can restore .ckpt file to the model when I want to continue to train the model from an old point.
But there is a problem when I test my trained model.
I use the code below directly without using tf.make_template:
train_output = drrn_model(train_input, is_training=False)
Then the terminal gave me a lots of NotFoundError like "Key LastLayer/Variable_2 not found in checkpoint".
But when I use
shared_model = tf.make_template('shared_model', drrn_model)
output_tensor = shared_model(input_tensor,is_training=False)
It can test normally.
So why we must use tf.make_template() again in testing stage. What is the difference between drrn_model and make_template when we construct our model.
And there is another question: the BN layer in tensorflow.
I have tried many ways but the outputs is always wrong(always worse then the version without BN layer).
There is my newest version of model with BN layer:
tensor = None
def drrn_model(input_tensor, is_training):
with tf.device("/gpu:0"):
with tf.variable_scope("FirstLayer"):
conv_0_w = tf.get_variable("conv_w", [3, 3, 1, 128], initializer=tf.random_normal_initializer(stddev=np.sqrt(2.0 / 9)))
tensor = tf.nn.conv2d(tf.nn.relu(batchnorm(input_tensor, is_training= is_training)), conv_0_w, strides=[1,1,1,1], padding="SAME")
first_layer = tensor
### recursion ###
with tf.variable_scope("recycle", reuse=False):
tensor = drrnblock(first_layer, tensor, is_training)
for i in range(1,10):
with tf.variable_scope("recycle", reuse=True):
tensor = drrnblock(first_layer, tensor, is_training)
### end layer ###
with tf.variable_scope("LastLayer"):
conv_end_w = tf.get_variable("conv_w", [3, 3, 128, 1], initializer=tf.random_normal_initializer(stddev=np.sqrt(2.0 / 9)))
conv_end_layer = tf.nn.conv2d(tf.nn.relu(batchnorm(tensor, is_training= is_training)), conv_end_w, strides=[1, 1, 1, 1], padding='SAME')
tensor = tf.add(input_tensor,conv_end_layer)
return tensor
def drrnblock(first_layer, input_layer, is_training):
conv1_w = tf.get_variable("conv1__w", [3, 3, 128, 128], initializer=tf.random_normal_initializer(stddev=np.sqrt(2.0 / 9)))
conv1_layer = tf.nn.conv2d(tf.nn.relu(batchnorm(input_layer, is_training= is_training)), conv1_w, strides=[1,1,1,1], padding= "SAME")
conv2_w = tf.get_variable("conv2__w", [3, 3, 128, 128], initializer=tf.random_normal_initializer(stddev=np.sqrt(2.0 / 9)))
conv2_layer = tf.nn.conv2d(tf.nn.relu(batchnorm(conv1_layer, is_training=is_training)), conv2_w, strides=[1, 1, 1, 1], padding="SAME")
tensor = tf.add(first_layer, conv2_layer)
return tensor
def batchnorm(inputs, is_training, decay = 0.999):# there is my BN layer
scale = tf.Variable(tf.ones([inputs.get_shape()[-1]]))
beta = tf.Variable(tf.zeros([inputs.get_shape()[-1]]))
pop_mean = tf.Variable(tf.zeros([inputs.get_shape()[-1]]), trainable=False)
pop_var = tf.Variable(tf.ones([inputs.get_shape()[-1]]), trainable=False)
if is_training:
batch_mean, batch_var = tf.nn.moments(inputs,[0,1,2])
print("batch_mean.shape: ", batch_mean.shape)
train_mean = tf.assign(pop_mean, pop_mean*decay+batch_mean*(1-decay))
train_var = tf.assign(pop_var, pop_var*decay+batch_var*(1-decay))
with tf.control_dependencies([train_mean, train_var]):
return tf.nn.batch_normalization(inputs,batch_mean,batch_var,beta,scale,variance_epsilon=1e-3)
else:
return tf.nn.batch_normalization(inputs,pop_mean,pop_var,beta,scale,variance_epsilon=1e-3)
Please tell me where is wrong in my code.
Thanks a lot!!
I'm trying to create a dice_loss function in Tensorflow.
I'm facing a trouble with tensorlfow. Executing the following code
import tensorflow as tf
import tensorlayer as tl
def conv3d(x, inChans, outChans, kernel_size, stride, padding):
weights = weight_variable([kernel_size, kernel_size, kernel_size, inChans, outChans])
biases = bias_variable([outChans])
conv = tf.nn.conv3d(x, weights, strides=[1, stride, stride, stride, 1], padding=padding)
return tf.nn.bias_add(conv, biases)
def train(loss_val, var_list):
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
grads = optimizer.compute_gradients(loss_val, var_list=var_list)
return optimizer.apply_gradients(grads)
def main(argv=None):
image = tf.placeholder(tf.float32, shape=[None, SLICE_SIZE, IMAGE_SIZE, IMAGE_SIZE, 1], name="input_image")
annotation = tf.placeholder(tf.float32, shape=[None, SLICE_SIZE, IMAGE_SIZE, IMAGE_SIZE, 1], name="annotation")
logits, pred_annotation = vnet.VNet(image)
loss = 1 - tl.cost.dice_coe(output=pred_annotation, target=annotation, axis=[1,2,3,4])
trainable_var = tf.trainable_variables()
train_op = train(loss, trainable_var)
sess = tf.Session()
...
...
def VNet(x):
...
out = tf.nn.elu(BatchNorm3d(conv3d(x, inChans, 2, kernel_size=5, stride=1, padding="SAME")))
out = conv3d(out, 2, 2, kernel_size=1, stride=1, padding="SAME")
annotation_pred = tf.to_float(tf.argmax(out, dimension=4, name='prediction'))
return out, tf.expand_dims(annotation_pred, dim=4)
I get the following error:
ValueError: No gradients provided for any variable: ...
Someone can help me?
When you do annotation_pred = tf.to_float(tf.argmax(out, dimension=4, name='prediction')), you get an index of the max value in your tensor. This index can't be derivated, thus the gradient can't flow throught this operation.
So as your loss is only defined by this value, and the gradient can't flow throught it, no gradient can be calculated for your network.
I don't know specificately how the dice loss work, but maybe you wanted to use tf.max instead of tf.argmax, or you have to find a way to use an operation that can let the gradient flow.