Is it somehow possible in GPflow to get FLOPS? I have found an example using Tensorflow but have no idea how to use this in a GPflow context:
g = tf.Graph()
run_meta = tf.RunMetadata()
with g.as_default():
A = tf.Variable(tf.random_normal( [25,16] ))
B = tf.Variable(tf.random_normal( [16,9] ))
C = tf.matmul(A,B)
opts = tf.profiler.ProfileOptionBuilder.float_operation()
flops = tf.profiler.profile(g, run_meta=run_meta, cmd='op', options=opts)
if flops is not None:
print('TF stats gives',flops.total_float_ops)
I dig a little bit in the source code of GPFlow. The key to make it work is to intercept the Tensorflow op that you want to profile before GPFlow's AutoFlows creates a new graph.
In my case, I wanted to profile the predict() function. The function you need is model._build_predict() (there's one equivalent for log-likelihood).
Here's how it works:
gpflow.reset_default_graph_and_session()
kernel = gpflow.kernels.RBF(1)
model = gpflow.models.GPR(X, Y, kernel)
run_metadata = tf.RunMetadata()
with model.enquire_session(session=None) as tf_session:
predict_op = model._build_predict(X)
tf_session.run(predict_op, options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE),
run_metadata=run_metadata)
opts = tf.profiler.ProfileOptionBuilder.float_operation()
prof = tf.profiler.profile(tf_session.graph, run_meta=run_metadata,
cmd='op', options=opts)
print('FOps: ', prof.total_float_ops)
Related
I have managed this implementation on retraining frozen graph in tensorflow 1 according to this wonderful detail topic. Basically, the methodology is described:
Load frozen model
Replace the constant frozen node with variable node.
The newly replaced variable node then will be redirected to the corresponding output of the frozen node.
This works in tensorflow 1.x by checking the tf.compat.v1.trainable_variables. However, in tensorflow 2.x, it can't work anymore.
Below is the code snippet:
1/ Load frozen model
frozen_path = '...'
detection_graph = tf.Graph()
with detection_graph.as_default():
od_graph_def = tf.compat.v1.GraphDef()
with tf.compat.v1.io.gfile.GFile(frozen_path, 'rb') as fid:
serialized_graph = fid.read()
od_graph_def.ParseFromString(serialized_graph)
tf.graph_util.import_graph_def(od_graph_def, name='')
2/ Create a clone
with detection_graph.as_default():
const_var_name_pairs = {}
probable_variables = [op for op in detection_graph.get_operations() if op.type == "Const"]
available_names = [op.name for op in detection_graph.get_operations()]
for op in probable_variables:
name = op.name
if name+'/read' not in available_names:
continue
tensor = detection_graph.get_tensor_by_name('{}:0'.format(name))
with tf.compat.v1.Session() as s:
tensor_as_numpy_array = s.run(tensor)
var_shape = tensor.get_shape()
# Give each variable a name that doesn't already exist in the graph
var_name = '{}_turned_var'.format(name)
var = tf.Variable(name=var_name, dtype=op.outputs[0].dtype, initial_value=tensor_as_numpy_array,trainable=True, shape=var_shape)
const_var_name_pairs[name] = var_name
3/ Relace frozen node by Graph Editor
import graph_def_editor as ge
ge_graph = ge.Graph(detection_graph.as_graph_def())
name_to_op = dict([(n.name, n) for n in ge_graph.nodes])
for const_name, var_name in const_var_name_pairs.items():
const_op = name_to_op[const_name+'/read']
var_reader_op = name_to_op[var_name + '/Read/ReadVariableOp']
ge.swap_outputs(ge.sgv(const_op), ge.sgv(var_reader_op))
detection_training_graph = ge_graph.to_tf_graph()
with detection_training_graph.as_default():
writer = tf.compat.v1.summary.FileWriter('remap', detection_training_graph )
writer.close
The problem was my Graph Editor when I import the tf.graph_def instead of the original tf.graph that has Variables.
Quickly solve by fixing step 3
Sol1: Using Graph Editor
ge_graph = ge.Graph(detection_graph)
for const_name, var_name in const_var_name_pairs.items():
const_op = ge_graph._node_name_to_node[const_name+'/read']
var_reader_op = ge_graph._node_name_to_node[var_name+'/Read/ReadVariableOp']
ge.swap_outputs(ge.sgv(const_op), ge.sgv(var_reader_op))
However, this requires disable eager execution. To work around with eager execution, you should attach the MetaGraphDef to Graph Editor as below
with detection_graph.as_default():
meta_saver = tf.compat.v1.train.Saver()
meta = meta_saver.export_meta_graph()
ge_graph = ge.Graph(detection_graph,collections=ge.graph._extract_collection_defs(meta))
However, this is the trickest to make the model trainable in tf2.x
Instead of using Graph Editor to export directly the graph, we should export ourselves. The reason is that the Graph Editor make the Variables data type to be resources. Therefore, we should export the graph as graphdef and import the variable def to the graph:
test_graph = tf.Graph()
with test_graph.as_default():
tf.import_graph_def(ge_graph.to_graph_def(), name="")
for var_name in ge_graph.variable_names:
var = ge_graph.get_variable_by_name(var_name)
ret = variable_pb2.VariableDef()
ret.variable_name = var._variable_name
ret.initial_value_name = var._initial_value_name
ret.initializer_name = var._initializer_name
ret.snapshot_name = var._snapshot_name
ret.trainable = var._trainable
ret.is_resource = True
tf_var = tf.Variable(variable_def=ret,dtype=tf.float32)
test_graph.add_to_collections(var.collection_names, tf_var)
Sol2: Manually map by Graphdef
with detection_graph.as_default() as graph:
training_graph_def = remap_input_node(detection_graph.as_graph_def(),const_var_name_pairs)
current_var = (tf.compat.v1.trainable_variables())
assert len(current_var)>0, "no training variables"
detection_training_graph = tf.Graph()
with detection_training_graph.as_default():
tf.graph_util.import_graph_def(training_graph_def, name='')
for var in current_var:
ret = variable_pb2.VariableDef()
ret.variable_name = var.name
ret.initial_value_name = var.name[:-2] + '/Initializer/initial_value:0'
ret.initializer_name = var.name[:-2] + '/Assign'
ret.snapshot_name = var.name[:-2] + '/Read/ReadVariableOp:0'
ret.trainable = True
ret.is_resource = True
tf_var = tf.Variable(variable_def=ret,dtype=tf.float32)
detection_training_graph.add_to_collections({'trainable_variables', 'variables'}, tf_var)
current_var = (tf.compat.v1.trainable_variables())
assert len(current_var)>0, "no training variables"
I am having a problem calculating the gradient in TensorFlow 1.15. I think it's something related context manager or keras session, but I am not sure about it. Following is the code I have written:
def create_adversarial_pattern_CW(input_patch, input_label, target_label):
input_patch_T = tf.cast(input_patch,tf.float32)
with tf.GradientTape() as tape:
tape.watch(input_patch_T)
patch_pred = model(input_patch_T)
loss_input_label = soft_dice_loss(input_label, patch_pred[0])
loss_target_label = soft_dice_loss(target_label, patch_pred[0])
f = loss_input_label - loss_target_label
f_grad = tape.gradient(f, input_patch_T)
#-------------------------#
print(type(f_grad))
#-------------------------#
f_grad_sign = tf.sign(f_grad)
return f_grad_sign
def DAG():
sess = K.get_session()
with sess.as_default() as sess:
adv_x_old = tf.cast(X,dtype=tf.float32)
for i in range(iters):
#-------------------------#
#y_pred = model(adv_x_old) -> If I uncomment this line the value of f_grad returned is None, otherwise it works fine, but I need this line
#-------------------------#
perturbations = create_adversarial_pattern_CW(adv_x_old, y, y_target)
adv_x_new = adv_x_old - alpha*perturbations
adv_x_old = adv_x_new
adv_patch_pred = model(adv_x_old)
To fix it, I tried to wrap the commented line as :
with tf.GradientTape() as tape:
with tape.stop_recording():
y_pred = model(adv_x_old)
but I still get the value of f_grad as None.
Let's say, some built-in function returns train_op and summary_op where summary_op is defined by tf.summary.merge(summaries, name='summary_op'), and I cannot touch the function.
Also, let's say, I am going to use the built-in slim.learning.train which takes train_op and summary_op as input arguments.
# -- typical
train_op, summary_op = model_fn(image)
slim.learning.train(train_op, summary_op=summary_op)
# -- my question
train_op, summary_op = model_fn(image)
some_other_summary_list = some_another_function()
summary_op_ = ... # is it possible to append some_other_summary_list to summary_op?
slim.learning.train(train_op, summary_op=summary_op_)
How I can combine summaries in already-merged summary_op and newly-collected summaries some_other_summary_list?
-- If I do tf.merge_all(tf.GraphKeys.SUMMARIES) actually there will be too many summaries since, in model_fn() collect only useful and necessary summaries.
-- I can think of defining separate summary_op2 and define train_step_fn as in:
from tensorflow.contrib.slim.python.slim.learning import train_step
def train_step_fn(...):
... = train_step(...)
if iteration % 100 == 0:
summaries = session.run(summary_op2)
summary_writer.add_summary(summaries, iteration)
slim.learning.train(train_op, summary_op=summary_op, train_step_fn=train_step_fn)
However, this seems too much if I can simply somehow append new summaries to summary_op. Is it possible?
If both "summary_op and newly-collected summaries some_other_summary_list" are created by tf.summary.merge, you can simply merge them again by tf.summary.merge([summary_op, summaries some_other_summary_list]), as demonstrated by this code:
import tensorflow as tf
a = tf.summary.scalar('a', tf.constant(0))
b = tf.summary.scalar('b', tf.constant(1))
c = tf.summary.scalar('c', tf.constant(2))
d = tf.summary.scalar('d', tf.constant(3))
ab = tf.summary.merge([a, b])
cd = tf.summary.merge([c, d])
abcd = tf.summary.merge([ab, cd])
with tf.Session() as sess:
writer = tf.summary.FileWriter('.', sess.graph)
summary = sess.run(abcd)
writer.add_summary(summary)
I would like to use the sparse_softmax_cross_entropy_with_logits
with the julia TensorFlow wrapper.
The operations is defined in the code here.
Basically, as I understand it the first argument should be logits, that would normally be fed to softmax to get them to be category probabilities (~1hot output).
And the second should be the correct labels as label ids.
I have adjusted the example code from the TensorFlow.jl readme
See below:
using Distributions
using TensorFlow
# Generate some synthetic data
x = randn(100, 50)
w = randn(50, 10)
y_prob = exp(x*w)
y_prob ./= sum(y_prob,2)
function draw(probs)
y = zeros(size(probs))
for i in 1:size(probs, 1)
idx = rand(Categorical(probs[i, :]))
y[i, idx] = 1
end
return y
end
y = draw(y_prob)
# Build the model
sess = Session(Graph())
X = placeholder(Float64)
Y_obs = placeholder(Float64)
Y_obs_lbl = indmax(Y_obs, 2)
variable_scope("logisitic_model", initializer=Normal(0, .001)) do
global W = get_variable("weights", [50, 10], Float64)
global B = get_variable("bias", [10], Float64)
end
L = X*W + B
Y=nn.softmax(L)
#costs = log(Y).*Y_obs #Dense (Orginal) way
costs = nn.sparse_softmax_cross_entropy_with_logits(L, Y_obs_lbl+1) #sparse way
Loss = -reduce_sum(costs)
optimizer = train.AdamOptimizer()
minimize_op = train.minimize(optimizer, Loss)
saver = train.Saver()
# Run training
run(sess, initialize_all_variables())
cur_loss, _ = run(sess, [Loss, minimize_op], Dict(X=>x, Y_obs=>y))
When I run it however, I get an error:
Tensorflow error: Status: Incompatible shapes: [1,100] vs. [100,10]
[[Node: gradients/SparseSoftmaxCrossEntropyWithLogits_10_grad/mul = Mul[T=DT_DOUBLE, _class=[], _device="/job:localhost/replica:0/task:0/cpu:0"](gradients/SparseSoftmaxCrossEntropyWithLogits_10_grad/ExpandDims, SparseSoftmaxCrossEntropyWithLogits_10:1)]]
in check_status(::TensorFlow.Status) at /home/ubuntu/.julia/v0.5/TensorFlow/src/core.jl:101
in run(::TensorFlow.Session, ::Array{TensorFlow.Port,1}, ::Array{Any,1}, ::Array{TensorFlow.Port,1}, ::Array{Ptr{Void},1}) at /home/ubuntu/.julia/v0.5/TensorFlow/src/run.jl:96
in run(::TensorFlow.Session, ::Array{TensorFlow.Tensor,1}, ::Dict{TensorFlow.Tensor,Array{Float64,2}}) at /home/ubuntu/.julia/v0.5/TensorFlow/src/run.jl:143
This only happens when I try to train it.
If I don't include an optimise function/output then it works fine.
So I am doing something that screws up the gradient math.
i wanna display some python variables in tensorboard, but i dont get it done.
My code so far, display only a line in tensorboard for the lines with static number, if i use the outcommented-lines, it does not work ? It then prints:
ValueError: Shapes () and (?,) are not compatible
Someone has an idea?
import tensorflow as tf
step = 0
session = tf.Session()
tensorboardVar = tf.Variable(0, "tensorboardVar")
pythonVar = tf.placeholder("int32", [None])
#update_tensorboardVar = tensorboardVar.assign(pythonVar)
update_tensorboardVar = tensorboardVar.assign(4)
tf.scalar_summary("myVar", update_tensorboardVar)
merged = tf.merge_all_summaries()
sum_writer = tf.train.SummaryWriter('/tmp/train/c/', session.graph)
session.run(tf.initialize_all_variables())
for i in range(100):
_, result = session.run([update_tensorboardVar, merged])
#_, result = session.run([update_tensorboardVar, merged], feed_dict={pythonVar: i})
sum_writer.add_summary(result, step)
step += 1
this is working:
import tensorflow as tf
import numpy as np
step = 0
session = tf.Session()
tensorboardVar = tf.Variable(0, "tensorboardVar")
pythonVar = tf.placeholder("int32", [])
update_tensorboardVar = tensorboardVar.assign(pythonVar)
tf.scalar_summary("myVar", update_tensorboardVar)
merged = tf.merge_all_summaries()
sum_writer = tf.train.SummaryWriter('/tmp/train/c/', session.graph)
session.run(tf.initialize_all_variables())
for i in range(100):
#_, result = session.run([update_tensorboardVar, merged])
j = np.array(i)
_, result = session.run([update_tensorboardVar, merged], feed_dict={pythonVar: j})
sum_writer.add_summary(result, step)
step += 1
An alternative way can be found in the second answer to Computing exact moving average over multiple batches in tensorflow. There it is shown how you can create custom summaries.