I need to loop in graph for different times. I am unable to pass the integer variable through feed_dict.
rough1 = tf.Graph()
with rough1.as_default():
st = tf.placeholder(tf.float32,shape = ())
d = tf.Variable(0)
for i in range(st):
d = tf.add(d,1)
with tf.Session(graph = rough1) as sess:
sess.run(tf.initialize_all_variables())
s = sess.run([d], feed_dict={st:3})
print s
When you build a Tensorflow graph, the graph isn't actually evaluated until you call tf.Session.run. A consequence of this is that Python constructs like range cannot inspect the value of a Tensor except by calling run(). Here, range(st) requires that the value of st be an integer known to Python, but the value of st is not known until the graph actually executes.
Put another way: the code that constructs the graph must not depend on the evaluation of that graph.
But here, you can just use a Python integer rather than a Tensor and everything works fine:
rough1 = tf.Graph()
with rough1.as_default():
d = tf.Variable(0)
for i in range(3):
d = tf.add(d,1)
with tf.Session(graph = rough1) as sess:
sess.run(tf.initialize_all_variables())
s = sess.run([d])
print s
Hope that helps!
Try this and see if it is what you are looking for.
st=1
rough1 = tf.Graph()
with rough1.as_default():
d = tf.Variable(0)
for i in range(st):
d = tf.add(d,1)
with tf.Session(graph = rough1) as sess:
sess.run(tf.initialize_all_variables())
s = sess.run([d])
print s
Related
My goal is to build a script to change an operation into another one using TF's graph editor. So far I tried making a script that just changes the input kernel weights of a Conv2D, but to no avail, as the interface is pretty confusing.
with tf.Session() as sess:
model_filename = sys.argv[1]
with gfile.FastGFile(model_filename, 'r') as f:
graph_def = graph_pb2.GraphDef()
text_format.Merge(f.read(), graph_def)
importer.import_graph_def(graph_def)
#my_sgv = ge.sgv("Conv2D", graph=tf.get_default_graph())
#print my_sgv
convs = find_conv2d_ops(tf.get_default_graph())
print convs
my_sgv = ge.sgv(convs)
print my_sgv
conv_tensor = tf.get_default_graph().get_tensor_by_name(convs[0].name + ':0')
conv_weights_input = tf.get_default_graph().get_tensor_by_name(convs[0].inputs[1].name)
weights_new = tf.Variable(tf.truncated_normal([1, 1, 1, 8], stddev=0.03),
name='Wnew')
ge.graph_replace(conv_tensor, {conv_weights_input: weights_new})
The error is "input needs to be a Tensor: ". Can someone please provide some insights?
Since you are dealing with a tf.Variable you don't need to use graph editor. tf.assign will be sufficient.
You can use it like the following:
assign_op = tf.assign(conv_weights_input, weights_new)
with tf.Session() as sess:
sess.run(assign_op)
If you are looking to sub out operations and not weights. Consider the following example (modified from this example):
import tensorflow as tf
import tensorflow.contrib.graph_editor as ge
def build():
a_pl = tf.placeholder(dtype=tf.float32, name="a")
b_pl = tf.placeholder(dtype=tf.float32, name="b")
c = tf.add(a_pl, b_pl, name="c")
build() #or load graph from disc
a = tf.constant(1.0, shape=[2, 3], name="a_const")
b = tf.constant(2.0, shape=[2, 3], name="b_const")
a_pl = tf.get_default_graph().get_tensor_by_name("a:0")
b_pl = tf.get_default_graph().get_tensor_by_name("b:0")
c = tf.get_default_graph().get_tensor_by_name("c:0")
c_ = ge.graph_replace(c, {a_pl: a, b_pl: b})
with tf.Session() as sess:
#no need for placeholders
print(sess.run(c_))
#will give error since a_pl and b_pl have no value
print(sess.run(c))
The issue with your code is that you're dealing with wights, and not tensors. The crux of the above example is that the first argument is the target tensor (output tensor) that have the to be replaced tensors as dependencies. The second argument are the actual tensors you want to replace.
It's also worth noting that conv_weights_input is actually a tensor, where weights_new is a tf.Variable. I believe what you want is to replace weights_new with a new conv operation with random weight initialisation.
I am able to perform addition using constant but not able to perform through tf.Variable
Below code is working fine when I am using constant for addition.
import tensorflow as tf
a = tf.constant(5)
b = tf.constant(6)
sess = tf.Session()
result = sess.run(a + b)
print(result)
But when I tried with tf.Variable it is not working, Here is mine code
import tensorflow as tf
a = tf.Variable(5)
b = tf.Variable(6)
sess = tf.Session()
result = sess.run(a + b)
print(result)
Can somebody help me ? Thanks !!!
You need to initialize the variables firstly:
import tensorflow as tf
a = tf.Variable(5)
b = tf.Variable(6)
sess = tf.Session()
Initialize variables:
sess.run(tf.global_variables_initializer())
result = sess.run(a + b)
print(result)
11
You can read more about variable initialization here, which says Unlike tf.Tensor objects, a tf.Variable exists outside the context of a single session.run call. So Before you can use a variable, it must be initialized. The initialization is Session specific, which means whenever you start a new session, and want to use these variables, you'll have to initialize them firstly.
Below is the code that I am running where in I am implementing a paper. I take two matrices, multiply them and then perform clustering. What am I doing wrong?
import tensorflow as tf
from sklearn.cluster import KMeans
import numpy as np
a = np.random.rand(10,10)
b = np.random.rand(10,5)
F = tf.placeholder("float", [None, 10], name='F')
mask = tf.placeholder("float", [None, 5], name='mask')
def getZfeature(F,mask):
return tf.matmul(F,mask)
def cluster(zfeature):
#km = KMeans(n_clusters=3)
#km.fit(x)
#mu = km.cluster_centers_
return zfeature
def computeQ(zfeature,mu):
print "computing q matrix"
print type(zfeature), type(mu)
#construct model
zfeature = getZfeature(F,mask)
mu = cluster(zfeature)
q = computeQ(zfeature,mu)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
sess.run(q, feed_dict={F: a, mask: b})
Working code below. Your problem is that q and mu don't do anything. q is a reference to the function computeQ as it doesn't return anything. mu doesn't do anything so in this answer I have evaluated zfeature. You can do more tensor operations in these two functions but you need to return a tensor for it to work.
import tensorflow as tf
from sklearn.cluster import KMeans
import numpy as np
a = np.random.rand(10,10)
b = np.random.rand(10,5)
F = tf.placeholder("float", [None, 10], name='F')
mask = tf.placeholder("float", [None, 5], name='mask')
def getZfeature(F,mask):
return tf.matmul(F,mask)
def cluster(zfeature):
#km = KMeans(n_clusters=3)
#km.fit(x)
#mu = km.cluster_centers_
return zfeature
def computeQ(zfeature,mu):
print ("computing q matrix")
print (type(zfeature), type(mu))
#construct model
zfeature = getZfeature(F,mask)
mu = cluster(zfeature)
q = computeQ(zfeature,mu)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
result=sess.run(zfeature, feed_dict={F: a, mask: b})
print(result)
Here is the code for k-means clustering using tensorflow from https://codesachin.wordpress.com/2015/11/14/k-means-clustering-with-tensorflow/
import tensorflow as tf
from random import choice, shuffle
from numpy import array
def TFKMeansCluster(vectors, noofclusters):
"""
K-Means Clustering using TensorFlow.
'vectors' should be a n*k 2-D NumPy array, where n is the number
of vectors of dimensionality k.
'noofclusters' should be an integer.
"""
noofclusters = int(noofclusters)
assert noofclusters < len(vectors)
#Find out the dimensionality
dim = len(vectors[0])
#Will help select random centroids from among the available vectors
vector_indices = list(range(len(vectors)))
shuffle(vector_indices)
#GRAPH OF COMPUTATION
#We initialize a new graph and set it as the default during each run
#of this algorithm. This ensures that as this function is called
#multiple times, the default graph doesn't keep getting crowded with
#unused ops and Variables from previous function calls.
graph = tf.Graph()
with graph.as_default():
#SESSION OF COMPUTATION
sess = tf.Session()
##CONSTRUCTING THE ELEMENTS OF COMPUTATION
##First lets ensure we have a Variable vector for each centroid,
##initialized to one of the vectors from the available data points
centroids = [tf.Variable((vectors[vector_indices[i]]))
for i in range(noofclusters)]
##These nodes will assign the centroid Variables the appropriate
##values
centroid_value = tf.placeholder("float64", [dim])
cent_assigns = []
for centroid in centroids:
cent_assigns.append(tf.assign(centroid, centroid_value))
##Variables for cluster assignments of individual vectors(initialized
##to 0 at first)
assignments = [tf.Variable(0) for i in range(len(vectors))]
##These nodes will assign an assignment Variable the appropriate
##value
assignment_value = tf.placeholder("int32")
cluster_assigns = []
for assignment in assignments:
cluster_assigns.append(tf.assign(assignment,
assignment_value))
##Now lets construct the node that will compute the mean
#The placeholder for the input
mean_input = tf.placeholder("float", [None, dim])
#The Node/op takes the input and computes a mean along the 0th
#dimension, i.e. the list of input vectors
mean_op = tf.reduce_mean(mean_input, 0)
##Node for computing Euclidean distances
#Placeholders for input
v1 = tf.placeholder("float", [dim])
v2 = tf.placeholder("float", [dim])
euclid_dist = tf.sqrt(tf.reduce_sum(tf.pow(tf.sub(
v1, v2), 2)))
##This node will figure out which cluster to assign a vector to,
##based on Euclidean distances of the vector from the centroids.
#Placeholder for input
centroid_distances = tf.placeholder("float", [noofclusters])
cluster_assignment = tf.argmin(centroid_distances, 0)
##INITIALIZING STATE VARIABLES
##This will help initialization of all Variables defined with respect
##to the graph. The Variable-initializer should be defined after
##all the Variables have been constructed, so that each of them
##will be included in the initialization.
init_op = tf.initialize_all_variables()
#Initialize all variables
sess.run(init_op)
##CLUSTERING ITERATIONS
#Now perform the Expectation-Maximization steps of K-Means clustering
#iterations. To keep things simple, we will only do a set number of
#iterations, instead of using a Stopping Criterion.
noofiterations = 100
for iteration_n in range(noofiterations):
##EXPECTATION STEP
##Based on the centroid locations till last iteration, compute
##the _expected_ centroid assignments.
#Iterate over each vector
for vector_n in range(len(vectors)):
vect = vectors[vector_n]
#Compute Euclidean distance between this vector and each
#centroid. Remember that this list cannot be named
#'centroid_distances', since that is the input to the
#cluster assignment node.
distances = [sess.run(euclid_dist, feed_dict={
v1: vect, v2: sess.run(centroid)})
for centroid in centroids]
#Now use the cluster assignment node, with the distances
#as the input
assignment = sess.run(cluster_assignment, feed_dict = {
centroid_distances: distances})
#Now assign the value to the appropriate state variable
sess.run(cluster_assigns[vector_n], feed_dict={
assignment_value: assignment})
##MAXIMIZATION STEP
#Based on the expected state computed from the Expectation Step,
#compute the locations of the centroids so as to maximize the
#overall objective of minimizing within-cluster Sum-of-Squares
for cluster_n in range(noofclusters):
#Collect all the vectors assigned to this cluster
assigned_vects = [vectors[i] for i in range(len(vectors))
if sess.run(assignments[i]) == cluster_n]
#Compute new centroid location
new_location = sess.run(mean_op, feed_dict={
mean_input: array(assigned_vects)})
#Assign value to appropriate variable
sess.run(cent_assigns[cluster_n], feed_dict={
centroid_value: new_location})
#Return centroids and assignments
centroids = sess.run(centroids)
assignments = sess.run(assignments)
return centroids, assignments
In my problem I need run GD with 1 example from data on each training step. It's known problem that session.run() has overhead and therefore it is too long to train model.
In attempt to avoid overhead I tried to use while_loop and train model on all data with one run() call. But it approach don't work and train_op don't execute even ones. Below simple example of what I'm doing:
data = [k*1. for k in range(10)]
tf.reset_default_graph()
i = tf.Variable(0, name='loop_i')
q_x = tf.FIFOQueue(100000, tf.float32)
q_y = tf.FIFOQueue(100000, tf.float32)
x = q_x.dequeue()
y = q_y.dequeue()
w = tf.Variable(0.)
b = tf.Variable(0.)
loss = (tf.add(tf.mul(x, w), b) - y)**2
gs = tf.Variable(0)
train_op = tf.train.GradientDescentOptimizer(0.05).minimize(loss, global_step=gs)
s = tf.Session()
s.run(tf.initialize_all_variables())
def cond(i):
return i < 10
def body(i):
return tf.tuple([tf.add(i, 1)], control_inputs=[train_op])
loop = tf.while_loop(cond, body, [i])
for _ in range(1):
s.run(q_x.enqueue_many((data, )))
s.run(q_y.enqueue_many((data, )))
s.run(loop)
s.close()
What I'm doing wrong? Or there is another solution of this problem with too expensive overhead?
Thanks!
The reason the model does not appear to train is because the input reading, gradient calculation, and the minimize() call are all defined outside (and hence, in dataflow terms, before) the body of the tf.while_loop(). This means that all of these parts of the model run only once, before the loop executes, and the loop itself has no effect.
A slight refactoring—to move the dequeue() operations, gradient calculation, and minimize() call inside the loop—fixes the problem and allows your program to train:
optimizer = tf.train.GradientDescentOptimizer(0.05)
def cond(i):
return i < 10
def body(i):
# Dequeue a new example each iteration.
x = q_x.dequeue()
y = q_y.dequeue()
# Compute the loss and gradient update based on the current example.
loss = (tf.add(tf.mul(x, w), b) - y)**2
train_op = optimizer.minimize(loss, global_step=gs)
# Ensure that the update is applied before continuing.
return tf.tuple([tf.add(i, 1)], control_inputs=[train_op])
loop = tf.while_loop(cond, body, [i])
UPDATE: Here's a complete program the executes the while loop, based on the code in your question:
import tensorflow as tf
# Define a single queue with two components to store the input data.
q_data = tf.FIFOQueue(100000, [tf.float32, tf.float32])
# We will use these placeholders to enqueue input data.
placeholder_x = tf.placeholder(tf.float32, shape=[None])
placeholder_y = tf.placeholder(tf.float32, shape=[None])
enqueue_data_op = q_data.enqueue_many([placeholder_x, placeholder_y])
gs = tf.Variable(0)
w = tf.Variable(0.)
b = tf.Variable(0.)
optimizer = tf.train.GradientDescentOptimizer(0.05)
# Construct the while loop.
def cond(i):
return i < 10
def body(i):
# Dequeue a single new example each iteration.
x, y = q_data.dequeue()
# Compute the loss and gradient update based on the current example.
loss = (tf.add(tf.multiply(x, w), b) - y) ** 2
train_op = optimizer.minimize(loss, global_step=gs)
# Ensure that the update is applied before continuing.
with tf.control_dependencies([train_op]):
return i + 1
loop = tf.while_loop(cond, body, [tf.constant(0)])
data = [k * 1. for k in range(10)]
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for _ in range(1):
# NOTE: Constructing the enqueue op ahead of time avoids adding
# (potentially many) copies of `data` to the graph.
sess.run(enqueue_data_op,
feed_dict={placeholder_x: data, placeholder_y: data})
print (sess.run([gs, w, b])) # Prints before-loop values.
sess.run(loop)
print (sess.run([gs, w, b])) # Prints after-loop values.
I am trying to initialize zero vectors in tensorflow as follow:
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
init = tf.initialize_all_variables()
# Tensorflow run
sess = tf.Session()
sess.run(init)
However, I am getting the following error:
InvalidArgumentError: dims must be a vector of int32.
Can you please help me fixing this problem?
Thanks
It works for me if you do this
W = tf.zeros([784, 10])
b = tf.zeros([10])
init = tf.initialize_all_variables()
# Tensorflow run
sess = tf.Session()
sess.run(init)
Also if you do it the way you're doing it. You'd still need to initialize W and b later on anyways as W below won't be initialized by the zeros tensor.
W = tf.Variable(tf.zeros([3,4]), name='x')
b = tf.Variable(x + 6, name='y')
model = tf.initialize_all_variables()
with tf.Session() as session:
session.run(model)
#Error: Attempting to use uninitialized value b
The example above will give an error but the one below won't and will give the correct answer.
W = tf.zeros([3,4], name='x')
b = tf.Variable(x + 6, name='y')
model = tf.initialize_all_variables()
with tf.Session() as session:
session.run(model)
If you want to do it the way you're with weights and biases (I'm guessing W and b stand for) I suggest looking here.
https://www.tensorflow.org/versions/r0.9/how_tos/variable_scope/index.html#variable-scope-example
Let me know if you still have any questions.