How do I reuse variables in TensorFlow? I want to reuse the tf.contrib.layers.linear
with tf.variable_scope("root") as varscope:
inputs_1 = tf.constant(0.5, shape=[2, 3, 4])
inputs_2 = tf.constant(0.5, shape=[2, 3, 4])
outputs_1 = tf.contrib.layers.linear(inputs_1, 5)
varscope.reuse_variables()
outputs_2 = tf.contrib.layers.linear(inputs_2, 5)
But it gives me the following result
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-51-a40b9ec68e25> in <module>()
5 outputs_1 = tf.contrib.layers.linear(inputs_1, 5)
6 varscope.reuse_variables()
----> 7 outputs_2 = tf.contrib.layers.linear(inputs_2, 5)
...
ValueError: Variable root/fully_connected_1/weights does not exist, or was not created with tf.get_variable(). Did you mean to set reuse=None in VarScope?
The problem is tf.contrib.layers.linear automatically creates a new set of linear layers with its own scope. When calling scope.reuse() there's nothing to be reused because those are new variables.
Try to do something like this instead
def function():
with tf.variable_scope("root") as varscope:
inputs = tf.constant(0.5, shape=[2, 3, 4])
outputs = tf.contrib.layers.linear(inputs, 5)
return outputs
result_1 = function()
tf.get_variable_scope().reuse_variables()
result_2 = function()
sess = tf.InteractiveSession()
sess.run(tf.initialize_all_variables())
a = sess.run(result_1)
b = sess.run(result_2)
np.all(a == b) # ==> True
you just need to modify linear(inputs_1, 5) to linear(inputs_1, 5, scope="linear")
with tf.variable_scope("root") as varscope:
inputs_1 = tf.constant(0.5, shape=[2, 3, 4])
inputs_2 = tf.constant(0.5, shape=[2, 3, 4])
outputs_1 = tf.contrib.layers.linear(inputs_1, 5, scope="linear")
varscope.reuse_variables()
outputs_2 = tf.contrib.layers.linear(inputs_2, 5, scope="linear")
Related
I am trying to build a windowed dataset from a univariate time series.
The idea is if the series looks like [1, 2, 3, 4, 5, 6] and the window length was 2, then
I'd take windows of length 3 to account for 2 X features and Y target output, so
[[1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 6]] then I'll shuffle them up to avoid bias from that, and split out the input features from target output for each window: [[[1, 2], [3]], [[2, 3], [4]], [[3, 4], [5]], [[4, 5], [6]]]
def windowed_dataset(series):
# Initially the data is (N,) expand dims to (N, 1)
series = tf.expand_dims(series, axis=-1)
# Tensorflow Dataset from the array
ds = tf.data.Dataset.from_tensor_slices(series)
# Create the windows that will serve as input features and label (hence +1)
ds = ds.window(window_len + 1, shift=1, drop_remainder=True)
ds = ds.flat_map(lambda w: w.batch(window_len + 1))
# randomize order
ds = ds.shuffle(shuffle_buffer)
# Separate the inputs and the target output(label)
ds = ds.map(lambda w: (w[:-1], w[-1]))
return ds.batch(batch_size).prefetch(1)
However I'd like to add some normalization. For example if my window is w=[1, 2, 3] then I'd like to normalize according to [p/w[0] - 1 for p in w]
I thought I could achieve this with ds.map and
def normalize_window(w):
return [((i/w[0]) -1) for i in w]
ds = ds.map(normalize_window)
because map is supposed to apply the function to each window in the dataset, but this didn't work. All the example in tensorflow dataset docs use map with lambda functions but I presume it works with regular functions too
Does anyone know how it should be done?
EDIT
The traceback I get is
<ipython-input-39-929295e1b775> in <module>()
----> 1 dataset = model_forecast_datasets(btc_model, np_data[:6])
11 frames
/usr/local/lib/python3.6/dist-packages/tensorflow/python/autograph/impl/api.py in wrapper(*args, **kwargs)
263 except Exception as e: # pylint:disable=broad-except
264 if hasattr(e, 'ag_error_metadata'):
--> 265 raise e.ag_error_metadata.to_exception(e)
266 else:
267 raise
OperatorNotAllowedInGraphError: in user code:
<ipython-input-38-b3d0f7e17689>:12 normalize_window *
return [(i/w[0] -1) for i in w]
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py:561 __iter__
self._disallow_iteration()
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py:557 _disallow_iteration
self._disallow_in_graph_mode("iterating over `tf.Tensor`")
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py:537 _disallow_in_graph_mode
" this function with #tf.function.".format(task))
OperatorNotAllowedInGraphError: iterating over `tf.Tensor` is not allowed in Graph execution. Use Eager execution or decorate this function with #tf.function.
You would need a function that vectorizes the calculation, something like
def normalize(data):
mean = tf.math.reduce_mean(data)
std = tf.math.reduce_std(data)
data = tf.subtract(data, mean)
data = tf.divide(data, std)
return data
ds = ds.map(normalize)
Edit: For your specific normalization this may work:
def normalize(data):
data1 = tf.subtract(data, tf.constant(1))
data1 = tf.divide(data1, data[0])
return data1
(this would have to go after batching ds = ds.flat_map(...)
deconv_shape1 = layer3.get_shape()
de_W1 = tf.Variable(tf.truncated_normal(shape=(4, 4, 4,
deconv_shape1[4].value, 2), mean = mu, stddev = sigma))
de_b1 = tf.Variable(tf.zeros(deconv_shape1[4].value))
output_shape=x.get_shape().as_list()
output_shape[1] *= 2
output_shape[2] *= 2
output_shape[3] *= 2
output_shape[4] = deconv_shape1[4].value
output_shape=np.asarray(output_shape)
output_shape=tfConv3DBackpropInputV2.convert_to_tensor(output_shape)
print(type(output_shape))
x = tf.nn.conv3d_transpose(x, de_W1, output_shape, strides=[1, 2, 2, 2, 1], padding="SAME")
x = tf.nn.bias_add(x,de_b1)
first_down_layer=x
x is of type int32.
I get the error as mentioned in tensorflow. What I am doing wrong, as I am not even calling Conv3DBackpropInputV2().
I am a newbie to tensorflow, please help!!
Just as a prefix, why don't you use the ready-made conv3d_transpose layer, tf.layers.conv3d_transpose(), why are you trying to put it together yourself with all these moving parts? But hey, maybe you have a good reason. So:
output_shape is of type int64. Run this code:
import tensorflow as tf
import numpy as np
a = tf.zeros( ( 5, 5, 5, 5, 5 ) )
b = a.get_shape().as_list()
c = np.asarray( b )
print( c.dtype )
will output
int64
So do this when converting to array:
output_shape = np.asarray( output_shape, dtype = np.int32 )
That should fix it.
Consider the following toy TensorFlow code. The fit method of LinearRegressor works properly and finds the right coefficients (i.e. y = x1 + x2), but evaluate (see the last print statement) hangs. Any idea what's wrong?
import tensorflow as tf
x1 = [1, 3, 4, 5, 1, 6, -1, -3]
x2 = [5, 2, 1, 5, 0, 2, 4, 2]
y = [6, 5,5, 10, 1, 8, 3, -1]
def train_fn():
return {'x1': tf.constant(x1), 'x2':tf.constant(x2)}, tf.constant(y)
features = [tf.contrib.layers.real_valued_column('x1', dimension=1),
tf.contrib.layers.real_valued_column('x2', dimension=1)]
estimator = tf.contrib.learn.LinearRegressor(feature_columns=features)
estimator.fit(input_fn=train_fn, steps=10000)
for vn in estimator.get_variable_names():
print('variable name', vn, estimator.get_variable_value(vn))
print(estimator.evaluate(input_fn=train_fn))
estimator.evaluate() takes a parameter steps, which defaults to None, which is interpreted as "infinity". It therefore never ends.
To make it end, pass steps=1 explicitly:
estimator.evaluate(input_fn=your_input_fn, steps=1)
I am looking for a TensorFlow way of implementing something similar to Python's list.index() function.
Given a matrix and a value to find, I want to know the first occurrence of the value in each row of the matrix.
For example,
m is a <batch_size, 100> matrix of integers
val = 23
result = [0] * batch_size
for i, row_elems in enumerate(m):
result[i] = row_elems.index(val)
I cannot assume that 'val' appears only once in each row, otherwise I would have implemented it using tf.argmax(m == val). In my case, it is important to get the index of the first occurrence of 'val' and not any.
It seems that tf.argmax works like np.argmax (according to the test), which will return the first index when there are multiple occurrences of the max value.
You can use tf.argmax(tf.cast(tf.equal(m, val), tf.int32), axis=1) to get what you want. However, currently the behavior of tf.argmax is undefined in case of multiple occurrences of the max value.
If you are worried about undefined behavior, you can apply tf.argmin on the return value of tf.where as #Igor Tsvetkov suggested.
For example,
# test with tensorflow r1.0
import tensorflow as tf
val = 3
m = tf.placeholder(tf.int32)
m_feed = [[0 , 0, val, 0, val],
[val, 0, val, val, 0],
[0 , val, 0, 0, 0]]
tmp_indices = tf.where(tf.equal(m, val))
result = tf.segment_min(tmp_indices[:, 1], tmp_indices[:, 0])
with tf.Session() as sess:
print(sess.run(result, feed_dict={m: m_feed})) # [2, 0, 1]
Note that tf.segment_min will raise InvalidArgumentError when there is some row containing no val. In your code row_elems.index(val) will raise exception too when row_elems don't contain val.
Looks a little ugly but works (assuming m and val are both tensors):
idx = list()
for t in tf.unpack(m, axis=0):
idx.append(tf.reduce_min(tf.where(tf.equal(t, val))))
idx = tf.pack(idx, axis=0)
EDIT:
As Yaroslav Bulatov mentioned, you could achieve the same result with tf.map_fn:
def index1d(t):
return tf.reduce_min(tf.where(tf.equal(t, val)))
idx = tf.map_fn(index1d, m, dtype=tf.int64)
Here is another solution to the problem, assuming there is a hit on every row.
import tensorflow as tf
val = 3
m = tf.constant([
[0 , 0, val, 0, val],
[val, 0, val, val, 0],
[0 , val, 0, 0, 0]])
# replace all entries in the matrix either with its column index, or out-of-index-number
match_indices = tf.where( # [[5, 5, 2, 5, 4],
tf.equal(val, m), # [0, 5, 2, 3, 5],
x=tf.range(tf.shape(m)[1]) * tf.ones_like(m), # [5, 1, 5, 5, 5]]
y=(tf.shape(m)[1])*tf.ones_like(m))
result = tf.reduce_min(match_indices, axis=1)
with tf.Session() as sess:
print(sess.run(result)) # [2, 0, 1]
Here is a solution which also considers the case the element is not included by the matrix (solution from github repository of DeepMind)
def get_first_occurrence_indices(sequence, eos_idx):
'''
args:
sequence: [batch, length]
eos_idx: scalar
'''
batch_size, maxlen = sequence.get_shape().as_list()
eos_idx = tf.convert_to_tensor(eos_idx)
tensor = tf.concat(
[sequence, tf.tile(eos_idx[None, None], [batch_size, 1])], axis = -1)
index_all_occurrences = tf.where(tf.equal(tensor, eos_idx))
index_all_occurrences = tf.cast(index_all_occurrences, tf.int32)
index_first_occurrences = tf.segment_min(index_all_occurrences[:, 1],
index_all_occurrences[:, 0])
index_first_occurrences.set_shape([batch_size])
index_first_occurrences = tf.minimum(index_first_occurrences + 1, maxlen)
return index_first_occurrences
And:
import tensorflow as tf
mat = tf.Variable([[1,2,3,4,5], [2,3,4,5,6], [3,4,5,6,7], [0,0,0,0,0]], dtype = tf.int32)
idx = 3
first_occurrences = get_first_occurrence_indices(mat, idx)
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
sess.run(first_occurrence) # [3, 2, 1, 5]
Let's say that I have an arbitrary number of iterables, all of which can be assumed to be sorted, and contain elements all of the same type (integers, for illustration's sake).
a = (1, 2, 3, 4, 5)
b = (2, 4, 5)
c = (1, 2, 3, 5)
I would like to write a generator function yielding the following:
(1, None, 1)
(2, 2, 2)
(3, None, 3)
(4, 4, None)
(5, 5, 5)
In other words, progressively yield sorted tuples with gaps where elements are missing from the input iterables.
My take on this, using only iterators, not heaps:
a = (1, 2, 4, 5)
b = (2, 5)
c = (1, 2, 6)
d = (1,)
inputs = [iter(x) for x in (a, b, c, d)]
def minwithreplacement(currents, inputs, minitem, done):
for i in xrange(len(currents)):
if currents[i] == minitem:
try:
currents[i] = inputs[i].next()
except StopIteration:
currents[i] = None
done[0] += 1
yield minitem
else:
yield None
def dothing(inputs):
currents = [it.next() for it in inputs]
done = [0]
while done[0] != len(currents):
yield minwithreplacement(currents, inputs, min(x for x in currents if x), done)
print [list(x) for x in dothing(inputs)] #Consuming iterators for display purposes
>>>[[1, None, 1, 1], [2, 2, 2, None], [4, None, None, None], [5, 5, None, None], [None, None, 6, None]]
We first need a variation of heapq.merge which also yields the index. You can get that by copy-pasting heapq.merge, and replacing each yield v with yield itnum, v. (I omit that part from my answer for readability).
Now we can do:
from collections import deque, OrderedDict
def f(*iterables):
pending = OrderedDict()
for i, v in merge(iterables):
if (not pending) or pending.keys()[-1] < v:
# a new greatest value
pending[v] = [None] * len(iterables)
pending[v][i] = v
# yield all values smaller than v
while len(pending) > 1 and pending.keys()[0] < v:
yield pending.pop(pending.keys()[0])
# yield remaining
while pending:
yield pending.pop(pending.keys()[0])
print list(f((1,2,3,4,5), (2,4,5), (1,2,3,5)))
=> [[1, None, 1], [2, 2, 2], [3, None, 3], [4, 4, None], [5, 5, 5]]