Converting from feed_dict to queues results in increasing loss - tensorflow

Question

I have a working tensorflow model that I am trying to convert to using queues. It may not be the best function but it works.
The data comes in as a list(dict()) called 'rows' from a processing pipeline outside of TF in the form format [{'y1': 1, 'y2': 0, 'y3':1, 'y4':0, 'x1':...'x1182': 0}] (SPECIAL_FIELD_CHAR is 'y', meaning it's calculated from the 'xN' data). The features_outputs() just returns the xs and the ys as ['y1', 'y2', 'y3', 'y4'] and ['x1', ..., 'x1182']. The idea is that the xs determine the ys. There are 4 independent ys that are calculated per row of xs.
def train_rows(initial_weights, weights_filename, rows):
(features, outputs ) = features_outputs(rows[0].keys())
x_true = [ [float(row[feature]) for feature in features] for row in rows]
try:
y_true = [ [float(row[output]) for output in outputs] for row in rows ]
except Exception as e:
print [row[output] for output in outputs], e
w_true = np.random.rand(len(features), 1) # init weights
b_true = np.random.rand(1) # init bias
x_in = tf.placeholder(tf.float32, [None, len(features)], "x_in")
if initial_weights is None:
w = tf.Variable(tf.random_normal((len(features), len(outputs))), name="w")
b = tf.Variable(tf.constant(0.1, shape=[len(outputs)]), name="b")
else:
w = tf.Variable(weights['w'], name="w")
b = tf.Variable(weights['b'], name="b")
h = tf.add(tf.matmul(x_in, w), b, name="h")
y_in = tf.placeholder(tf.float32, [None, len(outputs)], "y_in")
loss_op = tf.reduce_mean(tf.square(tf.subtract(y_in, h)), name="loss")
#train_op = tf.train.AdamOptimizer(0.01).minimize(loss_op)
train_op = tf.train.GradientDescentOptimizer(0.3).minimize(loss_op)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
last_error = 1.7976931348623157e+308
this_error = 1.7976931348623157e+307
diff = 1
iteration = initial_weights['iteration'] if initial_weights is not None and 'iteration' in initial_weights else 0
while diff > 0:
iteration += 1
last_error = this_error
for step in range(1000):
sess.run(train_op, feed_dict={
x_in: x_true,
y_in: y_true
})
w_computed = sess.run(w)
b_computed = sess.run(b)
pred = tf.add(tf.matmul(x_in, w), b)
results = sess.run(pred, feed_dict={x_in: x_true})
error = tf.losses.mean_squared_error(y_true, results)
this_error = float(error.eval())
(diff, locs) = compare(y_true, results)
if locs < 50:
print "iteration:", iteration, "error:",this_error, "diff:", diff, "locs:", locs
This produces a model that converges. However with the queue based version it does not, and error increases rapidly:
def multithreaded_train_rows(initial_weights, weights_filename, rows):
(features, outputs ) = features_outputs(rows[0].keys())
x_true = np.array([ [float(row[feature]) for feature in features] for row in rows])
y_true = np.array([ [float(row[output]) for output in outputs] for row in rows ])
#queue
q = tf.FIFOQueue(capacity=len(rows), dtypes=tf.float32)
#enq_op = q.enqueue_many(x_true)
enq_op = q.enqueue_many(np.array( [ [float(row[f]) for f in sorted(row.keys())] for row in rows] ))
qr = tf.train.QueueRunner(q, [enq_op] * 1)
tf.train.add_queue_runner(qr)
keys = sorted(row.keys())
x_indices = np.array([[i] for i in range(len(keys)) if not keys[i].startswith(SPECIAL_FIELD_CHAR)])
y_indices = np.array([[i] for i in range(len(keys)) if keys[i].startswith(SPECIAL_FIELD_CHAR)])
input = q.dequeue()
x_in = tf.transpose(tf.gather(input, x_indices))
y_in = tf.gather(input, y_indices)
if initial_weights is None:
print 'Creating weights', len(x_indices), len(y_indices)
w = tf.Variable(tf.random_normal((len(x_indices), len(y_indices))), name="w")
b = tf.Variable(tf.constant(0.1, shape=[len(y_indices)]), name="b")
else:
print 'Using supplied weights', len(weights['w']), len(weights['w'][0])
w = tf.Variable(weights['w'], name="w")
b = tf.Variable(weights['b'], name="b")
y = tf.add(tf.matmul(x_in, w), b, name="y")
loss_op = tf.reduce_mean(tf.squared_difference(y_in, y), name="loss")
train_op = tf.train.GradientDescentOptimizer(0.3).minimize(loss_op)
print 'Starting session'
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
last_error = 1.7976931348623157e+308
this_error = 1.7976931348623157e+307
diff = 1
iteration = initial_weights['iteration'] if initial_weights is not None and 'iteration' in initial_weights else 0
while diff > 0:
iteration += 1
last_error = this_error
for step in range(100):
sess.run([train_op, loss_op])
w_computed = sess.run(w)
b_computed = sess.run(b)
pred = tf.add(tf.matmul(x_in, w), b)
results = sess.run(y, feed_dict={x_in: x_true})
error = tf.losses.mean_squared_error(y_true, results)
this_error = float(error.eval())
(diff, locs) = compare(y_true, results)
if locs < 50:
print "iteration:", iteration, "error:",this_error, "diff:", diff, "locs:", locs
coord.request_stop()
coord.join(threads)
They are meant to be the same, but I've had to change a few things:
1. Add a tf.transpose() to the x_in for the matmul()
2. Queue the entire row of xs and ys, then pull apart using tf.gather().
I've searched a lot for examples that match mine, and I can find no documentation on how to restart a queue and continue the training from the beginning. It'll seemly train forever(not sure why, who is replenishing the queue?) It'll also never stop.
But most of all I have no idea why given the exact same data, the first converges and the second does not?

Answer 1

None of the gather marshalling is needed. Queue only the inputs (x_true), then
evaluate y against y_true, not y_in
loss_op = tf.reduce_mean(tf.square(y - y_true), name="loss")