Generate variable length data with Tensorflow ops - tensorflow

Question

I am trying to learn a classifier on audio files. I read my WAV files and convert them to a sequence of spectrogram images for training in a custom Python function. The function is called with tf.py_func and returns an array of images with the same shape. In other words the image shape is well defined, yet the number of images is dynamic. (e.g. 3 spectrograms for short audio snippet, 15 for a long snippet)
Is there a way to unpack the resulting list for further processing / enqueueing in tf.train.batch_join()? The undefined sequence length seems to be a problem for many TF ops. Can the length be inferred somehow?
...
// Read the audio file name and label from a CSV file
audio_file, label = tf.decode_csv(csv_content)
def read_audio(audio_file):
signal = read_wav(audio_file)
images = [generate_image(segment) for segment in split_audio(signal)]
// This output is of varying length depending on the length of audio file.
return images
// Convert audio file to a variable length sequence of images
// Shape: <unknown>, which is to be expected from tf.py_func
image_sequence = tf.py_func(wav_to_spectrogram, [audio_file], [tf.float32])[0]
// Auxilliary to set a shape for the images defined in tf.py_func
def process_image(in_image):
image = tf.image.convert_image_dtype(in_image, dtype=tf.float32)
image.set_shape([600, 39, 1])
return (image, label)
// Shape: (?, 600, 39, 1)
images_labels = tf.map_fn(process_image, image_sequence, dtype=(tf.float32, tf.int32))
// This will not work. 'images_and_labels' needs to be a list
images, label_index_batch = tf.train.batch_join(
images_and_labels,
batch_size=batch_size,
capacity=2 * num_preprocess_threads * batch_size,
shapes=[data_shape, []],
)

Answer 1

You can use variable size Tensor as input and enqueue_many to treat this tensor as a variable size input batch.
Below is an example of py_func generating variable size batches and batch with enqueue_many converting it to constant size batches.
import tensorflow as tf
tf.reset_default_graph()
# start with time-out to prevent hangs when experimenting
config = tf.ConfigProto()
config.operation_timeout_in_ms=2000
sess = tf.InteractiveSession(config=config)
# initialize first queue with 1, 2, 1, 2
queue1 = tf.FIFOQueue(capacity=4, dtypes=[tf.int32])
queue1_input = tf.placeholder(tf.int32)
queue1_enqueue = queue1.enqueue(queue1_input)
sess.run(queue1_enqueue, feed_dict={queue1_input: 1})
sess.run(queue1_enqueue, feed_dict={queue1_input: 2})
sess.run(queue1_enqueue, feed_dict={queue1_input: 1})
sess.run(queue1_enqueue, feed_dict={queue1_input: 2})
sess.run(queue1.close())
# call_func will produce variable size tensors
def range_func(x):
return np.array(range(x), dtype=np.int32)
[call_func] = tf.py_func(range_func, [queue1.dequeue()], [tf.int32])
queue2_dequeue = tf.train.batch([call_func], batch_size=3, shapes=[[]], enqueue_many=True)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
try:
while True:
print sess.run(queue2_dequeue)
except tf.errors.OutOfRangeError:
pass
finally:
coord.request_stop()
coord.join(threads)
sess.close()
You should see
[0 0 1]
[0 0 1]