How can I create a multihot embedding layer in Keras? - tensorflow

Question

I have sequential data where each element is a vector as follows:
x_i = [ 0. , 0. , 0. , 0.03666667, 0. ,
0. , 0.95666667, 0. , 0. , 0. ,
0. , 0. , 0. , 0. , 0. ,
0. , 0. , 0. , 0. , 0. ,
0. , 0. , 0. , 0. , 0. ,
0. , 0. , 0. , 0.00666667, 0. ,
0. , 0. , 0. , 0. , 0. ,
0. , 0. , 0. , 0. , 0. ,
0. ]
The vector represents the distribution of time (over a 5-minute block, for example) a user spent on a set of activities. The task is to predict the distribution of the tasks over the next time step t+1 given the previous N steps (t-N : t). Consequently, my input shape is:
X.shape = (batch_size, timesteps, input_length), and an example would be (32, 10, 41) where we have a batch size of 32, 10 timesteps in the past and the each element has a dimenionsality of 41.
To do this I'm using an LSTM built using Keras. Before passing this input to the LSTM though, I would like to create something similar to an Embedding layer that converts this representation into a dense high-dimensional vector similar to what's done in NLP and embedding one-hot vectors of words into an embedding space using the embedding layer. However, the Embedding layer in Keras only accepts integer inputs (or one-hot representations), and in my case what I would like to achieve is a matrix product between the input vector X (which is composed of several x_i as it represents time-series data) and an Embedding Matrix V. To illustrate:
X.shape = (10, 41)
Embedding matrix shape = (41, 100)
The role is to convert every element in X from it's 41 dimenional sparse representation into 100 dimensions via the matrix multiplication, and this should be done for all elements in the batch input.
To do that I've done the following
class EmbeddingMatrix(Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(EmbeddingMatrix, self).__init__(**kwargs)
def build(self, input_shape):
# Create a trainable weight variable for this layer.
self.kernel = self.add_weight(name='kernel',
shape=(input_shape[2], self.output_dim),
initializer='uniform',
trainable=True)
super(EmbeddingMatrix, self).build(input_shape) # Be sure to call this somewhere!
def call(self, x, mask=None):
return K.dot(x, self.kernel)
def compute_output_shape(self, input_shape):
return (input_shape[0], input_shape[1], self.output_dim)
And the LSTM network I'm using is as follows:
inputs = Input(shape=(FLAGS.look_back, FLAGS.inputlength))
inputs_embedded = EmbeddingMatrix(N_EMBEDDING)(inputs)
encoded = LSTM(N_HIDDEN, dropout=0.2, recurrent_dropout=0.2)(inputs_embedded)
dense = TimeDistributed(Dense(N_DENSE, activation='sigmoid'))(dropout)
dense_output = TimeDistributed(Dense(FLAGS.inputlength, activation='softmax'))(dense)
embedder = Model(inputs, inputs_embedded)
model = Model(inputs, dense_output)
model.compile(loss='mean_squared_error', optimizer = RMSprop(lr=LEARNING_RATE, clipnorm=5))
However, when running I get the following error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-24-5a28b4f3b6b9> in <module>()
5 inputs_embedded = EmbeddingMatrix(N_EMBEDDING)(inputs)
6
----> 7 encoded = LSTM(N_HIDDEN, dropout=0.2, recurrent_dropout=0.2)(inputs_embedded)
8
9 dense = TimeDistributed(Dense(N_DENSE, activation='sigmoid'))(dropout)
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/keras/layers/recurrent.py in __call__(self, inputs, initial_state, **kwargs)
260 # modify the input spec to include the state.
261 if initial_state is None:
--> 262 return super(Recurrent, self).__call__(inputs, **kwargs)
263
264 if not isinstance(initial_state, (list, tuple)):
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/keras/engine/topology.py in __call__(self, inputs, **kwargs)
567 '`layer.build(batch_input_shape)`')
568 if len(input_shapes) == 1:
--> 569 self.build(input_shapes[0])
570 else:
571 self.build(input_shapes)
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/keras/layers/recurrent.py in build(self, input_shape)
1041 initializer=bias_initializer,
1042 regularizer=self.bias_regularizer,
-> 1043 constraint=self.bias_constraint)
1044 else:
1045 self.bias = None
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
85 warnings.warn('Update your `' + object_name +
86 '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 87 return func(*args, **kwargs)
88 wrapper._original_function = func
89 return wrapper
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/keras/engine/topology.py in add_weight(self, name, shape, dtype, initializer, regularizer, trainable, constraint)
389 if dtype is None:
390 dtype = K.floatx()
--> 391 weight = K.variable(initializer(shape), dtype=dtype, name=name)
392 if regularizer is not None:
393 self.add_loss(regularizer(weight))
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/keras/layers/recurrent.py in bias_initializer(shape, *args, **kwargs)
1033 self.bias_initializer((self.units,), *args, **kwargs),
1034 initializers.Ones()((self.units,), *args, **kwargs),
-> 1035 self.bias_initializer((self.units * 2,), *args, **kwargs),
1036 ])
1037 else:
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py in concatenate(tensors, axis)
1721 return tf.sparse_concat(axis, tensors)
1722 else:
-> 1723 return tf.concat([to_dense(x) for x in tensors], axis)
1724
1725
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/tensorflow/python/ops/array_ops.py in concat(concat_dim, values, name)
1073 ops.convert_to_tensor(concat_dim,
1074 name="concat_dim",
-> 1075 dtype=dtypes.int32).get_shape(
1076 ).assert_is_compatible_with(tensor_shape.scalar())
1077 return identity(values[0], name=scope)
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/tensorflow/python/framework/ops.py in convert_to_tensor(value, dtype, name, as_ref, preferred_dtype)
667
668 if ret is None:
--> 669 ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
670
671 if ret is NotImplemented:
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/tensorflow/python/framework/constant_op.py in _constant_tensor_conversion_function(v, dtype, name, as_ref)
174 as_ref=False):
175 _ = as_ref
--> 176 return constant(v, dtype=dtype, name=name)
177
178
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/tensorflow/python/framework/constant_op.py in constant(value, dtype, shape, name, verify_shape)
163 tensor_value = attr_value_pb2.AttrValue()
164 tensor_value.tensor.CopyFrom(
--> 165 tensor_util.make_tensor_proto(value, dtype=dtype, shape=shape, verify_shape=verify_shape))
166 dtype_value = attr_value_pb2.AttrValue(type=tensor_value.tensor.dtype)
167 const_tensor = g.create_op(
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_util.py in make_tensor_proto(values, dtype, shape, verify_shape)
365 nparray = np.empty(shape, dtype=np_dt)
366 else:
--> 367 _AssertCompatible(values, dtype)
368 nparray = np.array(values, dtype=np_dt)
369 # check to them.
/Users/asturkmani/anaconda3/lib/python3.5/site-packages/tensorflow/python/framework/tensor_util.py in _AssertCompatible(values, dtype)
300 else:
301 raise TypeError("Expected %s, got %s of type '%s' instead." %
--> 302 (dtype.name, repr(mismatch), type(mismatch).__name__))
303
304
TypeError: Expected int32, got list containing Tensors of type '_Message' instead.
What could be causing this and what would be the best way to implement such a weighted embedding matrix?