The truth value of an array with more than one element is ambiguous. Reinforcement Learning Agent - tensorflow2.0

Question

I'm starting in DQN, creating an agent to run in a custom environment :
class ReplayBuffer:
def __init__(self, capacity=10000):
self.buffer = deque(maxlen=capacity)
def store(self, state, action, reward, next_state, done):
self.buffer.append([state, action, reward, next_state, done])
def sample(self):
sample = random.sample(self.buffer, args.batch_size)
states, actions, rewards, next_states, done = map(np.asarray, zip(*sample))
states = np.array(states).reshape(args.batch_size, -1)
next_states = np.array(next_states).reshape(args.batch_size, -1)
return states, actions, rewards, next_states, done
def size(self):
return len(self.buffer)
class DQN:
def __init__(self, state_img, state_legal_actions,
state_price, state_risk_reward,
action_dim, state_dim):
self.state_img = state_img
self.state_legal_actions = state_legal_actions
self.state_price = state_price
self.state_risk_reward = state_risk_reward
self.action_dim = action_dim
self.state_dim = state_dim
self.epsilon = args.eps
# Define the input layers
self.df_img_input = self.state_img.shape
self.rr_input = self.state_risk_reward.shape
self.legal_actions_input = self.state_legal_actions.shape
self.price_input = self.state_price.shape
self.model = self.nn_model()
def nn_model(self):
# Create input layers
input_img = Input(shape = self.df_img_input)
input_numeric1 = Input(shape = self.price_input)
input_numeric2 = Input(shape = self.legal_actions_input)
input_numeric3 = Input(shape = self.rr_input)
# Create the CNN for the image input
cnn = Conv2D(64, (3,3), activation='relu',
padding='same')(input_img)
cnn = MaxPooling2D()(cnn)
cnn = Dropout(0.25)(cnn)
cnn = Conv2D(128, (3,3), activation='relu',
padding='same')(input_img)
cnn = MaxPooling2D()(cnn)
cnn = Dropout(0.25)(cnn)
cnn = Conv2D(128, (3,3), activation='relu',
padding='same')(input_img)
cnn = Dropout(0.2)(cnn)
cnn = Flatten()(cnn)
# Create the DNN for the numeric inputs
dnn1 = Dense(64, activation='relu')(input_numeric1)
dnn1 = Dropout(0.25)(dnn1)
dnn2 = Dense(32, activation='relu')(input_numeric2)
dnn2 = Dropout(0.25)(dnn2)
dnn3 = Dense(32, activation='relu')(input_numeric3)
dnn3 = Dropout(0.25)(dnn3)
# Concatenate the outputs from the CNN and DNN
concat = Concatenate()([cnn, dnn1, dnn2, dnn3])
# Create the output layers
dense1 = Dense(128, activation='relu')(concat)
dense1 = Dropout(0.25)(dense1)
dense2 = Dense(128, activation='relu')(dense1)
dense2 = Dropout(0.25)(dense2)
dense3 = Dense(self.action_dim, activation='sigmoid')(dense2)
# Create the model
model = Model(inputs=[input_img, input_numeric1, input_numeric2, input_numeric3], outputs=dense3)
# Compile the model
model.compile(optimizer='adam',
loss=tf.keras.losses.MeanSquaredError(),
metrics=['accuracy'])
return model
def predict(self, input_img,input_numeric1,input_numeric2, input_numeric3):
return self.model.predict(input_img,input_numeric1,input_numeric2, input_numeric3)
def get_action(self, state):
state = [state['df_img'], state['price'], state['legal_actions'], state['risk_reward']]
self.epsilon *= args.eps_decay
self.epsilon = max(self.epsilon, args.eps_min)
q_value = self.predict(state[0],state[1],state[2],state[3])
if np.random.random() < self.epsilon:
return random.randint(0, self.action_dim - 1)
return np.argmax(q_value)
def train(self, states, targets):
self.model.fit(states, targets, epochs=2)
class Agent:
def __init__(self, env):
self.env = env
self.obs_dim = ['df_img', 'legal_actions', 'price', 'risk_reward']
self.state_img = self.env.observation_space[self.obs_dim[0]]
self.state_legal_actions = self.env.observation_space[self.obs_dim[1]]
self.state_price = self.env.observation_space[self.obs_dim[2]]
self.state_risk_reward = self.env.observation_space[self.obs_dim[3]]
self.state_dim = np.hstack((self.state_img, self.state_risk_reward,
self.state_price, self.state_legal_actions))
self.action_dim = self.env.action_space.n
self.model = DQN(self.state_img, self.state_legal_actions,
self.state_price,self.state_risk_reward,
self.action_dim, self.state_dim)
self.target_model = DQN(self.state_img, self.state_legal_actions,
self.state_price,self.state_risk_reward,
self.action_dim, self.state_dim)
self.update_target()
self.buffer = ReplayBuffer()
def update_target(self):
weights = self.model.model.get_weights()
self.target_model.model.set_weights(weights)
def replay_experience(self):
for _ in range(10):
states, actions, rewards, next_states, done = self.buffer.sample()
targets = self.model.predict(states)
next_q_values = self.target_model.predict(next_states).max(axis=1)
targets[range(args.batch_size), actions] = (
rewards + (1 - done) * next_q_values * args.gamma
)
self.model.train(states, targets)
def train(self, max_episodes=1000):
for ep in range(max_episodes):
done, episode_reward = False, 0
observation = self.env.reset()
#print(observation[0])
while not done:
action = self.model.get_action(observation)
next_observation, reward, done, _ = self.env.step(action)
self.buffer.store(
observation, action, reward * 0.01, next_observation, done
)
episode_reward += reward
observation = next_observation
if self.buffer.size() >= args.batch_size:
self.replay_experience()
self.update_target()
print(f"Episode#{ep} Reward:{episode_reward}")
tf.summary.scalar("episode_reward", episode_reward, step=ep)
but when i run it with :
if __name__ == "__main__":
env = TradingEnv(df_)
agent = Agent(env)
agent.train(max_episodes=2)
i receive the folllowing error:
--------------------------------------------------------------------------- ValueError Traceback (most recent call
last) /tmp/ipykernel_3672/1951022009.py in
2 env = ForexTradingEnv(df_)
3 agent = Agent(env)
----> 4 agent.train(max_episodes=2) # Increase max_episodes value
5
/tmp/ipykernel_3672/3112839645.py in train(self, max_episodes)
43
44 while not done:
---> 45 action = self.model.get_action(observation)
46 next_observation, reward, done, _ = self.env.step(action)
47 self.buffer.store(
/tmp/ipykernel_3672/2209466094.py in get_action(self, state)
78 self.epsilon *= args.eps_decay
79 self.epsilon = max(self.epsilon, args.eps_min)
---> 80 q_value = self.predict(state[0],state[1],state[2],state[3])
81 if np.random.random() < self.epsilon:
82 return random.randint(0, self.action_dim - 1)
/tmp/ipykernel_3672/2209466094.py in predict(self, input_img,
input_numeric1, input_numeric2, input_numeric3)
71
72 def predict(self, input_img,input_numeric1,input_numeric2, input_numeric3):
---> 73 return self.model.predict(input_img,input_numeric1,input_numeric2,
input_numeric3)
74
75 def get_action(self, state):
~/miniconda3/envs/tf/lib/python3.9/site-packages/keras/utils/traceback_utils.py
in error_handler(*args, **kwargs)
68 # To get the full stack trace, call:
69 # tf.debugging.disable_traceback_filtering()
---> 70 raise e.with_traceback(filtered_tb) from None
71 finally:
72 del filtered_tb
~/miniconda3/envs/tf/lib/python3.9/site-packages/keras/engine/data_adapter.py
in init(self, x, y, sample_weights, sample_weight_modes,
batch_size, epochs, steps, shuffle, **kwargs)
262 # If batch_size is not passed but steps is, calculate from the input
263 # data. Default to 32 for backwards compat.
--> 264 if not batch_size:
265 batch_size = int(math.ceil(num_samples / steps)) if steps else 32
266
ValueError: The truth value of an array with more than one element is
ambiguous. Use a.any() or a.all()
if you think you can help here would be really nice!
thanks in advance
best regards