Spring AMQP RabbitMQ RPC Priority Queue - rabbitmq

I am trying to have a RPC queue where messages from the Producer/client are prioritized to the consumer. I want all priority 2 messages processed before any priority one messages. I also want each consumer to be able to handle 10 messages at a time. I am able to get each consumer to process 10 messages at a time. I am NOT able to get the prioritization of messages to work. Below is my setup:
Config Files:
#Configuration
public class QueueConfig {
public static final String QUEUE_NAME = "requests";
private int maxPriority = 2;
#Autowired
private ConnectionFactory connectionFactory;
#Bean
public Queue requests() {
Map<String, Object> args = new HashMap<String, Object>();
args.put("x-max-priority", maxPriority);
return new Queue(QUEUE_NAME,true,false,false, args);
}
#Bean
public Queue replies() {
Map<String, Object> args = new HashMap<String, Object>();
args.put("x-max-priority", maxPriority);
return new Queue("replies",true,false,false, args);
}
#Bean
public SimpleMessageListenerContainer simpleMessageListenerContainer() {
SimpleMessageListenerContainer container = new SimpleMessageListenerContainer(connectionFactory);
container.setQueueNames(replies().getName());
return container;
}
#Bean
public RabbitTemplate template() {
RabbitTemplate rabbitTemplate = new RabbitTemplate(connectionFactory);
rabbitTemplate.setRoutingKey(requests().getName());
return rabbitTemplate;
}
#Bean
public AsyncRabbitTemplate asyncRabbitTemplate(RabbitTemplate rabbitTemplate, SimpleMessageListenerContainer container) {
AsyncRabbitTemplate asyncRabbitTemplate = new AsyncRabbitTemplate(rabbitTemplate, container);
asyncRabbitTemplate.setReceiveTimeout(90000);
return asyncRabbitTemplate;
}
}
Client/Producer:
#Component
public class Client {
#Autowired
private AsyncRabbitTemplate template;
public void sendHigh(String name) {
MessagePostProcessor messageProcessor = new MessagePostProcessor() {
#Override
public Message postProcessMessage(Message message) throws AmqpException {
message.getMessageProperties().setPriority(2);
return message;
}
};
ListenableFuture<String> response = template.convertSendAndReceive(QueueConfig.QUEUE_NAME, (Object) name,(MessagePostProcessor) messageProcessor);
try {
response.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
public void sendLow(String name) {
MessagePostProcessor messageProcessor = new MessagePostProcessor() {
#Override
public Message postProcessMessage(Message message) throws AmqpException {
message.getMessageProperties().setPriority(1);
return message;
}
};
ListenableFuture<String> response = template.convertSendAndReceive(QueueConfig.QUEUE_NAME, (Object) name,(MessagePostProcessor) messageProcessor);
try {
response.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
config file 2:
#Configuration
#EnableAsync
public class ServiceConfig implements AsyncConfigurer {
#Override
#Bean
public Executor getAsyncExecutor() {
return new SimpleAsyncTaskExecutor();
}
}
Consumer:
#Component
public class Consumer {
#RabbitListener(queues = QueueConfig.QUEUE_NAME)
public String consume(#Payload String name) {
System.out.println("Request Consumer " + name);
String result = name;
if(result.equals("john") || result.equals("john1")) {
try {
Thread.sleep(22000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return result;
}
Application.properties:
spring.rabbitmq.dynamic=true
spring.rabbitmq.port=5672
spring.rabbitmq.username=guest
spring.rabbitmq.password=guest
spring.rabbitmq.host=localhost
spring.rabbitmq.listener.simple.concurrency=10
spring.rabbitmq.listener.direct.acknowledge-mode=MANUAL
Junit:
I would expect all john and john1's to be received by the consumer before jeff, but that is not the behavior that I am seeing. Basically im looking at this println and expecting all john and john1's to print before jeff System.out.println("Request Consumer " + name);
#RunWith(SpringRunner.class)
#ComponentScan(basePackages = "com.test.test")
#EnableAutoConfiguration
#SpringBootTest
public class ApplicationTests {
#Autowired AsyncClass asyncClass;
#Test
public void contextLoads() throws InterruptedException, ExecutionException {
List<Future<String>> futures = new ArrayList<>();
futures.add(asyncClass.runAsyncHigh("john"));
futures.add(asyncClass.runAsyncHigh("john"));
futures.add(asyncClass.runAsyncHigh("john"));
futures.add(asyncClass.runAsyncHigh("john"));
futures.add(asyncClass.runAsyncHigh("john"));
futures.add(asyncClass.runAsyncHigh("john"));
futures.add(asyncClass.runAsyncHigh("john"));
futures.add(asyncClass.runAsyncHigh("john"));
futures.add(asyncClass.runAsyncHigh("john"));
futures.add(asyncClass.runAsyncHigh("john"));
Thread.sleep(1000);
futures.add(asyncClass.runAsyncLow("jeff"));
futures.add(asyncClass.runAsyncHigh("john1"));
futures.add(asyncClass.runAsyncHigh("john1"));
futures.add(asyncClass.runAsyncHigh("john1"));
futures.add(asyncClass.runAsyncHigh("john1"));
futures.add(asyncClass.runAsyncHigh("john1"));
futures.add(asyncClass.runAsyncHigh("john1"));
futures.add(asyncClass.runAsyncHigh("john1"));
futures.add(asyncClass.runAsyncHigh("john1"));
futures.add(asyncClass.runAsyncHigh("john1"));
futures.add(asyncClass.runAsyncHigh("john1"));
for(Future<String> future : futures) {
future.get();
}
}
#Component
public class AsyncClass {
#Autowired Client client;
#Async
public Future<String> runAsyncHigh(String name){
client.sendHigh(name);
return new AsyncResult<String>(name);
}
#Async
public Future<String> runAsyncLow(String name){
client.sendLow(name);
return new AsyncResult<String>(name);
}
}
Thanks,
Brian

Related

Converter issue with TestRabbitTemplate (Spring AMQP)

I am trying to run an integration test for my RabbitListener in Spring AMQP while the broker is not running so I am using TestRabbitTemplate. I am using a Jacksonized object and the Jackson2JsonMessageConverter was set for the TestRabbitTemplate. However upon sending a message to the exchange, I am getting the following exception.
org.springframework.amqp.rabbit.support.ListenerExecutionFailedException: Listener method could not be invoked with the incoming message
Caused by: org.springframework.messaging.converter.MessageConversionException: Cannot convert from [[B] to [com.murex.em.demo.springamqpdemo.message.CustomMessage] for GenericMessage [payload=byte[27], headers={amqp_contentEncoding=UTF-8, amqp_contentLength=27, amqp_replyTo=testRabbitTemplateReplyTo, id=fb0352d2-2abb-ae06-9b2e-03da1fc19e43, amqp_lastInBatch=false, contentType=application/json, __TypeId__=com.murex.em.demo.springamqpdemo.message.CustomMessage, timestamp=1676625212607}], failedMessage=GenericMessage [payload=byte[27], headers={amqp_contentEncoding=UTF-8, amqp_contentLength=27, amqp_replyTo=testRabbitTemplateReplyTo, id=fb0352d2-2abb-ae06-9b2e-03da1fc19e43, amqp_lastInBatch=false, contentType=application/json, __TypeId__=com.murex.em.demo.springamqpdemo.message.CustomMessage, timestamp=1676625212607}]
at org.springframework.messaging.handler.annotation.support.PayloadMethodArgumentResolver.resolveArgument(PayloadMethodArgumentResolver.java:145)
at org.springframework.amqp.rabbit.annotation.RabbitListenerAnnotationBeanPostProcessor$OptionalEmptyAwarePayloadArgumentResolver.resolveArgument(RabbitListenerAnnotationBeanPostProcessor.java:1053)
at org.springframework.messaging.handler.invocation.HandlerMethodArgumentResolverComposite.resolveArgument(HandlerMethodArgumentResolverComposite.java:118)
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.getMethodArgumentValues(InvocableHandlerMethod.java:147)
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:115)
at org.springframework.amqp.rabbit.listener.adapter.HandlerAdapter.invoke(HandlerAdapter.java:77)
at org.springframework.amqp.rabbit.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:263)
... 78 more
Here are the relevant chunks of my code:
#Jacksonized
#Builder
#Value
public class CustomMessage {
String id;
String name;
}
#Component
public class InwardMessageConsumer {
#RabbitListener(bindings = #QueueBinding(
value = #Queue(name = "inward.messageQueue", durable = "true"),
exchange = #Exchange(value = "inward", type = "topic"),
key = "inwardRoutingKey")
)
public String processMessage(CustomMessage customMessage) {
return customMessage.getName();
}
}
#SpringJUnitConfig
#SpringBootTest
class BrokerNotRunningIT {
#Autowired
private TestRabbitTemplate template;
#Test
public void testSendAndReceive() {
CustomMessage customMessage = CustomMessage.builder()
.id("123")
.name("name1")
.build();
assertThat(template.convertSendAndReceive("inward", "inward.messageQueue", customMessage)).isEqualTo("name1");
}
#TestConfiguration
public static class RabbitTestConfiguration {
#Bean
public TestRabbitTemplate testRabbitTemplate(
ConnectionFactory mockConnectionFactory,
Jackson2JsonMessageConverter jackson2JsonMessageConverter
) {
TestRabbitTemplate testRabbitTemplate = new TestRabbitTemplate(mockConnectionFactory);
testRabbitTemplate.setMessageConverter(jackson2JsonMessageConverter);
return testRabbitTemplate;
}
#Bean
public ConnectionFactory mockConnectionFactory() throws IOException {
ConnectionFactory factory = mock(ConnectionFactory.class);
Connection connection = mock(Connection.class);
Channel channel = mock(Channel.class);
AMQP.Queue.DeclareOk declareOk = mock(AMQP.Queue.DeclareOk.class);
willReturn(connection).given(factory).createConnection();
willReturn(channel).given(connection).createChannel(anyBoolean());
given(channel.isOpen()).willReturn(true);
given(channel.queueDeclare(anyString(), anyBoolean(), anyBoolean(), anyBoolean(), anyMap()))
.willReturn(declareOk);
return factory;
}
#Bean
public SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory(ConnectionFactory connectionFactory) {
SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory();
factory.setConnectionFactory(connectionFactory);
return factory;
}
#Bean
public SimpleMessageListenerContainer simpleMessageListenerContainer(ConnectionFactory connectionFactory) {
return new SimpleMessageListenerContainer(connectionFactory);
}
}
}
Apparently I needed to set the message converter in the RabbitListenerContainerFactory as well like this:
#Bean
public SimpleRabbitListenerContainerFactory rabbitListenerContainerFactory(ConnectionFactory connectionFactory, Jackson2JsonMessageConverter jackson2JsonMessageConverter) {
SimpleRabbitListenerContainerFactory factory = new SimpleRabbitListenerContainerFactory();
factory.setConnectionFactory(connectionFactory);
factory.setMessageConverter(jackson2JsonMessageConverter);
return factory;
}

How to improve Jackson objectMapper deserialize performance?

I use kafka send quote message, qps is up to 50w, the consumer can handle it, but just only deserialize message, the CPU utilization up to 70!
Anybody knows how to optimize it?
Here is the deserialize code:
public class QuoteMessageDeserializer implements Deserializer<RawQuote> {
private final ObjectMapper objectMapper;
public QuoteMessageDeserializer() {
objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
#Override
public void configure(Map<String, ?> configs, boolean isKey) {
}
#Override
public RawQuote deserialize(String topic, byte[] data) {
try {
QuoteMessage msg = objectMapper.readValue(data, QuoteMessage.class);
return msg.getData();
} catch (Exception e) {
……
}
}
#Override
public void close() {
}
}
I try to use AfterBurner,but it doesn't work to lower cpu utilization
public QuoteMessageDeserializer() {
objectMapper = new ObjectMapper();
objectMapper.registerModule(new AfterburnerModule());
objectMapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
thanks.

Data type is changed when listening message from message queue in RabbitMQ 2.3.6

I am using Spring Rabbit 2.3.6 for my spring application. I have list of Comment entity and send to message queue via this function
public void deletePost(long postId){
Post post = postRepository.findById(postId)
.orElseThrow(() -> new PostNotFoundException(String.valueOf(postId)));
List<Comment> comments = postBase.getComments(post);
postRepository.delete(post);
//post event
Map<String, Object> inputs= new HashMap<>();
inputs.put(InputParam.OBJECT_ID, postId);
inputs.put(InputParam.COMMENTS, comments);
messagingUtil.postEvent(SnwObjectType.POST, SnwActionType.DELETE, inputs);
}
Listening message
#RabbitListener(queues = MessagingConfig.QUEUE)
public void consumeMessageFromQueue(Map<String, Object> inputs) {
}
And this is data I received after listening message. Data type of each element in comment list have been changed, and I can't use it as Comment.
Have you any solution to data type from not being changed?
Updated: Messaging config
#Configuration
public class MessagingConfig {
public static final String QUEUE = "javatechie_queue";
public static final String EXCHANGE = "javatechie_exchange";
public static final String ROUTING_KEY = "javatechie_routingKey";
#Bean
public Queue queue() {
return new Queue(QUEUE);
}
#Bean
public TopicExchange exchange() {
return new TopicExchange(EXCHANGE);
}
#Bean
public Binding binding(Queue queue, TopicExchange exchange) {
return BindingBuilder.bind(queue).to(exchange).with(ROUTING_KEY);
}
#Bean
public MessageConverter converter() {
return new Jackson2JsonMessageConverter();
}
#Bean
public AmqpTemplate template(ConnectionFactory connectionFactory) {
final RabbitTemplate rabbitTemplate = new RabbitTemplate(connectionFactory);
rabbitTemplate.setMessageConverter(converter());
return rabbitTemplate;
}
}

How can I configure a Redis instance on Cloudfoundry in slave mode?

I'm trying to configure a master/slave replication on Cloudfoundry but I always fall with a 'ERR unknown command 'SLAVEOF''. I'm using Jedis as the underlying client of spring-data-reids with this code:
#Configuration
#ComponentScan(basePackages = { Constants.REDIS_PACKAGE })
#Order(1)
public class KeyValueConfig {
#Inject
#Named("redisMasterConnectionFactory")
private RedisConnectionFactory redisMasterConnectionFactory;
#Inject
#Named("redisSlaveConnectionFactory")
private RedisConnectionFactory redisSlaveConnectionFactory;
#Inject
#Named("redisCacheConnectionFactory")
private RedisConnectionFactory redisCacheConnectionFactory;
#Bean
public StringRedisTemplate redisMasterTemplate() {
StringRedisTemplate stringRedisTemplate = new StringRedisTemplate(
redisMasterConnectionFactory);
HealthChecks.register(new RedisHealthCheck(stringRedisTemplate,
"master"));
return stringRedisTemplate;
}
#Bean
public StringRedisTemplate redisSlaveTemplate() {
StringRedisTemplate stringRedisTemplate = new StringRedisTemplate(
redisSlaveConnectionFactory);
HealthChecks
.register(new RedisHealthCheck(stringRedisTemplate, "slave"));
return stringRedisTemplate;
}
#Bean
public StringRedisTemplate redisCacheTemplate() {
StringRedisTemplate stringRedisTemplate = new StringRedisTemplate(
redisCacheConnectionFactory);
HealthChecks
.register(new RedisHealthCheck(stringRedisTemplate, "cache"));
return stringRedisTemplate;
}
/**
* Properties to support the local and test mode of operation.
*/
#Configuration
#Profile({ Profiles.LOCAL, Profiles.PROD, Profiles.TEST })
static class Default implements MasterSlaveConfig {
#Inject
private Environment environment;
#Bean
public RedisConnectionFactory redisMasterConnectionFactory() {
JedisConnectionFactory redis = new JedisConnectionFactory();
redis.setHostName(environment.getProperty("redis.master.hostname"));
redis.setPort(environment.getProperty("redis.master.port",
Integer.class));
redis.setPassword(environment.getProperty("redis.master.password"));
redis.setUsePool(true);
return redis;
}
#Bean
public RedisConnectionFactory redisSlaveConnectionFactory() {
JedisConnectionFactory redis = new JedisConnectionFactory();
redis.setHostName(environment.getProperty("redis.slave.hostname"));
redis.setPort(environment.getProperty("redis.slave.port",
Integer.class));
redis.setPassword(environment.getProperty("redis.slave.password"));
redis.setUsePool(true);
return redis;
}
#Bean
public RedisConnectionFactory redisCacheConnectionFactory()
throws Exception {
JedisConnectionFactory redis = new JedisConnectionFactory();
redis.setHostName(environment.getProperty("redis.cache.hostname"));
redis.setPort(environment.getProperty("redis.cache.port",
Integer.class));
redis.setPassword(environment.getProperty("redis.cache.password"));
redis.setUsePool(true);
return redis;
}
}
/**
* Properties to support the cloud mode of operation.
*/
#Configuration
#Profile(Profiles.CLOUDFOUNDRY)
static class Cloud implements MasterSlaveConfig {
#Bean
public RedisConnectionFactory redisMasterConnectionFactory()
throws Exception {
CloudPoolConfiguration cloudPoolConfiguration = new CloudPoolConfiguration();
cloudPoolConfiguration.setPoolSize("3-5");
CloudRedisConnectionFactoryBean factory = new CloudRedisConnectionFactoryBean();
factory.setCloudPoolConfiguration(cloudPoolConfiguration);
factory.setServiceName("redis-master");
factory.afterPropertiesSet();
return factory.getObject();
}
#Bean
public RedisConnectionFactory redisSlaveConnectionFactory()
throws Exception {
CloudPoolConfiguration cloudPoolConfiguration = new CloudPoolConfiguration();
cloudPoolConfiguration.setPoolSize("3-5");
CloudRedisConnectionFactoryBean factory = new CloudRedisConnectionFactoryBean();
factory.setCloudPoolConfiguration(cloudPoolConfiguration);
factory.setServiceName("redis-slave");
factory.afterPropertiesSet();
RedisConnectionFactory redisSlaveConnectionFactory = initSlaveOnMaster(factory);
return redisSlaveConnectionFactory;
}
private RedisConnectionFactory initSlaveOnMaster(
CloudRedisConnectionFactoryBean factory) throws Exception {
RedisConnectionFactory redisSlaveConnectionFactory = factory
.getObject();
RedisServiceInfo serviceInfo = new CloudEnvironment()
.getServiceInfo("redis-master", RedisServiceInfo.class);
Jedis jedis = (Jedis) redisSlaveConnectionFactory.getConnection()
.getNativeConnection();
jedis.slaveof(serviceInfo.getHost(), serviceInfo.getPort());
return redisSlaveConnectionFactory;
}
#Bean
public RedisConnectionFactory redisCacheConnectionFactory()
throws Exception {
CloudPoolConfiguration cloudPoolConfiguration = new CloudPoolConfiguration();
cloudPoolConfiguration.setPoolSize("3-5");
CloudRedisConnectionFactoryBean factory = new CloudRedisConnectionFactoryBean();
factory.setCloudPoolConfiguration(cloudPoolConfiguration);
factory.setServiceName("redis-cache");
factory.afterPropertiesSet();
return factory.getObject();
}
}
interface MasterSlaveConfig {
RedisConnectionFactory redisMasterConnectionFactory() throws Exception;
RedisConnectionFactory redisSlaveConnectionFactory() throws Exception;
RedisConnectionFactory redisCacheConnectionFactory() throws Exception;
}
}
Ok... After checking the code, I found this:
rename-command SLAVEOF "" in redis.conf.erb (https://github.com/cloudfoundry/vcap-services/blob/master/redis/resources/redis.conf.erb)
So the SLAVEOF command is disable on Cloudfoudry, and so:
MONITOR
BGSAVE
BGREWRITEAOF
SLAVEOF
DEBUG
SYNC

question about simple MINA client and server

I am just trying to create a simple MINA server and client to evaluate. Here is my code.
public class Server {
private static final int PORT = 8080;
static class ServerHandler extends IoHandlerAdapter {
#Override
public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
cause.printStackTrace();
}
#Override
public void sessionCreated(IoSession session) {
System.out.println("session is created");
session.write("Thank you");
}
#Override
public void sessionClosed(IoSession session) throws Exception {
System.out.println("session is closed.");
}
#Override
public void messageReceived(IoSession session, Object message) {
System.out.println("message=" + message);
session.write("Reply="+message);
}
}
/**
* #param args
*/
public static void main(String[] args) throws Exception {
SocketAcceptor acceptor = new NioSocketAcceptor();
acceptor.getFilterChain().addLast( "logger", new LoggingFilter() );
acceptor.getFilterChain().addLast( "codec", new ProtocolCodecFilter( new TextLineCodecFactory( Charset.forName( "UTF-8" ))));
acceptor.setHandler(new Server.ServerHandler());
acceptor.getSessionConfig().setReadBufferSize( 2048 );
acceptor.getSessionConfig().setIdleTime( IdleStatus.BOTH_IDLE, 10 );
acceptor.bind(new InetSocketAddress(PORT));
System.out.println("Listening on port " + PORT);
for (;;) {
Thread.sleep(3000);
}
}
}
public class Client {
private static final int PORT = 8080;
private IoSession session;
private ClientHandler handler;
public Client() {
super();
}
public void initialize() throws Exception {
handler = new ClientHandler();
NioSocketConnector connector = new NioSocketConnector();
connector.getFilterChain().addLast( "codec", new ProtocolCodecFilter( new TextLineCodecFactory( Charset.forName( "UTF-8" ))));
connector.getFilterChain().addLast("logger", new LoggingFilter());
connector.setHandler(handler);
for (;;) {
try {
ConnectFuture future = connector.connect(new InetSocketAddress(PORT));
future.awaitUninterruptibly();
session = future.getSession();
break;
} catch (RuntimeIoException e) {
System.err.println("Failed to connect.");
e.printStackTrace();
Thread.sleep(5000);
}
}
if (session == null) {
throw new Exception("Unable to get session");
}
Sender sender = new Sender();
sender.start();
session.getCloseFuture().awaitUninterruptibly();
connector.dispose();
System.out.println("client is done.");
}
/**
* #param args
*/
public static void main(String[] args) throws Exception {
Client client = new Client();
client.initialize();
}
class Sender extends Thread {
#Override
public void run() {
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
handler.messageSent(session, "message");
}
}
class ClientHandler extends IoHandlerAdapter {
#Override
public void sessionOpened(IoSession session) {
}
#Override
public void messageSent(IoSession session, Object message) {
System.out.println("message sending=" + message);
session.write(message);
}
#Override
public void messageReceived(IoSession session, Object message) {
System.out.println("message receiving "+ message);
}
#Override
public void exceptionCaught(IoSession session, Throwable cause) {
cause.printStackTrace();
}
}
}
When I execute this code, the Client seems to keep sending a message instead of stopping after it sends. It looks to me that there is a recursive call in underlying MINA code. I know that I am doing something wrong.
Can somebody tell me how to fix this?
Thanks.
Try to initialize and start your sender and use the session within sessionOpened (ClientHandler)