Running my Docker image, this error is coming up:
2021-May-12 04:03:02.327 [org.springframework.amqp.rabbit.RabbitListenerEndpointContainer#0-1] ERROR org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer - [] Failed to check/redeclare auto-delete queue(s).
org.springframework.amqp.AmqpConnectException: java.net.ConnectException: Connection refused (Connection refused)
at org.springframework.amqp.rabbit.support.RabbitExceptionTranslator.convertRabbitAccessException(RabbitExceptionTranslator.java:61)
at org.springframework.amqp.rabbit.connection.AbstractConnectionFactory.createBareConnection(AbstractConnectionFactory.java:524)
at org.springframework.amqp.rabbit.connection.CachingConnectionFactory.createConnection(CachingConnectionFactory.java:751)
at org.springframework.amqp.rabbit.connection.ConnectionFactoryUtils.createConnection(ConnectionFactoryUtils.java:214)
at org.springframework.amqp.rabbit.core.RabbitTemplate.doExecute(RabbitTemplate.java:2089)
at org.springframework.amqp.rabbit.core.RabbitTemplate.execute(RabbitTemplate.java:2062)
at org.springframework.amqp.rabbit.core.RabbitTemplate.execute(RabbitTemplate.java:2042)
at org.springframework.amqp.rabbit.core.RabbitAdmin.getQueueInfo(RabbitAdmin.java:407)
at org.springframework.amqp.rabbit.core.RabbitAdmin.getQueueProperties(RabbitAdmin.java:391)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.attemptDeclarations(AbstractMessageListenerContainer.java:1859)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.redeclareElementsIfNecessary(AbstractMessageListenerContainer.java:1840)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$AsyncMessageProcessingConsumer.initialize(SimpleMessageListenerContainer.java:1354)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$AsyncMessageProcessingConsumer.run(SimpleMessageListenerContainer.java:1200)
at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: java.net.ConnectException: Connection refused (Connection refused)
at java.base/java.net.PlainSocketImpl.socketConnect(Native Method)
at java.base/java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:399)
at java.base/java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:242)
at java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:224)
at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:403)
at java.base/java.net.Socket.connect(Socket.java:609)
at com.rabbitmq.client.impl.SocketFrameHandlerFactory.create(SocketFrameHandlerFactory.java:60)
at com.rabbitmq.client.ConnectionFactory.newConnection(ConnectionFactory.java:1137)
at com.rabbitmq.client.ConnectionFactory.newConnection(ConnectionFactory.java:1087)
at org.springframework.amqp.rabbit.connection.AbstractConnectionFactory.connectAddresses(AbstractConnectionFactory.java:560)
at org.springframework.amqp.rabbit.connection.AbstractConnectionFactory.connect(AbstractConnectionFactory.java:533)
at org.springframework.amqp.rabbit.connection.AbstractConnectionFactory.createBareConnection(AbstractConnectionFactory.java:487)
... 12 common frames omitted
E/AndroidRuntime: FATAL EXCEPTION: main
Process: com.example.poster, PID: 23677
java.lang.RuntimeException: Unable to start activity ComponentInfo{com.example.poster/com.example.poster.MainActivity}: com.google.firebase.database.DatabaseException: Failed to get FirebaseDatabase instance: Specify DatabaseURL within FirebaseApp or from your getInstance() call.
at android.app.ActivityThread.performLaunchActivity(ActivityThread.java:2984)
at android.app.ActivityThread.handleLaunchActivity(ActivityThread.java:3045)
at android.app.ActivityThread.-wrap14(ActivityThread.java)
at android.app.ActivityThread$H.handleMessage(ActivityThread.java:1642)
at android.os.Handler.dispatchMessage(Handler.java:102)
at android.os.Looper.loop(Looper.java:154)
at android.app.ActivityThread.main(ActivityThread.java:6776)
at java.lang.reflect.Method.invoke(Native Method)
at com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:1496)
at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:1386)
Caused by: com.google.firebase.database.DatabaseException: Failed to get FirebaseDatabase instance: Specify DatabaseURL within FirebaseApp or from your getInstance() call.
at com.google.firebase.database.FirebaseDatabase.getInstance(com.google.firebase:firebase-database##16.0.4:114)
at com.google.firebase.database.FirebaseDatabase.getInstance(com.google.firebase:firebase-database##16.0.4:71)
at com.example.poster.MainActivity.onCreate(MainActivity.java:48)
at android.app.Activity.performCreate(Activity.java:6955)
at android.app.Instrumentation.callActivityOnCreate(Instrumentation.java:1126)
at android.app.ActivityThread.performLaunchActivity(ActivityThread.java:2927)
at android.app.ActivityThread.handleLaunchActivity(ActivityThread.java:3045)
at android.app.ActivityThread.-wrap14(ActivityThread.java)
at android.app.ActivityThread$H.handleMessage(ActivityThread.java:1642)
at android.os.Handler.dispatchMessage(Handler.java:102)
at android.os.Looper.loop(Looper.java:154)
at android.app.ActivityThread.main(ActivityThread.java:6776)
at java.lang.reflect.Method.invoke(Native Method)
at com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:1496)
at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:1386)
add these dipendencies for java
implementation platform('com.google.firebase:firebase-bom:28.4.2')
implementation 'com.google.firebase:firebase-database'
for kotlin
implementation platform('com.google.firebase:firebase-bom:28.4.2')
implementation 'com.google.firebase:firebase-database-ktx'
then if using instance outside us1 for the database add url
FirebaseDatabase database = FirebaseDatabase.getInstance("**url here**);
DatabaseReference myRef = database.getReference("**path here**");
We have an application that reads RabbitMQ messages from a number of queues using Spring AMPQ and a RabbitListener. Something like this
#RabbitListener(queues = {"#{'${rabbit.queues}'.split(',')}"})
public void processRabbitMessage(#Payload String data, #Header(AmqpHeaders.CONSUMER_QUEUE) String queue, #Header(AmqpHeaders.MESSAGE_ID) String messageId) throws Exception {
// Do some stuff
}
However I'm getting an intermittent error where the message header isn't set.
org.springframework.amqp.rabbit.listener.exception.ListenerExecutionFailedException: Listener method could not be invoked with the incoming message
Endpoint handler details:
Method [public void com.service.RabbitService.processRabbitMessage(java.lang.String,java.lang.String,java.lang.String) throws java.lang.Exception]
Bean [com.service.RabbitService#5b7a7f33]
at org.springframework.amqp.rabbit.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:135)
at org.springframework.amqp.rabbit.listener.adapter.MessagingMessageListenerAdapter.onMessage(MessagingMessageListenerAdapter.java:106)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:822)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:745)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.access$001(SimpleMessageListenerContainer.java:97)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$1.invokeListener(SimpleMessageListenerContainer.java:189)
at sun.reflect.GeneratedMethodAccessor90.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:333)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at org.springframework.retry.interceptor.RetryOperationsInterceptor$1.doWithRetry(RetryOperationsInterceptor.java:91)
at org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:286)
at org.springframework.retry.support.RetryTemplate.execute(RetryTemplate.java:179)
at org.springframework.retry.interceptor.RetryOperationsInterceptor.invoke(RetryOperationsInterceptor.java:115)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
at com.sun.proxy.$Proxy110.invokeListener(Unknown Source)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.invokeListener(SimpleMessageListenerContainer.java:1276)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.executeListener(AbstractMessageListenerContainer.java:726)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.doReceiveAndExecute(SimpleMessageListenerContainer.java:1219)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.receiveAndExecute(SimpleMessageListenerContainer.java:1189)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.access$1500(SimpleMessageListenerContainer.java:97)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$AsyncMessageProcessingConsumer.run(SimpleMessageListenerContainer.java:1421)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.springframework.messaging.MessageHandlingException: Missing header 'amqp_consumerQueue' for method parameter type [class java.lang.String]
at org.springframework.messaging.handler.annotation.support.HeaderMethodArgumentResolver.handleMissingValue(HeaderMethodArgumentResolver.java:100)
at org.springframework.messaging.handler.annotation.support.AbstractNamedValueMethodArgumentResolver.resolveArgument(AbstractNamedValueMethodArgumentResolver.java:103)
at org.springframework.messaging.handler.invocation.HandlerMethodArgumentResolverComposite.resolveArgument(HandlerMethodArgumentResolverComposite.java:112)
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.getMethodArgumentValues(InvocableHandlerMethod.java:135)
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:107)
at org.springframework.amqp.rabbit.listener.adapter.HandlerAdapter.invoke(HandlerAdapter.java:49)
at org.springframework.amqp.rabbit.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:126)
... 25 common frames omitted
All the messages are written to the queues in exactly the same way (as below) and come from the same source so are generally the same structure.
MessageProperties msgprop = new MessageProperties()
msgprop.setContentType(MessageProperties.CONTENT_TYPE_TEXT_PLAIN)
msgprop.setAppId(routing)
rabbitTemplate.send(exchange, routing, new Message(message.getBytes(), msgprop))
We're not loosing data since we can catch all the rejected messages in a DLX but it's difficult to reprocess when we don't know what queue they came from.
We know that none of the messages are null/malformed (even the ones generating errors) and I was under the impression that this was a header set by Rabbit based on the queue a message is written to so how can it be null?
Any ideas as to how this header is null for some and not all messages and/or a possible fix to insure the headers are set?
EDIT: Added some DEBUG logs of the last attempt at reading in the message. We have a stateless retry policy in place so it's tried twice already.
00:46:17.577 [SimpleAsyncTaskExecutor-1051] DEBUG o.s.retry.support.RetryTemplate - Checking for rethrow: count=3
00:46:17.577 [SimpleAsyncTaskExecutor-1051] DEBUG o.s.retry.support.RetryTemplate - Retry failed last attempt: count=3
00:46:17.577 [SimpleAsyncTaskExecutor-1051] WARN o.s.a.r.r.RejectAndDontRequeueRecoverer - Retries exhausted for message (Body:'{Message:Body}' MessageProperties [headers={}, timestamp=null, messageId=a.b-209491726, userId=null, receivedUserId=null, appId=a.b, clusterId=null, type=null, correlationId=null, correlationIdString=null, replyTo=null, contentType=text/plain, contentEncoding=null, contentLength=0, deliveryMode=null, receivedDeliveryMode=PERSISTENT, expiration=null, priority=0, redelivered=false, receivedExchange=exch, receivedRoutingKey=a.b, receivedDelay=null, deliveryTag=561, messageCount=0, consumerTag=amq.ctag-fpVPl0EjHwdKew6eTEMoWA, consumerQueue=null])org.springframework.amqp.rabbit.listener.exception.ListenerExecutionFailedException: Listener method could not be invoked with the incoming message Endpoint handler details:
Method [public void com.service.RabbitService.processRabbitMessage(java.lang.String,java.lang.String,java.lang.String) throws java.lang.Exception]Bean [com.service.RabbitService#5b7a7f33]
at org.springframework.amqp.rabbit.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:135)
at org.springframework.amqp.rabbit.listener.adapter.MessagingMessageListenerAdapter.onMessage(MessagingMessageListenerAdapter.java:106)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:822)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:745)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.access$001(SimpleMessageListenerContainer.java:97)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$1.invokeListener(SimpleMessageListenerContainer.java:189)
at sun.reflect.GeneratedMethodAccessor90.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:333)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at org.springframework.retry.interceptor.RetryOperationsInterceptor$1.doWithRetry(RetryOperationsInterceptor.java:91)
at org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:286)
at org.springframework.retry.support.RetryTemplate.execute(RetryTemplate.java:179)
at org.springframework.retry.interceptor.RetryOperationsInterceptor.invoke(RetryOperationsInterceptor.java:115)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
at com.sun.proxy.$Proxy110.invokeListener(Unknown Source)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.invokeListener(SimpleMessageListenerContainer.java:1276)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.executeListener(AbstractMessageListenerContainer.java:726)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.doReceiveAndExecute(SimpleMessageListenerContainer.java:1219)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.receiveAndExecute(SimpleMessageListenerContainer.java:1189)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.access$1500(SimpleMessageListenerContainer.java:97)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$AsyncMessageProcessingConsumer.run(SimpleMessageListenerContainer.java:1421)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.springframework.messaging.MessageHandlingException: Missing header 'amqp_consumerQueue' for method parameter type [class java.lang.String]
at org.springframework.messaging.handler.annotation.support.HeaderMethodArgumentResolver.handleMissingValue(HeaderMethodArgumentResolver.java:100)
at org.springframework.messaging.handler.annotation.support.AbstractNamedValueMethodArgumentResolver.resolveArgument(AbstractNamedValueMethodArgumentResolver.java:103)
at org.springframework.messaging.handler.invocation.HandlerMethodArgumentResolverComposite.resolveArgument(HandlerMethodArgumentResolverComposite.java:112)
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.getMethodArgumentValues(InvocableHandlerMethod.java:135)
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:107)
at org.springframework.amqp.rabbit.listener.adapter.HandlerAdapter.invoke(HandlerAdapter.java:49)
at org.springframework.amqp.rabbit.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:126)
... 25 common frames omitted
00:46:17.577 [SimpleAsyncTaskExecutor-1051] WARN o.s.a.r.l.ConditionalRejectingErrorHandler - Execution of Rabbit message listener failed.
org.springframework.amqp.rabbit.listener.exception.ListenerExecutionFailedException: Retry Policy Exhausted
at org.springframework.amqp.rabbit.retry.RejectAndDontRequeueRecoverer.recover(RejectAndDontRequeueRecoverer.java:45)
at org.springframework.amqp.rabbit.config.StatelessRetryOperationsInterceptorFactoryBean$1.recover(StatelessRetryOperationsInterceptorFactoryBean.java:66)
at org.springframework.amqp.rabbit.config.StatelessRetryOperationsInterceptorFactoryBean$1.recover(StatelessRetryOperationsInterceptorFactoryBean.java:59)
at org.springframework.retry.interceptor.RetryOperationsInterceptor$ItemRecovererCallback.recover(RetryOperationsInterceptor.java:141)
at org.springframework.retry.support.RetryTemplate.handleRetryExhausted(RetryTemplate.java:512)
at org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:350)
at org.springframework.retry.support.RetryTemplate.execute(RetryTemplate.java:179)
at org.springframework.retry.interceptor.RetryOperationsInterceptor.invoke(RetryOperationsInterceptor.java:115)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
at com.sun.proxy.$Proxy110.invokeListener(Unknown Source)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.invokeListener(SimpleMessageListenerContainer.java:1276)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.executeListener(AbstractMessageListenerContainer.java:726)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.doReceiveAndExecute(SimpleMessageListenerContainer.java:1219)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.receiveAndExecute(SimpleMessageListenerContainer.java:1189)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.access$1500(SimpleMessageListenerContainer.java:97)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$AsyncMessageProcessingConsumer.run(SimpleMessageListenerContainer.java:1421)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.springframework.amqp.AmqpRejectAndDontRequeueException: org.springframework.amqp.rabbit.listener.exception.ListenerExecutionFailedException: Listener method could not be invoked with the incoming message
Endpoint handler details:
Method [public void com.service.RabbitService.processRabbitMessage(java.lang.String,java.lang.String,java.lang.String) throws java.lang.Exception]
Bean [com.service.RabbitService#5b7a7f33]
... 18 common frames omitted
Caused by: org.springframework.amqp.rabbit.listener.exception.ListenerExecutionFailedException: Listener method could not be invoked with the incoming message
Endpoint handler details:
Method [public void com.service.RabbitService.processRabbitMessage(java.lang.String,java.lang.String,java.lang.String) throws java.lang.Exception]
Bean [com.service.RabbitService#5b7a7f33]
at org.springframework.amqp.rabbit.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:135)
at org.springframework.amqp.rabbit.listener.adapter.MessagingMessageListenerAdapter.onMessage(MessagingMessageListenerAdapter.java:106)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:822)
at org.springframework.amqp.rabbit.listener.AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:745)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer.access$001(SimpleMessageListenerContainer.java:97)
at org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer$1.invokeListener(SimpleMessageListenerContainer.java:189)
at sun.reflect.GeneratedMethodAccessor90.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:333)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at org.springframework.retry.interceptor.RetryOperationsInterceptor$1.doWithRetry(RetryOperationsInterceptor.java:91)
at org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:286)
... 12 common frames omitted
Caused by: org.springframework.messaging.MessageHandlingException: Missing header 'amqp_consumerQueue' for method parameter type [class java.lang.String]
at org.springframework.messaging.handler.annotation.support.HeaderMethodArgumentResolver.handleMissingValue(HeaderMethodArgumentResolver.java:100)
at org.springframework.messaging.handler.annotation.support.AbstractNamedValueMethodArgumentResolver.resolveArgument(AbstractNamedValueMethodArgumentResolver.java:103)
at org.springframework.messaging.handler.invocation.HandlerMethodArgumentResolverComposite.resolveArgument(HandlerMethodArgumentResolverComposite.java:112)
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.getMethodArgumentValues(InvocableHandlerMethod.java:135)
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:107)
at org.springframework.amqp.rabbit.listener.adapter.HandlerAdapter.invoke(HandlerAdapter.java:49)
at org.springframework.amqp.rabbit.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:126)
... 25 common frames omitted
00:46:17.577 [SimpleAsyncTaskExecutor-1051] DEBUG o.s.a.r.l.BlockingQueueConsumer - Rejecting messages (requeue=false)
00:46:17.577 [SimpleAsyncTaskExecutor-1051] DEBUG o.s.a.r.l.SimpleMessageListenerContainer - Cancelling Consumer#2f48a506: tags=[{}], channel=Cached Rabbit Channel: AMQChannel(amqp://exch#1.1.1.1//exch,1055), conn: Proxy#79239726 Shared Rabbit Connection: SimpleConnection#7d1c1c5d [delegate=amqp://exch#10.136.229.112:5672//exch, localPort= 45298], acknowledgeMode=AUTO local queue size=0
00:46:17.577 [SimpleAsyncTaskExecutor-1051] DEBUG o.s.a.r.l.BlockingQueueConsumer - Closing Rabbit Channel: Cached Rabbit Channel: AMQChannel(amqp://exch#10.136.229.112:5672//exch,1055), conn: Proxy#79239726 Shared Rabbit Connection: SimpleConnection#7d1c1c5d [delegate=amqp://exch#10.136.229.112:5672//exch, localPort= 45298]
I don't see how that's possible; we unconditionally set the consumerQueue property in the received message - which is the source for that header. We maintain a map of consumerTag -> queue name for this purpose.
A DEBUG log for a message delivery that exhibits this behavior would be useful.
We see this error a lot on our Test server. This happens usually after application has been into background for a while. Once this error occurs user is not able to do any thing. This issue goes off once App is killed and restarted.
[4/24/16 19:08:23:195 EDT] 0000927c com.ibm.ws.webcontainer.extension W SRVE0190E: File not found: /apps/services/my_custom_auth_request_url
[4/24/16 19:08:23:195 EDT] 0000927c com.worklight.core.auth.impl.AuthenticationFilter E FWLSE0048E: Unhandled exception caught: SRVE0190E: File not found: /apps/services/my_custom_auth_request_url
java.io.FileNotFoundException: SRVE0190E: File not found: /apps/services/my_custom_auth_request_url
at com.ibm.ws.webcontainer.extension.DefaultExtensionProcessor.handleRequest(DefaultExtensionProcessor.java:528)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.invokeTarget(WebAppFilterChain.java:150)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:93)
at com.worklight.core.auth.impl.AuthenticationFilter$1.execute(AuthenticationFilter.java:224)
at com.worklight.core.auth.impl.AuthenticationServiceBean.accessResource(AuthenticationServiceBean.java:82)
at com.worklight.core.auth.impl.AuthenticationFilter.doFilter(AuthenticationFilter.java:229)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:207)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:90)
at com.worklight.analytics.AnalyticsFilter.doFilter(AnalyticsFilter.java:124)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:207)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:90)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:1020)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1142)
at com.ibm.ws.webcontainer.webapp.WebApp.handleRequest(WebApp.java:4867)
at com.ibm.ws.webcontainer.osgi.DynamicVirtualHost$2.handleRequest(DynamicVirtualHost.java:297)
at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:997)
at com.ibm.ws.webcontainer.osgi.DynamicVirtualHost$2.run(DynamicVirtualHost.java:262)
at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink$TaskWrapper.run(HttpDispatcherLink.java:955)
at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink.ready(HttpDispatcherLink.java:341)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:470)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.handleNewRequest(HttpInboundLink.java:404)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.processRequest(HttpInboundLink.java:284)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.ready(HttpInboundLink.java:255)
at com.ibm.ws.channel.ssl.internal.SSLConnectionLink.determineNextChannel(SSLConnectionLink.java:1017)
at com.ibm.ws.channel.ssl.internal.SSLConnectionLink$MyReadCompletedCallback.complete(SSLConnectionLink.java:626)
at com.ibm.ws.channel.ssl.internal.SSLReadServiceContext$SSLReadCompletedCallback.complete(SSLReadServiceContext.java:1766)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.requestComplete(WorkQueueManager.java:504)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.attemptIO(WorkQueueManager.java:574)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.workerRun(WorkQueueManager.java:929)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager$Worker.run(WorkQueueManager.java:1018)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1156)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:626)
at java.lang.Thread.run(Thread.java:780)
[project MyProject]
java.io.FileNotFoundException: SRVE0190E: File not found: /apps/services/my_custom_auth_request_url
at com.ibm.ws.webcontainer.extension.DefaultExtensionProcessor.handleRequest(DefaultExtensionProcessor.java:528)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.invokeTarget(WebAppFilterChain.java:150)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:93)
at com.worklight.core.auth.impl.AuthenticationFilter$1.execute(AuthenticationFilter.java:224)
at com.worklight.core.auth.impl.AuthenticationServiceBean.accessResource(AuthenticationServiceBean.java:82)
at com.worklight.core.auth.impl.AuthenticationFilter.doFilter(AuthenticationFilter.java:229)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:207)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:90)
at com.worklight.analytics.AnalyticsFilter.doFilter(AnalyticsFilter.java:124)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:207)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:90)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:1020)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1142)
at com.ibm.ws.webcontainer.webapp.WebApp.handleRequest(WebApp.java:4867)
at com.ibm.ws.webcontainer.osgi.DynamicVirtualHost$2.handleRequest(DynamicVirtualHost.java:297)
at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:997)
at com.ibm.ws.webcontainer.osgi.DynamicVirtualHost$2.run(DynamicVirtualHost.java:262)
at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink$TaskWrapper.run(HttpDispatcherLink.java:955)
at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink.ready(HttpDispatcherLink.java:341)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:470)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.handleNewRequest(HttpInboundLink.java:404)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.processRequest(HttpInboundLink.java:284)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.ready(HttpInboundLink.java:255)
at com.ibm.ws.channel.ssl.internal.SSLConnectionLink.determineNextChannel(SSLConnectionLink.java:1017)
at com.ibm.ws.channel.ssl.internal.SSLConnectionLink$MyReadCompletedCallback.complete(SSLConnectionLink.java:626)
at com.ibm.ws.channel.ssl.internal.SSLReadServiceContext$SSLReadCompletedCallback.complete(SSLReadServiceContext.java:1766)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.requestComplete(WorkQueueManager.java:504)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.attemptIO(WorkQueueManager.java:574)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.workerRun(WorkQueueManager.java:929)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager$Worker.run(WorkQueueManager.java:1018)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1156)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:626)
at java.lang.Thread.run(Thread.java:780)
[4/24/16 19:08:23:197 EDT] 0000927c com.ibm.ws.webcontainer.webapp E SRVE0293E: [Servlet Error]-[FWLSE0304E: Unhandled exception caught: SRVE0190E: File not found: /apps/services/my_custom_auth_request_url [project MyProject]]: com.ibm.ws.webcontainer.webapp.WebAppErrorReport: FWLSE0304E: Unhandled exception caught: SRVE0190E: File not found: /apps/services/my_custom_auth_request_url [project MyProject]
at com.ibm.ws.webcontainer.webapp.WebAppDispatcherContext.sendError(WebAppDispatcherContext.java:629)
at com.ibm.ws.webcontainer.webapp.WebAppDispatcherContext.sendError(WebAppDispatcherContext.java:647)
at com.ibm.ws.webcontainer.srt.SRTServletResponse.sendError(SRTServletResponse.java:1303)
at com.worklight.core.auth.impl.AuthenticationFilter.doFilter(AuthenticationFilter.java:289)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:207)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:90)
at com.worklight.analytics.AnalyticsFilter.doFilter(AnalyticsFilter.java:124)
at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:207)
at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:90)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:1020)
at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1142)
at com.ibm.ws.webcontainer.webapp.WebApp.handleRequest(WebApp.java:4867)
at com.ibm.ws.webcontainer.osgi.DynamicVirtualHost$2.handleRequest(DynamicVirtualHost.java:297)
at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:997)
at com.ibm.ws.webcontainer.osgi.DynamicVirtualHost$2.run(DynamicVirtualHost.java:262)
at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink$TaskWrapper.run(HttpDispatcherLink.java:955)
at com.ibm.ws.http.dispatcher.internal.channel.HttpDispatcherLink.ready(HttpDispatcherLink.java:341)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:470)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.handleNewRequest(HttpInboundLink.java:404)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.processRequest(HttpInboundLink.java:284)
at com.ibm.ws.http.channel.internal.inbound.HttpInboundLink.ready(HttpInboundLink.java:255)
at com.ibm.ws.channel.ssl.internal.SSLConnectionLink.determineNextChannel(SSLConnectionLink.java:1017)
at com.ibm.ws.channel.ssl.internal.SSLConnectionLink$MyReadCompletedCallback.complete(SSLConnectionLink.java:626)
at com.ibm.ws.channel.ssl.internal.SSLReadServiceContext$SSLReadCompletedCallback.complete(SSLReadServiceContext.java:1766)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.requestComplete(WorkQueueManager.java:504)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.attemptIO(WorkQueueManager.java:574)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager.workerRun(WorkQueueManager.java:929)
at com.ibm.ws.tcpchannel.internal.WorkQueueManager$Worker.run(WorkQueueManager.java:1018)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1156)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:626)
at java.lang.Thread.run(Thread.java:780)
I assume that you are using a custom authenticator, and that your challenge handler uses submitLoginForm with the url my_custom_auth_request_url.
First, make sure that of course your custom authenticator knows how to respond to my_custom_auth_request_url.
But most likely, the error you are seeing is because you are trying to do a submitLoginForm outside of a challenge context.
Meaning, either:
you are trying to login before receiving any challenge.
you received a challenge, let the session expire, and then try to answer it too late.
You did not make sure that every single challenge you receive either ends by a submitLoginForm or a submitFailure.
It could also be related to your other question (MFP 7.1 - FWLSE0342E: Grant code validation failed: Grant code was already used). If your sessions are not handled correctly, you lose the context of the challenge.
Jboss 7.1.1 server is throwing hornetq reconnect errors on one of the queue I configured. I have removed messages from the queue using CLI and deleted the data folder as well, but this error is not going away. Appreciate any help.
CLI command
[standalone#localhost:9999 /] /subsystem=messaging/hornetq-server=default/jms-queue=myQueueC:remove-messages
{
"outcome" => "success",
"result" => 0
}
Exception stack trace from the server log
10:53:47,505 INFO [org.hornetq.ra.inflow.HornetQActivation] (default-short-running-threads-threads - 1) Attempting to reconnect org.hornetq.ra.inflow.HornetQActivationSpec(ra=org.hornetq.ra.HornetQResourceAdapter#20ebddc6 destination=java:/queue/C destinationType=javax.jms.Queue ack=Auto-acknowledge durable=false clientID=null user=null maxSession=15)
10:53:47,518 ERROR [org.hornetq.ra.inflow.HornetQActivation] (default-short-running-threads-threads - 1) Unable to reconnect org.hornetq.ra.inflow.HornetQActivationSpec(ra=org.hornetq.ra.HornetQResourceAdapter#20ebddc6 destination=java:/queue/C destinationType=javax.jms.Queue ack=Auto-acknowledge durable=false clientID=null user=null maxSession=15): java.lang.IllegalArgumentException: interface javax.resource.spi.endpoint.MessageEndpoint is not visible from class loader
at java.lang.reflect.Proxy$ProxyClassFactory.apply(Proxy.java:616) [rt.jar:1.7.0_67]
at java.lang.reflect.Proxy$ProxyClassFactory.apply(Proxy.java:592) [rt.jar:1.7.0_67]
at java.lang.reflect.WeakCache$Factory.get(WeakCache.java:244) [rt.jar:1.7.0_67]
at java.lang.reflect.WeakCache.get(WeakCache.java:141) [rt.jar:1.7.0_67]
at java.lang.reflect.Proxy.getProxyClass0(Proxy.java:455) [rt.jar:1.7.0_67]
at java.lang.reflect.Proxy.newProxyInstance(Proxy.java:738) [rt.jar:1.7.0_67]
at org.jboss.as.ejb3.inflow.JBossMessageEndpointFactory.createEndpoint(JBossMessageEndpointFactory.java:56)
at org.jboss.as.ejb3.inflow.JBossMessageEndpointFactory.createEndpoint(JBossMessageEndpointFactory.java:49)
at org.hornetq.ra.inflow.HornetQMessageHandler.setup(HornetQMessageHandler.java:185) [hornetq-ra-2.2.13.Final.jar:]
at org.hornetq.ra.inflow.HornetQActivation.setup(HornetQActivation.java:296) [hornetq-ra-2.2.13.Final.jar:]
at org.hornetq.ra.inflow.HornetQActivation.handleFailure(HornetQActivation.java:566) [hornetq-ra-2.2.13.Final.jar:]
at org.hornetq.ra.inflow.HornetQActivation$SetupActivation.run(HornetQActivation.java:609) [hornetq-ra-2.2.13.Final.jar:]
at org.jboss.jca.core.workmanager.WorkWrapper.run(WorkWrapper.java:212)
at org.jboss.threads.SimpleDirectExecutor.execute(SimpleDirectExecutor.java:33)
at org.jboss.threads.QueueExecutor.runTask(QueueExecutor.java:801)
at org.jboss.threads.QueueExecutor.access$100(QueueExecutor.java:45)
at org.jboss.threads.QueueExecutor$Worker.run(QueueExecutor.java:821)
at java.lang.Thread.run(Thread.java:745) [rt.jar:1.7.0_67]
at org.jboss.threads.JBossThread.run(JBossThread.java:122)
If you are using custom JBoss module, make sure that your module has dependency to javax.resource.api.
<module>
<dependencies>
<module name="javax.resource.api" />
</dependencies>
</module>
Another solution is: Add Dependencies: javax.resource.api line to your MyApp.jar/META-INF/MANIFEST.MF file.