Logstash - RabbitMQ connection Timeout error - rabbitmq

Question

I have installed logstash on 2 nodes to send the logs to RabbitMQ. SSL is configured on RabbitMQ listening 5671 port. I have configured both the logstash to push the logs to rabbitmq server on the 5671 port.
This is my configuration.
input {
file {
path => "/var/log/messages"
start_position => "beginning"
}
}
filter {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:system_auth_timestamp} %{SYSLOGHOST:system_auth_hostname} %{GREEDYDATA:command_issued}: %{GREEDYDATA:message}" }
add_tag => "syslog"
}
}
output {
rabbitmq {
exchange => "elasticsearch-exchange"
exchange_type => "direct"
key => "logstash-routing_key"
ssl => true
#verify_ssl => true
ssl_certificate_password => 'Password'
ssl_certificate_path => 'certfile'
ssl_version => "TLSv1.2"
host => "10.2.0.0"
vhost => "es_vhost"
durable => true
persistent => true
port => 5671
user => "admin"
password => "password"
heartbeat => "5"
}
stdout {
codec => rubydebug
}
}
This is the error I am getting in the logstash log.
{:timestamp=>"2017-12-26T07:22:32.708000+0000", :message=>"Pipeline aborted due to error", :exception=>java.util.concurrent.TimeoutException, :backtrace=>["com.rabbitmq.utility.BlockingCell.get(com/rabbitmq/utility/BlockingCell.java:77)", "com.rabbitmq.utility.BlockingCell.uninterruptibleGet(com/rabbitmq/utility/BlockingCell.java:111)", "com.rabbitmq.utility.BlockingValueOrException.uninterruptibleGetValue(com/rabbitmq/utility/BlockingValueOrException.java:37)", "com.rabbitmq.client.impl.AMQChannel$BlockingRpcContinuation.getReply(com/rabbitmq/client/impl/AMQChannel.java:367)", "com.rabbitmq.client.impl.AMQConnection.start(com/rabbitmq/client/impl/AMQConnection.java:293)", "com.rabbitmq.client.ConnectionFactory.newConnection(com/rabbitmq/client/ConnectionFactory.java:648)", "com.rabbitmq.client.ConnectionFactory.newConnection(com/rabbitmq/client/ConnectionFactory.java:678)", "java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:498)", "RUBY.new_connection_impl(/opt/logstash/vendor/bundle/jruby/1.9/gems/march_hare-2.15.0-java/lib/march_hare/session.rb:505)", "org.jruby.RubyProc.call(org/jruby/RubyProc.java:281)", "RUBY.converting_rjc_exceptions_to_ruby(/opt/logstash/vendor/bundle/jruby/1.9/gems/march_hare-2.15.0-java/lib/march_hare/session.rb:467)", "RUBY.new_connection_impl(/opt/logstash/vendor/bundle/jruby/1.9/gems/march_hare-2.15.0-java/lib/march_hare/session.rb:500)", "RUBY.initialize(/opt/logstash/vendor/bundle/jruby/1.9/gems/march_hare-2.15.0-java/lib/march_hare/session.rb:136)", "RUBY.connect(/opt/logstash/vendor/bundle/jruby/1.9/gems/march_hare-2.15.0-java/lib/march_hare/session.rb:109)", "RUBY.connect(/opt/logstash/vendor/bundle/jruby/1.9/gems/march_hare-2.15.0-java/lib/march_hare.rb:20)", "RUBY.connect(/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-mixin-rabbitmq_connection-4.1.1-java/lib/logstash/plugin_mixins/rabbitmq_connection.rb:174)", "RUBY.connect!(/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-mixin-rabbitmq_connection-4.1.1-java/lib/logstash/plugin_mixins/rabbitmq_connection.rb:131)", "RUBY.register(/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-rabbitmq-3.1.0-java/lib/logstash/outputs/rabbitmq.rb:40)", "RUBY.register(/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/output_delegator.rb:75)", "RUBY.start_workers(/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:181)", "org.jruby.RubyArray.each(org/jruby/RubyArray.java:1613)", "RUBY.start_workers(/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:181)", "RUBY.run(/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/pipeline.rb:136)", "RUBY.start_pipeline(/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.4-java/lib/logstash/agent.rb:473)", "java.lang.Thread.run(java/lang/Thread.java:745)"], :level=>:error}
{:timestamp=>"2017-12-26T07:22:35.710000+0000", :message=>"stopping pipeline", :id=>"main"}
This is the error I am getting in RabbitMQ logs.
=INFO REPORT==== 27-Dec-2017::05:44:27 ===
accepting AMQP connection <0.1228.0> (10.2.0.0:42187 -> 10.24.168.17:5601)
=WARNING REPORT==== 27-Dec-2017::05:44:35 ===
closing AMQP connection <0.1228.0> (10.2.0.0:42187 -> 10.24.168.17:5601):
client unexpectedly closed TCP connection
This is RabbitMQ conf
% This file managed by Puppet
% Template Path: rabbitmq/templates/rabbitmq.config
[
{rabbit, [
{cluster_nodes, {[rabbit#node01, rabbitmq#node02, rabbit#node03], disc}},
{cluster_partition_handling, ignore},
{tcp_listen_options,
[binary,
{packet, raw},
{reuseaddr, true},
{backlog, 128},
{nodelay, true},
{exit_on_close, false}]
},
{default_user, <<"admin">>},
{default_pass, <<"passowrd">>},
{handshake_timeout, 60000},
{tcp_listeners, []},
{ssl_listeners, [5671]},
{ssl_options, [{cacertfile,"/etc/rabbitmq/ssl_cert/testca/cacert.pem"},
{certfile,"/etc/rabbitmq/ssl_cert/server/cert.pem"},
{keyfile,"/etc/rabbitmq/ssl_cert/server/key.pem"},
{password, "Password"},
{verify,verify_peer},
{versions, ['tlsv1.2']},
{fail_if_no_peer_cert,false}]},
{ssl_handshake_timeout, 5000}
{log_levels, [{autocluster, debug}, {connection, info}]}
]},
{kernel, [
]},
{rabbitmq_management, [
{listener, [
{port, 15672}
]}
]}
].
% EOF
I have even changed the SSL listener port to 5601 and tried just to make sure that this is not port conflict. I am hitting the wall everytime here.

Answer 1

There was a mismatch in hostname. I have resolved it by proving an FQDN in the /etc/hosts file. SSL is working fine now.