ActiveMQ cookbook triggers the activemq service, but the service is not started - activemq

Activemq service is triggered via cookbook but it does not runs:
activemq/attributes/default.rb
default['activemq']['version'] = '5.11.0'
src_filename="apache-activemq-#{node['activemq']['version']}-bin.tar.gz"
src_filepath = "#{Chef::Config['file_cache_path']}/#{src_filename}"
default['activemq']['src_filepath'] = "#{src_filepath}"
default['activemq']['tar_filepath'] = "http://xxx-xx-xx-xxx-xx:8091/3rdparty/activemq/#{src_filename}"
default['activemq']['dir'] = "/usr/local/apache-activemq-#{node['activemq']['version']}"
default['activemq']['wrapper']['max_memory'] = '1024'
default['activemq']['wrapper']['useDedicatedTaskRunner'] = 'true'
default['activemq']['zooKeeper']['address']="xxx.xx.xx.xx:2181"
default['activemq']['zooKeeper']['hostname']="xxx.xx.xx.xx"
activemq/recipes/default.rb
include_recipe "lgjava"
activemq_home= node['activemq']['dir']
remote_file "#{node['activemq']['src_filepath']}" do
mode 0755
source node['activemq']['tar_filepath']
action :create
notifies :create, "directory[apache-activemq-#{node['activemq']['version']}]", :immediately
notifies :run, "execute[untar-activemq]", :immediately
end
directory "apache-activemq-#{node['activemq']['version']}" do
path "#{activemq_home}"
mode 0755
recursive true
end
execute "untar-activemq" do
cwd Chef::Config[:file_cache_path]
command <<-EOF
tar xvzf apache-activemq-#{node['activemq']['version']}-bin.tar.gz -C #{node['activemq']['dir'] } --strip 1
EOF
action :run
end
file "#{activemq_home}/bin/activemq" do
owner 'root'
group 'root'
mode '0755'
end
arch = node['kernel']['machine'] == 'x86_64' ? 'x86-64' : 'x86-32'
link '/etc/init.d/activemq' do
to "#{activemq_home}/bin/linux-#{arch}/activemq"
end
template "jetty-realm.properties" do
source "jetty-realm.properties.erb"
mode "0755"
path "#{activemq_home}/conf/jetty-realm.properties"
action :create
notifies :restart, 'service[activemq]'
end
template "activemq.xml" do
source "activemq.xml.erb"
mode "0755"
path "#{activemq_home}/conf/activemq.xml"
action :create
notifies :restart, 'service[activemq]'
end
service 'activemq' do
supports :restart => true, :status => true
action [:enable, :start]
end
# symlink so the default wrapper.conf can find the native wrapper library
link "#{activemq_home}/bin/linux" do
to "#{activemq_home}/bin/linux-#{arch}"
end
# symlink the wrapper's pidfile location into /var/run
link '/var/run/activemq.pid' do
to "#{activemq_home}/bin/linux/ActiveMQ.pid"
not_if 'test -f /var/run/activemq.pid'
end
template "#{activemq_home}/bin/linux/wrapper.conf" do
source 'wrapper.conf.erb'
mode '0644'
notifies :restart, 'service[activemq]'
end
activemq/templates/default/activemq.xml.erb
<!-- START SNIPPET: example -->
<beans
xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd">
<!-- Allows us to use system properties as variables in this configuration file -->
<bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="locations">
<value>file:${activemq.conf}/credentials.properties</value>
</property>
</bean>
<!-- Allows accessing the server log -->
<bean id="logQuery" class="io.fabric8.insight.log.log4j.Log4jLogQuery"
lazy-init="false" scope="singleton"
init-method="start" destroy-method="stop">
</bean>
<!--
The <broker> element is used to configure the ActiveMQ broker.
-->
<broker xmlns="http://activemq.apache.org/schema/core" brokerName="localhost" dataDirectory="${activemq.data}">
<destinationPolicy>
<policyMap>
<policyEntries>
<policyEntry topic=">" >
<!-- The constantPendingMessageLimitStrategy is used to prevent
slow topic consumers to block producers and affect other consumers
by limiting the number of messages that are retained
For more information, see:
http://activemq.apache.org/slow-consumer-handling.html
-->
<pendingMessageLimitStrategy>
<constantPendingMessageLimitStrategy limit="1000"/>
</pendingMessageLimitStrategy>
</policyEntry>
</policyEntries>
</policyMap>
</destinationPolicy>
<!--
The managementContext is used to configure how ActiveMQ is exposed in
JMX. By default, ActiveMQ uses the MBean server that is started by
the JVM. For more information, see:
http://activemq.apache.org/jmx.html
-->
<managementContext>
<managementContext createConnector="false"/>
</managementContext>
<!--
Configure message persistence for the broker. The default persistence
mechanism is the KahaDB store (identified by the kahaDB tag).
For more information, see:
http://activemq.apache.org/persistence.html
-->
<!--
<persistenceAdapter>
<kahaDB directory="${activemq.data}/kahadb"/>
</persistenceAdapter> -->
<persistenceAdapter>
<replicatedLevelDB directory="activemq-data"
replicas="2"
bind="tcp://0.0.0.0:0"
zkAddress=<%=node['activemq']['zooKeeper']['address']%>
zkPath="/activemq/leveldb-stores"
hostname=<%=node['activemq']['zooKeeper']['hostname']%> />
</persistenceAdapter>
<!--
The systemUsage controls the maximum amount of space the broker will
use before disabling caching and/or slowing down producers. For more information, see:
http://activemq.apache.org/producer-flow-control.html
-->
<systemUsage>
<systemUsage>
<memoryUsage>
<memoryUsage percentOfJvmHeap="70" />
</memoryUsage>
<storeUsage>
<storeUsage limit="100 gb"/>
</storeUsage>
<tempUsage>
<tempUsage limit="50 gb"/>
</tempUsage>
</systemUsage>
</systemUsage>
<!--
The transport connectors expose ActiveMQ over a given protocol to
clients and other brokers. For more information, see:
http://activemq.apache.org/configuring-transports.html
-->
<transportConnectors>
<!-- DOS protection, limit concurrent connections to 1000 and frame size to 100MB -->
<transportConnector name="openwire" uri="tcp://0.0.0.0:61616?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="amqp" uri="amqp://0.0.0.0:5672?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="stomp" uri="stomp://0.0.0.0:61613?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="mqtt" uri="mqtt://0.0.0.0:1883?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="ws" uri="ws://0.0.0.0:61614?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
</transportConnectors>
<!-- destroy the spring context on shutdown to stop jetty -->
<shutdownHooks>
<bean xmlns="http://www.springframework.org/schema/beans" class="org.apache.activemq.hooks.SpringContextHook" />
</shutdownHooks>
</broker>
<!--
Enable web consoles, REST and Ajax APIs and demos
The web consoles requires by default login, you can disable this in the jetty.xml file
Take a look at ${ACTIVEMQ_HOME}/conf/jetty.xml for more details
-->
<import resource="jetty.xml"/>
</beans>
activemq/templates/default/wrapper.conf.erb
#********************************************************************
# Wrapper Properties
#********************************************************************
#wrapper.debug=TRUE
set.default.ACTIVEMQ_HOME=../..
set.default.ACTIVEMQ_BASE=../..
set.default.ACTIVEMQ_CONF=%ACTIVEMQ_BASE%/conf
set.default.ACTIVEMQ_DATA=%ACTIVEMQ_BASE%/data
wrapper.working.dir=.
# Java Application
wrapper.java.command=java
# Java Main class. This class must implement the WrapperListener interface
# or guarantee that the WrapperManager class is initialized. Helper
# classes are provided to do this for you. See the Integration section
# of the documentation for details.
wrapper.java.mainclass=org.tanukisoftware.wrapper.WrapperSimpleApp
# Java Classpath (include wrapper.jar) Add class path elements as
# needed starting from 1
wrapper.java.classpath.1=%ACTIVEMQ_HOME%/bin/wrapper.jar
wrapper.java.classpath.2=%ACTIVEMQ_HOME%/bin/activemq.jar
# Java Library Path (location of Wrapper.DLL or libwrapper.so)
wrapper.java.library.path.1=%ACTIVEMQ_HOME%/bin/linux-x86-64/
# Java Additional Parameters
# note that n is the parameter number starting from 1.
wrapper.java.additional.1=-Dactivemq.home=%ACTIVEMQ_HOME%
wrapper.java.additional.2=-Dactivemq.base=%ACTIVEMQ_BASE%
wrapper.java.additional.3=-Djavax.net.ssl.keyStorePassword=password
wrapper.java.additional.4=-Djavax.net.ssl.trustStorePassword=password
wrapper.java.additional.5=-Djavax.net.ssl.keyStore=%ACTIVEMQ_CONF%/broker.ks
wrapper.java.additional.6=-Djavax.net.ssl.trustStore=%ACTIVEMQ_CONF%/broker.ts
wrapper.java.additional.7=-Dcom.sun.management.jmxremote
wrapper.java.additional.8=-Dorg.apache.activemq.UseDedicatedTaskRunner=<%= node['activemq']['wrapper']['useDedicatedTaskRunner'] %>
wrapper.java.additional.9=-Djava.util.logging.config.file=logging.properties
wrapper.java.additional.10=-Dactivemq.conf=%ACTIVEMQ_CONF%
wrapper.java.additional.11=-Dactivemq.data=%ACTIVEMQ_DATA%
wrapper.java.additional.12=-Djava.security.auth.login.config=%ACTIVEMQ_CONF%/login.config
# Uncomment to enable jmx
#wrapper.java.additional.n=-Dcom.sun.management.jmxremote.port=1616
#wrapper.java.additional.n=-Dcom.sun.management.jmxremote.authenticate=false
#wrapper.java.additional.n=-Dcom.sun.management.jmxremote.ssl=false
# Uncomment to enable YourKit profiling
#wrapper.java.additional.n=-Xrunyjpagent
# Uncomment to enable remote debugging
#wrapper.java.additional.n=-Xdebug -Xnoagent -Djava.compiler=NONE
#wrapper.java.additional.n=-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005
# Initial Java Heap Size (in MB)
#wrapper.java.initmemory=3
# Maximum Java Heap Size (in MB)
wrapper.java.maxmemory=<%= node['activemq']['wrapper']['max_memory'] %>
# Application parameters. Add parameters as needed starting from 1
wrapper.app.parameter.1=org.apache.activemq.console.Main
wrapper.app.parameter.2=start
#********************************************************************
# Wrapper Logging Properties
#********************************************************************
# Format of output for the console. (See docs for formats)
wrapper.console.format=PM
# Log Level for console output. (See docs for log levels)
wrapper.console.loglevel=INFO
# Log file to use for wrapper output logging.
wrapper.logfile=%ACTIVEMQ_DATA%/wrapper.log
# Format of output for the log file. (See docs for formats)
wrapper.logfile.format=LPTM
# Log Level for log file output. (See docs for log levels)
wrapper.logfile.loglevel=INFO
# Maximum size that the log file will be allowed to grow to before
# the log is rolled. Size is specified in bytes. The default value
# of 0, disables log rolling. May abbreviate with the 'k' (kb) or
# 'm' (mb) suffix. For example: 10m = 10 megabytes.
wrapper.logfile.maxsize=0
# Maximum number of rolled log files which will be allowed before old
# files are deleted. The default value of 0 implies no limit.
wrapper.logfile.maxfiles=0
# Log Level for sys/event log output. (See docs for log levels)
wrapper.syslog.loglevel=NONE
#********************************************************************
# Wrapper Windows Properties
#********************************************************************
# Title to use when running as a console
wrapper.console.title=ActiveMQ
#********************************************************************
# Wrapper Windows NT/2000/XP Service Properties
#********************************************************************
# WARNING - Do not modify any of these properties when an application
# using this configuration file has been installed as a service.
# Please uninstall the service before modifying this section. The
# service can then be reinstalled.
# Name of the service
wrapper.ntservice.name=ActiveMQ
# Display name of the service
wrapper.ntservice.displayname=ActiveMQ
# Description of the service
wrapper.ntservice.description=ActiveMQ Broker
# Service dependencies. Add dependencies as needed starting from 1
wrapper.ntservice.dependency.1=
# Mode in which the service is installed. AUTO_START or DEMAND_START
wrapper.ntservice.starttype=AUTO_START
# Allow the service to interact with the desktop.
wrapper.ntservice.interactive=false
Executing chef-client I get the following:
Recipe: activemq::default
* remote_file[/var/chef/cache/apache-activemq-5.11.0-bin.tar.gz] action create (up to date)
* directory[apache-activemq-5.11.0] action create (up to date)
* execute[untar-activemq] action run
- execute tar xvzf apache-activemq-5.11.0-bin.tar.gz -C /usr/local/apache-activemq-5.11.0 --strip 1
* file[/usr/local/apache-activemq-5.11.0/bin/activemq] action create (up to date)
* link[/etc/init.d/activemq] action create (up to date)
* template[jetty-realm.properties] action create
- update content in file /usr/local/apache-activemq-5.11.0/conf/jetty-realm.properties from 827a97 to 96c9a9
--- /usr/local/apache-activemq-5.11.0/conf/jetty-realm.properties 2015-01-30 13:13:51.000000000 +0000
+++ /tmp/chef-rendered-template20150211-9315-ul0l7k 2015-02-11 07:03:37.711054777 +0000
## -5,9 +5,9 ##
## The ASF licenses this file to You under the Apache License, Version 2.0
## (the "License"); you may not use this file except in compliance with
## the License. You may obtain a copy of the License at
-##
+##
## http://www.apache.org/licenses/LICENSE-2.0
-##
+##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## -17,6 +17,6 ##
# Defines users that can access the web (console, demo, etc.)
# username: *****
-admin: *****, *****
-user: ****, *****
+admin: *****, ****
+user: *****, ****
- change mode from '0644' to '0755'
- restore selinux security context
* template[activemq.xml] action create
- update content in file /usr/local/apache-activemq-5.11.0/conf/activemq.xml from ca8528 to 219698
--- /usr/local/apache-activemq-5.11.0/conf/activemq.xml 2015-01-30 13:13:51.000000000 +0000
+++ /tmp/chef-rendered-template20150211-9315-8syvv7 2015-02-11 07:03:37.736055052 +0000
## -78,11 +78,20 ##
http://activemq.apache.org/persistence.html
-->
+ <!--
<persistenceAdapter>
<kahaDB directory="${activemq.data}/kahadb"/>
- </persistenceAdapter>
+ </persistenceAdapter> -->
+ <persistenceAdapter>
+ <replicatedLevelDB directory="activemq-data"
+ replicas="2"
+ bind="tcp://x.x.x.x:x"
+ zkAddress=xxxxxxx
+ zkPath=xxxxxx
+ hostname=xxxxxx />
+ </persistenceAdapter>
<!--
The systemUsage controls the maximum amount of space the broker will
use before disabling caching and/or slowing down producers. For more information, see:
## -133,5 +142,4 ##
<import resource="jetty.xml"/>
</beans>
-<!-- END SNIPPET: example -->
- change mode from '0644' to '0755'
- restore selinux security context
* service[activemq] action enable (up to date)
* service[activemq] action start (up to date)
* link[/usr/local/apache-activemq-5.11.0/bin/linux] action create (up to date)
* link[/var/run/activemq.pid] action create (up to date)
* template[/usr/local/apache-activemq-5.11.0/bin/linux/wrapper.conf] action create (up to date)
Recipe: lgtomcat::default
* service[tomcat7] action restart
- restart service service[tomcat7]
Recipe: activemq::default
* service[activemq] action restart
- restart service service[activemq]
But on checking for activemq process, there is none started:
ps -eaf| grep activemq
root 9867 9301 0 07:07 pts/1 00:00:00 grep --color=auto activemq

Related

ActiveMQ is not shutting down properly -INFO: Regular shutdown not successful, sending SIGKILL to process

Whenever I try to stop ActiveMQ installed on RHEL server, it doesn't stops gracefully. Does anyone knows why? I am not sure why it tries to connect to JMX broker as shown below and fails. What do I need to fix these issues?
[activemq#myserver apache-activemq-5.15.11]$ bin/activemq stop
INFO: Loading '/web/servers/apache-activemq-5.15.11//bin/env'
INFO: Using java '/bin/java'
INFO: Waiting at least 30 seconds for regular process termination of pid '2963' :
Java Runtime: Oracle Corporation 1.8.0_252 /usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre
Heap sizes: current=62976k free=61991k max=932352k
JVM args: -Xms64M -Xmx1G -Djava.util.logging.config.file=logging.properties -Djava.security.auth.login.config=/web/servers/apache-activemq-5.15.11//conf/login.config -Dactivemq.classpath=/web/servers/apache-activemq-5.15.11//conf:/web/servers/apache-activemq-5.15.11//../lib/: -Dactivemq.home=/web/servers/apache-activemq-5.15.11/ -Dactivemq.base=/web/servers/apache-activemq-5.15.11/ -Dactivemq.conf=/web/servers/apache-activemq-5.15.11//conf -Dactivemq.data=/web/servers/apache-activemq-5.15.11//data
Extensions classpath:
[/web/servers/apache-activemq-5.15.11/lib,/web/servers/apache-activemq-5.15.11/lib/camel,/web/servers/apache-activemq-5.15.11/lib/optional,/web/servers/apache-activemq-5.15.11/lib/web,/web/servers/apache-activemq-5.15.11/lib/extra]
ACTIVEMQ_HOME: /web/servers/apache-activemq-5.15.11
ACTIVEMQ_BASE: /web/servers/apache-activemq-5.15.11
ACTIVEMQ_CONF: /web/servers/apache-activemq-5.15.11/conf
ACTIVEMQ_DATA: /web/servers/apache-activemq-5.15.11/data
Connecting to pid: 2963
INFO: failed to resolve jmxUrl for pid:2963, using default JMX url
Connecting to JMX URL: service:jmx:rmi:///jndi/rmi://localhost:1099/jmxrmi
INFO: Broker not available at: service:jmx:rmi:///jndi/rmi://localhost:1099/jmxrmi
...............................
INFO: Regular shutdown not successful, sending SIGKILL to process
INFO: sending SIGKILL to pid '2963'
EDIT:
Adding activemq.xml configuration file
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- START SNIPPET: example -->
<beans
xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd">
<!-- Allows us to use system properties as variables in this configuration file -->
<bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="locations">
<value>file:${activemq.conf}/credentials.properties</value>
</property>
</bean>
<!-- Allows accessing the server log -->
<bean id="logQuery" class="io.fabric8.insight.log.log4j.Log4jLogQuery"
lazy-init="false" scope="singleton"
init-method="start" destroy-method="stop">
</bean>
<!--
The <broker> element is used to configure the ActiveMQ broker.
-->
<broker xmlns="http://activemq.apache.org/schema/core" brokerName="localhost" dataDirectory="${activemq.data}">
<destinationPolicy>
<policyMap>
<policyEntries>
<policyEntry topic=">" >
<!-- The constantPendingMessageLimitStrategy is used to prevent
slow topic consumers to block producers and affect other consumers
by limiting the number of messages that are retained
For more information, see:
http://activemq.apache.org/slow-consumer-handling.html
-->
<pendingMessageLimitStrategy>
<constantPendingMessageLimitStrategy limit="1000"/>
</pendingMessageLimitStrategy>
</policyEntry>
</policyEntries>
</policyMap>
</destinationPolicy>
<!--
The managementContext is used to configure how ActiveMQ is exposed in
JMX. By default, ActiveMQ uses the MBean server that is started by
the JVM. For more information, see:
http://activemq.apache.org/jmx.html
-->
<managementContext>
<managementContext createConnector="false"/>
</managementContext>
<!--
Configure message persistence for the broker. The default persistence
mechanism is the KahaDB store (identified by the kahaDB tag).
For more information, see:
http://activemq.apache.org/persistence.html
-->
<persistenceAdapter>
<kahaDB directory="${activemq.data}/kahadb"/>
</persistenceAdapter>
<!--
The systemUsage controls the maximum amount of space the broker will
use before disabling caching and/or slowing down producers. For more information, see:
http://activemq.apache.org/producer-flow-control.html
-->
<systemUsage>
<systemUsage>
<memoryUsage>
<memoryUsage percentOfJvmHeap="70" />
</memoryUsage>
<storeUsage>
<storeUsage limit="100 gb"/>
</storeUsage>
<tempUsage>
<tempUsage limit="50 gb"/>
</tempUsage>
</systemUsage>
</systemUsage>
<!--
The transport connectors expose ActiveMQ over a given protocol to
clients and other brokers. For more information, see:
http://activemq.apache.org/configuring-transports.html
-->
<transportConnectors>
<!-- DOS protection, limit concurrent connections to 1000 and frame size to 100MB -->
<transportConnector name="openwire" uri="tcp://0.0.0.0:61616?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="amqp" uri="amqp://0.0.0.0:5672?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="stomp" uri="stomp://0.0.0.0:61613?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="mqtt" uri="mqtt://0.0.0.0:1883?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="ws" uri="ws://0.0.0.0:61614?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
</transportConnectors>
<!-- destroy the spring context on shutdown to stop jetty -->
<shutdownHooks>
<bean xmlns="http://www.springframework.org/schema/beans" class="org.apache.activemq.hooks.SpringContextHook" />
</shutdownHooks>
</broker>
<!--
Enable web consoles, REST and Ajax APIs and demos
The web consoles requires by default login, you can disable this in the jetty.xml file
Take a look at ${ACTIVEMQ_HOME}/conf/jetty.xml for more details
-->
<import resource="jetty.xml"/>
</beans>
<!-- END SNIPPET: example -->
Adding activemq.log file from /web/servers/apache-activemq-5.15.11/data
Pasting shareable link below:
https://drive.google.com/file/d/1vQ6HkOu53mzMi-GqbGxaQP6EJNC4RT4G/view?usp=sharing
Don't know if this is the issue for you, but after some reading it sounded like activemq uses jmx for its shutdown hook, but the default config that ships with activemq does not setup the jmx connector.
For me, activemq was able to shutdown if I changed:
<managementContext>
<managementContext createConnector="false"/>
</managementContext>
To specify a jmx connectorPort:
<managementContext>
<managementContext connectorPort="1099"/>
</managementContext>
Some references:
Activemq Shutdown fails and then kills process
https://issues.apache.org/jira/browse/AMQ-6927

jmx doesnt seem to be working with activeMQ

I'm trying to use JMX with activeMQ for monitoring so far I've been using this and this as a reference but so far I'm unable to connect to jmx remotely and also I don't see any mention of jmx url in activemq logs. I'm wondering if there is another way to make sure jmx is working? is it supposed to be indicated in activemq logs?
PS I'm using jdk1.7 and activeMQ 5.14.2.
Thanks in advance!
EDIT
I set useJmx="true" in my activemq.xml file:
<broker xmlns="http://activemq.apache.org/schema/core" brokerName="primary" useJmx="true" dataDirectory="${activemq.data}">
I tried two steps:
FIRST
I tried changing management context from createConnector="false" to :
<managementContext>
<managementContext createConnector="true" connectorPort="1099"/>
</managementContext>
FOR FIRST TIME THE PORT IS OPEN AND ACTIVEMQ RUNS FINE AND JMX URL GETS REPORTED IN LOGS ALTHOUGH I CAN NOT CONNECT IT TO IT REMOTLEY BUT IM ASSUMING ITS WORKING
SECOND
I reverted back the changes I made for managmentContext and I tried setting:
ACTIVEMQ_SUNJMX_START="-Dcom.sun.management.jmxremote.port=1099 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.password.file=${ACTIVEMQ_BASE}/jmx.password -Dcom.sun.management.jmxremote.access.file=${ACTIVEMQ_BASE}/jmx.access"
in bin/activemq script and I set a username in conf/jmx.access file as:
admin readwrite
And also have set a password in conf/jmx.password:
admin activemq
NOW ACTIVEMQ IS NOT RUNNINT AT ALL BUT IT WILL RUN IF I SET
AUTHENTICATE=FALSE AND DELETE JMX.ACCESS AND JMX.PASSWORD
CONFIGURATION IN BIN/ACTIVEMQ FILE BUT I NEED USER NAME AND PASSWORD
FOR SECURITY REASONS
I found this post which has the exact same issue as mine. any ideas?
Password authentication for remote monitoring is enabled by default. To disable it, set the following system property when you start the JVM:
-Dcom.sun.management.jmxremote.authenticate=false like you done in second test but you need to add system property -Dcom.sun.management.jmxremote
Try to add these jvm param to env file and update host ip
-Djava.net.preferIPv4Stack=true -Djava.rmi.server.hostname=X.X.X.X
UPDATE
SO, to resume, i think that the FIRST step you tried is the best, for making it working these are the steps :
revert all jmx env file changes, like this :
# ACTIVEMQ_SUNJMX_START="$ACTIVEMQ_SUNJMX_START -Dcom.sun.management.jmxremote.port=1099 "
# ACTIVEMQ_SUNJMX_START="$ACTIVEMQ_SUNJMX_START -Dcom.sun.management.jmxremote.password.file=${ACTIVEMQ_CONF}/jmx.password"
# ACTIVEMQ_SUNJMX_START="$ACTIVEMQ_SUNJMX_START -Dcom.sun.management.jmxremote.access.file=${ACTIVEMQ_CONF}/jmx.access"
# ACTIVEMQ_SUNJMX_START="$ACTIVEMQ_SUNJMX_START -Dcom.sun.management.jmxremote.ssl=false"
ACTIVEMQ_SUNJMX_START="$ACTIVEMQ_SUNJMX_START -Dcom.sun.management.jmxremote"
<broker useJmx="true" ...
<managementContext>
<managementContext createConnector="true" connectorPort="1099" />
</managementContext>
verify that in AMQ logs you have
INFO | JMX consoles can connect to
service:jmx:rmi:///jndi/rmi://localhost:1099/jmxrmi |
org.apache.activemq.broker.jmx.ManagementContext | JMX connector
NOTE : Assuming that 10.10.10.16 is the IP of AMQ host.
try to connect with jconsole from another machine than AMQ host with url "service:jmx:rmi:///jndi/rmi://10.10.10.16:1099/jmxrmi" without user/pwd.
if you cannot connect, try like this :
<managementContext>
<managementContext createConnector="true" connectorPort="1099" connectorHost="10.10.10.16" />
</managementContext>
verify that in AMQ logs you have
INFO | JMX consoles can connect to
service:jmx:rmi:///jndi/rmi://10.10.10.16:1099/jmxrmi |
org.apache.activemq.broker.jmx.ManagementContext | JMX connector
retry to connect, step 4
at this step normally you can connect with jconsole.
if you want to add security and authorizations, use this :
<managementContext>
<managementContext createConnector="true" connectorPort="1099" connectorHost="10.10.10.16" >
<property xmlns="http://www.springframework.org/schema/beans" name="environment">
<map xmlns="http://www.springframework.org/schema/beans">
<entry xmlns="http://www.springframework.org/schema/beans" key="jmx.remote.x.password.file"
value="${activemq.conf}/jmx.password"/>
<entry xmlns="http://www.springframework.org/schema/beans" key="jmx.remote.x.access.file"
value="${activemq.conf}/jmx.access"/>
</map>
</property>
</managementContext>
</managementContext>
Please try these steps and let me know in which one you fails to connect and provide error message from jconsole.
A couple troubleshooting steps:
Start jconsole or visualvm on the same system and connect using the "pid" attach method. Browse the MBeans and confirm org.apache.activemq beans are present
Run netstat -na and confirm ports 1099 (and 44444) are in LISTEN
Look at logs and confirm you do not have any "java.net.BindException: Address already in use.." messages that indicate a port conflict with an already running Java process.
Edit bin/env to configure JMX (this disables requiring SSL, sets the port to 1099 and disables requiring username and password.
ACTIVEMQ_SUNJMX_START="$ACTIVEMQ_SUNJMX_START -Dcom.sun.management.jmxremote.port=1099 "
ACTIVEMQ_SUNJMX_START="$ACTIVEMQ_SUNJMX_START -Dcom.sun.management.jmxremote.ssl=false "
ACTIVEMQ_SUNJMX_START="$ACTIVEMQ_SUNJMX_START -Dcom.sun.management.jmxremote "
# ACTIVEMQ_SUNJMX_START="$ACTIVEMQ_SUNJMX_START -Dcom.sun.management.jmxremote.password.file=${ACTIVEMQ_CONF}/jmx.password"`
# ACTIVEMQ_SUNJMX_START="$ACTIVEMQ_SUNJMX_START -Dcom.sun.management.jmxremote.access.file=${ACTIVEMQ_CONF}/jmx.access"

Tomcat Clustering Multicast throws error - Unable to perform failure detection check, assuming member down

Hi I am new to clustering concept, so I tried to establish a Tomcat cluster with 2 instances. I followed the examples, mainly at http://www.mulesoft.com/tcat/tomcat-clustering and few other web sources.
My sample set up is as,
in workers.properties
# Define worker names
worker.list=jkstatus, loadbalancer
# Create virtual workers
worker.jkstatus.type=status
worker.loadbalancer.type=lb
# Declare Tomcat server worker 1
worker.worker1.type=ajp13
worker.worker1.host=localhost
worker.worker1.port=7009
# Declare Tomcat server worker 2
worker.worker2.type=ajp13
worker.worker2.port=9009
worker.worker2.host=localhost
# Associate real workers with virtual LoadBalancer worker
worker.loadbalancer.balance_workers=worker1,worker2
And Apache httpd.conf as,
# ADDED CLUSTER CONFIG
# Load module
LoadModule jk_module modules/mod_jk.so
# Specify path to worker configuration file
JkWorkersFile C:/tomcat_clustered/workers.properties
# Configure logging and memory
JkShmFile C:/tomcat_clustered/log/location/mod_jk.shm
JkLogFile C:/tomcat_clustered/log/location/mod_jk.log
JkLogLevel info
# Configure monitoring
JkMount /jkmanager/* jkstatus
<Location /jkmanager>
Order deny,allow
deny from all
allow from localhost
</Location>
# Configure applications
JkMount /* loadbalancer
# END CLUSTER CONFIG
With each tomcat(worker) server.xml as,
<Engine name="Catalina" defaultHost="localhost" jvmRoute="worker1">
<!--For clustering, please take a look at documentation at:
/docs/cluster-howto.html (simple how to)
/docs/config/cluster.html (reference documentation) -->
<!--
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster"/>
-->
<!-- CLUSTER BEGIN -->
<Cluster className="org.apache.catalina.ha.tcp.SimpleTcpCluster" channelSendOptions="8">
<Manager className="org.apache.catalina.ha.session.DeltaManager"
expireSessionsOnShutdown="false"
notifyListenersOnReplication="true"/>
<Channel className="org.apache.catalina.tribes.group.GroupChannel">
<!---->
<Membership className="org.apache.catalina.tribes.membership.McastService"
address="239.0.0.1"
port="45564" frequency="500"
dropTime="3000"/>
<Sender className="org.apache.catalina.tribes.transport.ReplicationTransmitter">
<Transport className="org.apache.catalina.tribes.transport.nio.PooledParallelSender"/>
</Sender>
<Receiver className="org.apache.catalina.tribes.transport.nio.NioReceiver"
address="auto" port="4000" autoBind="100"
selectorTimeout="5000" maxThreads="6"/>
<Interceptor className="org.apache.catalina.tribes.group.interceptors.TcpFailureDetector"/>
<Interceptor className="org.apache.catalina.tribes.group.interceptors.MessageDispatch15Interceptor"/>
</Channel>
<Valve className="org.apache.catalina.ha.tcp.ReplicationValve" filter=""/>
<Valve className="org.apache.catalina.ha.session.JvmRouteBinderValve"/>
<ClusterListener className="org.apache.catalina.ha.session.JvmRouteSessionIDBinderListener"/>
<ClusterListener className="org.apache.catalina.ha.session.ClusterSessionListener"/>
</Cluster>
<!-- CLUSTER END -->
<!-- Use the LockOutRealm to prevent attempts to guess user passwords
via a brute-force attack -->
<Realm className="org.apache.catalina.realm.LockOutRealm">
For worker1, worker2. I am working on Windows 7. It looks like some multicast option is needed but I am not sure. On starting up of tomcat, one is starting well, and on starting second(say worker2) both tomcat console's are throwing errors as,
at org.apache.catalina.tribes.group.ChannelInterceptorBase.heartbeat(Cha
nelInterceptorBase.java:103)
at org.apache.catalina.tribes.group.GroupChannel.heartbeat(GroupChannel.
ava:155)
at org.apache.catalina.tribes.group.GroupChannel$HeartbeatThread.run(Gro
pChannel.java:690)
ep 15, 2014 10:11:03 AM org.apache.catalina.tribes.group.interceptors.TcpFailur
Detector memberAlive
EVERE: Unable to perform failure detection check, assuming member down.
ava.net.SocketException: Permission denied: connect
at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method)
at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketI
pl.java:85)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.ja
a:339)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocket
mpl.java:200)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java
Not sure if its due to some multicast permissions on my Windows or any config related issues. Please suggest some good pointers in this regard
You need to enable multicast. see this post how to enable multicast is windows, https://serverfault.com/questions/262634/how-do-i-know-if-ip-multicasting-is-enabled-on-my-network-in-windows
In Linux Environment most of the system kernel is capable to process the multicast address.
but we need to add route entry in kernel routing table.
sudo route add -net 224.0.0.0 netmask 240.0.0.0 dev eth0
The error that you're reporting looks like it might be OS / networking related. Hard to say exactly. I can tell you this though.
You've declared a TcpFailureDetector.
<Interceptor className="org.apache.catalina.tribes.group.interceptors.TcpFailureDetector"/>
This is part of Tomcat's cluster configuration and it's responsibility is to make a TCP connection to any node that is suspected of failure. If the listener can't connect to the node, it'll be marked as down. If it can connect, then the node remains active.
In this case, the listener is attempting to make a connection to one of your nodes and it's failing with an error from the JDK. Googling that specific error turns up some suggestions, one of which is to use "-Djava.net.preferIPv4Stack=true".
I get java.net.SocketException: Permission denied: connect when sending an email in Jenkins
On a separate note, if you've note seen the official documentation on Tomcat clustering, I would strongly recommend you check that out. Here's the link.
http://tomcat.apache.org/tomcat-7.0-doc/cluster-howto.html

ActiveMQ 5.8 network of broker with custom jmx port

I am trying to run a network of broker with 2 brokers on the same network but on 2 different virtual machines.
Because of some internal constraints I have to use a custom jmx port.
I am using the Tanuki wrapper to launch ActiveMQ on an Ubuntu server.
Here is the relevant part of my activemq.xml
<broker xmlns="http://activemq.apache.org/schema/core" advisorySupport="false" useJmx="true" brokerName="test1" dataDirectory=".../data/activemq">
<networkConnectors>
<networkConnector uri="multicast://1.2.3.4:101234?group=test"
dynamicOnly="true"
networkTTL="3"
prefetchSize="1"
decreaseNetworkConsumerPriority="true"
userName="user"
password="password"/>
</networkConnectors>
<transportConnectors>
<transportConnector name="openwire" uri="tcp://0.0.0.0:61616" rebalanceClusterClients="true" updateClusterClients="true" />
<transportConnector name="nio" uri="nio://0.0.0.0:61617" rebalanceClusterClients="true" updateClusterClients="true" discoveryUri="multicast://1.2.3.4:101234?group=test" />
</transportConnectors>
...
<managementContext>
<managementContext createConnector="false"/>
</managementContext>
...
</broker>
Here is the relevant part of wrapper.conf:
# Uncomment to enable jmx
wrapper.java.additional.1=-Dcom.sun.management.jmxremote
wrapper.java.additional.2=-Dcom.sun.management.jmxremote.port=4321
wrapper.java.additional.3=-Dcom.sun.management.jmxremote.authenticate=false
wrapper.java.additional.4=-Dcom.sun.management.jmxremote.ssl=false
When running ActiveMQ on both broker, I see the process with the expected options:
activemq 30682 30680 3 13:27 ? 00:00:30 java -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=4321 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djavax.net.ssl.keyStore=../../conf/broker.ks -Djavax.net.ssl.trustStore=../../conf/broker.ts -Dcom.sun.management.jmxremote -Dorg.apache.activemq.UseDedicatedTaskRunner=true -Djava.util.logging.config.file=logging.properties -Dactivemq.conf=../../conf -Dactivemq.data=../../data -Xms2048m -Xmx2048m -Djava.library.path=../../bin/linux-x86-64/ -classpath ../../bin/wrapper.jar:../../bin/activemq.jar -Dwrapper.key=y4TuwO32Hj6kN7w8 -Dwrapper.port=32000 -Dwrapper.jvm.port.min=31000 -Dwrapper.jvm.port.max=31999 -Dwrapper.pid=30680 -Dwrapper.version=3.2.3 -Dwrapper.native_library=wrapper -Dwrapper.service=TRUE -Dwrapper.cpu.timeout=10 -Dwrapper.jvmid=1 org.tanukisoftware.wrapper.WrapperSimpleApp org.apache.activemq.console.Main start
The port is open on the running shorewall.
The network of broker is up but I cannot connect to the jmx using jvisualvm with server_dns:4321. It returns the error
"cannot connect to server_dns:4321 using service:jmx:rmi:///jndi/rmi://server_dns:4321/jmxrmi
I can see some traffic on the port via tcpdump.
Could anybody tell me what I am doing wrong or how I should use ActiveMQ as a Network Of Broker with a custom jmx port?
JMX needs 2 open ports. An extra one is necessary for rmi.
I figured it out thanks to this post: Apache ActiveMQ browser can't connect to JMX console
In my case the fix is to change the configuration of my wrapper to expose the rmi port and open the port on the firewall
# Uncomment to enable jmx
wrapper.java.additional.1=-Dcom.sun.management.jmxremote
wrapper.java.additional.2=-Dcom.sun.management.jmxremote.port=4321
wrapper.java.additional.3=-Dcom.sun.management.jmxremote.port=8765
wrapper.java.additional.4=-Dcom.sun.management.jmxremote.authenticate=false
wrapper.java.additional.5=-Dcom.sun.management.jmxremote.ssl=false

ActiveMQ 5.x - XInclude doesn't seem to work in configuration files

I'm using ActiveMQ 5.x, and I'd like to pull out some of the stuff in activemq.xml brokers into separate files. Note: I cannot use <import> it won't work.
I would like to use XInclude in the activemq.xml configuration file.
However when I try to do it, I get an error "cannot find element xi:include"
Here is the activemq.xml file I am using. Can you please help me figure out how to get external files included?
activemq.xml
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- START SNIPPET: example -->
<beans
xmlns="http://www.springframework.org/schema/beans"
xmlns:amq="http://activemq.apache.org/schema/core"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:xi="http://www.w3.org/2003/XInclude"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://activemq.apache.org/schema/core http://activemq.apache.org/schema/core/activemq-core.xsd">
<!-- Allows us to use system properties as variables in this configuration file -->
<bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="locations">
<value>file:${activemq.base}/conf/credentials.properties</value>
</property>
</bean>
<!--
The <broker> element is used to configure the ActiveMQ broker.
-->
<broker xmlns="http://activemq.apache.org/schema/core" brokerName="localhost" dataDirectory="${activemq.base}/data" destroyApplicationContextOnStop="true">
<!--
For better performances use VM cursor and small memory limit.
For more information, see:
http://activemq.apache.org/message-cursors.html
Also, if your producer is "hanging", it's probably due to producer flow control.
For more information, see:
http://activemq.apache.org/producer-flow-control.html
-->
<destinationPolicy>
<policyMap>
<policyEntries>
<policyEntry topic=">" producerFlowControl="true" memoryLimit="1mb">
<pendingSubscriberPolicy>
<vmCursor />
</pendingSubscriberPolicy>
</policyEntry>
<policyEntry queue=">" producerFlowControl="true" memoryLimit="1mb">
<!-- Use VM cursor for better latency
For more information, see:
http://activemq.apache.org/message-cursors.html
<pendingQueuePolicy>
<vmQueueCursor/>
</pendingQueuePolicy>
-->
</policyEntry>
</policyEntries>
</policyMap>
</destinationPolicy>
<!--
The managementContext is used to configure how ActiveMQ is exposed in
JMX. By default, ActiveMQ uses the MBean server that is started by
the JVM. For more information, see:
http://activemq.apache.org/jmx.html
-->
<managementContext>
<managementContext createConnector="false"/>
</managementContext>
<!--
Configure message persistence for the broker. The default persistence
mechanism is the KahaDB store (identified by the kahaDB tag).
For more information, see:
http://activemq.apache.org/persistence.html
-->
<persistenceAdapter>
<kahaDB directory="${activemq.base}/data/kahadb"/>
</persistenceAdapter>
<!--
The systemUsage controls the maximum amount of space the broker will
use before slowing down producers. For more information, see:
http://activemq.apache.org/producer-flow-control.html
<systemUsage>
<systemUsage>
<memoryUsage>
<memoryUsage limit="20 mb"/>
</memoryUsage>
<storeUsage>
<storeUsage limit="1 gb"/>
</storeUsage>
<tempUsage>
<tempUsage limit="100 mb"/>
</tempUsage>
</systemUsage>
</systemUsage>
-->
<!--
The transport connectors expose ActiveMQ over a given protocol to
clients and other brokers. For more information, see:
http://activemq.apache.org/configuring-transports.html
-->
<transportConnectors>
<transportConnector name="openwire" uri="tcp://0.0.0.0:61616"/>
</transportConnectors>
<xi:include href="transport-connectors.xml" parse="xml"/>
</broker>
<!--
Enable web consoles, REST and Ajax APIs and demos
Take a look at ${ACTIVEMQ_HOME}/conf/jetty.xml for more details
-->
<import resource="jetty.xml"/>
</beans>
<!-- END SNIPPET: example -->
transport-connectors.xml
<transportConnectors>
<transportConnector name="openwire" uri="tcp://0.0.0.0:61616"/>
</transportConnectors>
This does not work. Here is the error I get.
Caused by: org.xml.sax.SAXParseException: cvc-complex-type.2.4.c: The
matching wildcard is strict, but no declaration can be found for
element 'xi:include'.
I also tried using entity definitions, and just could not figure out how to get this to work.
Please recommend changes to activemq.xml that can get this to work. Thanks!!
The answer is - there is no support for this in activemq.
After investigation, I see Activemq does not have XInclude supported/turned-on in Spring 3.0 and it requires extending Spring XML handler classes that alter the DocBuilder to allow XIncludes.
I created a new feature request to suggest this be added https://issues.apache.org/jira/browse/AMQ-3886
Bummer.