JMX issue with DataStax OpsCenter Agent - datastax

Question

We're running an 8 node DSE cluster (just Cassandra) divided among two datacenters. Everything is working fine apart from the agent on one node which stubbornly refuses to cooperate.
Here is the version info:
Cassandra 3.0.9.1346
DSE 5.0.3
OpsCenter 6.03
All nodes have upgraded SSTables and have been repaired.
Here is the log:
INFO [async-dispatch-47] 2016-11-08 14:33:13,811 Starting system.
INFO [async-dispatch-47] 2016-11-08 14:33:13,832 Starting DynamicEnvironmentComponent
WARN [async-dispatch-47] 2016-11-08 14:33:13,845 Exception while processing JMX data: java.lang.NullPointerException
ERROR [async-dispatch-47] 2016-11-08 14:33:13,845 Error starting DynamicEnvironmentComponent.
java.lang.NullPointerException
at clojure.java.io$as_relative_path.invoke(io.clj:404)
at clojure.java.io$file.invoke(io.clj:416)
at opsagent.environment.collection$cassandra_yaml_location__GT_install_location.invoke(collection.clj:118)
at opsagent.environment.dynamic$dynamic_env_state.invoke(dynamic.clj:61)
at clojure.core$partial$fn__4527.invoke(core.clj:2492)
at opsagent.jmx$create_jmx_pool_with_config$wrapper__11504.doInvoke(jmx.clj:221)
at clojure.lang.RestFn.invoke(RestFn.java:410)
at clojure.lang.AFn.applyToHelper(AFn.java:154)
at clojure.lang.RestFn.applyTo(RestFn.java:132)
at clojure.core$apply.invoke(core.clj:630)
at opsagent.environment.dynamic$add_dynamic_state.invoke(dynamic.clj:143)
at opsagent.environment.dynamic.DynamicEnvironmentComponent.start(dynamic.clj:168)
at com.stuartsierra.component$fn__8838$G__8832__8840.invoke(component.clj:4)
at com.stuartsierra.component$fn__8838$G__8831__8843.invoke(component.clj:4)
at clojure.lang.Var.invoke(Var.java:379)
at clojure.lang.AFn.applyToHelper(AFn.java:154)
at clojure.lang.Var.applyTo(Var.java:700)
at clojure.core$apply.invoke(core.clj:632)
at com.stuartsierra.component$try_action.invoke(component.clj:116)
at clojure.lang.Var.invoke(Var.java:401)
at opsagent.config_service$update_system$fn__20056.invoke(config_service.clj:200)
at clojure.lang.ArraySeq.reduce(ArraySeq.java:114)
at clojure.core$reduce.invoke(core.clj:6518)
at opsagent.config_service$update_system.doInvoke(config_service.clj:194)
at clojure.lang.RestFn.invoke(RestFn.java:425)
at opsagent.config_service$start_system_BANG_.invoke(config_service.clj:219)
at opsagent.config_service$fn__20133$fn__20134$state_machine__4719__auto____20135$fn__20137.invoke(config_service.clj:245)
at opsagent.config_service$fn__20133$fn__20134$state_machine__4719__auto____20135.invoke(config_service.clj:242)
at clojure.core.async.impl.ioc_macros$run_state_machine.invoke(ioc_macros.clj:940)
at clojure.core.async.impl.ioc_macros$run_state_machine_wrapped.invoke(ioc_macros.clj:944)
at clojure.core.async$ioc_alts_BANG_$fn__4884.invoke(async.clj:362)
at clojure.core.async$do_alts$fn__4838$fn__4841.invoke(async.clj:231)
at clojure.core.async.impl.channels.ManyToManyChannel$fn__1215.invoke(channels.clj:262)
at clojure.lang.AFn.run(AFn.java:22)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
INFO [async-dispatch-47] 2016-11-08 14:33:13,848 Finished starting system.
Any ideas?
Edit: here is the requested additional info.
The node's addresses are as follows:
listen: 10.2.2.22
rpc: 10.1.2.22
All ips are private and SSL is disabled because the DCs are connected via a VPN.
# nodetool status
Status=Up/Down
|/ State=Normal/Leaving/Joining/Moving
-- Address Load Tokens Owns Host ID Rack
UN 192.168.1.72 12.14 GB 256 ? a15c57e1-3c53-4d4f-9df9-29945b9f1c88 RAC1
UN 192.168.1.92 11.36 GB 256 ? 9820b96a-a3c6-460f-839b-5dabc89313a0 RAC1
UN 192.168.1.82 11.67 GB 256 ? f9c13cb0-ee44-4ce2-ac7e-14ec1f7c1d23 RAC1
Datacenter: DC2
===============
Status=Up/Down
|/ State=Normal/Leaving/Joining/Moving
-- Address Load Tokens Owns Host ID Rack
UN 10.2.2.32 11.04 GB 256 ? bfe86bb3-d272-4946-ac8a-e176fe9f8e64 RAC3
UN 10.2.2.22 11.29 GB 256 ? c8694e93-0d8a-41b4-82a3-6c450497e8ec RAC2
UN 10.2.2.52 11.46 GB 256 ? e941faf1-ad5b-46a7-8857-bdf9dd2a3459 RAC5
UN 10.2.2.42 10.9 GB 256 ? 7bbd2397-a3bc-4cfe-9c03-334186e7e0dd RAC4
UN 10.2.2.12 5.29 GB 256 ? bf7a0587-2b09-47d6-b6d5-24e1422318b9 RAC1
# address.yaml
stomp_interface: 192.168.1.31
use_ssl: 0
cassandara_conf: /etc/dse/cassandra
root#anc-t2:~# netstat -lnt Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address
State tcp 0 0 10.1.2.22:9160 0.0.0.0:*
LISTEN tcp 0 0 10.1.2.22:9042 0.0.0.0:*
LISTEN tcp 0 0 0.0.0.0:22 0.0.0.0:*
LISTEN tcp 0 0 10.2.2.22:7000 0.0.0.0:*
LISTEN tcp 0 0 0.0.0.0:17500 0.0.0.0:*
LISTEN tcp 0 0 127.0.0.1:7199 0.0.0.0:*
LISTEN tcp 0 0 127.0.0.1:17600 0.0.0.0:*
LISTEN tcp 0 0 10.2.2.22:8609 0.0.0.0:*
LISTEN tcp 0 0 127.0.0.1:54882 0.0.0.0:*
LISTEN tcp 0 0 127.0.0.1:17603 0.0.0.0:*
LISTEN tcp6 0 0 :::6900 :::*
LISTEN tcp6 0 0 :::61621 :::*
LISTEN tcp6 0 0 :::9910 :::*
LISTEN tcp6 0 0 :::22 :::*
LISTEN tcp6 0 0 :::17500 :::*
LISTEN
# opscenterd.conf
[webserver] port = 8888 interface = 0.0.0.0
# The following settings can be used to enable ssl support for the opscenter
# web application. Change these values to point to the ssl certificate and key
# that you wish to use for your OpsCenter install, as well as the port you would like
# to serve ssl traffic from.
#ssl_keyfile = /var/lib/opscenter/ssl/opscenter.key
#ssl_certfile = /var/lib/opscenter/ssl/opscenter.pem
#ssl_port = 8443
[authentication]
# Set this option to True to enable OpsCenter authentication. A default admin
# account will be created with the username "admin" and password "admin".
# Accounts and roles can then be created and modified from within the web UI. enabled = False
# To help us better understand the needs of users and to improve OpsCenter, OpsCenter
# reports information about itself and the clusters it manages to a central DataStax
# server. This information is reported anonymously, and potentially sensitive
# information, such as IP addresses, are hashed in a non-reversible way:
# http://www.datastax.com/documentation/opscenter/help/statsReporterProperties.html [stat_reporter]
# The interval setting determines how often statistics are reported. To disable
# reporting, set to 0
# interval = 86400 # 24 hours
# cluster.conf
[jmx]
username =
password =
port = 7199
[agents]
[cassandra]
username =
seed_hosts = 192.168.1.72
password =
cql_port = 9042
cqlsh:OpsCenter> desc KEYSPACE;
CREATE KEYSPACE "OpsCenter" WITH replication = {'class': 'NetworkTopologyStrategy', 'DC1': '1', 'DC2': '1'} AND durable_writes = true;
CREATE TABLE "OpsCenter".rollup_state (
node text,
name text,
res int,
avg float,
histogram blob,
max float,
min float,
ts timestamp,
type int,
value float,
weight float,
PRIMARY KEY (node, name, res)
) WITH CLUSTERING ORDER BY (name ASC, res ASC)
AND bloom_filter_fp_chance = 0.1
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"version": [5, 2, 1], "info": "OpsCenter management data."}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.0
AND default_time_to_live = 0
AND gc_grace_seconds = 0
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99PERCENTILE';
CREATE TABLE "OpsCenter".events_timeline (
key text,
column1 bigint,
value blob,
PRIMARY KEY (key, column1)
) WITH COMPACT STORAGE
AND CLUSTERING ORDER BY (column1 ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"info": "OpsCenter management data.", "version": [5, 2, 1]}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '8', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 864000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.25
AND speculative_retry = '99PERCENTILE';
CREATE TABLE "OpsCenter".settings (
key blob,
column1 blob,
value blob,
PRIMARY KEY (key, column1)
) WITH COMPACT STORAGE
AND CLUSTERING ORDER BY (column1 ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"info": "OpsCenter management data.", "version": [5, 2, 1]}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '12', 'min_threshold': '8'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 864000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 1.0
AND speculative_retry = '99PERCENTILE';
CREATE TABLE "OpsCenter".rollups60 (
key text,
timestamp varint,
value blob,
PRIMARY KEY (key, timestamp)
) WITH COMPACT STORAGE
AND CLUSTERING ORDER BY (timestamp ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"info": "OpsCenter management data.", "version": [5, 2, 1]}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 0
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.25
AND speculative_retry = '99PERCENTILE';
CREATE TABLE "OpsCenter".backup_reports (
week text,
event_time timestamp,
backup_id text,
type text,
destination text,
deleted_at timestamp,
full_status text,
keyspaces text,
status text,
PRIMARY KEY (week, event_time, backup_id, type, destination)
) WITH CLUSTERING ORDER BY (event_time DESC, backup_id ASC, type ASC, destination ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"info": "OpsCenter management data.", "version": [5, 2, 1]}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 864000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.0
AND speculative_retry = '99PERCENTILE';
CREATE TABLE "OpsCenter".rollups86400 (
key text,
timestamp varint,
value blob,
PRIMARY KEY (key, timestamp)
) WITH COMPACT STORAGE
AND CLUSTERING ORDER BY (timestamp ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"info": "OpsCenter management data.", "version": [5, 2, 1]}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '8', 'min_threshold': '2'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 0
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.25
AND speculative_retry = '99PERCENTILE';
CREATE TABLE "OpsCenter".bestpractice_results (
key text,
column1 varint,
value blob,
PRIMARY KEY (key, column1)
) WITH COMPACT STORAGE
AND CLUSTERING ORDER BY (column1 DESC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"info": "OpsCenter management data.", "version": [5, 2, 1]}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 0
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.25
AND speculative_retry = '99PERCENTILE';
CREATE TABLE "OpsCenter".pdps (
key text,
column1 text,
value blob,
PRIMARY KEY (key, column1)
) WITH COMPACT STORAGE
AND CLUSTERING ORDER BY (column1 ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"info": "OpsCenter management data.", "version": [5, 2, 1]}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 0
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.25
AND speculative_retry = '99PERCENTILE';
CREATE TABLE "OpsCenter".rollups7200 (
key text,
timestamp varint,
value blob,
PRIMARY KEY (key, timestamp)
) WITH COMPACT STORAGE
AND CLUSTERING ORDER BY (timestamp ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"info": "OpsCenter management data.", "version": [5, 2, 1]}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '8', 'min_threshold': '2'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 0
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.25
AND speculative_retry = '99PERCENTILE';
CREATE TABLE "OpsCenter".events (
key text PRIMARY KEY,
action bigint,
api_source_ip text,
column_family text,
event_source text,
"keyspace" text,
level bigint,
message text,
source_node text,
success boolean,
target_node text,
time bigint,
user text
) WITH COMPACT STORAGE
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"info": "OpsCenter management data.", "version": [5, 2, 1]}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '12', 'min_threshold': '8'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 864000
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.25
AND speculative_retry = '99PERCENTILE';
CREATE TABLE "OpsCenter".rollups300 (
key text,
timestamp varint,
value blob,
PRIMARY KEY (key, timestamp)
) WITH COMPACT STORAGE
AND CLUSTERING ORDER BY (timestamp ASC)
AND bloom_filter_fp_chance = 0.01
AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
AND comment = '{"info": "OpsCenter management data.", "version": [5, 2, 1]}'
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '16', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND crc_check_chance = 1.0
AND dclocal_read_repair_chance = 0.1
AND default_time_to_live = 0
AND gc_grace_seconds = 0
AND max_index_interval = 2048
AND memtable_flush_period_in_ms = 0
AND min_index_interval = 128
AND read_repair_chance = 0.25
AND speculative_retry = '99PERCENTILE';