while stress test, all requests goes to only 1 node - apache

Question

kindly i need someone to help me in weird issue i am facing.
i am deploying a web application on Jboss eap 6.
i configured two JBoss nodes over the same machine ( server 1 - server 2 ) and configured them to become cluster environment.
also i configured Apache mod_cluster to work as load balancer.
when i tried to test the environment by different machines that call my application, everything worked fine, two nodes worked together and the load evenly distributed over the two nodes.
the problem occured when started using microsoft visual studio 2012 web load, it simulates concurrent request to the application. what i noticed that all requests recieved by 1 node while the other nodes stays idle !.
i tried to some trouble shooting like editing the standalone-ha.xml in both nodes and edited the following lines
<subsystem xmlns="urn:jboss:domain:modcluster:1.1">
<mod-cluster-config advertise-socket="modcluster" connector="ajp">
<simple-load-provider/>
<dynamic-load-provider history="20">
<load-metric type="busyness" weight="3"/>
<load-metric type="sessions" weight="3" capacity="2"/>
<load-metric type="heap" weight="2"/>
</dynamic-load-provider>
</mod-cluster-config>
</subsystem>
nothing changed
this is exactly my standalone-ha.xml file for 1 node, infact both nodes same file
<?xml version='1.0' encoding='UTF-8'?>
<server xmlns="urn:jboss:domain:1.5">
<extensions>
<extension module="org.jboss.as.clustering.infinispan"/>
<extension module="org.jboss.as.clustering.jgroups"/>
<extension module="org.jboss.as.connector"/>
<extension module="org.jboss.as.deployment-scanner"/>
<extension module="org.jboss.as.ee"/>
<extension module="org.jboss.as.ejb3"/>
<extension module="org.jboss.as.jaxrs"/>
<extension module="org.jboss.as.jdr"/>
<extension module="org.jboss.as.jmx"/>
<extension module="org.jboss.as.jpa"/>
<extension module="org.jboss.as.jsf"/>
<extension module="org.jboss.as.logging"/>
<extension module="org.jboss.as.mail"/>
<extension module="org.jboss.as.modcluster"/>
<extension module="org.jboss.as.naming"/>
<extension module="org.jboss.as.pojo"/>
<extension module="org.jboss.as.remoting"/>
<extension module="org.jboss.as.sar"/>
<extension module="org.jboss.as.security"/>
<extension module="org.jboss.as.threads"/>
<extension module="org.jboss.as.transactions"/>
<extension module="org.jboss.as.web"/>
<extension module="org.jboss.as.webservices"/>
<extension module="org.jboss.as.weld"/>
</extensions>
<system-properties>
<property name="org.apache.catalina.connector.URI_ENCODING" value="UTF-8"/>
<property name="org.apache.catalina.connector.USE_BODY_ENCODING_FOR_QUERY_STRING" value="true"/>
</system-properties>
<management>
<security-realms>
<security-realm name="ManagementRealm">
<authentication>
<local default-user="$local"/>
<properties path="mgmt-users.properties" relative-to="jboss.server.config.dir"/>
</authentication>
<authorization map-groups-to-roles="false">
<properties path="mgmt-groups.properties" relative-to="jboss.server.config.dir"/>
</authorization>
</security-realm>
<security-realm name="ApplicationRealm">
<authentication>
<local default-user="$local" allowed-users="*"/>
<properties path="application-users.properties" relative-to="jboss.server.config.dir"/>
</authentication>
<authorization>
<properties path="application-roles.properties" relative-to="jboss.server.config.dir"/>
</authorization>
</security-realm>
</security-realms>
<audit-log>
<formatters>
<json-formatter name="json-formatter"/>
</formatters>
<handlers>
<file-handler name="file" formatter="json-formatter" path="audit-log.log" relative-to="jboss.server.data.dir"/>
</handlers>
<logger log-boot="true" log-read-only="false" enabled="false">
<handlers>
<handler name="file"/>
</handlers>
</logger>
</audit-log>
<management-interfaces>
<native-interface security-realm="ManagementRealm">
<socket-binding native="management-native"/>
</native-interface>
<http-interface security-realm="ManagementRealm">
<socket-binding http="management-http"/>
</http-interface>
</management-interfaces>
<access-control provider="simple">
<role-mapping>
<role name="SuperUser">
<include>
<user name="$local"/>
</include>
</role>
</role-mapping>
</access-control>
</management>
<profile>
<subsystem xmlns="urn:jboss:domain:logging:1.3">
<console-handler name="CONSOLE">
<level name="INFO"/>
<formatter>
<pattern-formatter pattern="%K{level}%d{HH:mm:ss,SSS} %-5p [%c] (%t) %s%E%n"/>
</formatter>
</console-handler>
<periodic-rotating-file-handler name="FILE" autoflush="true">
<formatter>
<pattern-formatter pattern="%d{HH:mm:ss,SSS} %-5p [%c] (%t) %s%E%n"/>
</formatter>
<file relative-to="jboss.server.log.dir" path="server.log"/>
<suffix value=".yyyy-MM-dd"/>
<append value="true"/>
</periodic-rotating-file-handler>
<logger category="com.arjuna">
<level name="WARN"/>
</logger>
<logger category="org.apache.tomcat.util.modeler">
<level name="WARN"/>
</logger>
<logger category="org.jboss.as.config">
<level name="DEBUG"/>
</logger>
<logger category="sun.rmi">
<level name="WARN"/>
</logger>
<logger category="jacorb">
<level name="WARN"/>
</logger>
<logger category="jacorb.config">
<level name="ERROR"/>
</logger>
<root-logger>
<level name="INFO"/>
<handlers>
<handler name="CONSOLE"/>
<handler name="FILE"/>
</handlers>
</root-logger>
</subsystem>
<subsystem xmlns="urn:jboss:domain:datasources:1.1">
<datasources>
<datasource jndi-name="java:jboss/datasources/ExampleDS" pool-name="ExampleDS" enabled="true" use-java-context="true">
<connection-url>jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE</connection-url>
<driver>h2</driver>
<security>
<user-name>sa</user-name>
<password>sa</password>
</security>
</datasource>
<drivers>
<driver name="h2" module="com.h2database.h2">
<xa-datasource-class>org.h2.jdbcx.JdbcDataSource</xa-datasource-class>
</driver>
</drivers>
</datasources>
</subsystem>
<subsystem xmlns="urn:jboss:domain:deployment-scanner:1.1">
<deployment-scanner path="deployments" relative-to="jboss.server.base.dir" scan-interval="5000"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:ee:1.1">
<spec-descriptor-property-replacement>false</spec-descriptor-property-replacement>
<jboss-descriptor-property-replacement>true</jboss-descriptor-property-replacement>
</subsystem>
<subsystem xmlns="urn:jboss:domain:ejb3:1.4">
<session-bean>
<stateless>
<bean-instance-pool-ref pool-name="slsb-strict-max-pool"/>
</stateless>
<stateful default-access-timeout="5000" cache-ref="simple" clustered-cache-ref="clustered"/>
<singleton default-access-timeout="5000"/>
</session-bean>
<pools>
<bean-instance-pools>
<strict-max-pool name="slsb-strict-max-pool" max-pool-size="20" instance-acquisition-timeout="5" instance-acquisition-timeout-unit="MINUTES"/>
<strict-max-pool name="mdb-strict-max-pool" max-pool-size="20" instance-acquisition-timeout="5" instance-acquisition-timeout-unit="MINUTES"/>
</bean-instance-pools>
</pools>
<caches>
<cache name="simple" aliases="NoPassivationCache"/>
<cache name="passivating" passivation-store-ref="file" aliases="SimpleStatefulCache"/>
<cache name="clustered" passivation-store-ref="infinispan" aliases="StatefulTreeCache"/>
</caches>
<passivation-stores>
<file-passivation-store name="file"/>
<cluster-passivation-store name="infinispan" cache-container="ejb"/>
</passivation-stores>
<async thread-pool-name="default"/>
<timer-service thread-pool-name="default">
<data-store path="timer-service-data" relative-to="jboss.server.data.dir"/>
</timer-service>
<remote connector-ref="remoting-connector" thread-pool-name="default"/>
<thread-pools>
<thread-pool name="default">
<max-threads count="10"/>
<keepalive-time time="100" unit="milliseconds"/>
</thread-pool>
</thread-pools>
<default-security-domain value="other"/>
<default-missing-method-permissions-deny-access value="true"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:infinispan:1.4">
<cache-container name="singleton" aliases="cluster ha-partition" default-cache="default">
<transport lock-timeout="60000"/>
<replicated-cache name="default" mode="SYNC" batching="true">
<locking isolation="REPEATABLE_READ"/>
</replicated-cache>
</cache-container>
<cache-container name="web" aliases="standard-session-cache" default-cache="repl" module="org.jboss.as.clustering.web.infinispan">
<transport lock-timeout="60000"/>
<replicated-cache name="repl" mode="ASYNC" batching="true">
<file-store/>
</replicated-cache>
<replicated-cache name="sso" mode="SYNC" batching="true"/>
<distributed-cache name="dist" l1-lifespan="0" mode="ASYNC" batching="true">
<file-store/>
</distributed-cache>
</cache-container>
<cache-container name="ejb" aliases="sfsb sfsb-cache" default-cache="repl" module="org.jboss.as.clustering.ejb3.infinispan">
<transport lock-timeout="60000"/>
<replicated-cache name="repl" mode="ASYNC" batching="true">
<eviction strategy="LRU" max-entries="10000"/>
<file-store/>
</replicated-cache>
<replicated-cache name="remote-connector-client-mappings" mode="SYNC" batching="true"/>
<distributed-cache name="dist" l1-lifespan="0" mode="ASYNC" batching="true">
<eviction strategy="LRU" max-entries="10000"/>
<file-store/>
</distributed-cache>
</cache-container>
<cache-container name="hibernate" default-cache="local-query" module="org.jboss.as.jpa.hibernate:4">
<transport lock-timeout="60000"/>
<local-cache name="local-query">
<transaction mode="NONE"/>
<eviction strategy="LRU" max-entries="10000"/>
<expiration max-idle="100000"/>
</local-cache>
<invalidation-cache name="entity" mode="SYNC">
<transaction mode="NON_XA"/>
<eviction strategy="LRU" max-entries="10000"/>
<expiration max-idle="100000"/>
</invalidation-cache>
<replicated-cache name="timestamps" mode="ASYNC">
<transaction mode="NONE"/>
<eviction strategy="NONE"/>
</replicated-cache>
</cache-container>
</subsystem>
<subsystem xmlns="urn:jboss:domain:jaxrs:1.0"/>
<subsystem xmlns="urn:jboss:domain:jca:1.1">
<archive-validation enabled="true" fail-on-error="true" fail-on-warn="false"/>
<bean-validation enabled="true"/>
<default-workmanager>
<short-running-threads>
<core-threads count="50"/>
<queue-length count="50"/>
<max-threads count="50"/>
<keepalive-time time="10" unit="seconds"/>
</short-running-threads>
<long-running-threads>
<core-threads count="50"/>
<queue-length count="50"/>
<max-threads count="50"/>
<keepalive-time time="10" unit="seconds"/>
</long-running-threads>
</default-workmanager>
<cached-connection-manager/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:jdr:1.0"/>
<subsystem xmlns="urn:jboss:domain:jgroups:1.1" default-stack="udp">
<stack name="udp">
<transport type="UDP" socket-binding="jgroups-udp"/>
<protocol type="PING"/>
<protocol type="MERGE3"/>
<protocol type="FD_SOCK" socket-binding="jgroups-udp-fd"/>
<protocol type="FD"/>
<protocol type="VERIFY_SUSPECT"/>
<protocol type="pbcast.NAKACK"/>
<protocol type="UNICAST2"/>
<protocol type="pbcast.STABLE"/>
<protocol type="pbcast.GMS"/>
<protocol type="UFC"/>
<protocol type="MFC"/>
<protocol type="FRAG2"/>
<protocol type="RSVP"/>
</stack>
<stack name="tcp">
<transport type="TCP" socket-binding="jgroups-tcp"/>
<protocol type="TCPPING">
<property name="initial_hosts">
0.0.0.0[7600],0.0.0.0[7600]
</property>
<property name="num_initial_members">
2
</property>
<property name="port_range">
0
</property>
<property name="timeout">
2000
</property>
</protocol>
<protocol type="MPING" socket-binding="jgroups-mping"/>
<protocol type="MERGE2"/>
<protocol type="FD_SOCK" socket-binding="jgroups-tcp-fd"/>
<protocol type="FD"/>
<protocol type="VERIFY_SUSPECT"/>
<protocol type="pbcast.NAKACK"/>
<protocol type="UNICAST2"/>
<protocol type="pbcast.STABLE"/>
<protocol type="pbcast.GMS"/>
<protocol type="UFC"/>
<protocol type="MFC"/>
<protocol type="FRAG2"/>
<protocol type="RSVP"/>
</stack>
</subsystem>
<subsystem xmlns="urn:jboss:domain:jmx:1.3">
<expose-resolved-model/>
<expose-expression-model/>
<remoting-connector/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:jpa:1.1">
<jpa default-datasource="" default-extended-persistence-inheritance="DEEP"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:jsf:1.0"/>
<subsystem xmlns="urn:jboss:domain:mail:1.1">
<mail-session jndi-name="java:jboss/mail/Default">
<smtp-server outbound-socket-binding-ref="mail-smtp"/>
</mail-session>
</subsystem>
<subsystem xmlns="urn:jboss:domain:modcluster:1.1">
<mod-cluster-config advertise-socket="modcluster" connector="ajp">
<simple-load-provider/>
<dynamic-load-provider history="20">
<load-metric type="busyness" weight="3"/>
<load-metric type="sessions" weight="3" capacity="2"/>
<load-metric type="heap" weight="2"/>
</dynamic-load-provider>
</mod-cluster-config>
</subsystem>
<subsystem xmlns="urn:jboss:domain:naming:1.4">
<remote-naming/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:pojo:1.0"/>
<subsystem xmlns="urn:jboss:domain:remoting:1.1">
<connector name="remoting-connector" socket-binding="remoting" security-realm="ApplicationRealm"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:resource-adapters:1.1"/>
<subsystem xmlns="urn:jboss:domain:sar:1.0"/>
<subsystem xmlns="urn:jboss:domain:security:1.2">
<security-domains>
<security-domain name="other" cache-type="default">
<authentication>
<login-module code="Remoting" flag="optional">
<module-option name="password-stacking" value="useFirstPass"/>
</login-module>
<login-module code="RealmDirect" flag="required">
<module-option name="password-stacking" value="useFirstPass"/>
</login-module>
</authentication>
</security-domain>
<security-domain name="jboss-web-policy" cache-type="default">
<authorization>
<policy-module code="Delegating" flag="required"/>
</authorization>
</security-domain>
<security-domain name="jboss-ejb-policy" cache-type="default">
<authorization>
<policy-module code="Delegating" flag="required"/>
</authorization>
</security-domain>
</security-domains>
</subsystem>
<subsystem xmlns="urn:jboss:domain:threads:1.1"/>
<subsystem xmlns="urn:jboss:domain:transactions:1.4">
<core-environment>
<process-id>
<uuid/>
</process-id>
</core-environment>
<recovery-environment socket-binding="txn-recovery-environment" status-socket-binding="txn-status-manager"/>
<coordinator-environment default-timeout="300"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:web:1.5" default-virtual-server="default-host" native="false">
<connector name="http" protocol="HTTP/1.1" scheme="http" socket-binding="http"/>
<connector name="ajp" protocol="AJP/1.3" scheme="http" socket-binding="ajp"/>
<virtual-server name="default-host" enable-welcome-root="true">
<alias name="localhost"/>
<alias name="example.com"/>
</virtual-server>
</subsystem>
<subsystem xmlns="urn:jboss:domain:webservices:1.2">
<modify-wsdl-address>true</modify-wsdl-address>
<wsdl-host>${jboss.bind.address:127.0.0.1}</wsdl-host>
<endpoint-config name="Standard-Endpoint-Config"/>
<endpoint-config name="Recording-Endpoint-Config">
<pre-handler-chain name="recording-handlers" protocol-bindings="##SOAP11_HTTP ##SOAP11_HTTP_MTOM ##SOAP12_HTTP ##SOAP12_HTTP_MTOM">
<handler name="RecordingHandler" class="org.jboss.ws.common.invocation.RecordingServerHandler"/>
</pre-handler-chain>
</endpoint-config>
<client-config name="Standard-Client-Config"/>
</subsystem>
<subsystem xmlns="urn:jboss:domain:weld:1.0"/>
</profile>
<interfaces>
<interface name="management">
<inet-address value="${jboss.bind.address.management:127.0.0.1}"/>
</interface>
<interface name="public">
<inet-address value="${jboss.bind.address:127.0.0.1}"/>
</interface>
<interface name="unsecure">
<inet-address value="${jboss.bind.address.unsecure:127.0.0.1}"/>
</interface>
</interfaces>
<socket-binding-group name="standard-sockets" default-interface="public" port-offset="${jboss.socket.binding.port-offset:0}">
<socket-binding name="management-native" interface="management" port="${jboss.management.native.port:9999}"/>
<socket-binding name="management-http" interface="management" port="${jboss.management.http.port:9990}"/>
<socket-binding name="management-https" interface="management" port="${jboss.management.https.port:9443}"/>
<socket-binding name="ajp" port="8009"/>
<socket-binding name="http" port="8080"/>
<socket-binding name="https" port="8443"/>
<socket-binding name="jgroups-mping" port="0" multicast-address="${jboss.default.multicast.address:230.0.0.4}" multicast-port="45700"/>
<socket-binding name="jgroups-tcp" port="7600"/>
<socket-binding name="jgroups-tcp-fd" port="57600"/>
<socket-binding name="jgroups-udp" port="55200" multicast-address="${jboss.default.multicast.address:230.0.0.4}" multicast-port="45688"/>
<socket-binding name="jgroups-udp-fd" port="54200"/>
<socket-binding name="modcluster" port="0" multicast-address="224.0.1.105" multicast-port="23364"/>
<socket-binding name="remoting" port="4447"/>
<socket-binding name="txn-recovery-environment" port="4712"/>
<socket-binding name="txn-status-manager" port="4713"/>
<outbound-socket-binding name="mail-smtp">
<remote-destination host="localhost" port="25"/>
</outbound-socket-binding>
</socket-binding-group>
</server>
this is the starting batch for server 1
title server1
bin/standalone.bat -Djboss.node.name=server1 -server-config=standalone-ha.xml -b 0.0.0.0 -Djgroups.bind_addr=127.0.0.1
this is the starting batch for server 2
title server2
bin/standalone.bat -Djboss.socket.binding.port-offset=100 -Djboss.node.name=server2 -server-config=standalone-ha.xml -b 0.0.0.0 -Djgroups.bind_addr=127.0.0.1
please someone helps me finding out what's missing or wrong configured in my testing environment.
thanks

Answer 1

finally i solved my problem, it turned out that when recorded the scenario of testing in Microsoft load test , it added jsessionid to the URL, which caused the problem because of the same sessionid for all requests.
the solution was removing this parameter from testing URLs :)