Related
Looks like partition map exchange process and cache CRUD operations in parallel are causing deadlock or lock acquire failures.
What should be the strategy to handle this scenario ?
Ignite server has below errors:
Exception stack trace 1:
WARNING: Dumping the near node thread that started transaction [xidVer=GridCacheVersion [topVer=247332659, order=1635852705217, nodeOrder=1], nodeId=2735bef0-7404-41e3-843f-7043490c9d84]
Stack trace of the transaction owner thread:
Thread [name="client-connector-#56%perf-dn1%", id=93, state=WAITING, blockCnt=5023, waitCnt=36165]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304)
at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178)
at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141)
at o.a.i.i.processors.cache.GridCacheAdapter$41.op(GridCacheAdapter.java:3430)
at o.a.i.i.processors.cache.GridCacheAdapter$41.op(GridCacheAdapter.java:3423)
at o.a.i.i.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4480)
at o.a.i.i.processors.cache.GridCacheAdapter.remove0(GridCacheAdapter.java:3423)
at o.a.i.i.processors.cache.GridCacheAdapter.remove(GridCacheAdapter.java:3405)
at o.a.i.i.processors.cache.GridCacheAdapter.remove(GridCacheAdapter.java:3388)
at o.a.i.i.processors.cache.IgniteCacheProxyImpl.remove(IgniteCacheProxyImpl.java:1438)
at o.a.i.i.processors.cache.GatewayProtectedCacheProxy.remove(GatewayProtectedCacheProxy.java:964)
at o.a.i.i.processors.platform.client.cache.ClientCacheRemoveKeyRequest.process(ClientCacheRemoveKeyRequest.java:41)
at o.a.i.i.processors.platform.client.ClientRequestHandler.handle(ClientRequestHandler.java:77)
at o.a.i.i.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:204)
at o.a.i.i.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:55)
at o.a.i.i.util.nio.GridNioFilterChain$TailFilter.onMessageReceived(GridNioFilterChain.java:279)
at o.a.i.i.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
at o.a.i.i.util.nio.GridNioAsyncNotifyFilter$3.body(GridNioAsyncNotifyFilter.java:97)
at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120)
at o.a.i.i.util.worker.GridWorkerPool$1.run(GridWorkerPool.java:70)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Exception stack trace 2:
WARNING: >>> Transaction [startTime=11:39:27.214, curTime=11:40:36.277, systemTime=0, userTime=69063, tx=GridNearTxLocal [mappings=IgniteTxMappingsImpl [], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=null, hasRemoteLocks=false, trackTimeout=false, systemTime=44700, systemStartTime=0, prepareStartTime=0, prepareTime=0, commitOrRollbackStartTime=0, commitOrRollbackTime=0, lb=null, mvccOp=null, qryId=-1, crdVer=0, thread=client-connector-#57%perf-dn1%, mappings=IgniteTxMappingsImpl [], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, span=o.a.i.i.processors.tracing.NoopSpan#4a931268, nearNodes=KeySetView [], dhtNodes=KeySetView [], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxStateImpl [activeCacheIds=[], recovery=null, mvccEnabled=null, mvccCachingCacheIds=[], txMap=EmptySet []], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705226, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=95, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=SUSPENDED, timedOut=false, topVer=AffinityTopologyVersion [topVer=-1, minorTopVer=0], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69079ms, onePhaseCommit=false], size=0]]]]
Nov 2, 2021 11:40:36 AM org.apache.ignite.logger.java.JavaLogger warning
WARNING: First 10 long running cache futures [total=16]
Nov 2, 2021 11:40:36 AM org.apache.ignite.logger.java.JavaLogger warning
WARNING: >>> Future [startTime=11:39:27.324, curTime=11:40:36.277, fut=GridDhtLockFuture [span=o.a.i.i.processors.tracing.NoopSpan#4a931268, nearNodeId=2735bef0-7404-41e3-843f-7043490c9d84, nearLockVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], threadId=124, futId=be58a60ec71-1d64903c-c700-4deb-bace-cc5158713120, lockVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], read=false, err=null, timedOut=false, timeout=0, tx=GridNearTxLocal [mappings=IgniteTxMappingsImpl []dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]], nearLocallyMapped=false, colocatedLocallyMapped=true, needCheckBackup=null, hasRemoteLocks=false, trackTimeout=false, systemTime=75000, systemStartTime=971108549857700, prepareStartTime=0, prepareTime=0, commitOrRollbackStartTime=0, commitOrRollbackTime=0, lb=null, mvccOp=null, qryId=-1, crdVer=0, thread=client-connector-#84%perf-dn1%, mappings=IgniteTxMappingsImpl []dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, span=o.a.i.i.processors.tracing.NoopSpan#4a931268, nearNodes=KeySetView [], dhtNodes=KeySetView [], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxStateImpl [activeCacheIds=[585748697], recovery=false, mvccEnabled=false, mvccCachingCacheIds=[], txMap=ArrayList [IgniteTxEntry [txKey=IgniteTxKey [key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], cacheId=585748697], val=TxEntryValueHolder [val=null, op=DELETE], prevVal=TxEntryValueHolder [val=null, op=NOOP], oldVal=TxEntryValueHolder [val=null, op=NOOP], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=CacheEntryPredicate[] [], filtersPassed=false, filtersSet=true, entry=GridDhtCacheEntry [rdrs=ReaderId[] [], part=244, super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], val=null, ver=GridCacheVersion [topVer=247332659, order=1635852705229, nodeOrder=1], hash=1085684290, extras=GridCacheMvccEntryExtras [mvcc=GridCacheMvcc [locs=LinkedList [GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705207, nodeOrder=1], threadId=122, id=2104, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705207, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=1|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], threadId=124, id=2102, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705213, nodeOrder=1], threadId=122, id=2120, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705213, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705207, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705214, nodeOrder=1], threadId=123, id=2118, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705214, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705207, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705217, nodeOrder=1], threadId=93, id=2108, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705217, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705218, nodeOrder=1], threadId=115, id=2106, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705218, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705222, nodeOrder=1], threadId=95, id=2110, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705222, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705223, nodeOrder=1], threadId=120, id=2112, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705223, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705227, nodeOrder=1], threadId=118, id=2114, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705227, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]]], mapped=false, trackable=true, createTtl=-1, accessTtl=-1, needReturnVal=false, skipStore=false, keepBinary=true, innerFuts=EmptyList [], pendingLocks=HashSet [KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true]], super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=0, lsnrCalls=0, done=false, cancelled=false, err=null, futs=EmptyList []]]]]
Nov 2, 2021 11:40:36 AM org.apache.ignite.logger.java.JavaLogger warning
Exception stack trace 3 when TxTimeoutOnPartitionMapExchange is set to some value
Nov 2, 2021 1:11:19 PM org.apache.ignite.logger.java.JavaLogger warning
WARNING: The transaction was forcibly rolled back on partition map exchange because a timeout is reached: [tx=GridNearTxLocal[xid=e0e3db0ec71-00000000-0ebe-146c-0000-000000000001, xidVersion=GridCacheVersion [topVer=247338092, order=1635858071054, nodeOrder=1], nearXidVersion=GridCacheVersion [topVer=247338092, order=1635858071054, nodeOrder=1], concurrency=PESSIMISTIC, isolation=REPEATABLE_READ, state=ACTIVE, invalidate=false, rollbackOnly=false, nodeId=1ed7077a-5547-41c1-94ab-d246944ed4a9, timeout=0, startTime=1635858659324, duration=20140, label=null], topVer=AffinityTopologyVersion [topVer=1, minorTopVer=104]]
Nov 2, 2021 1:11:19 PM org.apache.ignite.logger.java.JavaLogger error
SEVERE: Failed to process client request [req=o.a.i.i.processors.platform.client.cache.ClientCacheRemoveKeyRequest#2fea4a3]
javax.cache.CacheException: class org.apache.ignite.transactions.TransactionRollbackException: Failed to finish transaction because it has been rolled back [timeout=0, tx=GridNearTxLocal[xid=f0e3db0ec71-00000000-0ebe-146c-0000-000000000001, xidVersion=GridCacheVersion [topVer=247338092, order=1635858071055, nodeOrder=1], nearXidVersion=GridCacheVersion [topVer=247338092, order=1635858071055, nodeOrder=1], concurrency=PESSIMISTIC, isolation=REPEATABLE_READ, state=ACTIVE, invalidate=false, rollbackOnly=false, nodeId=1ed7077a-5547-41c1-94ab-d246944ed4a9, timeout=0, startTime=1635858659324, duration=20140, label=null]]
at org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1266)
at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.cacheException(IgniteCacheProxyImpl.java:2084)
at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.remove(IgniteCacheProxyImpl.java:1441)
at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.remove(GatewayProtectedCacheProxy.java:964)
at org.apache.ignite.internal.processors.platform.client.cache.ClientCacheRemoveKeyRequest.process(ClientCacheRemoveKeyRequest.java:41)
at org.apache.ignite.internal.processors.platform.client.ClientRequestHandler.handle(ClientRequestHandler.java:77)
at org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:204)
at org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:55)
at org.apache.ignite.internal.util.nio.GridNioFilterChain$TailFilter.onMessageReceived(GridNioFilterChain.java:279)
at org.apache.ignite.internal.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
at org.apache.ignite.internal.util.nio.GridNioAsyncNotifyFilter$3.body(GridNioAsyncNotifyFilter.java:97)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at org.apache.ignite.internal.util.worker.GridWorkerPool$1.run(GridWorkerPool.java:70)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: class org.apache.ignite.transactions.TransactionRollbackException: Failed to finish transaction because it has been rolled back [timeout=0, tx=GridNearTxLocal[xid=f0e3db0ec71-00000000-0ebe-146c-0000-000000000001, xidVersion=GridCacheVersion [topVer=247338092, order=1635858071055, nodeOrder=1], nearXidVersion=GridCacheVersion [topVer=247338092, order=1635858071055, nodeOrder=1], concurrency=PESSIMISTIC, isolation=REPEATABLE_READ, state=ACTIVE, invalidate=false, rollbackOnly=false, nodeId=1ed7077a-5547-41c1-94ab-d246944ed4a9, timeout=0, startTime=1635858659324, duration=20140, label=null]]
at org.apache.ignite.internal.util.IgniteUtils$11.apply(IgniteUtils.java:974)
at org.apache.ignite.internal.util.IgniteUtils$11.apply(IgniteUtils.java:972)
... 16 more
This is the expected behavior:
When the partition map exchange starts, Ignite acquires a global lock
at a particular stage. The lock can’t be obtained while incomplete
transactions are running in parallel. These transactions prevent the
partition map exchange process from moving forward, thus, blocking
some operations such as a new node join process.
In other words, PME process should wait until the end of all active transactions to acquire a global lock. Therefore it's recommended to:
either set a global TX timeout (TransactionConfiguration#DefaultTxTimeout) or configure it individually for every transaction, i.e. make sure that you are not running it forever in a worst-case scenario.
configure TxTimeoutOnPartitionMapExchange to be non zero (default).
The concrete numbers are individual, but I suppose mostly you'd like to have a lower timeout on PME in comparison to a "normal" TX timeout for quicker cluster-wide updates.
HI I am facing a critical issue with Ignite in our production servers . We have 2 instances with heap sizes of 8gb each . Sometimes due to long gc pause or network issue one of our instances gets stopped . This causes aws auto-scaling to kick in and bring another instance up . This is fine but we have observed that in tis state the grid becomes unstable and our new ignite instaces are never able to join the topology and hang forever causing new autoscaled instances to come again and again .The workaround for this is to restart other instances in the cluster as doing so causes nodes to join again .But ideally in a prod environment this should happen automatically with auto scaling .
Had also added a longer failuredetection timeout but that also doesnt solve it completely and we still observe this sometimes .
The logs observed on the new instances not coming up is as below .Igite version use is 2.4 and off heap mode is used for partitioned caches .Our grid is setup using tcp discovery service using a s3 bucket .
I have some transactional caches as well which do lock based on
tryLocks.
evtLatch=0, remaining=[a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9], super=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=1272213534]]]
2018-07-18 16:34:10.534 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], node=7d5e83aa-736a-4190-8b64-7261db7382f6]. Dumping pending objects that might be the cause:
2018-07-18 16:34:20.534 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], node=7d5e83aa-736a-4190-8b64-7261db7382f6]. Dumping pending objects that might be the cause:
2018-07-18 16:34:20.534 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Ready affinity version: AffinityTopologyVersion [topVer=-1, minorTopVer=0]
2018-07-18 16:34:20.535 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Last exchange future: GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931660255, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=7d5e83aa, msg=null, type=NODE_JOINED, tstamp=1531931329481], crd=TcpDiscoveryNode [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, addrs=[10.83.87.131, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-87-131.ec2.internal/10.83.87.131:47500], discPort=47500, order=26, intOrder=14, lastExchangeTime=1531931329258, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], discoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931660255, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=7d5e83aa, msg=null, type=NODE_JOINED, tstamp=1531931329481], nodeId=7d5e83aa, evt=NODE_JOINED], added=true, initFut=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=true, hash=247159314], init=true, lastVer=null, partReleaseFut=PartitionReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[ExplicitLockReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], TxReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], AtomicUpdateReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], DataStreamerReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]]]], exchActions=ExchangeActions [startCaches=null, stopCaches=null, startGrps=[], stopGrps=[], resetParts=null, stateChangeRequest=null], affChangeMsg=null, initTs=1531931329576, centralizedAff=false, changeGlobalStateE=null, done=false, state=SRV, evtLatch=0, remaining=[a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9], super=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=1272213534]]
2018-07-18 16:34:20.535 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.a.i.i.p.c.GridCachePartitionExchangeManager - First 10 pending exchange futures [total=0]
2018-07-18 16:34:20.535 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Last 10 exchange futures (total: 1):
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - >>> GridDhtPartitionsExchangeFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], evt=NODE_JOINED, evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931660255, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], done=false]
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending transactions:
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending explicit locks:
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending cache futures:
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending atomic cache futures:
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending data streamer futures:
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending transaction deadlock detection futures:
2018-07-18 16:34:20.547 UTC [FDPS] [grid-nio-worker-tcp-comm-3-#28%fdps%] [INFO ] [,] o.apache.ignite.internal.diagnostic - Exchange future waiting for coordinator response [crd=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0]]
Remote node information:
General node info [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, client=false, discoTopVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], time=12:34:20.537]
Partitions exchange info [readyVer=AffinityTopologyVersion [topVer=29, minorTopVer=0]]
Last initialized exchange future: GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=ba6aba6c-7f5d-41bf-bfcc-5eefcad36b62, addrs=[10.83.85.122, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-85-122.ec2.internal/10.83.85.122:47500], discPort=47500, order=30, intOrder=16, lastExchangeTime=1531930705943, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=30, nodeId8=a450db0b, msg=Node joined: TcpDiscoveryNode [id=ba6aba6c-7f5d-41bf-bfcc-5eefcad36b62, addrs=[10.83.85.122, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-85-122.ec2.internal/10.83.85.122:47500], discPort=47500, order=30, intOrder=16, lastExchangeTime=1531930705943, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], type=NODE_JOINED, tstamp=1531930706210], crd=TcpDiscoveryNode [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, addrs=[10.83.87.131, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-87-131.ec2.internal/10.83.87.131:47500], discPort=47500, order=26, intOrder=14, lastExchangeTime=1531931660254, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], discoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=ba6aba6c-7f5d-41bf-bfcc-5eefcad36b62, addrs=[10.83.85.122, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-85-122.ec2.internal/10.83.85.122:47500], discPort=47500, order=30, intOrder=16, lastExchangeTime=1531930705943, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=30, nodeId8=a450db0b, msg=Node joined: TcpDiscoveryNode [id=ba6aba6c-7f5d-41bf-bfcc-5eefcad36b62, addrs=[10.83.85.122, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-85-122.ec2.internal/10.83.85.122:47500], discPort=47500, order=30, intOrder=16, lastExchangeTime=1531930705943, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], type=NODE_JOINED, tstamp=1531930706210], nodeId=ba6aba6c, evt=NODE_JOINED], added=true, initFut=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=1921954756], init=false, lastVer=GridCacheVersion [topVer=0, order=1531930704443, nodeOrder=0], partReleaseFut=PartitionReleaseFuture [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], futures=[ExplicitLockReleaseFuture [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], futures=[ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935479, nodeOrder=26], threadId=39726, id=559000, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=221, val=49583853497448469294730566354366524577617095530402283666, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547787212113, nodeOrder=26], threadId=39741, id=603904, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=288, val=49583853499611641578988037213538229804531966271996035234, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935487, nodeOrder=26], threadId=39740, id=558993, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=133, val=49583853497448469294730566354417299462040910024459419794, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935323, nodeOrder=26], threadId=39728, id=558949, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=1023, val=49583853497448469294730566353278491339963927967496667282, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935470, nodeOrder=26], threadId=39951, id=559009, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=556, val=49583853497448469294730566354226289182541798339977937042, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935497, nodeOrder=26], threadId=39683, id=558982, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=373, val=49583853497448469294730566354541818821461216966893109394, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935339, nodeOrder=26], threadId=39682, id=558941, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=156, val=49583853497448469294730566353353444740780034976328450194, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935358, nodeOrder=26], threadId=39936, id=558921, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=59, val=49583853497448469294730566353578304943228356208982229138, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandida... and 48550 skipped ...ead=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935486, nodeOrder=26], threadId=39894, id=558992, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=488, val=49583853497448469294730566354434224423515514832905306258, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935331, nodeOrder=26], threadId=39893, id=558948, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=570, val=49583853497448469294730566353289371672340459630069022866, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]]]], TxReleaseFuture [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], futures=[]], AtomicUpdateReleaseFuture [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], futures=[]], DataStreamerReleaseFuture [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], futures=[]]]], exchActions=null, affChangeMsg=null, initTs=1531930706210, centralizedAff=false, changeGlobalStateE=null, done=false, state=CRD, evtLatch=0, remaining=[ba6aba6c-7f5d-41bf-bfcc-5eefcad36b62], super=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=325602672]]
Communication SPI statistics [rmtNode=7d5e83aa-736a-4190-8b64-7261db7382f6]
Communication SPI recovery descriptors:
[key=ConnectionKey [nodeId=7d5e83aa-736a-4190-8b64-7261db7382f6, idx=0, connCnt=0], msgsSent=5, msgsAckedByRmt=0, msgsRcvd=7, lastAcked=0, reserveCnt=1, descIdHash=1972345954]
Communication SPI clients:
[node=7d5e83aa-736a-4190-8b64-7261db7382f6, client=GridTcpNioCommunicationClient [ses=GridSelectorNioSessionImpl [worker=DirectNioClientWorker [super=AbstractNioClientWorker [idx=3, bytesRcvd=5740, bytesSent=77322, bytesRcvd0=853, bytesSent0=0, select=true, super=GridWorker [name=grid-nio-worker-tcp-comm-3, igniteInstanceName=fdps, finished=false, hashCode=2068348067, interrupted=false, runner=grid-nio-worker-tcp-comm-3-#28%fdps%]]], writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], readBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], inRecovery=GridNioRecoveryDescriptor [acked=0, resendCnt=0, rcvCnt=7, sentCnt=5, reserved=true, lastAck=0, nodeLeft=false, node=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], connected=true, connectCnt=0, queueLimit=262144, reserveCnt=1, pairedConnections=false], outRecovery=GridNioRecoveryDescriptor [acked=0, resendCnt=0, rcvCnt=7, sentCnt=5, reserved=true, lastAck=0, nodeLeft=false, node=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], connected=true, connectCnt=0, queueLimit=262144, reserveCnt=1, pairedConnections=false], super=GridNioSessionImpl [locAddr=/10.83.87.131:47100, rmtAddr=/10.83.89.183:34664, createTime=1531931330498, closeTime=0, bytesSent=77322, bytesRcvd=5740, bytesSent0=0, bytesRcvd0=853, sndSchedTime=1531931330498, lastSndTime=1531931500547, lastRcvTime=1531931660527, readsPaused=false, filterChain=FilterChain[filters=[GridNioCodecFilter [parser=org.apache.ignite.internal.util.nio.GridDirectParser#665c2413, directMode=true], GridConnectionBytesVerifyFilter], accepted=true]], super=GridAbstractCommunicationClient [lastUsed=1531931330508, closed=false, connIdx=0]]]
NIO sessions statistics:
>> Selector info [idx=3, keysCnt=1, bytesRcvd=5740, bytesRcvd0=853, bytesSent=77322, bytesSent0=0]
Connection info [in=true, rmtAddr=/10.83.89.183:34664, locAddr=/10.83.87.131:47100, msgsSent=5, msgsAckedByRmt=0, descIdHash=1972345954, unackedMsgs=[IgniteDiagnosticMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage], msgsRcvd=7, lastAcked=0, descIdHash=1972345954, bytesRcvd=5740, bytesRcvd0=853, bytesSent=77322, bytesSent0=0, opQueueSize=0]
Exchange future: GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=a450db0b, msg=Node joined: TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], type=NODE_JOINED, tstamp=1531931329402], crd=null, exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], discoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=a450db0b, msg=Node joined: TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], type=NODE_JOINED, tstamp=1531931329402], nodeId=7d5e83aa, evt=NODE_JOINED], added=true, initFut=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=980776600], init=false, lastVer=GridCacheVersion [topVer=0, order=1531931327875, nodeOrder=0], partReleaseFut=null, exchActions=null, affChangeMsg=null, initTs=0, centralizedAff=false, changeGlobalStateE=null, done=false, state=null, evtLatch=0, remaining=[], super=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=2138568466]]
Local communication statistics:
Communication SPI statistics [rmtNode=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9]
Communication SPI recovery descriptors:
[key=ConnectionKey [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, idx=0, connCnt=-1], msgsSent=7, msgsAckedByRmt=0, msgsRcvd=6, lastAcked=0, reserveCnt=1, descIdHash=1891649612]
Communication SPI clients:
Communication SPI clients:
[node=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, client=GridTcpNioCommunicationClient [ses=GridSelectorNioSessionImpl [worker=DirectNioClientWorker [super=AbstractNioClientWorker [idx=0, bytesRcvd=92833, bytesSent=5698, bytesRcvd0=15539, bytesSent0=853, select=true, super=GridWorker [name=grid-nio-worker-tcp-comm-0, igniteInstanceName=fdps, finished=false, hashCode=2040212682, interrupted=false, runner=grid-nio-worker-tcp-comm-0-#25%fdps%]]], writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], readBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], inRecovery=GridNioRecoveryDescriptor [acked=0, resendCnt=0, rcvCnt=6, sentCnt=7, reserved=true, lastAck=0, nodeLeft=false, node=TcpDiscoveryNode [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, addrs=[10.83.87.131, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-87-131.ec2.internal/10.83.87.131:47500], discPort=47500, order=26, intOrder=14, lastExchangeTime=1531931329258, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], connected=false, connectCnt=1, queueLimit=262144, reserveCnt=1, pairedConnections=false], outRecovery=GridNioRecoveryDescriptor [acked=0, resendCnt=0, rcvCnt=6, sentCnt=7, reserved=true, lastAck=0, nodeLeft=false, node=TcpDiscoveryNode [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, addrs=[10.83.87.131, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-87-131.ec2.internal/10.83.87.131:47500], discPort=47500, order=26, intOrder=14, lastExchangeTime=1531931329258, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], connected=false, connectCnt=1, queueLimit=262144, reserveCnt=1, pairedConnections=false], super=GridNioSessionImpl [locAddr=/10.83.89.183:34664, rmtAddr=ip-10-83-87-131.ec2.internal/10.83.87.131:47100, createTime=1531931330468, closeTime=0, bytesSent=5698, bytesRcvd=92833, bytesSent0=853, bytesRcvd0=15539, sndSchedTime=1531931330468, lastSndTime=1531931660528, lastRcvTime=1531931660538, readsPaused=false, filterChain=FilterChain[filters=[GridNioCodecFilter [parser=org.apache.ignite.internal.util.nio.GridDirectParser#72024a61, directMode=true], GridConnectionBytesVerifyFilter], accepted=false]], super=GridAbstractCommunicationClient [lastUsed=1531931330468, closed=false, connIdx=0]]]
NIO sessions statistics:
>> Selector info [idx=0, keysCnt=1, bytesRcvd=92833, bytesRcvd0=15539, bytesSent=5698, bytesSent0=853]
Connection info [in=false, rmtAddr=ip-10-83-87-131.ec2.internal/10.83.87.131:47100, locAddr=/10.83.89.183:34664, msgsSent=7, msgsAckedByRmt=0, descIdHash=1891649612, unackedMsgs=[GridDhtPartitionsSingleMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage], msgsRcvd=6, lastAcked=0, descIdHash=1891649612, bytesRcvd=92833, bytesRcvd0=15539, bytesSent=5698, bytesSent0=853, opQueueSize=0]
2018-07-18 16:34:29.598 UTC [FDPS] [localhost-startStop-1] [WARN ] [,] o.a.i.i.p.c.GridCachePartitionExchangeManager - Still waiting for initial partition map exchange [fut=GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931669507, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=7d5e83aa, msg=null, type=NODE_JOINED, tstamp=1531931329481], crd=TcpDiscoveryNode [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, addrs=[10.83.87.131, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-87-131.ec2.internal/10.83.87.131:47500], discPort=47500, order=26, intOrder=14, lastExchangeTime=1531931329258, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], discoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931669507, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=7d5e83aa, msg=null, type=NODE_JOINED, tstamp=1531931329481], nodeId=7d5e83aa, evt=NODE_JOINED], added=true, initFut=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=true, hash=247159314], init=true, lastVer=null, partReleaseFut=PartitionReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[ExplicitLockReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], TxReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], AtomicUpdateReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], DataStreamerReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]]]], exchActions=ExchangeActions [startCaches=null, stopCaches=null, startGrps=[], stopGrps=[], resetParts=null, stateChangeRequest=null], affChangeMsg=null, initTs=1531931329576, centralizedAff=false, changeGlobalStateE=null, done=false, state=SRV, evtLatch=0, remaining=[a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9], super=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=1272213534]]]
2018-07-18 16:34:30.537 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], node=7d5e83aa-736a-4190-8b64-7261db7382f6]. Dumping pending objects that might be the cause:
2018-07-18 16:34:40.537 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], node=7d5e83aa-736a-4190-8b64-7261db7382f6]. Dumping pending objects that might be the cause:
Info about the other node 10-83-85-122
The other joining node never got started and was stuck in the ignite start phase . The logs also dont show the node to get up or the ip discovery to get kicked in . to eventually cause the node to be removed via autoscaling .
Transactional errors received
javax.cache.CacheException: Failed to acquire lock for keys (primary node left grid, retry transaction if possible) [keys=[UserKeyCacheObjectImpl [part=281,
Partition map exchange is a process of exchanging information between nodes where each piece of data is stored. It happens every time, when topology changes.
Every node sends a GridDhtPartitionsSingleMessage to a coordinator. Once the coordinator collected all such messages, it sends GridDhtPartitionsFullMessage back to other nodes. These messages are sent over communication SPI.
But if some of non-coordinator nodes don't send the SingleMessage to the coordinator, or if the coordinator doesn't send the FullMessage, then "Failed to wait for partition map exchange" error occurs.
Judging by the piece of log, that you provided, a node with ID=ba6aba6c didn't send the SingleMessage to the coordinator. It may mean, that communication SPI doesn't work there properly. Make sure, that ports, that are required for communication SPI are available. Usually it's 47100..47200.
Also joining node may be stuck on something. Look at its log to figure out, what happens there.
I have 3 node cluster with 20+ client and it's running in spark context.Initially it working fine but randomly get issue whenever new node i.e. client try to connect with cluster.The cluster getting inoperative.I have got following logs when its stuck.If I restart any Ignite server explicitly then its release and work fine.I have use Ignite 2.4.0 version. same issue produced in Ignite 2.5.0 version too.
client side Logs
Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=44, minorTopVer=0], node=4d885cfd-45ed-43a2-8088-f35c9469797f]. Dumping pending objects that might be the cause:
GridDhtPartitionsExchangeFuture [topVer=AffinityTopologyVersion [topVer=44, minorTopVer=0], evt=NODE_JOINED, evtNode=TcpDiscoveryNode [id=4d885cfd-45ed-43a2-8088-f35c9469797f, addrs=[0:0:0:0:0:0:0:1%lo, 10.13.10.179, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, hdn6.mstorm.com/10.13.10.179:0], discPort=0, order=44, intOrder=0, lastExchangeTime=1527651620413, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=true], done=false]
Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=44, minorTopVer=0], node=4d885cfd-45ed-43a2-8088-f35c9469797f]. Dumping pending objects that might be the cause:
GridDhtPartitionsExchangeFuture [topVer=AffinityTopologyVersion [topVer=44, minorTopVer=0], evt=NODE_JOINED, evtNode=TcpDiscoveryNode [id=4d885cfd-45ed-43a2-8088-f35c9469797f, addrs=[0:0:0:0:0:0:0:1%lo, 10.13.10.179, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, hdn6.mstorm.com/10.13.10.179:0], discPort=0, order=44, intOrder=0, lastExchangeTime=1527651620413, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=true], done=false]
Failed to wait for initial partition map exchange. Possible reasons are:
^-- Transactions in deadlock.
^-- Long running transactions (ignore if this is the case).
^-- Unreleased explicit locks.
Still waiting for initial partition map exchange [fut=GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=4d885cfd-45ed-43a2-8088-f35c9469797f, addrs=
Server Side Logs
Possible starvation in striped pool. Thread name: sys-stripe-0-#1 Queue: [Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtTxPrepareResponse [nearEvicted=null, futId=869dd4ca361-fe7e167d-4d80-4f57-b004-13359a9f2c11, miniId=1, super=GridDistributedTxPrepareResponse [txState=null, part=-1, err=null, super=GridDistributedBaseMessage [ver=GridCacheVersion [topVer=139084030, order=1527604094903, nodeOrder=1], committedVers=null, rolledbackVers=null, cnt=0, super=GridCacheIdMessage [cacheId=0]]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [part=984, val=null, hasValBytes=true], val=BinaryObjectImpl [arr= true, ctx=false, start=0], prevVal=null, super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null, nearFutId=0, flags=]]]], o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout#2735c674, Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtTxPrepareRequest [nearNodeId=628e3078-17fd-4e49-b9ae-ad94ad97a2f1, futId=6576e4ca361-6e7cdac2-d5a3-4624-9ad3-b93f25546cc3, miniId=1, topVer=AffinityTopologyVersion [topVer=20, minorTopVer=0], invalidateNearEntries={}, nearWrites=null, owned=null, nearXidVer=GridCacheVersion [topVer=139084030, order=1527604094933, nodeOrder=2], subjId=628e3078-17fd-4e49-b9ae-ad94ad97a2f1, taskNameHash=0, preloadKeys=null, super=GridDistributedTxPrepareRequest [threadId=86, concurrency=OPTIMISTIC, isolation=READ_COMMITTED, writeVer=GridCacheVersion [topVer=139084030, order=1527604094935, nodeOrder=2], timeout=0, reads=null, writes=[IgniteTxEntry [key=BinaryObjectImpl [arr= true, ctx=false, start=0], cacheId=-1755241537, txKey=null, val=[op=UPDATE, val=BinaryObjectImpl [arr= true, ctx=false, start=0]], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=null, filtersPassed=false, filtersSet=false, entry=null, prepared=0, locked=false, nodeId=null, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=null]], dhtVers=null, txSize=0, plc=2, txState=null, flags=onePhase|last, super=GridDistributedBaseMessage [ver=GridCacheVersion [topVer=139084030, order=1527604094933, nodeOrder=2], committedVers=null, rolledbackVers=null, cnt=0, super=GridCacheIdMessage [cacheId=0]]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicDeferredUpdateResponse [futIds=GridLongList [idx=2, arr=[65774,65775]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [part=1016, val=null, hasValBytes=true], parent=GridNearAtomicAbstractSingleUpdateRequest [nodeId=null, futId=49328, topVer=AffinityTopologyVersion [topVer=20, minorTopVer=0], parent=GridNearAtomicAbstractUpdateRequest [res=null, flags=needRes]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicDeferredUpdateResponse [futIds=GridLongList [idx=1, arr=[98591]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicDeferredUpdateResponse [futIds=GridLongList [idx=1, arr=[114926]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [part=1016, val=null, hasValBytes=true], parent=GridNearAtomicAbstractSingleUpdateRequest [nodeId=null, futId=32946, topVer=AffinityTopologyVersion [topVer=20, minorTopVer=0], parent=GridNear
Is there something missing from these steps to make wso2 identity server work properly?
Fresh download of wso2is-5.2.0.zip
unpacked
ran bin\wso2server.bat
browsed to https://localhost:9443/
attempted login with admin/admin
web ui frozen at "connecting to localhost"
Attempted to shutdown the server with ctrl+c at console.
Got this result:
[2016-10-19 10:37:05,427] ERROR {org.wso2.carbon.authenticator.proxy.AuthenticationAdminClient} - Error occurred while logging in
org.apache.axis2.AxisFault: Exception occurred while trying to invoke service method login
at org.apache.axis2.util.Utils.getInboundFaultFromMessageContext(Utils.java:531)
at org.apache.axis2.description.OutInAxisOperationClient.handleResponse(OutInAxisOperation.java:370)
at org.apache.axis2.description.OutInAxisOperationClient.send(OutInAxisOperation.java:445)
at org.apache.axis2.description.OutInAxisOperationClient.executeImpl(OutInAxisOperation.java:225)
at org.apache.axis2.client.OperationClient.execute(OperationClient.java:149)
at org.wso2.carbon.authenticator.stub.AuthenticationAdminStub.login(AuthenticationAdminStub.java:659)
at org.wso2.carbon.authenticator.proxy.AuthenticationAdminClient.login(AuthenticationAdminClient.java:64)
at org.wso2.carbon.ui.DefaultCarbonAuthenticator.doAuthentication(DefaultCarbonAuthenticator.java:119)
at org.wso2.carbon.ui.AbstractCarbonUIAuthenticator.handleSecurity(AbstractCarbonUIAuthenticator.java:218)
at org.wso2.carbon.ui.BasicAuthUIAuthenticator.authenticate(BasicAuthUIAuthenticator.java:83)
at org.wso2.carbon.ui.CarbonUILoginUtil.handleLogin(CarbonUILoginUtil.java:390)
at org.wso2.carbon.ui.CarbonSecuredHttpContext.handleSecurity(CarbonSecuredHttpContext.java:246)
at org.eclipse.equinox.http.servlet.internal.ServletRegistration.service(ServletRegistration.java:60)
at org.eclipse.equinox.http.servlet.internal.ProxyServlet.processAlias(ProxyServlet.java:128)
at org.eclipse.equinox.http.servlet.internal.ProxyServlet.service(ProxyServlet.java:68)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
at org.wso2.carbon.tomcat.ext.servlet.DelegationServlet.service(DelegationServlet.java:68)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:120)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.wso2.carbon.tomcat.ext.filter.CharacterSetFilter.doFilter(CharacterSetFilter.java:61)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:120)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:99)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:57)
at org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:62)
at org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:159)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
at org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:57)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:442)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1082)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:623)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1756)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1715)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
[2016-10-19 10:37:05,427] ERROR {org.wso2.carbon.authenticator.proxy.AuthenticationAdminClient} - Error occurred while logging in
org.apache.axis2.AxisFault: Exception occurred while trying to invoke service method login
at org.apache.axis2.util.Utils.getInboundFaultFromMessageContext(Utils.java:531)
at org.apache.axis2.description.OutInAxisOperationClient.handleResponse(OutInAxisOperation.java:370)
at org.apache.axis2.description.OutInAxisOperationClient.send(OutInAxisOperation.java:445)
at org.apache.axis2.description.OutInAxisOperationClient.executeImpl(OutInAxisOperation.java:225)
at org.apache.axis2.client.OperationClient.execute(OperationClient.java:149)
at org.wso2.carbon.authenticator.stub.AuthenticationAdminStub.login(AuthenticationAdminStub.java:659)
at org.wso2.carbon.authenticator.proxy.AuthenticationAdminClient.login(AuthenticationAdminClient.java:64)
at org.wso2.carbon.ui.DefaultCarbonAuthenticator.doAuthentication(DefaultCarbonAuthenticator.java:119)
at org.wso2.carbon.ui.AbstractCarbonUIAuthenticator.handleSecurity(AbstractCarbonUIAuthenticator.java:218)
at org.wso2.carbon.ui.BasicAuthUIAuthenticator.authenticate(BasicAuthUIAuthenticator.java:83)
at org.wso2.carbon.ui.CarbonUILoginUtil.handleLogin(CarbonUILoginUtil.java:390)
at org.wso2.carbon.ui.CarbonSecuredHttpContext.handleSecurity(CarbonSecuredHttpContext.java:246)
at org.eclipse.equinox.http.servlet.internal.ServletRegistration.service(ServletRegistration.java:60)
at org.eclipse.equinox.http.servlet.internal.ProxyServlet.processAlias(ProxyServlet.java:128)
at org.eclipse.equinox.http.servlet.internal.ProxyServlet.service(ProxyServlet.java:68)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
at org.wso2.carbon.tomcat.ext.servlet.DelegationServlet.service(DelegationServlet.java:68)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:120)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.wso2.carbon.tomcat.ext.filter.CharacterSetFilter.doFilter(CharacterSetFilter.java:61)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:120)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:99)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:57)
at org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:62)
at org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:159)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
at org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:57)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:442)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1082)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:623)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1756)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1715)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
[2016-10-19 10:37:05,740] INFO {org.wso2.carbon.core.services.util.CarbonAuthenticationUtil} - 'admin#carbon.super [-1234]' logged in at [2016-10-19 10:37:05,740-0400]
I have a problem with a webservices that I need to deploy on my server.
When I deploy it on my local environment on Tomcat and xercesImpl.jar I have no problem.
Tomcat 7, java 6, xercesImpl 2.9.1
The issue come up when I deploy the war on Weblogic.
I have tried to deploy without the xercesImpl.jar as I know that Weblogic has own implementation and I recive this issue:
Weblogic 12g Java 6
[2015-05-12 13:30:35,940] ERROR org.apache.axis2.engine.AxisEngine {[ACTIVE] ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)' } -
org.apache.axis2.AxisFault
at org.apache.rampart.handler.WSDoAllReceiver.processMessage(WSDoAllReceiver.java:92)
at org.apache.rampart.handler.WSDoAllHandler.invoke(WSDoAllHandler.java:72)
at org.apache.axis2.engine.Phase.invoke(Phase.java:317)
at org.apache.axis2.engine.AxisEngine.invoke(AxisEngine.java:264)
at org.apache.axis2.engine.AxisEngine.receive(AxisEngine.java:163)
at org.apache.axis2.transport.http.HTTPTransportUtils.processHTTPPostRequest(HTTPTransportUtils.java:275)
at org.apache.axis2.transport.http.AxisServlet.doPost(AxisServlet.java:133)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:751)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:844)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:280)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:254)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:136)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:341)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:238)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3366)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3333)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2220)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2146)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2124)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1564)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:254)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:295)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:254)
Caused by: java.lang.NullPointerException
at com.ciccio.ta.application.ServiceAppHelper.getSecurityService(ServiceAppHelper.java:171)
at com.ciccio.ta.application.Application.getSecurityService(Application.java:810)
at com.ciccio.ta.application.Application.getSecurityService(Application.java:794)
at com.ciccio.ta.kernel.TaContext.preProcessRequest(TaContext.java:1510)
at com.ciccio.ta.services2.security.PasswordCallBackHandlerServer.handle(PasswordCallBackHandlerServer.java:79)
at org.apache.ws.security.processor.UsernameTokenProcessor.handleUsernameToken(UsernameTokenProcessor.java:145)
at org.apache.ws.security.processor.UsernameTokenProcessor.handleToken(UsernameTokenProcessor.java:53)
at org.apache.ws.security.WSSecurityEngine.processSecurityHeader(WSSecurityEngine.java:311)
at org.apache.ws.security.WSSecurityEngine.processSecurityHeader(WSSecurityEngine.java:228)
at org.apache.rampart.handler.WSDoAllReceiver.processBasic(WSDoAllReceiver.java:211)
at org.apache.rampart.handler.WSDoAllReceiver.processMessage(WSDoAllReceiver.java:86)
... 25 more
[2015-05-12 13:30:36,303] ERROR org.apache.axis2.engine.AxisEngine {[ACTIVE] ExecuteThread: '0' for queue: 'weblogic.kernel.Default (self-tuning)' } -
org.apache.axis2.AxisFault
at org.apache.rampart.handler.WSDoAllReceiver.processMessage(WSDoAllReceiver.java:92)
at org.apache.rampart.handler.WSDoAllHandler.invoke(WSDoAllHandler.java:72)
at org.apache.axis2.engine.Phase.invoke(Phase.java:317)
at org.apache.axis2.engine.AxisEngine.invoke(AxisEngine.java:264)
at org.apache.axis2.engine.AxisEngine.receive(AxisEngine.java:163)
at org.apache.axis2.transport.http.HTTPTransportUtils.processHTTPPostRequest(HTTPTransportUtils.java:275)
at org.apache.axis2.transport.http.AxisServlet.doPost(AxisServlet.java:133)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:751)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:844)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:280)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:254)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:136)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:341)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:238)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3366)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3333)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2220)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2146)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2124)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1564)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:254)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:295)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:254)
Caused by: java.lang.NullPointerException
at com.ciccio.ta.application.ServiceAppHelper.getSecurityService(ServiceAppHelper.java:171)
at com.ciccio.ta.application.Application.getSecurityService(Application.java:810)
at com.ciccio.ta.application.Application.getSecurityService(Application.java:794)
at com.ciccio.ta.kernel.TaContext.preProcessRequest(TaContext.java:1510)
at com.ciccio.ta.services2.security.PasswordCallBackHandlerServer.handle(PasswordCallBackHandlerServer.java:79)
at org.apache.ws.security.processor.UsernameTokenProcessor.handleUsernameToken(UsernameTokenProcessor.java:145)
at org.apache.ws.security.processor.UsernameTokenProcessor.handleToken(UsernameTokenProcessor.java:53)
at org.apache.ws.security.WSSecurityEngine.processSecurityHeader(WSSecurityEngine.java:311)
at org.apache.ws.security.WSSecurityEngine.processSecurityHeader(WSSecurityEngine.java:228)
at org.apache.rampart.handler.WSDoAllReceiver.processBasic(WSDoAllReceiver.java:211)
at org.apache.rampart.handler.WSDoAllReceiver.processMessage(WSDoAllReceiver.java:86)
... 25 more
[2015-05-12 13:35:31,630] INFO startup.com.ciccio.ta.jmx.SystemInfoLogger {TASystemThread } - VMID= 4843173, 12/05/2015 13:35:31 IST, MaxMemory=932352, MaxMemoryUsedPercent=27, MemoryTotal=613376, MemoryUsed=258208, NumberBOEs=0,
If I deploy the war file with xercesImpl.jar and changing the weblogic.xml as follow:
<container-descriptor>
<servlet-reload-check-secs>-1</servlet-reload-check-secs>
<prefer-web-inf-classes>false</prefer-web-inf-classes>
<show-archived-real-path-enabled>true</show-archived-real-path-enabled>
</container-descriptor>
I receive a different error:
[2015-05-12 13:47:14,895] ERROR com.ciccio.reports.operationtypes.ReportingServicesMessageReceiverInOut {[ACTIVE] ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)' ff-sm-03, null, "null", PE:01000:0000012877, null, null, TaContext-2, 3, 0} - A marshalling exception occured for the type com.ciccio.reports.operationtypes.GetReportsResponse:
javax.xml.bind.MarshalException
- with linked exception:
[java.lang.IllegalArgumentException: Local name may not be null or empty]
at org.eclipse.persistence.jaxb.JAXBMarshaller.marshal(JAXBMarshaller.java:316)
at com.ciccio.ta.services2.validation.TaValidationHandler.validateMarshallerAgainstSchema(TaValidationHandler.java:103)
at com.ciccio.reports.operationtypes.ReportingServicesMessageReceiverInOut.toOM(ReportingServicesMessageReceiverInOut.java:1064)
at com.ciccio.reports.operationtypes.ReportingServicesMessageReceiverInOut.toEnvelope(ReportingServicesMessageReceiverInOut.java:1098)
at com.ciccio.reports.operationtypes.ReportingServicesMessageReceiverInOut.invokeBusinessLogic(ReportingServicesMessageReceiverInOut.java:157)
at org.apache.axis2.receivers.AbstractInOutMessageReceiver.invokeBusinessLogic(AbstractInOutMessageReceiver.java:40)
at org.apache.axis2.receivers.AbstractMessageReceiver.receive(AbstractMessageReceiver.java:100)
at org.apache.axis2.engine.AxisEngine.receive(AxisEngine.java:176)
at org.apache.axis2.transport.http.HTTPTransportUtils.processHTTPPostRequest(HTTPTransportUtils.java:275)
at org.apache.axis2.transport.http.AxisServlet.doPost(AxisServlet.java:133)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:751)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:844)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:280)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:254)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:136)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:341)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:238)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3366)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3333)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2220)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2146)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2124)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1564)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:254)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:295)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:254)
Caused by: java.lang.IllegalArgumentException: Local name may not be null or empty
at org.apache.axiom.om.impl.llom.OMAttributeImpl.<init>(OMAttributeImpl.java:56)
at org.apache.axiom.om.impl.llom.OMElementImpl.addAttribute(OMElementImpl.java:612)
at org.apache.axiom.om.impl.builder.SAXOMBuilder.startElement(SAXOMBuilder.java:111)
at org.eclipse.persistence.oxm.record.ContentHandlerRecord.openAndCloseStartElement(ContentHandlerRecord.java:186)
at org.eclipse.persistence.oxm.record.ContentHandlerRecord.openStartElement(ContentHandlerRecord.java:201)
at org.eclipse.persistence.oxm.record.ValidatingMarshalRecord.openStartElement(ValidatingMarshalRecord.java:170)
at org.eclipse.persistence.internal.oxm.XPathNode.startElement(XPathNode.java:395)
at org.eclipse.persistence.internal.oxm.XMLCompositeCollectionMappingNodeValue.marshalSingleValue(XMLCompositeCollectionMappingNodeValue.java:299)
at org.eclipse.persistence.internal.oxm.XMLCompositeCollectionMappingNodeValue.marshal(XMLCompositeCollectionMappingNodeValue.java:91)
at org.eclipse.persistence.internal.oxm.NodeValue.marshal(NodeValue.java:151)
at org.eclipse.persistence.internal.oxm.NodeValue.marshal(NodeValue.java:104)
at org.eclipse.persistence.internal.oxm.record.ObjectMarshalContext.marshal(ObjectMarshalContext.java:60)
at org.eclipse.persistence.internal.oxm.XPathNode.marshal(XPathNode.java:386)
at org.eclipse.persistence.internal.oxm.TreeObjectBuilder.buildRow(TreeObjectBuilder.java:488)
at org.eclipse.persistence.oxm.XMLMarshaller.marshal(XMLMarshaller.java:1240)
at org.eclipse.persistence.oxm.XMLMarshaller.marshal(XMLMarshaller.java:833)
at org.eclipse.persistence.oxm.XMLMarshaller.marshal(XMLMarshaller.java:788)
at org.eclipse.persistence.jaxb.JAXBMarshaller.marshal(JAXBMarshaller.java:314)
... 28 more
[2015-05-12 13:47:14,899] ERROR remote.com.ciccio.reports.operationtypes.ReportingServicesMessageReceiverInOut {[ACTIVE] ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)' ff-sm-03, null, "null", PE:01000:0000012877, null, null, TaContext-2, 3, 0} - A marshalling exception occured for the type com.ciccio.reports.operationtypes.GetReportsResponse:
javax.xml.bind.MarshalException
- with linked exception:
[java.lang.IllegalArgumentException: Local name may not be null or empty]
at org.eclipse.persistence.jaxb.JAXBMarshaller.marshal(JAXBMarshaller.java:316)
at com.ciccio.ta.services2.validation.TaValidationHandler.validateMarshallerAgainstSchema(TaValidationHandler.java:103)
at com.ciccio.reports.operationtypes.ReportingServicesMessageReceiverInOut.toOM(ReportingServicesMessageReceiverInOut.java:1064)
at com.ciccio.reports.operationtypes.ReportingServicesMessageReceiverInOut.toEnvelope(ReportingServicesMessageReceiverInOut.java:1098)
at com.ciccio.reports.operationtypes.ReportingServicesMessageReceiverInOut.invokeBusinessLogic(ReportingServicesMessageReceiverInOut.java:157)
at org.apache.axis2.receivers.AbstractInOutMessageReceiver.invokeBusinessLogic(AbstractInOutMessageReceiver.java:40)
at org.apache.axis2.receivers.AbstractMessageReceiver.receive(AbstractMessageReceiver.java:100)
at org.apache.axis2.engine.AxisEngine.receive(AxisEngine.java:176)
at org.apache.axis2.transport.http.HTTPTransportUtils.processHTTPPostRequest(HTTPTransportUtils.java:275)
at org.apache.axis2.transport.http.AxisServlet.doPost(AxisServlet.java:133)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:751)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:844)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:280)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:254)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:136)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:341)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:238)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3366)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3333)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2220)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2146)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2124)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1564)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:254)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:295)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:254)
Caused by: java.lang.IllegalArgumentException: Local name may not be null or empty
at org.apache.axiom.om.impl.llom.OMAttributeImpl.<init>(OMAttributeImpl.java:56)
at org.apache.axiom.om.impl.llom.OMElementImpl.addAttribute(OMElementImpl.java:612)
at org.apache.axiom.om.impl.builder.SAXOMBuilder.startElement(SAXOMBuilder.java:111)
at org.eclipse.persistence.oxm.record.ContentHandlerRecord.openAndCloseStartElement(ContentHandlerRecord.java:186)
at org.eclipse.persistence.oxm.record.ContentHandlerRecord.openStartElement(ContentHandlerRecord.java:201)
at org.eclipse.persistence.oxm.record.ValidatingMarshalRecord.openStartElement(ValidatingMarshalRecord.java:170)
at org.eclipse.persistence.internal.oxm.XPathNode.startElement(XPathNode.java:395)
at org.eclipse.persistence.internal.oxm.XMLCompositeCollectionMappingNodeValue.marshalSingleValue(XMLCompositeCollectionMappingNodeValue.java:299)
at org.eclipse.persistence.internal.oxm.XMLCompositeCollectionMappingNodeValue.marshal(XMLCompositeCollectionMappingNodeValue.java:91)
at org.eclipse.persistence.internal.oxm.NodeValue.marshal(NodeValue.java:151)
at org.eclipse.persistence.internal.oxm.NodeValue.marshal(NodeValue.java:104)
at org.eclipse.persistence.internal.oxm.record.ObjectMarshalContext.marshal(ObjectMarshalContext.java:60)
at org.eclipse.persistence.internal.oxm.XPathNode.marshal(XPathNode.java:386)
at org.eclipse.persistence.internal.oxm.TreeObjectBuilder.buildRow(TreeObjectBuilder.java:488)
at org.eclipse.persistence.oxm.XMLMarshaller.marshal(XMLMarshaller.java:1240)
at org.eclipse.persistence.oxm.XMLMarshaller.marshal(XMLMarshaller.java:833)
at org.eclipse.persistence.oxm.XMLMarshaller.marshal(XMLMarshaller.java:788)
at org.eclipse.persistence.jaxb.JAXBMarshaller.marshal(JAXBMarshaller.java:314)
... 28 more
[2015-05-12 13:47:14,930] ERROR org.apache.axis2.engine.AxisEngine {[ACTIVE] ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)' ff-sm-03, null, "null", PE:01000:0000012877, null, null, TaContext-2, 3, 0} - ValidationException
org.apache.axis2.AxisFault: ValidationException
at com.ciccio.reports.operationtypes.ReportingServicesMessageReceiverInOut.createAxisFault(ReportingServicesMessageReceiverInOut.java:1377)
at com.ciccio.reports.operationtypes.ReportingServicesMessageReceiverInOut.invokeBusinessLogic(ReportingServicesMessageReceiverInOut.java:187)
at org.apache.axis2.receivers.AbstractInOutMessageReceiver.invokeBusinessLogic(AbstractInOutMessageReceiver.java:40)
at org.apache.axis2.receivers.AbstractMessageReceiver.receive(AbstractMessageReceiver.java:100)
at org.apache.axis2.engine.AxisEngine.receive(AxisEngine.java:176)
at org.apache.axis2.transport.http.HTTPTransportUtils.processHTTPPostRequest(HTTPTransportUtils.java:275)
at org.apache.axis2.transport.http.AxisServlet.doPost(AxisServlet.java:133)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:751)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:844)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:280)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:254)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:136)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:341)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:238)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3366)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3333)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2220)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2146)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2124)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1564)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:254)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:295)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:254)
Any help???
Thanks