Transform xyz data into surface for Plotly - numpy

I have a text file with xyz data called elevation.txt.
29.6615067663688 -98.3933654143067 -0.772875302700243
29.6615067663688 -98.3933757636723 -0.71250410914962
29.6615065076668 -98.3933654143067 -0.757068728178126
29.6615032793597 -98.3933757636723 -0.761051993140882
29.6615067663688 -98.3933647490732 -0.762067716839908
29.6615067663688 -98.3933861130379 -0.6694827930163
29.6615039261939 -98.3933861130379 -0.700458102119342
29.6615067663688 -98.3933964624035 -0.616901857503478
29.661506592754 -98.3933964624035 -0.616901857503478
29.6615067663688 -98.3933969253778 -0.624557146812265
29.6615157595849 -98.3933447155754 -0.765732910338727
29.6615157595849 -98.393355064941 -0.70790301178198
29.6615151056918 -98.3933447155754 -0.764905434184072
29.66151057784 -98.393355064941 -0.775122479690196
29.6615157595849 -98.3933432209641 -0.769340655884378
29.6615157595849 -98.3933654143067 -0.65363454400682
29.6615157595849 -98.3933757636723 -0.607481276569502
29.6615157595849 -98.3933861130379 -0.576043135792759
29.6615157595849 -98.3933964624035 -0.562651209935325
29.6615157595849 -98.3934068117692 -0.574109217270234
29.6615106655842 -98.3934068117692 -0.614824188539728
29.6615157595849 -98.3934151474394 -0.590072189764953
29.6615247528009 -98.3933240168442 -0.716696572671196
29.6615247528009 -98.3933343662098 -0.686067695192037
29.661524433825 -98.3933240168442 -0.716696572671196
29.6615197295622 -98.3933343662098 -0.756041264529586
29.6615247528009 -98.393323315098 -0.727161408607287
29.6615247528009 -98.3933447155754 -0.638302943876681
29.6615247528009 -98.393355064941 -0.595028363722974
29.6615247528009 -98.3933654143067 -0.556618694163944
29.6615247528009 -98.3933757636723 -0.524177330077976
29.6615247528009 -98.3933861130379 -0.502490718796878
29.6615247528009 -98.3933964624035 -0.493857409456894
29.6615247528009 -98.3934068117692 -0.504201213967065
29.6615247528009 -98.3934171611348 -0.537894228748428
29.6615169901716 -98.3934171611348 -0.583259938640822
29.6615247528009 -98.3934272247451 -0.553953638867654
29.661533746017 -98.3933136674785 -0.65337903725532
29.661533746017 -98.3933240168442 -0.606100909015026
29.6615302013555 -98.3933136674785 -0.708003376811007
29.661533746017 -98.3933086036904 -0.674403153398373
29.661533746017 -98.3933343662098 -0.567366118147447
29.661533746017 -98.3933447155754 -0.534873445625749
29.661533746017 -98.393355064941 -0.508345134300043
29.661533746017 -98.3933654143067 -0.4861347257239
29.661533746017 -98.3933757636723 -0.467501455070533
29.661533746017 -98.3933861130379 -0.454825455237369
29.661533746017 -98.3933964624035 -0.450999899418036
29.661533746017 -98.3934068117692 -0.460597462045143
29.661533746017 -98.3934171611348 -0.485507380618685
29.661533746017 -98.3934275105004 -0.527974722804913
29.661525070305 -98.3934275105004 -0.556840681625602
29.661533746017 -98.3934344310752 -0.562287963820779
29.6615427392331 -98.3933033181129 -0.573540650163409
29.6615427392331 -98.3933136674785 -0.534670699160301
29.6615374459315 -98.3933033181129 -0.636710444804976
29.6615427392331 -98.393296254736 -0.60161550624673
29.6615427392331 -98.3933240168442 -0.504091160067674
29.6615427392331 -98.3933343662098 -0.481685184127628
29.6615427392331 -98.3933447155754 -0.464099499361851
29.6615427392331 -98.393355064941 -0.451925941678382
29.6615427392331 -98.3933654143067 -0.442387452656249
29.6615427392331 -98.3933757636723 -0.434505319003051
29.6615427392331 -98.3933861130379 -0.43005283413379
29.6615427392331 -98.3933964624035 -0.430975358433614
29.6615427392331 -98.3934068117692 -0.439570957852027
29.6615427392331 -98.3934171611348 -0.457793169386434
29.6615427392331 -98.3934275105004 -0.486800614061849
29.6615427392331 -98.393437859866 -0.526190392944102
29.6615399177873 -98.393437859866 -0.537130143062313
29.6615427392331 -98.3934387004372 -0.514852475111695
29.6615517324491 -98.3932929687473 -0.505021052994948
29.6615517324491 -98.3933033181129 -0.473109594469918
29.6615456966351 -98.3932929687473 -0.575492082966775
29.6615517324491 -98.3932864360227 -0.521250369085692
29.6615517324491 -98.3933136674785 -0.45228220458921
29.6615517324491 -98.3933240168442 -0.437672906029206
29.6615517324491 -98.3933343662098 -0.428764266490191
29.6615517324491 -98.3933447155754 -0.423388509125061
29.6615517324491 -98.393355064941 -0.42064725625545
29.6615517324491 -98.3933654143067 -0.42059184404286
29.6615517324491 -98.3933757636723 -0.421243310930151
29.6615517324491 -98.3933861130379 -0.423173771259656
29.6615517324491 -98.3933964624035 -0.428077085966038
29.6615517324491 -98.3934068117692 -0.436455120482265
29.6615517324491 -98.3934171611348 -0.449453737366639
29.6615517324491 -98.3934275105004 -0.467176208307023
29.6615517324491 -98.393437859866 -0.491158847783368
29.6615517324491 -98.3934410558634 -0.497183097610673
29.6615607256652 -98.3932826193817 -0.448472127921775
29.6615607256652 -98.3932929687473 -0.423026080899604
29.6615555491089 -98.3932826193817 -0.489444421659471
29.6615607256652 -98.3932774428508 -0.459300103364114
29.6615607256652 -98.3933033181129 -0.41007264067008
29.6615607256652 -98.3933136674785 -0.403379653580657
29.6615607256652 -98.3933240168442 -0.40160597586771
29.6615607256652 -98.3933343662098 -0.402303299508787
29.6615607256652 -98.3933447155754 -0.405026619865623
29.6615607256652 -98.393355064941 -0.409214051179902
29.6615607256652 -98.3933654143067 -0.415145859438724
29.6615607256652 -98.3933757636723 -0.421400604537439
29.6615607256652 -98.3933861130379 -0.428143788263469
29.6615607256652 -98.3933964624035 -0.436987817589795
29.6615607256652 -98.3934068117692 -0.44512400629128
29.6615607256652 -98.3934171611348 -0.454197381772929
29.6615607256652 -98.3934275105004 -0.463704639044295
29.6615607256652 -98.393437859866 -0.475074798026128
29.6615607256652 -98.3934419552231 -0.47966411974312
29.6615697188812 -98.393272270016 -0.406872175697117
29.6615697188812 -98.3932826193817 -0.381486406072099
29.6615665050732 -98.393272270016 -0.423858240303332
29.6615697188812 -98.3932696405537 -0.408051452734071
29.6615697188812 -98.3932929687473 -0.37507324528184
29.6615697188812 -98.3933033181129 -0.376443590970155
29.6615697188812 -98.3933136674785 -0.381102967001596
29.6615697188812 -98.3933240168442 -0.387274655391025
29.6615697188812 -98.3933343662098 -0.39395895958836
29.6615697188812 -98.3933447155754 -0.402001773351628
29.6615697188812 -98.393355064941 -0.410521732197196
29.6615697188812 -98.3933654143067 -0.42095882320541
29.6615697188812 -98.3933757636723 -0.430973881383409
29.6615697188812 -98.3933861130379 -0.441647395592099
29.6615697188812 -98.3933964624035 -0.452005925774586
29.6615697188812 -98.3934068117692 -0.461284006552319
29.6615697188812 -98.3934171611348 -0.467917848494494
29.6615697188812 -98.3934275105004 -0.471415514054002
29.6615697188812 -98.393437859866 -0.473245094507774
29.6615697188812 -98.3934419996021 -0.472659420054981
29.6615787120973 -98.393272270016 -0.355651132143748
29.6615787120973 -98.3932826193817 -0.347239308692565
29.6615787120973 -98.3932632737198 -0.35591381410751
29.6615787120973 -98.3932929687473 -0.35162619014784
29.6615787120973 -98.3933033181129 -0.361200636616275
29.6615787120973 -98.3933136674785 -0.371593174544898
29.6615787120973 -98.3933240168442 -0.382765752356477
29.6615787120973 -98.3933343662098 -0.395095946884567
29.6615787120973 -98.3933447155754 -0.407420962025753
29.6615787120973 -98.393355064941 -0.420616003396162
29.6615787120973 -98.3933654143067 -0.434115117961653
29.6615787120973 -98.3933757636723 -0.447571228535537
29.6615787120973 -98.3933861130379 -0.460083111502854
29.6615787120973 -98.3933964624035 -0.471282501279632
29.6615787120973 -98.3934068117692 -0.479825008942328
29.6615787120973 -98.3934171611348 -0.483889960099589
29.6615787120973 -98.3934275105004 -0.483803486838212
29.6615787120973 -98.393437859866 -0.479388198273796
29.6615787120973 -98.3934408449891 -0.478342909984406
29.6615877053134 -98.3932619206504 -0.352953163198265
29.6615877053134 -98.393272270016 -0.330661192674995
29.661580967234 -98.3932619206504 -0.367248936850558
I read the data using
elevation = np.loadtxt('elevation.txt')
x, y, z = elevation[:, 0], elevation[:, 1], elevation[:, 2]
Because each z point is an unique combination of the corresponding x and y point, they all have the same length, 150.
I want to post this surface using plotly, not matplotlib.
Their documentation says the following:
The data the describes the coordinates of the surface is set in z.
Data in z should be a 2D list. Coordinates in x and y can either be 1D
lists or 2D lists (e.g. to graph parametric surfaces).
However, the shape of z is equal to
(150, )
It needs to be equal to
(150, 150)
How can I achieve this?


The Surface class expects the points to be spaced linearly across both x and y (like those that would be created by np.linspace, examples here). It appears that your data is a set of 3d points which are not linearly spaced out in a grid as Surface expects.
Perhaps a better solution would be to build a 3d scatter plot, or a 3d mesh.

Related

Ignite Cache operations get stuck when multiple thin clients try to perform CRUD operations in parallel with partition map exchange

Looks like partition map exchange process and cache CRUD operations in parallel are causing deadlock or lock acquire failures.
What should be the strategy to handle this scenario ?
Ignite server has below errors:
Exception stack trace 1:
WARNING: Dumping the near node thread that started transaction [xidVer=GridCacheVersion [topVer=247332659, order=1635852705217, nodeOrder=1], nodeId=2735bef0-7404-41e3-843f-7043490c9d84]
Stack trace of the transaction owner thread:
Thread [name="client-connector-#56%perf-dn1%", id=93, state=WAITING, blockCnt=5023, waitCnt=36165]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:304)
at o.a.i.i.util.future.GridFutureAdapter.get0(GridFutureAdapter.java:178)
at o.a.i.i.util.future.GridFutureAdapter.get(GridFutureAdapter.java:141)
at o.a.i.i.processors.cache.GridCacheAdapter$41.op(GridCacheAdapter.java:3430)
at o.a.i.i.processors.cache.GridCacheAdapter$41.op(GridCacheAdapter.java:3423)
at o.a.i.i.processors.cache.GridCacheAdapter.syncOp(GridCacheAdapter.java:4480)
at o.a.i.i.processors.cache.GridCacheAdapter.remove0(GridCacheAdapter.java:3423)
at o.a.i.i.processors.cache.GridCacheAdapter.remove(GridCacheAdapter.java:3405)
at o.a.i.i.processors.cache.GridCacheAdapter.remove(GridCacheAdapter.java:3388)
at o.a.i.i.processors.cache.IgniteCacheProxyImpl.remove(IgniteCacheProxyImpl.java:1438)
at o.a.i.i.processors.cache.GatewayProtectedCacheProxy.remove(GatewayProtectedCacheProxy.java:964)
at o.a.i.i.processors.platform.client.cache.ClientCacheRemoveKeyRequest.process(ClientCacheRemoveKeyRequest.java:41)
at o.a.i.i.processors.platform.client.ClientRequestHandler.handle(ClientRequestHandler.java:77)
at o.a.i.i.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:204)
at o.a.i.i.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:55)
at o.a.i.i.util.nio.GridNioFilterChain$TailFilter.onMessageReceived(GridNioFilterChain.java:279)
at o.a.i.i.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
at o.a.i.i.util.nio.GridNioAsyncNotifyFilter$3.body(GridNioAsyncNotifyFilter.java:97)
at o.a.i.i.util.worker.GridWorker.run(GridWorker.java:120)
at o.a.i.i.util.worker.GridWorkerPool$1.run(GridWorkerPool.java:70)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Exception stack trace 2:
WARNING: >>> Transaction [startTime=11:39:27.214, curTime=11:40:36.277, systemTime=0, userTime=69063, tx=GridNearTxLocal [mappings=IgniteTxMappingsImpl [], nearLocallyMapped=false, colocatedLocallyMapped=false, needCheckBackup=null, hasRemoteLocks=false, trackTimeout=false, systemTime=44700, systemStartTime=0, prepareStartTime=0, prepareTime=0, commitOrRollbackStartTime=0, commitOrRollbackTime=0, lb=null, mvccOp=null, qryId=-1, crdVer=0, thread=client-connector-#57%perf-dn1%, mappings=IgniteTxMappingsImpl [], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, span=o.a.i.i.processors.tracing.NoopSpan#4a931268, nearNodes=KeySetView [], dhtNodes=KeySetView [], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxStateImpl [activeCacheIds=[], recovery=null, mvccEnabled=null, mvccCachingCacheIds=[], txMap=EmptySet []], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705226, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=95, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=SUSPENDED, timedOut=false, topVer=AffinityTopologyVersion [topVer=-1, minorTopVer=0], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69079ms, onePhaseCommit=false], size=0]]]]
Nov 2, 2021 11:40:36 AM org.apache.ignite.logger.java.JavaLogger warning
WARNING: First 10 long running cache futures [total=16]
Nov 2, 2021 11:40:36 AM org.apache.ignite.logger.java.JavaLogger warning
WARNING: >>> Future [startTime=11:39:27.324, curTime=11:40:36.277, fut=GridDhtLockFuture [span=o.a.i.i.processors.tracing.NoopSpan#4a931268, nearNodeId=2735bef0-7404-41e3-843f-7043490c9d84, nearLockVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], threadId=124, futId=be58a60ec71-1d64903c-c700-4deb-bace-cc5158713120, lockVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], read=false, err=null, timedOut=false, timeout=0, tx=GridNearTxLocal [mappings=IgniteTxMappingsImpl []dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]], nearLocallyMapped=false, colocatedLocallyMapped=true, needCheckBackup=null, hasRemoteLocks=false, trackTimeout=false, systemTime=75000, systemStartTime=971108549857700, prepareStartTime=0, prepareTime=0, commitOrRollbackStartTime=0, commitOrRollbackTime=0, lb=null, mvccOp=null, qryId=-1, crdVer=0, thread=client-connector-#84%perf-dn1%, mappings=IgniteTxMappingsImpl []dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, span=o.a.i.i.processors.tracing.NoopSpan#4a931268, nearNodes=KeySetView [], dhtNodes=KeySetView [], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxStateImpl [activeCacheIds=[585748697], recovery=false, mvccEnabled=false, mvccCachingCacheIds=[], txMap=ArrayList [IgniteTxEntry [txKey=IgniteTxKey [key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], cacheId=585748697], val=TxEntryValueHolder [val=null, op=DELETE], prevVal=TxEntryValueHolder [val=null, op=NOOP], oldVal=TxEntryValueHolder [val=null, op=NOOP], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=CacheEntryPredicate[] [], filtersPassed=false, filtersSet=true, entry=GridDhtCacheEntry [rdrs=ReaderId[] [], part=244, super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], val=null, ver=GridCacheVersion [topVer=247332659, order=1635852705229, nodeOrder=1], hash=1085684290, extras=GridCacheMvccEntryExtras [mvcc=GridCacheMvcc [locs=LinkedList [GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705207, nodeOrder=1], threadId=122, id=2104, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705207, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=1|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], threadId=124, id=2102, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705213, nodeOrder=1], threadId=122, id=2120, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705213, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705207, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705214, nodeOrder=1], threadId=123, id=2118, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705214, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705207, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705217, nodeOrder=1], threadId=93, id=2108, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705217, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705218, nodeOrder=1], threadId=115, id=2106, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705218, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705222, nodeOrder=1], threadId=95, id=2110, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705222, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705223, nodeOrder=1], threadId=120, id=2112, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705223, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null], GridCacheMvccCandidate [nodeId=2735bef0-7404-41e3-843f-7043490c9d84, ver=GridCacheVersion [topVer=247332659, order=1635852705227, nodeOrder=1], threadId=118, id=2114, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], reentry=null, otherNodeId=2735bef0-7404-41e3-843f-7043490c9d84, otherVer=GridCacheVersion [topVer=247332659, order=1635852705227, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]]]dNearNodes=null, ownerVer=GridCacheVersion [topVer=247332659, order=1635852705211, nodeOrder=1], serOrder=null, key=KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true], masks=local=1|owner=0|ready=1|reentry=0|used=0|tx=1|single_implicit=0|dht_local=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=3]]], prepared=0, locked=false, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=2, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1]]]], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=247332659, order=1635852705208, nodeOrder=1], writeVer=null, implicit=false, loc=true, threadId=124, startTime=1635853167214, nodeId=2735bef0-7404-41e3-843f-7043490c9d84, isolation=REPEATABLE_READ, concurrency=PESSIMISTIC, timeout=0, sysInvalidate=false, sys=false, plc=2, commitVer=null, finalizing=NONE, invalidParts=null, state=ACTIVE, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=162], mvccSnapshot=null, skipCompletedVers=false, parentTx=null, duration=69094ms, onePhaseCommit=false], size=1]]], mapped=false, trackable=true, createTtl=-1, accessTtl=-1, needReturnVal=false, skipStore=false, keepBinary=true, innerFuts=EmptyList [], pendingLocks=HashSet [KeyCacheObjectImpl [part=244, val=data=6ff0c60ec71-625345be-9a91-497a-895e-abbe5df9da3d], hasValBytes=true]], super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=0, lsnrCalls=0, done=false, cancelled=false, err=null, futs=EmptyList []]]]]
Nov 2, 2021 11:40:36 AM org.apache.ignite.logger.java.JavaLogger warning
Exception stack trace 3 when TxTimeoutOnPartitionMapExchange is set to some value
Nov 2, 2021 1:11:19 PM org.apache.ignite.logger.java.JavaLogger warning
WARNING: The transaction was forcibly rolled back on partition map exchange because a timeout is reached: [tx=GridNearTxLocal[xid=e0e3db0ec71-00000000-0ebe-146c-0000-000000000001, xidVersion=GridCacheVersion [topVer=247338092, order=1635858071054, nodeOrder=1], nearXidVersion=GridCacheVersion [topVer=247338092, order=1635858071054, nodeOrder=1], concurrency=PESSIMISTIC, isolation=REPEATABLE_READ, state=ACTIVE, invalidate=false, rollbackOnly=false, nodeId=1ed7077a-5547-41c1-94ab-d246944ed4a9, timeout=0, startTime=1635858659324, duration=20140, label=null], topVer=AffinityTopologyVersion [topVer=1, minorTopVer=104]]
Nov 2, 2021 1:11:19 PM org.apache.ignite.logger.java.JavaLogger error
SEVERE: Failed to process client request [req=o.a.i.i.processors.platform.client.cache.ClientCacheRemoveKeyRequest#2fea4a3]
javax.cache.CacheException: class org.apache.ignite.transactions.TransactionRollbackException: Failed to finish transaction because it has been rolled back [timeout=0, tx=GridNearTxLocal[xid=f0e3db0ec71-00000000-0ebe-146c-0000-000000000001, xidVersion=GridCacheVersion [topVer=247338092, order=1635858071055, nodeOrder=1], nearXidVersion=GridCacheVersion [topVer=247338092, order=1635858071055, nodeOrder=1], concurrency=PESSIMISTIC, isolation=REPEATABLE_READ, state=ACTIVE, invalidate=false, rollbackOnly=false, nodeId=1ed7077a-5547-41c1-94ab-d246944ed4a9, timeout=0, startTime=1635858659324, duration=20140, label=null]]
at org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1266)
at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.cacheException(IgniteCacheProxyImpl.java:2084)
at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.remove(IgniteCacheProxyImpl.java:1441)
at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.remove(GatewayProtectedCacheProxy.java:964)
at org.apache.ignite.internal.processors.platform.client.cache.ClientCacheRemoveKeyRequest.process(ClientCacheRemoveKeyRequest.java:41)
at org.apache.ignite.internal.processors.platform.client.ClientRequestHandler.handle(ClientRequestHandler.java:77)
at org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:204)
at org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.onMessage(ClientListenerNioListener.java:55)
at org.apache.ignite.internal.util.nio.GridNioFilterChain$TailFilter.onMessageReceived(GridNioFilterChain.java:279)
at org.apache.ignite.internal.util.nio.GridNioFilterAdapter.proceedMessageReceived(GridNioFilterAdapter.java:109)
at org.apache.ignite.internal.util.nio.GridNioAsyncNotifyFilter$3.body(GridNioAsyncNotifyFilter.java:97)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)
at org.apache.ignite.internal.util.worker.GridWorkerPool$1.run(GridWorkerPool.java:70)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: class org.apache.ignite.transactions.TransactionRollbackException: Failed to finish transaction because it has been rolled back [timeout=0, tx=GridNearTxLocal[xid=f0e3db0ec71-00000000-0ebe-146c-0000-000000000001, xidVersion=GridCacheVersion [topVer=247338092, order=1635858071055, nodeOrder=1], nearXidVersion=GridCacheVersion [topVer=247338092, order=1635858071055, nodeOrder=1], concurrency=PESSIMISTIC, isolation=REPEATABLE_READ, state=ACTIVE, invalidate=false, rollbackOnly=false, nodeId=1ed7077a-5547-41c1-94ab-d246944ed4a9, timeout=0, startTime=1635858659324, duration=20140, label=null]]
at org.apache.ignite.internal.util.IgniteUtils$11.apply(IgniteUtils.java:974)
at org.apache.ignite.internal.util.IgniteUtils$11.apply(IgniteUtils.java:972)
... 16 more

This is the expected behavior:
When the partition map exchange starts, Ignite acquires a global lock
at a particular stage. The lock can’t be obtained while incomplete
transactions are running in parallel. These transactions prevent the
partition map exchange process from moving forward​, thus, blocking
some operations such as a new node join process.
In other words, PME process should wait until the end of all active transactions to acquire a global lock. Therefore it's recommended to:
either set a global TX timeout (TransactionConfiguration#DefaultTxTimeout) or configure it individually for every transaction, i.e. make sure that you are not running it forever in a worst-case scenario.
configure TxTimeoutOnPartitionMapExchange to be non zero (default).
The concrete numbers are individual, but I suppose mostly you'd like to have a lower timeout on PME in comparison to a "normal" TX timeout for quicker cluster-wide updates.

pandas read csv does not separating columns, too many columns?

I have tried many ways, but it still did not work
below is the data of the first row, how can I seperate them? is it because the number of column exceeds the limit?
1, Car, 48.85, 9.770344, 37.977391, 23.737688, 4.9178, 0.0518, -0.0299, 0.000000, 37.977391, 23.737688, 4.9207, -0.0124, -0.0354, 0.040000, 37.977391, 23.737688, 4.9160, -0.0519, -0.0413, 0.080000, 37.977390, 23.737688, 4.9057, -0.0914, -0.0478, 0.120000, 37.977390, 23.737689, 4.8871, -0.1679, -0.0550, 0.160000, 37.977389, 23.737689, 4.8547, -0.2815, -0.0628, 0.200000, 37.977389, 23.737689, 4.8033, -0.4320, -0.0713, 0.240000, 37.977388, 23.737690, 4.7318, -0.5617, -0.0804, 0.280000, 37.977388, 23.737690, 4.6442, -0.6543, -0.0905, 0.320000, 37.977388, 23.737690, 4.5437, -0.7413, -0.1017, 0.360000, 37.977387, 23.737690, 4.4311, -0.8227, -0.1143, 0.400000, 37.977387, 23.737691, 4.3046, -0.9354, -0.1285, 0.440000, 37.977386, 23.737691, 4.1600, -1.0729, -0.1439, 0.480000, 37.977386, 23.737691, 4.4411, -0.5617, -0.0949, 0.520000, 37.977385, 23.737692, 4.3561, -0.6183, -0.1044, 0.560000, 37.977385, 23.737692, 4.2639, -0.6621, -0.1133, 0.600000, 37.977384, 23.737692, 4.1654, -0.7059, -0.1219, 0.640000, 37.977384, 23.737692, 4.0595, -0.7660, -0.1298, 0.680000, 37.977384, 23.737693, 3.9439, -0.8388, -0.1369, 0.720000, 37.977383, 23.737693, 3.8183, -0.9065, -0.1430, 0.760000, 37.977383, 23.737693, 3.6842, -0.9561, -0.1484, 0.800000, 37.977383, 23.737693, 3.5447, -0.9812, -0.1530, 0.840000, 37.977382, 23.737694, 3.4018, -1.0035, -0.1567, 0.880000, 37.977382, 23.737694, 3.2559, -1.0228, -0.1593, 0.920000, 37.977382, 23.737694, 3.1076, -1.0370, -0.1610, 0.960000, 37.977381, 23.737694, 2.9576, -1.0458, -0.1618, 1.000000, 37.977381, 23.737695, 2.8081, -1.0303, -0.1618, 1.040000, 37.977381, 23.737695, 2.6611, -1.0118, -0.1604, 1.080000, 37.977381, 23.737695, 2.5170, -0.9904, -0.1573, 1.120000, 37.977381, 23.737695, 2.3763, -0.9638, -0.1525, 1.160000, 37.977380, 23.737695, 2.2398, -0.9320, -0.1460, 1.200000, 37.977380, 23.737695, 2.1096, -0.8757, -0.1377, 1.240000, 37.977380, 23.737695, 1.9883, -0.8090, -0.1276, 1.280000, 37.977380, 23.737696, 1.8768, -0.7394, -0.1154, 1.320000, 37.977380, 23.737696, 1.7751, -0.6739, -0.1014, 1.360000, 37.977380, 23.737696, 1.6823, -0.6145, -0.0855, 1.400000, 37.977380, 23.737696, 1.5988, -0.5450, -0.0675, 1.440000, 37.977380, 23.737696, 1.5264, -0.4611, -0.0477, 1.480000, 37.977380, 23.737696, 1.4666, -0.3696, -0.0283, 1.520000, 37.977380, 23.737696, 1.4197, -0.2816, -0.0099, 1.560000, 37.977379, 23.737696, 1.3844, -0.2080, 0.0065, 1.600000, 37.977379, 23.737696, 1.3586, -0.1507, 0.0209, 1.640000, 37.977379, 23.737696, 1.3406, -0.0995, 0.0337, 1.680000, 37.977379, 23.737696, 1.3298, -0.0502, 0.0457, 1.720000, 37.977379, 23.737697, 1.3266, 0.0054, 0.0567, 1.760000, 37.977379, 23.737697, 1.3316, 0.0637, 0.0666, 1.800000, 37.977379, 23.737697, 1.3438, 0.1067, 0.0751, 1.840000, 37.977379, 23.737697, 1.3615, 0.1384, 0.0826, 1.880000, 37.977379, 23.737697, 1.3829, 0.1588, 0.0891, 1.920000, 37.977379, 23.737697, 1.4077, 0.1854, 0.0946, 1.960000, 37.977379, 23.737697, 1.4367, 0.2183, 0.0989, 2.000000, 37.977378, 23.737697, 1.4694, 0.2359, 0.1020, 2.040000, 37.977378, 23.737697, 1.5038, 0.2422, 0.1042, 2.080000, 37.977378, 23.737697, 1.5384, 0.2371, 0.1055, 2.120000, 37.977378, 23.737698, 1.5726, 0.2383, 0.1053, 2.160000, 37.977378, 23.737698, 1.6074, 0.2457, 0.1033, 2.200000, 37.977378, 23.737698, 1.6422, 0.2379, 0.0995, 2.240000, 37.977378, 23.737698, 1.6752, 0.2193, 0.0940, 2.280000, 37.977377, 23.737698, 1.7046, 0.1895, 0.0867, 2.320000, 37.977377, 23.737698, 1.7296, 0.1574, 0.0773, 2.360000, 37.977377, 23.737698, 1.7503, 0.1307, 0.0657, 2.400000, 37.977377, 23.737698, 1.7669, 0.0993, 0.0521, 2.440000, 37.977377, 23.737698, 1.7787, 0.0652, 0.0367, 2.480000, 37.977377, 23.737699, 1.7857, 0.0317, 0.0203, 2.520000, 37.977376, 23.737699, 1.7876, -0.0060, 0.0035, 2.560000, 37.977376, 23.737699, 1.7842, -0.0411, -0.0115, 2.600000, 37.977376, 23.737699, 1.7765, -0.0660, -0.0241, 2.640000, 37.977376, 23.737699, 1.7656, -0.0855, -0.0344, 2.680000, 37.977376, 23.737699, 1.7524, -0.0975, -0.0423, 2.720000, 37.977376, 23.737699, 1.7377, -0.1062, -0.0481, 2.760000, 37.977375, 23.737699, 1.7217, -0.1158, -0.0525, 2.800000, 37.977375, 23.737699, 1.7049, -0.1182, -0.0557, 2.840000, 37.977375, 23.737700, 1.6876, -0.1215, -0.0576, 2.880000, 37.977375, 23.737700, 1.6698, -0.1257, -0.0582, 2.920000, 37.977375, 23.737700, 1.6517, -0.1266, -0.0577, 2.960000, 37.977375, 23.737700, 1.6336, -0.1243, -0.0561, 3.000000, 37.977375, 23.737700, 1.6164, -0.1147, -0.0534, 3.040000, 37.977374, 23.737700, 1.6005, -0.1061, -0.0498, 3.080000, 37.977374, 23.737700, 1.5858, -0.0983, -0.0453, 3.120000, 37.977374, 23.737700, 1.5724, -0.0873, -0.0402, 3.160000, 37.977374, 23.737700, 1.5609, -0.0730, -0.0342, 3.200000, 37.977374, 23.737700, 1.5519, -0.0515, -0.0275, 3.240000, 37.977374, 23.737701, 1.5460, -0.0307, -0.0199, 3.280000, 37.977374, 23.737701, 1.5430, -0.0108, -0.0118, 3.320000, 37.977374, 23.737701, 1.5430, 0.0115, -0.0031, 3.360000, 37.977373, 23.737701, 1.5465, 0.0368, 0.0062, 3.400000, 37.977373, 23.737701, 1.5542, 0.0698, 0.0164, 3.440000, 37.977373, 23.737701, 1.5670, 0.1081, 0.0272, 3.480000, 37.977373, 23.737701, 1.5853, 0.1467, 0.0382, 3.520000, 37.977373, 23.737701, 1.6091, 0.1830, 0.0491, 3.560000, 37.977373, 23.737701, 1.6377, 0.2140, 0.0593, 3.600000, 37.977373, 23.737701, 1.6704, 0.2403, 0.0685, 3.640000, 37.977373, 23.737702, 1.7071, 0.2695, 0.0768, 3.680000, 37.977373, 23.737702, 1.7480, 0.2993, 0.0843, 3.720000, 37.977372, 23.737702, 1.7930, 0.3255, 0.0908, 3.760000, 37.977372, 23.737702, 1.8414, 0.3460, 0.0960, 3.800000, 37.977372, 23.737702, 1.8918, 0.3548, 0.1000, 3.840000, 37.977372, 23.737702, 1.9435, 0.3634, 0.1029, 3.880000, 37.977372, 23.737702, 1.9964, 0.3717, 0.1045, 3.920000, 37.977372, 23.737702, 2.0503, 0.3764, 0.1050, 3.960000, 37.977371, 23.737702, 2.1046, 0.3775, 0.1043, 4.000000, 37.977371, 23.737703, 2.1582, 0.3670, 0.1026, 4.040000, 37.977371, 23.737703, 2.2102, 0.3562, 0.0999, 4.080000, 37.977371, 23.737703, 2.2607, 0.3451, 0.0960, 4.120000, 37.977371, 23.737703, 2.3094, 0.3304, 0.0911, 4.160000, 37.977370, 23.737703, 2.3556, 0.3122, 0.0853, 4.200000, 37.977370, 23.737703, 2.3984, 0.2823, 0.0788, 4.240000, 37.977370, 23.737703, 2.4369, 0.2516, 0.0714, 4.280000, 37.977370, 23.737704, 2.4709, 0.2206, 0.0630, 4.320000, 37.977369, 23.737704, 2.5002, 0.1866, 0.0537, 4.360000, 37.977369, 23.737704, 2.5244, 0.1498, 0.0434, 4.400000, 37.977369, 23.737704, 2.5426, 0.1025, 0.0323, 4.440000, 37.977369, 23.737704, 2.5534, 0.0477, 0.0204, 4.480000, 37.977368, 23.737704, 2.5563, -0.0077, 0.0082, 4.520000, 37.977368, 23.737704, 2.5513, -0.0607, -0.0041, 4.560000, 37.977368, 23.737705, 2.5393, -0.1061, -0.0160, 4.600000, 37.977368, 23.737705, 2.5213, -0.1438, -0.0275, 4.640000, 37.977367, 23.737705, 2.4977, -0.1844, -0.0385, 4.680000, 37.977367, 23.737705, 2.4683, -0.2247, -0.0487, 4.720000, 37.977367, 23.737705, 2.4333, -0.2605, -0.0579, 4.760000, 37.977367, 23.737705, 2.3937, -0.2895, -0.0663, 4.800000, 37.977366, 23.737705, 2.3511, -0.3025, -0.0740, 4.840000, 37.977366, 23.737705, 2.3066, -0.3149, -0.0807, 4.880000, 37.977366, 23.737706, 2.2605, -0.3264, -0.0864, 4.920000, 37.977366, 23.737706, 2.2129, -0.3335, -0.0911, 4.960000, 37.977365, 23.737706, 2.1647, -0.3360, -0.0949, 5.000000, 37.977365, 23.737706, 2.1173, -0.3228, -0.0978, 5.040000, 37.977365, 23.737706, 2.0718, -0.3087, -0.0998, 5.080000, 37.977365, 23.737706, 2.0285, -0.2939, -0.1008, 5.120000, 37.977365, 23.737706, 1.9875, -0.2746, -0.1008, 5.160000, 37.977365, 23.737706, 1.9497, -0.2507, -0.0999, 5.200000, 37.977364, 23.737707, 1.9165, -0.2110, -0.0981, 5.240000, 37.977364, 23.737707, 1.8890, -0.1702, -0.0953, 5.280000, 37.977364, 23.737707, 1.8675, -0.1287, -0.0916, 5.320000, 37.977364, 23.737707, 1.8523, -0.0826, -0.0870, 5.360000, 37.977364, 23.737707, 1.8440, -0.0325, -0.0815, 5.400000, 37.977364, 23.737707, 1.8440, 0.0321, -0.0750, 5.440000, 37.977364, 23.737707, 1.8540, 0.1068, -0.0675, 5.480000, 37.977364, 23.737707, 1.8747, 0.1818, -0.0593, 5.520000, 37.977363, 23.737707, 1.9061, 0.2537, -0.0507, 5.560000, 37.977363, 23.737707, 1.9471, 0.3157, -0.0421, 5.600000, 37.977363, 23.737708, 1.9963, 0.3671, -0.0334, 5.640000, 37.977363, 23.737708, 2.0531, 0.4226, -0.0247, 5.680000, 37.977363, 23.737708, 2.1179, 0.4776, -0.0159, 5.720000, 37.977363, 23.737708, 2.1902, 0.5257, -0.0071, 5.760000, 37.977363, 23.737708, 2.2686, 0.5640, 0.0011, 5.800000, 37.977362, 23.737708, 2.3510, 0.5805, 0.0088, 5.840000, 37.977362, 23.737708, 2.4358, 0.5965, 0.0161, 5.880000, 37.977362, 23.737709, 2.5228, 0.6120, 0.0230, 5.920000, 37.977362, 23.737709, 2.6115, 0.6206, 0.0295, 5.960000, 37.977362, 23.737709, 2.7010, 0.6224, 0.0353, 6.000000, 37.977361, 23.737709, 2.7892, 0.6024, 0.0404, 6.040000, 37.977361, 23.737709, 2.8745, 0.5818, 0.0450, 6.080000, 37.977361, 23.737710, 2.9568, 0.5608, 0.0490, 6.120000, 37.977361, 23.737710, 3.0355, 0.5330, 0.0525, 6.160000, 37.977360, 23.737710, 3.1098, 0.4984, 0.0553, 6.200000, 37.977360, 23.737710, 3.1775, 0.4419, 0.0576, 6.240000, 37.977360, 23.737711, 3.2362, 0.3740, 0.0594, 6.280000, 37.977360, 23.737711, 3.2851, 0.3056, 0.0606, 6.320000, 37.977359, 23.737711, 3.3249, 0.2465, 0.0612, 6.360000, 37.977359, 23.737711, 3.3568, 0.1970, 0.0613, 6.400000, 37.977359, 23.737712, 3.3813, 0.1427, 0.0610, 6.440000, 37.977358, 23.737712, 3.3969, 0.0741, 0.0602, 6.480000, 37.977358, 23.737712, 3.4018, -0.0054, 0.0590, 6.520000, 37.977358, 23.737712, 3.3956, -0.0805, 0.0576, 6.560000, 37.977357, 23.737713, 3.3797, -0.1413, 0.0563, 6.600000, 37.977357, 23.737713, 3.3560, -0.1875, 0.0552, 6.640000, 37.977357, 23.737713, 3.3264, -0.2242, 0.0542, 6.680000, 37.977356, 23.737713, 3.2915, -0.2605, 0.0533, 6.720000, 37.977356, 23.737714, 3.2503, -0.3108, 0.0524, 6.760000, 37.977356, 23.737714, 3.2013, -0.3705, 0.0516, 6.800000, 37.977355, 23.737714, 3.1442, -0.4228, 0.0508, 6.840000, 37.977355, 23.737714, 3.0807, -0.4586, 0.0501, 6.880000, 37.977355, 23.737714, 3.0133, -0.4780, 0.0495, 6.920000, 37.977355, 23.737715, 2.9420, -0.5114, 0.0489, 6.960000, 37.977354, 23.737715, 2.8650, -0.5589, 0.0483, 7.000000, 37.977354, 23.737715, 2.7816, -0.5990, 0.0476, 7.040000, 37.977354, 23.737715, 2.6937, -0.6226, 0.0468, 7.080000, 37.977354, 23.737715, 2.6035, -0.6297, 0.0460, 7.120000, 37.977354, 23.737715, 2.5113, -0.6507, 0.0450, 7.160000, 37.977353, 23.737716, 2.4151, -0.6858, 0.0439, 7.200000, 37.977353, 23.737716, 2.3144, -0.7134, 0.0426, 7.240000, 37.977353, 23.737716, 2.2103, -0.7313, 0.0413, 7.280000, 37.977353, 23.737716, 2.1049, -0.7327, 0.0398, 7.320000, 37.977353, 23.737716, 1.9998, -0.7272, 0.0383, 7.360000, 37.977352, 23.737716, 1.8952, -0.7258, 0.0367, 7.400000, 37.977352, 23.737716, 1.7908, -0.7236, 0.0351, 7.440000, 37.977352, 23.737717, 1.6872, -0.7160, 0.0333, 7.480000, 37.977352, 23.737717, 1.5833, -0.7270, 0.0316, 7.520000, 37.977352, 23.737717, 1.4829, -0.6960, 0.0299, 7.560000, 37.977352, 23.737717, 1.3868, -0.6678, 0.0281, 7.600000, 37.977351, 23.737717, 1.2944, -0.6444, 0.0264, 7.640000, 37.977351, 23.737717, 1.2049, -0.6281, 0.0246, 7.680000, 37.977351, 23.737717, 1.1169, -0.6234, 0.0229, 7.720000, 37.977351, 23.737717, 1.0293, -0.6153, 0.0213, 7.760000, 37.977351, 23.737717, 0.9429, -0.6068, 0.0196, 7.800000, 37.977351, 23.737717, 0.8581, -0.5930, 0.0180, 7.840000, 37.977351, 23.737717, 0.7757, -0.5742, 0.0166, 7.880000, 37.977351, 23.737717, 0.6964, -0.5501, 0.0158, 7.920000, 37.977351, 23.737717, 0.6204, -0.5286, 0.0143, 7.960000, 37.977351, 23.737717, 0.5473, -0.5110, 0.0131, 8.000000, 37.977351, 23.737717, 0.4773, -0.4863, 0.0117, 8.040000, 37.977351, 23.737717, 0.4112, -0.4562, 0.0103, 8.080000, 37.977351, 23.737717, 0.3498, -0.4208, 0.0090, 8.120000, 37.977351, 23.737717, 0.2933, -0.3892, 0.0077, 8.160000, 37.977351, 23.737717, 0.2405, -0.3690, 0.0077, 8.200000, 37.977351, 23.737717, 0.1925, -0.3337, 0.0069, 8.240000, 37.977351, 23.737717, 0.1496, -0.2968, 0.0060, 8.280000, 37.977351, 23.737717, 0.1127, -0.2527, 0.0053, 8.320000, 37.977351, 23.737717, 0.0826, -0.2006, 0.0048, 8.360000, 37.977351, 23.737717, 0.0594, -0.1573, 0.0041, 8.400000, 37.977351, 23.737717, 0.0424, -0.0710, 0.0030, 8.440000, 37.977351, 23.737717, 0.0360, -0.0097, 0.0026, 8.480000, 37.977351, 23.737717, 0.0389, 0.0577, 0.0024, 8.520000, 37.977351, 23.737717, 0.0510, 0.1194, 0.0023, 8.560000, 37.977351, 23.737717, 0.0734, 0.1993, 0.0023, 8.600000, 37.977351, 23.737717, 0.1028, 0.2464, 0.0023, 8.640000, 37.977351, 23.737717, 0.1399, 0.3060, 0.0023, 8.680000, 37.977351, 23.737717, 0.1852, 0.3597, 0.0023, 8.720000, 37.977351, 23.737717, 0.2373, 0.4008, 0.0023, 8.760000, 37.977351, 23.737717, 0.2947, 0.4341, 0.0022, 8.800000, 37.977351, 23.737717, 0.3561, 0.4491, 0.0019, 8.840000, 37.977351, 23.737717, 0.4206, 0.4769, 0.0019, 8.880000, 37.977351, 23.737717, 0.4896, 0.5122, 0.0018, 8.920000, 37.977351, 23.737717, 0.5629, 0.5368, 0.0016, 8.960000, 37.977351, 23.737717, 0.6390, 0.5503, 0.0008, 9.000000, 37.977351, 23.737717, 0.7161, 0.5517, 0.0007, 9.040000, 37.977351, 23.737717, 0.7938, 0.5608, 0.0003, 9.080000, 37.977351, 23.737717, 0.8733, 0.5756, 0.0001, 9.120000, 37.977351, 23.737718, 0.9541, 0.5795, -0.0000, 9.160000, 37.977351, 23.737718, 1.0347, 0.5724, -0.0001, 9.200000, 37.977350, 23.737718, 1.1134, 0.5543, -0.0002, 9.240000, 37.977350, 23.737718, 1.1900, 0.5425, -0.0013, 9.280000, 37.977350, 23.737718, 1.2640, 0.5284, -0.0016, 9.320000, 37.977350, 23.737718, 1.3363, 0.5205, -0.0016, 9.360000, 37.977350, 23.737718, 1.4076, 0.5132, -0.0017, 9.400000, 37.977350, 23.737718, 1.4776, 0.5033, -0.0017, 9.440000, 37.977350, 23.737718, 1.5462, 0.4938, -0.0016, 9.480000, 37.977349, 23.737718, 1.6131, 0.4350, -0.0009, 9.520000, 37.977349, 23.737719, 1.6735, 0.4043, -0.0008, 9.560000, 37.977349, 23.737719, 1.7293, 0.3705, -0.0008, 9.600000, 37.977349, 23.737719, 1.7805, 0.3406, -0.0008, 9.640000, 37.977349, 23.737719, 1.8274, 0.3099, -0.0008, 9.680000, 37.977349, 23.737719, 1.8692, 0.2709, -0.0008, 9.720000, 37.977348, 23.737719, 1.9046, 0.2208, -0.0008, 9.760000, 37.977348, 23.737719, 1.9327, 0.1694, -0.0007, 9.800000, 37.977348, 23.737720, 1.9539, 0.1255, -0.0007, 9.840000, 37.977348, 23.737720, 1.9698, 0.0950, -0.0006, 9.880000, 37.977348, 23.737720, 1.9822, 0.0781, -0.0006, 9.920000, 37.977348, 23.737720, 1.9915, 0.0501, -0.0005, 9.960000, 37.977347, 23.737720, 1.9959, 0.0110, -0.0004, 10.000000, 37.977347, 23.737720, 1.9952, -0.0206, -0.0003, 10.040000, 37.977347, 23.737720, 1.9909, -0.0385, -0.0001, 10.080000, 37.977347, 23.737721, 1.9851, -0.0429, 0.0000, 10.120000, 37.977347, 23.737721, 1.9778, -0.0584, 0.0002, 10.160000, 37.977347, 23.737721, 1.9675, -0.0849, 0.0004, 10.200000, 37.977346, 23.737721, 1.9539, -0.1040, 0.0006, 10.240000, 37.977346, 23.737721, 1.9387, -0.1072, 0.0009, 10.280000, 37.977346, 23.737721, 1.9240, -0.0969, 0.0011, 10.320000, 37.977346, 23.737721, 1.9103, -0.0922, 0.0013, 10.360000, 37.977346, 23.737721, 1.8964, -0.1019, 0.0016, 10.400000, 37.977346, 23.737722, 1.8803, -0.1213, 0.0018, 10.440000, 37.977345, 23.737722, 1.8612, -0.1447, 0.0021, 10.480000, 37.977345, 23.737722, 1.8388, -0.1657, 0.0023, 10.520000, 37.977345, 23.737722, 1.8140, -0.1794, 0.0025, 10.560000, 37.977345, 23.737722, 1.7874, -0.1890, 0.0026, 10.600000, 37.977345, 23.737722, 1.7592, -0.2036, 0.0027, 10.640000, 37.977345, 23.737722, 1.7277, -0.2328, 0.0028, 10.680000, 37.977344, 23.737723, 1.6915, -0.2709, 0.0029, 10.720000, 37.977344, 23.737723, 1.6503, -0.3008, 0.0029, 10.760000, 37.977344, 23.737723, 1.6060, -0.3149, 0.0029, 10.800000, 37.977344, 23.737723, 1.5607, -0.3147, 0.0028, 10.840000, 37.977344, 23.737723, 1.5151, -0.3181, 0.0027, 10.880000, 37.977344, 23.737723, 1.4688, -0.3250, 0.0026, 10.920000, 37.977344, 23.737723, 1.4221, -0.3235, 0.0026, 10.960000, 37.977343, 23.737723, 1.3762, -0.3137, 0.0026, 11.000000, 37.977343, 23.737723, 1.3328, -0.2895, 0.0027, 11.040000, 37.977343, 23.737724, 1.2926, -0.2687, 0.0027, 11.080000, 37.977343, 23.737724, 1.2551, -0.2515, 0.0024, 11.120000, 37.977343, 23.737724, 1.2208, -0.2260, 0.0017, 11.160000, 37.977343, 23.737724, 1.1907, -0.1923, 0.0008, 11.200000, 37.977343, 23.737724, 1.1664, -0.1442, -0.0004, 11.240000, 37.977343, 23.737724, 1.1503, -0.0800, -0.0017, 11.280000, 37.977343, 23.737724, 1.1431, -0.0193, -0.0029, 11.320000, 37.977343, 23.737724, 1.1430, 0.0180, -0.0041, 11.360000, 37.977343, 23.737724, 1.1468, 0.0342, -0.0053, 11.400000, 37.977343, 23.737724, 1.1521, 0.0390, -0.0064, 11.440000, 37.977342, 23.737724, 1.1589, 0.0557, -0.0074, 11.480000, 37.977342, 23.737724, 1.1695, 0.0920, -0.0079, 11.520000, 37.977342, 23.737724, 1.1850, 0.1227, -0.0077, 11.560000, 37.977342, 23.737724, 1.2040, 0.1415, -0.0067, 11.600000, 37.977342, 23.737724, 1.2250, 0.1508, -0.0048, 11.640000, 37.977342, 23.737724, 1.2459, 0.1390, -0.0022, 11.680000, 37.977342, 23.737725, 1.2652, 0.1296, 0.0003, 11.720000, 37.977342, 23.737725, 1.2857, 0.1552, 0.0024, 11.760000, 37.977342, 23.737725, 1.3121, 0.2104, 0.0041, 11.800000, 37.977342, 23.737725, 1.3476, 0.2829, 0.0055, 11.840000, 37.977342, 23.737725, 1.3914, 0.3259, 0.0065, 11.880000, 37.977341, 23.737725, 1.4393, 0.3395, 0.0077, 11.920000, 37.977341, 23.737725, 1.4917, 0.3883, 0.0090, 11.960000, 37.977341, 23.737725, 1.5537, 0.4724, 0.0105, 12.000000, 37.977341, 23.737725, 1.6290, 0.5738, 0.0123, 12.040000, 37.977341, 23.737726, 1.7168, 0.6457, 0.0143, 12.080000, 37.977341, 23.737726, 1.8129, 0.6881, 0.0163, 12.120000, 37.977340, 23.737726, 1.9175, 0.7656, 0.0183, 12.160000, 37.977340, 23.737726, 2.0359, 0.8783, 0.0206, 12.200000, 37.977340, 23.737726, 2.1717, 1.0083, 0.0234, 12.240000, 37.977340, 23.737726, 2.3253, 1.1252, 0.0265, 12.280000, 37.977340, 23.737726, 2.4937, 1.2128, 0.0298, 12.320000, 37.977340, 23.737726, 2.6740, 1.2915, 0.0333, 12.360000, 37.977340, 23.737726, 2.8664, 1.3810, 0.0368, 12.400000, 37.977339, 23.737727, 3.0733, 1.4927, 0.0403, 12.440000, 37.977339, 23.737727, 3.2985, 1.6354, 0.0438, 12.480000, 37.977339, 23.737727, 3.5455, 1.7943, 0.0474, 12.520000, 37.977339, 23.737727, 3.8136, 1.9298, 0.0510, 12.560000, 37.977339, 23.737727, 4.0990, 2.0343, 0.0542, 12.600000, 37.977338, 23.737727, 4.3987, 2.1273, 0.0569, 12.640000, 37.977338, 23.737728, 4.7125, 2.2311, 0.0589, 12.680000, 37.977338, 23.737728, 5.0426, 2.3545, 0.0602, 12.720000, 37.977337, 23.737728, 5.3915, 2.4910, 0.0610, 12.760000, 37.977337, 23.737729, 5.7593, 2.6173, 0.0621, 12.800000, 37.977336, 23.737729, 6.1429, 2.7103, 0.0637, 12.840000, 37.977336, 23.737729, 6.5381, 2.7789, 0.0659, 12.880000, 37.977335, 23.737730, 6.9415, 2.8231, 0.0687, 12.920000, 37.977335, 23.737730, 7.3521, 2.8807, 0.0720, 12.960000, 37.977334, 23.737731, 7.7721, 2.9515, 0.0755, 13.000000, 37.977333, 23.737731, 8.1998, 2.9889, 0.0790, 13.040000, 37.977333, 23.737732, 8.6311, 3.0019, 0.0825, 13.080000, 37.977332, 23.737732, 9.0626, 2.9903, 0.0859, 13.120000, 37.977331, 23.737733, 9.4933, 2.9921, 0.0893, 13.160000, 37.977330, 23.737734, 9.9252, 3.0071, 0.0928, 13.200000, 37.977329, 23.737734, 10.3569, 2.9887, 0.0966, 13.240000, 37.977328, 23.737735, 10.7843, 2.9473, 0.1007, 13.280000, 37.977327, 23.737736, 11.2040, 2.8814, 0.1049, 13.320000, 37.977326, 23.737736, 11.6138, 2.8106, 0.1094, 13.360000, 37.977325, 23.737737, 12.0143, 2.7511, 0.1142, 13.400000, 37.977324, 23.737738, 12.4053, 2.6806, 0.1192, 13.440000, 37.977323, 23.737739, 12.7858, 2.6042, 0.1246, 13.480000, 37.977322, 23.737739, 13.1554, 2.5290, 0.1307, 13.520000, 37.977321, 23.737740, 13.5136, 2.4450, 0.1376, 13.560000, 37.977319, 23.737741, 13.8600, 2.3670, 0.1452, 13.600000, 37.977318, 23.737742, 14.1969, 2.3112, 0.1537, 13.640000, 37.977317, 23.737743, 14.5265, 2.2667, 0.1628, 13.680000, 37.977316, 23.737744, 14.8508, 2.2384, 0.1726, 13.720000, 37.977314, 23.737745, 15.1716, 2.2175, 0.1831, 13.760000, 37.977313, 23.737746, 15.4894, 2.1953, 0.1941, 13.800000, 37.977311, 23.737747, 15.8051, 2.1896, 0.2058, 13.840000, 37.977310, 23.737748, 16.1198, 2.1820, 0.2181, 13.880000, 37.977309, 23.737749, 16.4333, 2.1724, 0.2310, 13.920000, 37.977307, 23.737750, 16.7460, 2.1703, 0.2444, 13.960000, 37.977306, 23.737751, 17.0589, 2.1757, 0.2584, 14.000000, 37.977304, 23.737752, 17.3738, 2.1975, 0.2728, 14.040000, 37.977303, 23.737753, 17.6917, 2.2174, 0.2878, 14.080000, 37.977301, 23.737754, 18.0123, 2.2352, 0.3032, 14.120000, 37.977299, 23.737755, 18.3360, 2.2606, 0.3190, 14.160000, 37.977298, 23.737756, 18.6638, 2.2934, 0.3352, 14.200000, 37.977296, 23.737757, 18.9976, 2.3427, 0.3518, 14.240000, 37.977294, 23.737758, 19.3387, 2.3946, 0.3687, 14.280000, 37.977293, 23.737759, 19.6871, 2.4445, 0.3861, 14.320000, 37.977291, 23.737760, 20.0427, 2.4937, 0.4039, 14.360000, 37.977289, 23.737761, 20.4053, 2.5436, 0.4219, 14.400000, 37.977287, 23.737763, 20.7760, 2.6051, 0.4403, 14.440000, 37.977286, 23.737764, 21.1565, 2.6788, 0.4594, 14.480000, 37.977284, 23.737765, 21.5479, 2.7572, 0.4807, 14.520000, 37.977282, 23.737766, 21.9502, 2.8305, 0.5045, 14.560000, 37.977280, 23.737767, 22.3622, 2.8916, 0.5304, 14.600000, 37.977278, 23.737769, 22.7822, 2.9416, 0.5584, 14.640000, 37.977276, 23.737770, 23.2094, 2.9924, 0.5883, 14.680000, 37.977274, 23.737771, 23.6440, 3.0444, 0.6201, 14.720000, 37.977272, 23.737772, 24.0863, 3.0975, 0.6537, 14.760000, 37.977269, 23.737774, 24.5358, 3.1466, 0.6893, 14.800000, 37.977267, 23.737775, 24.9912, 3.1783, 0.7267, 14.840000, 37.977265, 23.737776, 25.4507, 3.2031, 0.7660, 14.880000, 37.977263, 23.737777, 25.9132, 3.2211, 0.8070, 14.920000, 37.977260, 23.737779, 26.3784, 3.2401, 0.8497, 14.960000, 37.977258, 23.737780, 26.8464, 3.2601, 0.8943, 15.000000, 37.977255, 23.737782, 27.3160, 3.2626, 0.9405, 15.040000, 37.977253, 23.737783, 27.7855, 3.2582, 0.9883, 15.080000, 37.977250, 23.737784, 28.2539, 3.2471, 1.0372, 15.120000, 37.977247, 23.737786, 28.7207, 3.2369, 1.0874, 15.160000, 37.977245, 23.737787, 29.1862, 3.2279, 1.1388, 15.200000, 37.977242, 23.737789, 29.6491, 3.2014, 1.1918, 15.240000, 37.977239, 23.737790, 30.1062, 3.1476, 1.2463, 15.280000, 37.977236, 23.737791, 30.5550, 3.0869, 1.3021, 15.320000, 37.977233, 23.737793, 30.9971, 3.0534, 1.3589, 15.360000, 37.977230, 23.737794, 31.4367, 3.0516, 1.4167, 15.400000, 37.977228, 23.737796, 31.8774, 3.0698, 1.4756, 15.440000, 37.977224, 23.737797, 32.3202, 3.0809, 1.5351, 15.480000, 37.977221, 23.737798, 32.7632, 3.0716, 1.5937, 15.520000, 37.977218, 23.737800, 33.2048, 3.0616, 1.6506, 15.560000, 37.977215, 23.737801, 33.6454, 3.0587, 1.7042, 15.600000, 37.977212, 23.737803, 34.0865, 3.0674, 1.7542, 15.640000, 37.977209, 23.737804, 34.5311, 3.1079, 1.8006, 15.680000, 37.977205, 23.737805, 34.9819, 3.1528, 1.8437, 15.720000, 37.977202, 23.737807, 35.4362, 3.1571, 1.8837, 15.760000, 37.977198, 23.737808, 35.8881, 3.1199, 1.9206, 15.800000, 37.977195, 23.737809, 36.3325, 3.0527, 1.9544, 15.840000, 37.977192, 23.737810, 36.7695, 3.0162, 1.9852, 15.880000, 37.977188, 23.737812, 37.2034, 3.0104, 2.0131, 15.920000, 37.977184, 23.737813, 37.6335, 2.9638, 2.0383, 15.960000, 37.977181, 23.737814, 38.0540, 2.8764, 2.0606, 16.000000, 37.977177, 23.737815, 38.4597, 2.7589, 2.0798, 16.040000, 37.977173, 23.737816, 38.8508, 2.6721, 2.0959, 16.080000, 37.977170, 23.737817, 39.2315, 2.6160, 2.1092, 16.120000, 37.977166, 23.737818, 39.6012, 2.5192, 2.1201, 16.160000, 37.977162, 23.737819, 39.9541, 2.3816, 2.1291, 16.200000, 37.977158, 23.737820, 40.2850, 2.2140, 2.1359, 16.240000, 37.977154, 23.737821, 40.5922, 2.0531, 2.1404, 16.280000, 37.977150, 23.737822, 40.8785, 1.9230, 2.1428, 16.320000, 37.977145, 23.737823, 41.1471, 1.8082, 2.1429, 16.360000, 37.977141, 23.737824, 41.3989, 1.6884, 2.1412, 16.400000, 37.977137, 23.737825, 41.6316, 1.5436, 2.1376, 16.440000, 37.977133, 23.737826, 41.8405, 1.3583, 2.1319, 16.480000, 37.977128, 23.737827, 42.0210, 1.1492, 2.1224, 16.520000, 37.977124, 23.737827, 42.1734, 0.9672, 2.1091, 16.560000, 37.977120, 23.737828, 42.3025, 0.8258, 2.0933, 16.600000, 37.977115, 23.737829, 42.4127, 0.7044, 2.0756, 16.640000, 37.977111, 23.737830, 42.5051, 0.5780, 2.0558, 16.680000, 37.977107, 23.737830, 42.5777, 0.4312, 2.0339, 16.720000, 37.977102, 23.737831, 42.6276, 0.2612, 2.0098, 16.760000, 37.977098, 23.737831, 42.6533, 0.0952, 1.9835, 16.800000, 37.977094, 23.737832, 42.6577, -0.0332, 1.9549, 16.840000, 37.977090, 23.737832, 42.6463, -0.1258, 1.9240, 16.880000, 37.977085, 23.737833, 42.6241, -0.1828, 1.8905, 16.920000, 37.977081, 23.737833, 42.5920, -0.2630, 1.8541, 16.960000, 37.977077, 23.737833, 42.5467, -0.3663, 1.8147, 17.000000, 37.977073, 23.737834, 42.4892, -0.4320, 1.7724, 17.040000, 37.977068, 23.737834, 42.4249, -0.4619, 1.7273, 17.080000, 37.977064, 23.737834, 42.3588, -0.4561, 1.6795, 17.120000, 37.977060, 23.737834, 42.2919, -0.4734, 1.6290, 17.160000, 37.977056, 23.737835, 42.2208, -0.5139, 1.5757, 17.200000, 37.977052, 23.737835, 42.1466, -0.5168, 1.5193, 17.240000, 37.977047, 23.737835, 42.0746, -0.4827, 1.4597, 17.280000, 37.977043, 23.737835, 42.0102, -0.4129, 1.3970, 17.320000, 37.977039, 23.737835, 41.9557, -0.3443, 1.3310, 17.360000, 37.977035, 23.737835, 41.9092, -0.3009, 1.2618, 17.400000, 37.977031, 23.737835, 41.8690, -0.2574, 1.1892, 17.440000, 37.977027, 23.737835, 41.8352, -0.2126, 1.1132, 17.480000, 37.977022, 23.737835, 41.8091, -0.1735, 1.0771, 17.520000, 37.977018, 23.737835, 41.7893, -0.1137, 1.0388, 17.560000, 37.977014, 23.737835, 41.7779, -0.0496, 0.9996, 17.600000, 37.977010, 23.737835, 41.7737, -0.0084, 0.9602, 17.640000, 37.977006, 23.737835, 41.7735, 0.0067, 0.9204, 17.680000, 37.977002, 23.737834, 41.7734, -0.0081, 0.8803, 17.720000, 37.976997, 23.737834, 41.7710, -0.0233, 0.8400, 17.760000, 37.976993, 23.737834, 41.7678, -0.0170, 0.7993, 17.800000, 37.976989, 23.737834, 41.7653, -0.0118, 0.7583, 17.840000, 37.976985, 23.737834, 41.7633, -0.0078, 0.7168, 17.880000, 37.976981, 23.737834, 41.7616, -0.0053, 0.6745, 17.920000, 37.976976, 23.737834, 41.7603, -0.0011, 0.6313, 17.960000, 37.976972, 23.737833, 41.7595, 0.0052, 0.5878, 18.000000,

Your input file contains a space after each comma.
To omit them gracefully while reading, pass skipinitialspace=True
to read_csv.
Another hint: If the row that you provided in your post is really
the first row of your file (it contains just data, not column names),
then your file has no row with column names.
To inform read_csv about it, pass either header=None
or names with the proper list of column names.
And the last detail: Probably the first column of your source should be
the index. If this is the case, pass index_col=[0].

Ignite issue when a node in the cluster becomes unstable unable to join the cluster and hangs indefinitely

HI I am facing a critical issue with Ignite in our production servers . We have 2 instances with heap sizes of 8gb each . Sometimes due to long gc pause or network issue one of our instances gets stopped . This causes aws auto-scaling to kick in and bring another instance up . This is fine but we have observed that in tis state the grid becomes unstable and our new ignite instaces are never able to join the topology and hang forever causing new autoscaled instances to come again and again .The workaround for this is to restart other instances in the cluster as doing so causes nodes to join again .But ideally in a prod environment this should happen automatically with auto scaling .
Had also added a longer failuredetection timeout but that also doesnt solve it completely and we still observe this sometimes .
The logs observed on the new instances not coming up is as below .Igite version use is 2.4 and off heap mode is used for partitioned caches .Our grid is setup using tcp discovery service using a s3 bucket .
I have some transactional caches as well which do lock based on
tryLocks.
evtLatch=0, remaining=[a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9], super=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=1272213534]]]
2018-07-18 16:34:10.534 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], node=7d5e83aa-736a-4190-8b64-7261db7382f6]. Dumping pending objects that might be the cause:
2018-07-18 16:34:20.534 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], node=7d5e83aa-736a-4190-8b64-7261db7382f6]. Dumping pending objects that might be the cause:
2018-07-18 16:34:20.534 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Ready affinity version: AffinityTopologyVersion [topVer=-1, minorTopVer=0]
2018-07-18 16:34:20.535 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Last exchange future: GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931660255, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=7d5e83aa, msg=null, type=NODE_JOINED, tstamp=1531931329481], crd=TcpDiscoveryNode [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, addrs=[10.83.87.131, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-87-131.ec2.internal/10.83.87.131:47500], discPort=47500, order=26, intOrder=14, lastExchangeTime=1531931329258, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], discoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931660255, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=7d5e83aa, msg=null, type=NODE_JOINED, tstamp=1531931329481], nodeId=7d5e83aa, evt=NODE_JOINED], added=true, initFut=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=true, hash=247159314], init=true, lastVer=null, partReleaseFut=PartitionReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[ExplicitLockReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], TxReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], AtomicUpdateReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], DataStreamerReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]]]], exchActions=ExchangeActions [startCaches=null, stopCaches=null, startGrps=[], stopGrps=[], resetParts=null, stateChangeRequest=null], affChangeMsg=null, initTs=1531931329576, centralizedAff=false, changeGlobalStateE=null, done=false, state=SRV, evtLatch=0, remaining=[a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9], super=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=1272213534]]
2018-07-18 16:34:20.535 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.a.i.i.p.c.GridCachePartitionExchangeManager - First 10 pending exchange futures [total=0]
2018-07-18 16:34:20.535 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Last 10 exchange futures (total: 1):
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - >>> GridDhtPartitionsExchangeFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], evt=NODE_JOINED, evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931660255, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], done=false]
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending transactions:
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending explicit locks:
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending cache futures:
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending atomic cache futures:
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending data streamer futures:
2018-07-18 16:34:20.536 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Pending transaction deadlock detection futures:
2018-07-18 16:34:20.547 UTC [FDPS] [grid-nio-worker-tcp-comm-3-#28%fdps%] [INFO ] [,] o.apache.ignite.internal.diagnostic - Exchange future waiting for coordinator response [crd=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0]]
Remote node information:
General node info [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, client=false, discoTopVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], time=12:34:20.537]
Partitions exchange info [readyVer=AffinityTopologyVersion [topVer=29, minorTopVer=0]]
Last initialized exchange future: GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=ba6aba6c-7f5d-41bf-bfcc-5eefcad36b62, addrs=[10.83.85.122, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-85-122.ec2.internal/10.83.85.122:47500], discPort=47500, order=30, intOrder=16, lastExchangeTime=1531930705943, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=30, nodeId8=a450db0b, msg=Node joined: TcpDiscoveryNode [id=ba6aba6c-7f5d-41bf-bfcc-5eefcad36b62, addrs=[10.83.85.122, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-85-122.ec2.internal/10.83.85.122:47500], discPort=47500, order=30, intOrder=16, lastExchangeTime=1531930705943, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], type=NODE_JOINED, tstamp=1531930706210], crd=TcpDiscoveryNode [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, addrs=[10.83.87.131, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-87-131.ec2.internal/10.83.87.131:47500], discPort=47500, order=26, intOrder=14, lastExchangeTime=1531931660254, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], discoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=ba6aba6c-7f5d-41bf-bfcc-5eefcad36b62, addrs=[10.83.85.122, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-85-122.ec2.internal/10.83.85.122:47500], discPort=47500, order=30, intOrder=16, lastExchangeTime=1531930705943, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=30, nodeId8=a450db0b, msg=Node joined: TcpDiscoveryNode [id=ba6aba6c-7f5d-41bf-bfcc-5eefcad36b62, addrs=[10.83.85.122, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-85-122.ec2.internal/10.83.85.122:47500], discPort=47500, order=30, intOrder=16, lastExchangeTime=1531930705943, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], type=NODE_JOINED, tstamp=1531930706210], nodeId=ba6aba6c, evt=NODE_JOINED], added=true, initFut=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=1921954756], init=false, lastVer=GridCacheVersion [topVer=0, order=1531930704443, nodeOrder=0], partReleaseFut=PartitionReleaseFuture [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], futures=[ExplicitLockReleaseFuture [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], futures=[ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935479, nodeOrder=26], threadId=39726, id=559000, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=221, val=49583853497448469294730566354366524577617095530402283666, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547787212113, nodeOrder=26], threadId=39741, id=603904, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=288, val=49583853499611641578988037213538229804531966271996035234, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935487, nodeOrder=26], threadId=39740, id=558993, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=133, val=49583853497448469294730566354417299462040910024459419794, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935323, nodeOrder=26], threadId=39728, id=558949, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=1023, val=49583853497448469294730566353278491339963927967496667282, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935470, nodeOrder=26], threadId=39951, id=559009, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=556, val=49583853497448469294730566354226289182541798339977937042, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935497, nodeOrder=26], threadId=39683, id=558982, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=373, val=49583853497448469294730566354541818821461216966893109394, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935339, nodeOrder=26], threadId=39682, id=558941, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=156, val=49583853497448469294730566353353444740780034976328450194, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935358, nodeOrder=26], threadId=39936, id=558921, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=59, val=49583853497448469294730566353578304943228356208982229138, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandida... and 48550 skipped ...ead=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935486, nodeOrder=26], threadId=39894, id=558992, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=488, val=49583853497448469294730566354434224423515514832905306258, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], ExplicitLockSpan [topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], firstCand=GridCacheMvccCandidate [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, ver=GridCacheVersion [topVer=141782290, order=1547786935331, nodeOrder=26], threadId=39893, id=558948, topVer=AffinityTopologyVersion [topVer=29, minorTopVer=0], reentry=null, otherNodeId=null, otherVer=null, mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=570, val=49583853497448469294730566353289371672340459630069022866, hasValBytes=false], masks=local=1|owner=0|ready=0|reentry=0|used=0|tx=0|single_implicit=0|dht_local=0|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]]]], TxReleaseFuture [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], futures=[]], AtomicUpdateReleaseFuture [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], futures=[]], DataStreamerReleaseFuture [topVer=AffinityTopologyVersion [topVer=30, minorTopVer=0], futures=[]]]], exchActions=null, affChangeMsg=null, initTs=1531930706210, centralizedAff=false, changeGlobalStateE=null, done=false, state=CRD, evtLatch=0, remaining=[ba6aba6c-7f5d-41bf-bfcc-5eefcad36b62], super=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=325602672]]
Communication SPI statistics [rmtNode=7d5e83aa-736a-4190-8b64-7261db7382f6]
Communication SPI recovery descriptors:
[key=ConnectionKey [nodeId=7d5e83aa-736a-4190-8b64-7261db7382f6, idx=0, connCnt=0], msgsSent=5, msgsAckedByRmt=0, msgsRcvd=7, lastAcked=0, reserveCnt=1, descIdHash=1972345954]
Communication SPI clients:
[node=7d5e83aa-736a-4190-8b64-7261db7382f6, client=GridTcpNioCommunicationClient [ses=GridSelectorNioSessionImpl [worker=DirectNioClientWorker [super=AbstractNioClientWorker [idx=3, bytesRcvd=5740, bytesSent=77322, bytesRcvd0=853, bytesSent0=0, select=true, super=GridWorker [name=grid-nio-worker-tcp-comm-3, igniteInstanceName=fdps, finished=false, hashCode=2068348067, interrupted=false, runner=grid-nio-worker-tcp-comm-3-#28%fdps%]]], writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], readBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], inRecovery=GridNioRecoveryDescriptor [acked=0, resendCnt=0, rcvCnt=7, sentCnt=5, reserved=true, lastAck=0, nodeLeft=false, node=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], connected=true, connectCnt=0, queueLimit=262144, reserveCnt=1, pairedConnections=false], outRecovery=GridNioRecoveryDescriptor [acked=0, resendCnt=0, rcvCnt=7, sentCnt=5, reserved=true, lastAck=0, nodeLeft=false, node=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], connected=true, connectCnt=0, queueLimit=262144, reserveCnt=1, pairedConnections=false], super=GridNioSessionImpl [locAddr=/10.83.87.131:47100, rmtAddr=/10.83.89.183:34664, createTime=1531931330498, closeTime=0, bytesSent=77322, bytesRcvd=5740, bytesSent0=0, bytesRcvd0=853, sndSchedTime=1531931330498, lastSndTime=1531931500547, lastRcvTime=1531931660527, readsPaused=false, filterChain=FilterChain[filters=[GridNioCodecFilter [parser=org.apache.ignite.internal.util.nio.GridDirectParser#665c2413, directMode=true], GridConnectionBytesVerifyFilter], accepted=true]], super=GridAbstractCommunicationClient [lastUsed=1531931330508, closed=false, connIdx=0]]]
NIO sessions statistics:
>> Selector info [idx=3, keysCnt=1, bytesRcvd=5740, bytesRcvd0=853, bytesSent=77322, bytesSent0=0]
Connection info [in=true, rmtAddr=/10.83.89.183:34664, locAddr=/10.83.87.131:47100, msgsSent=5, msgsAckedByRmt=0, descIdHash=1972345954, unackedMsgs=[IgniteDiagnosticMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage], msgsRcvd=7, lastAcked=0, descIdHash=1972345954, bytesRcvd=5740, bytesRcvd0=853, bytesSent=77322, bytesSent0=0, opQueueSize=0]
Exchange future: GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=a450db0b, msg=Node joined: TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], type=NODE_JOINED, tstamp=1531931329402], crd=null, exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], discoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=a450db0b, msg=Node joined: TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931329178, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], type=NODE_JOINED, tstamp=1531931329402], nodeId=7d5e83aa, evt=NODE_JOINED], added=true, initFut=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=980776600], init=false, lastVer=GridCacheVersion [topVer=0, order=1531931327875, nodeOrder=0], partReleaseFut=null, exchActions=null, affChangeMsg=null, initTs=0, centralizedAff=false, changeGlobalStateE=null, done=false, state=null, evtLatch=0, remaining=[], super=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=2138568466]]
Local communication statistics:
Communication SPI statistics [rmtNode=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9]
Communication SPI recovery descriptors:
[key=ConnectionKey [nodeId=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, idx=0, connCnt=-1], msgsSent=7, msgsAckedByRmt=0, msgsRcvd=6, lastAcked=0, reserveCnt=1, descIdHash=1891649612]
Communication SPI clients:
Communication SPI clients:
[node=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, client=GridTcpNioCommunicationClient [ses=GridSelectorNioSessionImpl [worker=DirectNioClientWorker [super=AbstractNioClientWorker [idx=0, bytesRcvd=92833, bytesSent=5698, bytesRcvd0=15539, bytesSent0=853, select=true, super=GridWorker [name=grid-nio-worker-tcp-comm-0, igniteInstanceName=fdps, finished=false, hashCode=2040212682, interrupted=false, runner=grid-nio-worker-tcp-comm-0-#25%fdps%]]], writeBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], readBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], inRecovery=GridNioRecoveryDescriptor [acked=0, resendCnt=0, rcvCnt=6, sentCnt=7, reserved=true, lastAck=0, nodeLeft=false, node=TcpDiscoveryNode [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, addrs=[10.83.87.131, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-87-131.ec2.internal/10.83.87.131:47500], discPort=47500, order=26, intOrder=14, lastExchangeTime=1531931329258, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], connected=false, connectCnt=1, queueLimit=262144, reserveCnt=1, pairedConnections=false], outRecovery=GridNioRecoveryDescriptor [acked=0, resendCnt=0, rcvCnt=6, sentCnt=7, reserved=true, lastAck=0, nodeLeft=false, node=TcpDiscoveryNode [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, addrs=[10.83.87.131, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-87-131.ec2.internal/10.83.87.131:47500], discPort=47500, order=26, intOrder=14, lastExchangeTime=1531931329258, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], connected=false, connectCnt=1, queueLimit=262144, reserveCnt=1, pairedConnections=false], super=GridNioSessionImpl [locAddr=/10.83.89.183:34664, rmtAddr=ip-10-83-87-131.ec2.internal/10.83.87.131:47100, createTime=1531931330468, closeTime=0, bytesSent=5698, bytesRcvd=92833, bytesSent0=853, bytesRcvd0=15539, sndSchedTime=1531931330468, lastSndTime=1531931660528, lastRcvTime=1531931660538, readsPaused=false, filterChain=FilterChain[filters=[GridNioCodecFilter [parser=org.apache.ignite.internal.util.nio.GridDirectParser#72024a61, directMode=true], GridConnectionBytesVerifyFilter], accepted=false]], super=GridAbstractCommunicationClient [lastUsed=1531931330468, closed=false, connIdx=0]]]
NIO sessions statistics:
>> Selector info [idx=0, keysCnt=1, bytesRcvd=92833, bytesRcvd0=15539, bytesSent=5698, bytesSent0=853]
Connection info [in=false, rmtAddr=ip-10-83-87-131.ec2.internal/10.83.87.131:47100, locAddr=/10.83.89.183:34664, msgsSent=7, msgsAckedByRmt=0, descIdHash=1891649612, unackedMsgs=[GridDhtPartitionsSingleMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage, IgniteDiagnosticMessage], msgsRcvd=6, lastAcked=0, descIdHash=1891649612, bytesRcvd=92833, bytesRcvd0=15539, bytesSent=5698, bytesSent0=853, opQueueSize=0]
2018-07-18 16:34:29.598 UTC [FDPS] [localhost-startStop-1] [WARN ] [,] o.a.i.i.p.c.GridCachePartitionExchangeManager - Still waiting for initial partition map exchange [fut=GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931669507, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=7d5e83aa, msg=null, type=NODE_JOINED, tstamp=1531931329481], crd=TcpDiscoveryNode [id=a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9, addrs=[10.83.87.131, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-87-131.ec2.internal/10.83.87.131:47500], discPort=47500, order=26, intOrder=14, lastExchangeTime=1531931329258, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], exchId=GridDhtPartitionExchangeId [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], discoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=7d5e83aa-736a-4190-8b64-7261db7382f6, addrs=[10.83.89.183, 127.0.0.1], sockAddrs=[/127.0.0.1:47500, ip-10-83-89-183.ec2.internal/10.83.89.183:47500], discPort=47500, order=32, intOrder=17, lastExchangeTime=1531931669507, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=false], topVer=32, nodeId8=7d5e83aa, msg=null, type=NODE_JOINED, tstamp=1531931329481], nodeId=7d5e83aa, evt=NODE_JOINED], added=true, initFut=GridFutureAdapter [ignoreInterrupts=false, state=DONE, res=true, hash=247159314], init=true, lastVer=null, partReleaseFut=PartitionReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[ExplicitLockReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], TxReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], AtomicUpdateReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]], DataStreamerReleaseFuture [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], futures=[]]]], exchActions=ExchangeActions [startCaches=null, stopCaches=null, startGrps=[], stopGrps=[], resetParts=null, stateChangeRequest=null], affChangeMsg=null, initTs=1531931329576, centralizedAff=false, changeGlobalStateE=null, done=false, state=SRV, evtLatch=0, remaining=[a450db0b-ce86-4f0b-a34b-a2f9c83bb3d9], super=GridFutureAdapter [ignoreInterrupts=false, state=INIT, res=null, hash=1272213534]]]
2018-07-18 16:34:30.537 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], node=7d5e83aa-736a-4190-8b64-7261db7382f6]. Dumping pending objects that might be the cause:
2018-07-18 16:34:40.537 UTC [FDPS] [exchange-worker-#35%fdps%] [WARN ] [,] o.apache.ignite.internal.diagnostic - Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=32, minorTopVer=0], node=7d5e83aa-736a-4190-8b64-7261db7382f6]. Dumping pending objects that might be the cause:
Info about the other node 10-83-85-122
The other joining node never got started and was stuck in the ignite start phase . The logs also dont show the node to get up or the ip discovery to get kicked in . to eventually cause the node to be removed via autoscaling .
Transactional errors received
javax.cache.CacheException: Failed to acquire lock for keys (primary node left grid, retry transaction if possible) [keys=[UserKeyCacheObjectImpl [part=281,

Partition map exchange is a process of exchanging information between nodes where each piece of data is stored. It happens every time, when topology changes.
Every node sends a GridDhtPartitionsSingleMessage to a coordinator. Once the coordinator collected all such messages, it sends GridDhtPartitionsFullMessage back to other nodes. These messages are sent over communication SPI.
But if some of non-coordinator nodes don't send the SingleMessage to the coordinator, or if the coordinator doesn't send the FullMessage, then "Failed to wait for partition map exchange" error occurs.
Judging by the piece of log, that you provided, a node with ID=ba6aba6c didn't send the SingleMessage to the coordinator. It may mean, that communication SPI doesn't work there properly. Make sure, that ports, that are required for communication SPI are available. Usually it's 47100..47200.
Also joining node may be stuck on something. Look at its log to figure out, what happens there.

Ignite cache fails after Failed to process selector key...java.io.IOException: Broken pipe exception

We are running Ignite 2.4 & have 2 server nodes & 30 odd client nodes. We use zookeeper discovery & the nodes are deployed in a Docker swarm environment.
After a while of running i saw the below exception in one of the ignite clients & the caches no longer seem to work,
service-be - [INFO ] 2018-06-15 02:01:52.256 [grid-timeout-worker-#55] org.apache.ignite.internal.IgniteKernal -
Metrics for local node (to disable set 'metricsLogFrequency' to 0)
^-- Node [id=5249f20c, uptime=02:49:02.178]
^-- H/N/C [hosts=34, nodes=34, CPUs=816]
^-- CPU [cur=24.2%, avg=0.27%, GC=0%]
^-- PageMemory [pages=0]
^-- Heap [used=848MB, free=17.19%, comm=1024MB]
^-- Non heap [used=241MB, free=84.12%, comm=251MB]
^-- Outbound messages queue [size=4]
^-- Public thread pool [active=0, idle=0, qSize=0]
^-- System thread pool [active=0, idle=24, qSize=0]
service-be - [INFO ] 2018-06-15 02:01:52.432 [grid-nio-worker-tcp-comm-2-#59] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Accepted incoming communication connection [locAddr=/10.11.0.7:47100, rmtAddr=/10.11.0.75:59204]
service-be - [INFO ] 2018-06-15 02:01:52.433 [grid-nio-worker-tcp-comm-2-#59] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Received incoming connection when already connected to this node, rejecting [locNode=5249f20c-456b-4b6f-ab41-f5cd5c3c05ba, rmtNode=6739c9af-42d1-4aad-ac9c-ac738ed13534]
service-be - [INFO ] 2018-06-15 02:01:52.634 [grid-nio-worker-tcp-comm-3-#60] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Accepted incoming communication connection [locAddr=/10.11.0.7:47100, rmtAddr=/10.11.0.75:59206]
service-be - [INFO ] 2018-06-15 02:01:52.635 [grid-nio-worker-tcp-comm-3-#60] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Received incoming connection when already connected to this node, rejecting [locNode=5249f20c-456b-4b6f-ab41-f5cd5c3c05ba, rmtNode=6739c9af-42d1-4aad-ac9c-ac738ed13534]
service-be - [INFO ] 2018-06-15 02:01:52.836 [grid-nio-worker-tcp-comm-4-#61] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Accepted incoming communication connection [locAddr=/10.11.0.7:47100, rmtAddr=/10.11.0.75:59208]
service-be - [INFO ] 2018-06-15 02:01:52.837 [grid-nio-worker-tcp-comm-4-#61] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Received incoming connection when already connected to this node, rejecting [locNode=5249f20c-456b-4b6f-ab41-f5cd5c3c05ba, rmtNode=6739c9af-42d1-4aad-ac9c-ac738ed13534]
service-be - [INFO ] 2018-06-15 02:01:53.038 [grid-nio-worker-tcp-comm-5-#62] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Accepted incoming communication connection [locAddr=/10.11.0.7:47100, rmtAddr=/10.11.0.75:59210]
service-be - [INFO ] 2018-06-15 02:01:53.039 [grid-nio-worker-tcp-comm-5-#62] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Received incoming connection when already connected to this node, rejecting [locNode=5249f20c-456b-4b6f-ab41-f5cd5c3c05ba, rmtNode=6739c9af-42d1-4aad-ac9c-ac738ed13534]
service-be - [ERROR] 2018-06-15 02:01:53.231 [grid-nio-worker-tcp-comm-0-#57] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Failed to process selector key [ses=GridSelectorNioSessionImpl [worker=DirectNioClientWorker [super=AbstractNioClientWorker [idx=0, bytesRcvd=70700138, bytesSent=18478193, bytesRcvd0=0, bytesSent0=0, select=true, super=GridWorker [name=grid-nio-worker-tcp-comm-0, igniteInstanceName=null, finished=false, hashCode=30436088, interrupted=false, runner=grid-nio-worker-tcp-comm-0-#57]]], writeBuf=java.nio.DirectByteBuffer[pos=0 lim=186 cap=32768], readBuf=java.nio.DirectByteBuffer[pos=0 lim=32768 cap=32768], inRecovery=GridNioRecoveryDescriptor [acked=48224, resendCnt=0, rcvCnt=111504, sentCnt=48229, reserved=true, lastAck=111488, nodeLeft=false, node=TcpDiscoveryNode [id=6739c9af-42d1-4aad-ac9c-ac738ed13534, addrs=[10.11.0.74, 10.11.0.75, 127.0.0.1, 172.18.0.22], sockAddrs=[/172.18.0.22:47500, bdd554c3dc77/10.11.0.75:47500, /10.11.0.74:47500, /127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1529039549468, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], connected=false, connectCnt=1, queueLimit=131072, reserveCnt=2, pairedConnections=false], outRecovery=GridNioRecoveryDescriptor [acked=48224, resendCnt=0, rcvCnt=111504, sentCnt=48229, reserved=true, lastAck=111488, nodeLeft=false, node=TcpDiscoveryNode [id=6739c9af-42d1-4aad-ac9c-ac738ed13534, addrs=[10.11.0.74, 10.11.0.75, 127.0.0.1, 172.18.0.22], sockAddrs=[/172.18.0.22:47500, bdd554c3dc77/10.11.0.75:47500, /10.11.0.74:47500, /127.0.0.1:47500], discPort=47500, order=1, intOrder=1, lastExchangeTime=1529039549468, loc=false, ver=2.4.0#20180305-sha1:aa342270, isClient=false], connected=false, connectCnt=1, queueLimit=131072, reserveCnt=2, pairedConnections=false], super=GridNioSessionImpl [locAddr=/10.11.0.7:42970, rmtAddr=bdd554c3dc77/10.11.0.75:47100, createTime=1529039561958, closeTime=0, bytesSent=18478193, bytesRcvd=70700138, bytesSent0=0, bytesRcvd0=0, sndSchedTime=1529044007457, lastSndTime=1529049712225, lastRcvTime=1529049712225, readsPaused=false, filterChain=FilterChain[filters=[GridNioCodecFilter [parser=o.a.i.i.util.nio.GridDirectParser#7a15b36, directMode=true], GridConnectionBytesVerifyFilter], accepted=false]]]
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:51)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
at org.apache.ignite.internal.util.nio.GridNioServer$DirectNioClientWorker.processWrite0(GridNioServer.java:1636)
at org.apache.ignite.internal.util.nio.GridNioServer$DirectNioClientWorker.processWrite(GridNioServer.java:1293)
at org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.processSelectedKeysOptimized(GridNioServer.java:2307)
at org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.bodyInternal(GridNioServer.java:2080)
at org.apache.ignite.internal.util.nio.GridNioServer$AbstractNioClientWorker.body(GridNioServer.java:1749)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:110)
at java.lang.Thread.run(Thread.java:748)
service-be - [WARN ] 2018-06-15 02:01:53.231 [grid-nio-worker-tcp-comm-0-#57] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Closing NIO session because of unhandled exception [cls=class o.a.i.i.util.nio.GridNioException, msg=Broken pipe]
service-be - [INFO ] 2018-06-15 02:01:53.240 [grid-nio-worker-tcp-comm-6-#63] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Accepted incoming communication connection [locAddr=/10.11.0.7:47100, rmtAddr=/10.11.0.75:59212]
service-be - [WARN ] 2018-06-15 02:02:03.253 [tcp-comm-worker-#1] org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi - Connect timed out (consider increasing 'failureDetectionTimeout' configuration property) [addr=/172.18.0.22:47100, failureDetectionTimeout=10000]
On searching with the remote node with which there seems to be trouble connecting (as mentioned in the trace above) I also see these warnings in some of the other client nodes aswell.
Any obvious pointers on what could be going wrong?. From what i have searched one suggestion was to use ipv4 but the docker overlay has enableipv6 as disabled in our case..so i am not sure how much that will help.
[root#rhel743411 logs]# egrep -i "6739c9af-42d1-4aad-ac9c-ac738ed13534" *
service1-mw.log:service1-mw - [WARN ] 2018-06-16 00:27:55.884 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:26:02.991, curTime=00:27:55.876, fut=GridDhtColocatedLockFuture [threadId=39579, keys=[UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=776a8520461-6a403605-a8fd-4ed1-bd45-92e648929a2a, lockVer=GridCacheVersion [topVer=140519300, order=1529059257539, nodeOrder=6], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service1-mw.log:service1-mw - [WARN ] 2018-06-16 00:27:55.884 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:25:55.893, curTime=00:27:55.876, fut=GridDhtColocatedLockFuture [threadId=297, keys=[UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=f03a8520461-6a403605-a8fd-4ed1-bd45-92e648929a2a, lockVer=GridCacheVersion [topVer=140519300, order=1529059253553, nodeOrder=6], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service1-mw.log:service1-mw - [WARN ] 2018-06-16 00:27:55.884 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:26:51.661, curTime=00:27:55.876, fut=GridDhtColocatedLockFuture [threadId=38749, keys=[UserKeyCacheObjectImpl [part=7, val=7, hasValBytes=false], UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=354b8520461-6a403605-a8fd-4ed1-bd45-92e648929a2a, lockVer=GridCacheVersion [topVer=140519300, order=1529059268380, nodeOrder=6], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service1-mw.log:service1-mw - [WARN ] 2018-06-16 00:27:55.885 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:26:51.772, curTime=00:27:55.876, fut=GridDhtColocatedLockFuture [threadId=343, keys=[UserKeyCacheObjectImpl [part=7, val=7, hasValBytes=false]], futId=125b8520461-6a403605-a8fd-4ed1-bd45-92e648929a2a, lockVer=GridCacheVersion [topVer=140519300, order=1529059268816, nodeOrder=6], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:01:10.227 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=23:59:12.637, curTime=00:01:10.221, fut=GridDhtColocatedLockFuture [threadId=21129, keys=[UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=f5216120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529058842000, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:09:10.242 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:07:30.520, curTime=00:09:10.239, fut=GridDhtColocatedLockFuture [threadId=21304, keys=[UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=42176120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529058982457, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:13:10.269 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:11:32.462, curTime=00:13:10.268, fut=GridDhtColocatedLockFuture [threadId=21368, keys=[UserKeyCacheObjectImpl [part=7, val=7, hasValBytes=false]], futId=c0f96120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529059041395, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:15:10.281 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:13:43.800, curTime=00:15:10.279, fut=GridDhtColocatedLockFuture [threadId=172, keys=[UserKeyCacheObjectImpl [part=7, val=7, hasValBytes=false], UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=49ab6120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529059079186, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:17:10.289 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:15:44.860, curTime=00:17:10.287, fut=GridDhtColocatedLockFuture [threadId=172, keys=[UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=a3ec6120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529059106786, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:20:10.299 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:18:51.741, curTime=00:20:10.298, fut=GridDhtColocatedLockFuture [threadId=172, keys=[UserKeyCacheObjectImpl [part=7, val=7, hasValBytes=false], UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=8ace6120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529059136637, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:21:10.308 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:19:19.018, curTime=00:21:10.304, fut=GridDhtColocatedLockFuture [threadId=21484, keys=[UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=bd7f6120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529059155514, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:24:10.326 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:23:03.860, curTime=00:24:10.323, fut=GridDhtColocatedLockFuture [threadId=21544, keys=[UserKeyCacheObjectImpl [part=7, val=7, hasValBytes=false]], futId=f3e17120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529059200701, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:24:10.326 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:22:52.783, curTime=00:24:10.323, fut=GridDhtColocatedLockFuture [threadId=172, keys=[UserKeyCacheObjectImpl [part=7, val=7, hasValBytes=false], UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=edc17120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529059199113, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:26:10.330 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:24:59.321, curTime=00:26:10.328, fut=GridDhtColocatedLockFuture [threadId=172, keys=[UserKeyCacheObjectImpl [part=7, val=7, hasValBytes=false]], futId=74737120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529059232146, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]
service2-mw.log:service2y-mw - [WARN ] 2018-06-16 00:29:10.349 [grid-timeout-worker-#55] org.apache.ignite.internal.diagnostic - Found long running cache future [startTime=00:27:32.480, curTime=00:29:10.347, fut=GridDhtColocatedLockFuture [threadId=21621, keys=[UserKeyCacheObjectImpl [part=8, val=8, hasValBytes=false]], futId=1fe57120461-0c4dcfda-c90b-42a3-83c4-8d2f8ecb6ab1, lockVer=GridCacheVersion [topVer=140519300, order=1529059289421, nodeOrder=17], read=false, retval=true, err=null, timeout=120000, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], done=0, trackable=true, createTtl=-1, accessTtl=-1, skipStore=false, keepBinary=false, recovery=false, miniId=1, topVer=AffinityTopologyVersion [topVer=34, minorTopVer=0], innerFuts=[[node=6739c9af-42d1-4aad-ac9c-ac738ed13534, rcvRes=false, loc=false, done=false]], inTx=false, super=GridCompoundIdentityFuture [super=GridCompoundFuture [rdc=Bool reducer: true, initFlag=1, lsnrCalls=0, done=false, cancelled=false, err=null, futs=[false]]]]]

Ignite Cluster getting stuck when new node Join or release

I have 3 node cluster with 20+ client and it's running in spark context.Initially it working fine but randomly get issue whenever new node i.e. client try to connect with cluster.The cluster getting inoperative.I have got following logs when its stuck.If I restart any Ignite server explicitly then its release and work fine.I have use Ignite 2.4.0 version. same issue produced in Ignite 2.5.0 version too.
client side Logs
Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=44, minorTopVer=0], node=4d885cfd-45ed-43a2-8088-f35c9469797f]. Dumping pending objects that might be the cause:
GridDhtPartitionsExchangeFuture [topVer=AffinityTopologyVersion [topVer=44, minorTopVer=0], evt=NODE_JOINED, evtNode=TcpDiscoveryNode [id=4d885cfd-45ed-43a2-8088-f35c9469797f, addrs=[0:0:0:0:0:0:0:1%lo, 10.13.10.179, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, hdn6.mstorm.com/10.13.10.179:0], discPort=0, order=44, intOrder=0, lastExchangeTime=1527651620413, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=true], done=false]
Failed to wait for partition map exchange [topVer=AffinityTopologyVersion [topVer=44, minorTopVer=0], node=4d885cfd-45ed-43a2-8088-f35c9469797f]. Dumping pending objects that might be the cause:
GridDhtPartitionsExchangeFuture [topVer=AffinityTopologyVersion [topVer=44, minorTopVer=0], evt=NODE_JOINED, evtNode=TcpDiscoveryNode [id=4d885cfd-45ed-43a2-8088-f35c9469797f, addrs=[0:0:0:0:0:0:0:1%lo, 10.13.10.179, 127.0.0.1], sockAddrs=[/0:0:0:0:0:0:0:1%lo:0, /127.0.0.1:0, hdn6.mstorm.com/10.13.10.179:0], discPort=0, order=44, intOrder=0, lastExchangeTime=1527651620413, loc=true, ver=2.4.0#20180305-sha1:aa342270, isClient=true], done=false]
Failed to wait for initial partition map exchange. Possible reasons are:
^-- Transactions in deadlock.
^-- Long running transactions (ignore if this is the case).
^-- Unreleased explicit locks.
Still waiting for initial partition map exchange [fut=GridDhtPartitionsExchangeFuture [firstDiscoEvt=DiscoveryEvent [evtNode=TcpDiscoveryNode [id=4d885cfd-45ed-43a2-8088-f35c9469797f, addrs=
Server Side Logs
Possible starvation in striped pool. Thread name: sys-stripe-0-#1 Queue: [Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtTxPrepareResponse [nearEvicted=null, futId=869dd4ca361-fe7e167d-4d80-4f57-b004-13359a9f2c11, miniId=1, super=GridDistributedTxPrepareResponse [txState=null, part=-1, err=null, super=GridDistributedBaseMessage [ver=GridCacheVersion [topVer=139084030, order=1527604094903, nodeOrder=1], committedVers=null, rolledbackVers=null, cnt=0, super=GridCacheIdMessage [cacheId=0]]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [part=984, val=null, hasValBytes=true], val=BinaryObjectImpl [arr= true, ctx=false, start=0], prevVal=null, super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=null, nearFutId=0, flags=]]]], o.a.i.i.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$DeferredUpdateTimeout#2735c674, Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtTxPrepareRequest [nearNodeId=628e3078-17fd-4e49-b9ae-ad94ad97a2f1, futId=6576e4ca361-6e7cdac2-d5a3-4624-9ad3-b93f25546cc3, miniId=1, topVer=AffinityTopologyVersion [topVer=20, minorTopVer=0], invalidateNearEntries={}, nearWrites=null, owned=null, nearXidVer=GridCacheVersion [topVer=139084030, order=1527604094933, nodeOrder=2], subjId=628e3078-17fd-4e49-b9ae-ad94ad97a2f1, taskNameHash=0, preloadKeys=null, super=GridDistributedTxPrepareRequest [threadId=86, concurrency=OPTIMISTIC, isolation=READ_COMMITTED, writeVer=GridCacheVersion [topVer=139084030, order=1527604094935, nodeOrder=2], timeout=0, reads=null, writes=[IgniteTxEntry [key=BinaryObjectImpl [arr= true, ctx=false, start=0], cacheId=-1755241537, txKey=null, val=[op=UPDATE, val=BinaryObjectImpl [arr= true, ctx=false, start=0]], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=null, filtersPassed=false, filtersSet=false, entry=null, prepared=0, locked=false, nodeId=null, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=null]], dhtVers=null, txSize=0, plc=2, txState=null, flags=onePhase|last, super=GridDistributedBaseMessage [ver=GridCacheVersion [topVer=139084030, order=1527604094933, nodeOrder=2], committedVers=null, rolledbackVers=null, cnt=0, super=GridCacheIdMessage [cacheId=0]]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicDeferredUpdateResponse [futIds=GridLongList [idx=2, arr=[65774,65775]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [part=1016, val=null, hasValBytes=true], parent=GridNearAtomicAbstractSingleUpdateRequest [nodeId=null, futId=49328, topVer=AffinityTopologyVersion [topVer=20, minorTopVer=0], parent=GridNearAtomicAbstractUpdateRequest [res=null, flags=needRes]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicDeferredUpdateResponse [futIds=GridLongList [idx=1, arr=[98591]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridDhtAtomicDeferredUpdateResponse [futIds=GridLongList [idx=1, arr=[114926]]]]], Message closure [msg=GridIoMessage [plc=2, topic=TOPIC_CACHE, topicOrd=8, ordered=false, timeout=0, skipOnTimeout=false, msg=GridNearAtomicSingleUpdateRequest [key=KeyCacheObjectImpl [part=1016, val=null, hasValBytes=true], parent=GridNearAtomicAbstractSingleUpdateRequest [nodeId=null, futId=32946, topVer=AffinityTopologyVersion [topVer=20, minorTopVer=0], parent=GridNear