3 Replies Latest reply: Apr 20, 2011 10:32 AM by 701681 RSS

    deadlock on cluster startup - push replication + coherence 3.6.1.2

    701681
      Has anything changed in 3.6.1.2 that would give a deadlock on startup?

      Push replication is making a call to this on CoherencePushReplicationProvider.registerPublisher(...):

      NamedCache destinationCache = CacheFactory.getCache("coherence.messagingpattern.destinations");

      interesting area of thread dump:

      2011-04-13 15:55:35,293 ERROR [Logger@9240861 3.6.1.2] Coherence: 2011-04-13 15:55:35.292/363.217 Oracle Coherence GE 3.6.1.2 <Error> (thread=Cluster, member
      =7): Detected soft timeout) of {WrapperGuardable Guard{Daemon=DistributedCache:StaticDataCacheStoreService} Service=PartitionedCache{Name=StaticDataCacheStor
      eService, State=(SERVICE_STARTED), LocalStorage=enabled, PartitionCount=257, BackupCount=1, AssignedPartitions=257, BackupPartitions=0}}
      2011-04-13 15:55:35,294 WARN [Logger@9240861 3.6.1.2] Coherence: 2011-04-13 15:55:35.294/363.219 Oracle Coherence GE 3.6.1.2 <Warning> (thread=Recovery Thre
      ad, member=7): Logging stacktrace due to soft-timeout
      2011-04-13 15:55:35,314 ERROR [Logger@9240861 3.6.1.2] Coherence: 2011-04-13 15:55:35.310/363.235 Oracle Coherence GE 3.6.1.2 <Error> (thread=Recovery Thread
      , member=7): Full Thread Dump

      Thread[DistributedCache:StaticDataCacheStoreService,5,Cluster]
      java.lang.Object.wait(Native Method)
      java.lang.Object.wait(Object.java:485)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.poll(Grid.CDB:32)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.poll(Grid.CDB:11)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$BinaryMap.invoke(PartitionedCache.CDB:30
      )
      com.tangosol.util.ConverterCollections$ConverterInvocableMap.invoke(ConverterCollections.java:2156)
      com.tangosol.util.ConverterCollections$ConverterNamedCache.invoke(ConverterCollections.java:2622)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$ViewMap.invoke(PartitionedCache.CDB:11)
      com.tangosol.coherence.component.util.SafeNamedCache.invoke(SafeNamedCache.CDB:1)
      com.oracle.coherence.patterns.pushreplication.providers.coherence.CoherencePushReplicationProvider.registerPublisher(CoherencePushReplicationProvider
      .java:172)
      com.rbsfm.hft.staticdata.coherence.pushreplication.HFTPushReplicationProvider.registerCachePublishers(HFTPushReplicationProvider.java:114)
      com.rbsfm.hft.staticdata.coherence.pushreplication.HFTPushReplicationProvider.establishPublishingInfrastructureFor(HFTPushReplicationProvider.java:71
      )
      com.oracle.coherence.patterns.pushreplication.PublishingCacheStore.<init>(PublishingCacheStore.java:196)
      sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
      sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
      sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
      java.lang.reflect.Constructor.newInstance(Constructor.java:513)
      com.tangosol.util.ClassHelper.newInstance(ClassHelper.java:615)
      com.tangosol.run.xml.XmlHelper.createInstance(XmlHelper.java:2501)
      com.tangosol.run.xml.XmlHelper.createInstance(XmlHelper.java:2426)
      com.tangosol.net.DefaultConfigurableCacheFactory.instantiateAny(DefaultConfigurableCacheFactory.java:3256)
      com.oracle.coherence.environment.extensible.ExtensibleEnvironment.instantiateAny(ExtensibleEnvironment.java:720)
      com.tangosol.net.DefaultConfigurableCacheFactory.instantiateCacheStore(DefaultConfigurableCacheFactory.java:3106)
      com.tangosol.net.DefaultConfigurableCacheFactory.instantiateReadWriteBackingMap(DefaultConfigurableCacheFactory.java:1674)
      com.tangosol.net.DefaultConfigurableCacheFactory.configureBackingMap(DefaultConfigurableCacheFactory.java:1429)
      com.tangosol.net.DefaultConfigurableCacheFactory$Manager.instantiateBackingMap(DefaultConfigurableCacheFactory.java:3904)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$Storage.instantiateResourceMap(Partition
      edCache.CDB:22)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$Storage.setCacheName(PartitionedCache.CD
      B:25)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache$ServiceConfig$ConfigListener.entryInsert
      ed(PartitionedCache.CDB:17)
      com.tangosol.util.MapEvent.dispatch(MapEvent.java:266)
      com.tangosol.util.MapEvent.dispatch(MapEvent.java:226)
      com.tangosol.util.MapListenerSupport.fireEvent(MapListenerSupport.java:556)
      com.tangosol.util.ObservableHashMap.dispatchEvent(ObservableHashMap.java:229)
      com.tangosol.util.ObservableHashMap$Entry.onAdd(ObservableHashMap.java:270)
      com.tangosol.util.SafeHashMap.put(SafeHashMap.java:244)
      com.tangosol.coherence.component.util.ServiceConfig$Map.updateInternal(ServiceConfig.CDB:30)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid$ConfigUpdate.onReceived(Grid.CDB:42)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.onMessage(Grid.CDB:11)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.onNotify(Grid.CDB:33)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.PartitionedService.onNotify(PartitionedService.CDB:3)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache.onNotify(PartitionedCache.CDB:3)
      com.tangosol.coherence.component.util.Daemon.run(Daemon.CDB:42)
      java.lang.Thread.run(Thread.java:662)

      Thread[Main-Load-Thread,5,main]
      java.lang.Object.wait(Native Method)
      java.lang.Object.wait(Object.java:485)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.poll(Grid.CDB:32)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.Grid.poll(Grid.CDB:11)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache.ensureCache(PartitionedCache.CDB:29)
      com.tangosol.coherence.component.util.daemon.queueProcessor.service.grid.partitionedService.PartitionedCache.ensureCache(PartitionedCache.CDB:36)
      com.tangosol.coherence.component.util.safeService.SafeCacheService.ensureCache$Router(SafeCacheService.CDB:1)
      com.tangosol.coherence.component.util.safeService.SafeCacheService.ensureCache(SafeCacheService.CDB:33)
      com.tangosol.net.DefaultConfigurableCacheFactory.ensureCache(DefaultConfigurableCacheFactory.java:875)
      com.tangosol.net.DefaultConfigurableCacheFactory.configureCache(DefaultConfigurableCacheFactory.java:1231)
      com.tangosol.net.DefaultConfigurableCacheFactory.ensureCache(DefaultConfigurableCacheFactory.java:290)
      com.tangosol.net.CacheFactory.getCache(CacheFactory.java:735)
      com.tangosol.net.CacheFactory.getCache(CacheFactory.java:712)
      ---main.StorageMain.registerCachesWithCluster(StorageMain.java:174)
      ---.main.StorageMain.loadCaches(StorageMain.java:117)
      ---.StorageMain.start(StorageMain.java:76)
      ---.MainController$1.run(MainController.java:106)
      java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
      java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
      java.lang.Thread.run(Thread.java:662)

      Cheers,
      Neville.