3 Replies Latest reply on Sep 15, 2014 3:44 PM by 900104

    WebLogic getting persistent store handle errors during startup and deployment

    SteveElkind

      We're getting errors flooding the domain log during startup, and errors using weblogic.Deployer, around persistent file store invalid handles.  It's happening for multiple deployed apps.  I'm guessing somehow the admin server's default store got corrupted. 

       

      Searching the KB results in, seemingly, things that don't apply, e.g., using NFS mounts for filestores, we have (virtual) SAN on VMWare.  We did not use the filestore compression utlity.  I'm not finding any hints on how to recover.

       

      Can anyone provide some insights on what might have caused this, and more importantly, how to recover?

       

      WebLogic 10.3.5, JDK 1.6.0 (64 bit) on Linux (RHEL 5.7).  Domain log file samples below.

       

      Thanks!

       

      ####<Jun 25, 2014 7:13:44 PM GMT> <Error> <Deployer> <pqa-sg-wl2    > <prp_appsvcs_admin> <[STANDBY] ExecuteThread: '2' for queue: 'weblogic.kernel.Default (self-tuning)'> <<WLS Kernel>> <> <> <1403723624884> <BEA-149607> <Failed to persist state for deployment with name dashboard. weblogic.management.ManagementException: [Deployer:149612]There were problems saving to the persistent store for 'AppRTState[appId=dashboard,retireTimeoutSecs=-1,retireTimeMillis=-1,

      Module(s) State:{ROOT_MODULE={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}, dashboard.war={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=dashboard.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=dashboard.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}},

      AppTarget(s) State:{prp_appsvcs_cluster_sg=weblogic.management.deploy.internal.AppTargetState@103bc934(appName=dashboard, target=prp_appsvcs_cluster_sg, state=STATE_ACTIVE, stagingState=-1)}]'.

      weblogic.store.PersistentStoreException: [Store:280039]The persistent store handle 25 is not valid

              at weblogic.store.io.file.FileStoreIO.create(FileStoreIO.java:204)

              at weblogic.store.io.file.FileStoreIO.update(FileStoreIO.java:280)

              at weblogic.store.internal.UpdateRequest.run(UpdateRequest.java:34)

              at weblogic.store.internal.StoreRequest.doTheIO(StoreRequest.java:64)

              at weblogic.store.internal.PersistentStoreImpl.run(PersistentStoreImpl.java:729)

              at java.lang.Thread.run(Thread.java:662)

       

      weblogic.management.ManagementException: [Deployer:149612]There were problems saving to the persistent store for 'AppRTState[appId=dashboard,retireTimeoutSecs=-1,retireTimeMillis=-1,

      Module(s) State:{ROOT_MODULE={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}, dashboard.war={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=dashboard.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=dashboard.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}},

      AppTarget(s) State:{prp_appsvcs_cluster_sg=weblogic.management.deploy.internal.AppTargetState@103bc934(appName=dashboard, target=prp_appsvcs_cluster_sg, state=STATE_ACTIVE, stagingState=-1)}]'.

      weblogic.store.PersistentStoreException: [Store:280039]The persistent store handle 25 is not valid

              at weblogic.store.io.file.FileStoreIO.create(FileStoreIO.java:204)

              at weblogic.store.io.file.FileStoreIO.update(FileStoreIO.java:280)

              at weblogic.store.internal.UpdateRequest.run(UpdateRequest.java:34)

              at weblogic.store.internal.StoreRequest.doTheIO(StoreRequest.java:64)

              at weblogic.store.internal.PersistentStoreImpl.run(PersistentStoreImpl.java:729)

              at java.lang.Thread.run(Thread.java:662)

       

              at weblogic.management.deploy.internal.AppRuntimeStateManager.save(AppRuntimeStateManager.java:287)

              at weblogic.management.deploy.internal.AppRuntimeStateManager.updateState(AppRuntimeStateManager.java:778)

              at weblogic.deploy.internal.adminserver.DeploymentManager.updateRuntimeState(DeploymentManager.java:2169)

              at weblogic.deploy.internal.adminserver.DeploymentManager.handleReceivedStatus(DeploymentManager.java:1123)

              at weblogic.deploy.internal.adminserver.DeploymentServiceDriver.receivedStatusFrom(DeploymentServiceDriver.java:174)

              at weblogic.deploy.service.internal.adminserver.DeploymentServiceCallbackDeliverer.statusReceived(DeploymentServiceCallbackDeliverer.java:120)

              at weblogic.deploy.service.internal.adminserver.StatusDeliverer$2.run(StatusDeliverer.java:86)

              at weblogic.work.SelfTuningWorkManagerImpl$WorkAdapterImpl.run(SelfTuningWorkManagerImpl.java:528)

              at weblogic.work.ExecuteThread.execute(ExecuteThread.java:209)

              at weblogic.work.ExecuteThread.run(ExecuteThread.java:178)

      Caused By: weblogic.store.PersistentStoreException: [Store:280039]The persistent store handle 25 is not valid

              at weblogic.store.io.file.FileStoreIO.create(FileStoreIO.java:204)

              at weblogic.store.io.file.FileStoreIO.update(FileStoreIO.java:280)

              at weblogic.store.internal.UpdateRequest.run(UpdateRequest.java:34)

              at weblogic.store.internal.StoreRequest.doTheIO(StoreRequest.java:64)

              at weblogic.store.internal.PersistentStoreImpl.run(PersistentStoreImpl.java:729)

              at java.lang.Thread.run(Thread.java:662)

      >

      ####<Jun 25, 2014 7:13:44 PM GMT> <Error> <Deployer> <pqa-sg-wl2    > <prp_appsvcs_admin> <[STANDBY] ExecuteThread: '8' for queue: 'weblogic.kernel.Default (self-tuning)'> <<WLS Kernel>> <> <> <1403723624885> <BEA-149607> <Failed to persist state for deployment with name ums. weblogic.management.ManagementException: [Deployer:149612]There were problems saving to the persistent store for 'AppRTState[appId=ums,retireTimeoutSecs=-1,retireTimeMillis=-1,

      Module(s) State:{ROOT_MODULE={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}, ums.war={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=ums.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=ums.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}},

      AppTarget(s) State:{prp_appsvcs_cluster_sg=weblogic.management.deploy.internal.AppTargetState@77127f51(appName=ums, target=prp_appsvcs_cluster_sg, state=STATE_ACTIVE, stagingState=-1)}]'.

      weblogic.store.PersistentStoreException: [Store:280039]The persistent store handle 20 is not valid

              at weblogic.store.io.file.FileStoreIO.create(FileStoreIO.java:204)

              at weblogic.store.io.file.FileStoreIO.update(FileStoreIO.java:280)

              at weblogic.store.internal.UpdateRequest.run(UpdateRequest.java:34)

              at weblogic.store.internal.StoreRequest.doTheIO(StoreRequest.java:64)

              at weblogic.store.internal.PersistentStoreImpl.run(PersistentStoreImpl.java:729)

              at java.lang.Thread.run(Thread.java:662)

       

      weblogic.management.ManagementException: [Deployer:149612]There were problems saving to the persistent store for 'AppRTState[appId=ums,retireTimeoutSecs=-1,retireTimeMillis=-1,

      Module(s) State:{ROOT_MODULE={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}, ums.war={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=ums.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=ums.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}},

      AppTarget(s) State:{prp_appsvcs_cluster_sg=weblogic.management.deploy.internal.AppTargetState@77127f51(appName=ums, target=prp_appsvcs_cluster_sg, state=STATE_ACTIVE, stagingState=-1)}]'.

      weblogic.store.PersistentStoreException: [Store:280039]The persistent store handle 20 is not valid

              at weblogic.store.io.file.FileStoreIO.create(FileStoreIO.java:204)

              at weblogic.store.io.file.FileStoreIO.update(FileStoreIO.java:280)

              at weblogic.store.internal.UpdateRequest.run(UpdateRequest.java:34)

              at weblogic.store.internal.StoreRequest.doTheIO(StoreRequest.java:64)

              at weblogic.store.internal.PersistentStoreImpl.run(PersistentStoreImpl.java:729)

              at java.lang.Thread.run(Thread.java:662)

       

              at weblogic.management.deploy.internal.AppRuntimeStateManager.save(AppRuntimeStateManager.java:287)

              at weblogic.management.deploy.internal.AppRuntimeStateManager.updateState(AppRuntimeStateManager.java:778)

              at weblogic.deploy.internal.adminserver.DeploymentManager.updateRuntimeState(DeploymentManager.java:2169)

              at weblogic.deploy.internal.adminserver.DeploymentManager.handleReceivedStatus(DeploymentManager.java:1123)

              at weblogic.deploy.internal.adminserver.DeploymentServiceDriver.receivedStatusFrom(DeploymentServiceDriver.java:174)

              at weblogic.deploy.service.internal.adminserver.DeploymentServiceCallbackDeliverer.statusReceived(DeploymentServiceCallbackDeliverer.java:120)

              at weblogic.deploy.service.internal.adminserver.StatusDeliverer$2.run(StatusDeliverer.java:86)

              at weblogic.work.SelfTuningWorkManagerImpl$WorkAdapterImpl.run(SelfTuningWorkManagerImpl.java:528)

              at weblogic.work.ExecuteThread.execute(ExecuteThread.java:209)

              at weblogic.work.ExecuteThread.run(ExecuteThread.java:178)

      Caused By: weblogic.store.PersistentStoreException: [Store:280039]The persistent store handle 20 is not valid

              at weblogic.store.io.file.FileStoreIO.create(FileStoreIO.java:204)

              at weblogic.store.io.file.FileStoreIO.update(FileStoreIO.java:280)

              at weblogic.store.internal.UpdateRequest.run(UpdateRequest.java:34)

              at weblogic.store.internal.StoreRequest.doTheIO(StoreRequest.java:64)

              at weblogic.store.internal.PersistentStoreImpl.run(PersistentStoreImpl.java:729)

              at java.lang.Thread.run(Thread.java:662)

      >

      ####<Jun 25, 2014 7:13:44 PM GMT> <Error> <Deployer> <pqa-sg-wl2    > <prp_appsvcs_admin> <[STANDBY] ExecuteThread: '7' for queue: 'weblogic.kernel.Default (self-tuning)'> <<WLS Kernel>> <> <> <1403723624886> <BEA-149607> <Failed to persist state for deployment with name umsUI. weblogic.management.ManagementException: [Deployer:149612]There were problems saving to the persistent store for 'AppRTState[appId=umsUI,retireTimeoutSecs=-1,retireTimeMillis=-1,

      Module(s) State:{ROOT_MODULE={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}, umsUI.war={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=umsUI.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=umsUI.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}},

      AppTarget(s) State:{prp_appsvcs_cluster_sg=weblogic.management.deploy.internal.AppTargetState@74fad37e(appName=umsUI, target=prp_appsvcs_cluster_sg, state=STATE_ACTIVE, stagingState=-1)}]'.

      weblogic.store.PersistentStoreException: [Store:280039]The persistent store handle 29 is not valid

              at weblogic.store.io.file.FileStoreIO.create(FileStoreIO.java:204)

              at weblogic.store.io.file.FileStoreIO.update(FileStoreIO.java:280)

              at weblogic.store.internal.UpdateRequest.run(UpdateRequest.java:34)

              at weblogic.store.internal.StoreRequest.doTheIO(StoreRequest.java:64)

              at weblogic.store.internal.PersistentStoreImpl.run(PersistentStoreImpl.java:729)

              at java.lang.Thread.run(Thread.java:662)

       

      weblogic.management.ManagementException: [Deployer:149612]There were problems saving to the persistent store for 'AppRTState[appId=umsUI,retireTimeoutSecs=-1,retireTimeMillis=-1,

      Module(s) State:{ROOT_MODULE={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=ROOT_MODULE,TargetName=prp_appsvcs_cluster_sg/*[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}, umsUI.war={prp_appsvcs_cluster_sg={prp_appsvcs_2_sg={prp_appsvcs_2_sg=TargetModuleState[ModuleId=umsUI.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_2_sg],State=STATE_ACTIVE}, prp_appsvcs_1_sg={prp_appsvcs_1_sg=TargetModuleState[ModuleId=umsUI.war,TargetName=prp_appsvcs_cluster_sg/Cluster[prp_appsvcs_1_sg],State=STATE_ACTIVE}}}},

      AppTarget(s) State:{prp_appsvcs_cluster_sg=weblogic.management.deploy.internal.AppTargetState@74fad37e(appName=umsUI, target=prp_appsvcs_cluster_sg, state=STATE_ACTIVE, stagingState=-1)}]'.

      weblogic.store.PersistentStoreException: [Store:280039]The persistent store handle 29 is not valid

              at weblogic.store.io.file.FileStoreIO.create(FileStoreIO.java:204)

              at weblogic.store.io.file.FileStoreIO.update(FileStoreIO.java:280)

              at weblogic.store.internal.UpdateRequest.run(UpdateRequest.java:34)

              at weblogic.store.internal.StoreRequest.doTheIO(StoreRequest.java:64)

              at weblogic.store.internal.PersistentStoreImpl.run(PersistentStoreImpl.java:729)

              at java.lang.Thread.run(Thread.java:662)

       

              at weblogic.management.deploy.internal.AppRuntimeStateManager.save(AppRuntimeStateManager.java:287)

              at weblogic.management.deploy.internal.AppRuntimeStateManager.updateState(AppRuntimeStateManager.java:778)

              at weblogic.deploy.internal.adminserver.DeploymentManager.updateRuntimeState(DeploymentManager.java:2169)

              at weblogic.deploy.internal.adminserver.DeploymentManager.handleReceivedStatus(DeploymentManager.java:1123)

              at weblogic.deploy.internal.adminserver.DeploymentServiceDriver.receivedStatusFrom(DeploymentServiceDriver.java:174)

              at weblogic.deploy.service.internal.adminserver.DeploymentServiceCallbackDeliverer.statusReceived(DeploymentServiceCallbackDeliverer.java:120)

              at weblogic.deploy.service.internal.adminserver.StatusDeliverer$2.run(StatusDeliverer.java:86)

              at weblogic.work.SelfTuningWorkManagerImpl$WorkAdapterImpl.run(SelfTuningWorkManagerImpl.java:528)

              at weblogic.work.ExecuteThread.execute(ExecuteThread.java:209)

              at weblogic.work.ExecuteThread.run(ExecuteThread.java:178)

      Caused By: weblogic.store.PersistentStoreException: [Store:280039]The persistent store handle 29 is not valid

              at weblogic.store.io.file.FileStoreIO.create(FileStoreIO.java:204)

              at weblogic.store.io.file.FileStoreIO.update(FileStoreIO.java:280)

              at weblogic.store.internal.UpdateRequest.run(UpdateRequest.java:34)

              at weblogic.store.internal.StoreRequest.doTheIO(StoreRequest.java:64)

              at weblogic.store.internal.PersistentStoreImpl.run(PersistentStoreImpl.java:729)

              at java.lang.Thread.run(Thread.java:662)

      >