Hi,
We needed to rebuild our index's so we set index.recovery.mode=TRUE
The indexes got rebuild but after they finished the alfresco did not start and just hang for few hours (~16 hours) until we got exceptions.
We use Alfresco community 2.1
Here are the logs:
2009-06-10 09:58:03,244 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - Index recovery started: 320,858 transactions.
2009-06-10 10:05:13,257 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - 10 % complete.
2009-06-10 10:11:34,465 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - 20 % complete.
2009-06-10 10:19:29,978 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - 30 % complete.
2009-06-10 10:29:41,511 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - 40 % complete.
2009-06-10 10:41:53,544 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - 50 % complete.
2009-06-10 11:01:54,129 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - 60 % complete.
2009-06-10 12:02:27,520 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - 70 % complete.
2009-06-10 12:09:47,174 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - 80 % complete.
2009-06-10 12:17:11,416 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - 90 % complete.
2009-06-10 12:27:59,403 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - 100 % complete.
2009-06-10 12:27:59,406 INFO [main]: org.alfresco.repo.node.index.FullIndexRecoveryComponent - Index recovery completed.
2009-06-11 04:13:32,915 ERROR [main]: org.hibernate.transaction.JDBCTransaction - JDBC rollback failed
com.mysql.jdbc.exceptions.MySQLNonTransientConnectionException: Connection.close() has already been called. Invalid operation in this state.
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:888)
at com.mysql.jdbc.Connection.getMutex(Connection.java:3588)
at com.mysql.jdbc.Connection.rollback(Connection.java:5035)
at org.apache.commons.dbcp.DelegatingConnection.rollback(DelegatingConnection.java:265)
at org.apache.commons.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.rollback(PoolingDataSource.java:288)
at org.hibernate.transaction.JDBCTransaction.rollbackAndResetAutoCommit(JDBCTransaction.java:183)
at org.hibernate.transaction.JDBCTransaction.rollback(JDBCTransaction.java:162)
at org.springframework.orm.hibernate3.HibernateTransactionManager.doRollback(HibernateTransactionManager.java:581)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.doRollbackOnCommitException(AbstractPlatformTransactionManager.java:795)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:682)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:624)
at org.springframework.transaction.interceptor.TransactionAspectSupport.commitTransactionAfterReturning(TransactionAspectSupport.java:307)
at org.alfresco.util.transaction.SpringAwareUserTransaction.commit(SpringAwareUserTransaction.java:420)
at org.alfresco.repo.transaction.RetryingTransactionHelper.doInTransaction(RetryingTransactionHelper.java:239)
at org.alfresco.repo.transaction.RetryingTransactionHelper.doInTransaction(RetryingTransactionHelper.java:155)
at org.alfresco.repo.node.index.AbstractReindexComponent.reindex(AbstractReindexComponent.java:220)
at org.alfresco.repo.node.index.IndexRecoveryBootstrapBean.onBootstrap(IndexRecoveryBootstrapBean.java:35)
at org.alfresco.util.AbstractLifecycleBean.onApplicationEvent(AbstractLifecycleBean.java:62)
at org.springframework.context.event.SimpleApplicationEventMulticaster$1.run(SimpleApplicationEventMulticaster.java:77)
at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:49)
at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:75)
at org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:241)
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:349)
at org.springframework.web.context.support.AbstractRefreshableWebApplicationContext.refresh(AbstractRefreshableWebApplicationContext.java:156)
at org.springframework.web.context.ContextLoader.createWebApplicationContext(ContextLoader.java:246)
at org.springframework.web.context.ContextLoader.initWebApplicationContext(ContextLoader.java:184)
at org.springframework.web.context.ContextLoaderListener.contextInitialized(ContextLoaderListener.java:49)
at org.apache.catalina.core.StandardContext.listenerStart(StandardContext.java:3764)
at org.apache.catalina.core.StandardContext.start(StandardContext.java:4216)
at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:760)
at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:740)
at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:544)
at org.apache.catalina.startup.HostConfig.deployWAR(HostConfig.java:825)
at org.apache.catalina.startup.HostConfig.deployWARs(HostConfig.java:714)
at org.apache.catalina.startup.HostConfig.deployApps(HostConfig.java:490)
at org.apache.catalina.startup.HostConfig.start(HostConfig.java:1138)
at org.apache.catalina.startup.HostConfig.lifecycleEvent(HostConfig.java:311)
at org.apache.catalina.util.LifecycleSupport.fireLifecycleEvent(LifecycleSupport.java:120)
at org.apache.catalina.core.ContainerBase.start(ContainerBase.java:1022)
at org.apache.catalina.core.StandardHost.start(StandardHost.java:736)
at org.apache.catalina.core.ContainerBase.start(ContainerBase.java:1014)
at org.apache.catalina.core.StandardEngine.start(StandardEngine.java:443)
at org.apache.catalina.core.StandardService.start(StandardService.java:448)
at org.apache.catalina.core.StandardServer.start(StandardServer.java:700)
at org.apache.catalina.startup.Catalina.start(Catalina.java:552)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:295)
at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:433)
2009-06-11 04:13:32,921 ERROR [main]: org.springframework.orm.hibernate3.HibernateTransactionManager - Commit exception overridden by rollback exception
org.springframework.dao.DataAccessResourceFailureException: could not load an entity: [org.alfresco.repo.domain.hibernate.NodeStatusImpl#component[protocol,identifier,guid]{protocol=archive, guid=9088b445-7bfe-11dc-81de-9b84088d742a, identifier=SpacesStore}]; nested exception is org.hibernate.exception.JDBCConnectionException: could not load an entity: [org.alfresco.repo.domain.hibernate.NodeStatusImpl#component[protocol,identifier,guid]{protocol=archive, guid=9088b445-7bfe-11dc-81de-9b84088d742a, identifier=SpacesStore}]
Caused by:
org.hibernate.exception.JDBCConnectionException: could not load an entity: [org.alfresco.repo.domain.hibernate.NodeStatusImpl#component[protocol,identifier,guid]{protocol=archive, guid=9088b445-7bfe-11dc-81de-9b84088d742a, identifier=SpacesStore}]
at org.hibernate.exception.SQLStateConverter.convert(SQLStateConverter.java:74)
at org.hibernate.exception.JDBCExceptionHelper.convert(JDBCExceptionHelper.java:43)
at org.hibernate.loader.Loader.loadEntity(Loader.java:1865)
at org.hibernate.loader.entity.AbstractEntityLoader.load(AbstractEntityLoader.java:48)
at org.hibernate.loader.entity.AbstractEntityLoader.load(AbstractEntityLoader.java:42)
at org.hibernate.persister.entity.AbstractEntityPersister.load(AbstractEntityPersister.java:2992)
at org.hibernate.event.def.DefaultLoadEventListener.loadFromDatasource(DefaultLoadEventListener.java:395)
at org.hibernate.event.def.DefaultLoadEventListener.doLoad(DefaultLoadEventListener.java:375)
at org.hibernate.event.def.DefaultLoadEventListener.load(DefaultLoadEventListener.java:139)
at org.hibernate.event.def.DefaultLoadEventListener.proxyOrLoad(DefaultLoadEventListener.java:195)
at org.hibernate.event.def.DefaultLoadEventListener.onLoad(DefaultLoadEventListener.java:103)
at org.hibernate.impl.SessionImpl.fireLoad(SessionImpl.java:878)
at org.hibernate.impl.SessionImpl.get(SessionImpl.java:815)
at org.hibernate.impl.SessionImpl.get(SessionImpl.java:808)
at org.springframework.orm.hibernate3.HibernateTemplate$1.doInHibernate(HibernateTemplate.java:465)
at org.springframework.orm.hibernate3.HibernateTemplate.execute(HibernateTemplate.java:367)
at org.springframework.orm.hibernate3.HibernateTemplate.get(HibernateTemplate.java:459)
at org.springframework.orm.hibernate3.HibernateTemplate.get(HibernateTemplate.java:453)
at org.alfresco.repo.node.db.hibernate.HibernateNodeDaoServiceImpl.getNodeStatus(HibernateNodeDaoServiceImpl.java:368)
at org.alfresco.repo.node.db.hibernate.HibernateNodeDaoServiceImpl.getNode(HibernateNodeDaoServiceImpl.java:465)
at sun.reflect.GeneratedMethodAccessor115.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:281)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:187)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:154)
at org.alfresco.repo.transaction.TransactionalDaoInterceptor.invoke(TransactionalDaoInterceptor.java:68)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:176)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:210)
at $Proxy1.getNode(Unknown Source)
at org.alfresco.repo.node.db.DbNodeServiceImpl.getNodeNotNull(DbNodeServiceImpl.java:137)
at org.alfresco.repo.node.db.DbNodeServiceImpl.getParentAssocs(DbNodeServiceImpl.java:1218)
at org.alfresco.repo.node.AbstractNodeServiceImpl.getParentAssocs(AbstractNodeServiceImpl.java:580)
at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:281)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:187)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:154)
at org.alfresco.repo.transaction.TransactionResourceInterceptor.invoke(TransactionResourceInterceptor.java:144)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:176)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:210)
at $Proxy2.getParentAssocs(Unknown Source)
at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.alfresco.repo.service.StoreRedirectorProxyFactory$RedirectorInvocationHandler.invoke(StoreRedirectorProxyFactory.java:221)
at $Proxy3.getParentAssocs(Unknown Source)
at org.alfresco.repo.search.impl.lucene.ADMLuceneIndexerImpl.getNodeCounts(ADMLuceneIndexerImpl.java:1029)
at org.alfresco.repo.search.impl.lucene.ADMLuceneIndexerImpl.createDocuments(ADMLuceneIndexerImpl.java:518)
at org.alfresco.repo.search.impl.lucene.AbstractLuceneIndexerImpl.indexImpl(AbstractLuceneIndexerImpl.java:631)
at org.alfresco.repo.search.impl.lucene.AbstractLuceneIndexerImpl.indexImpl(AbstractLuceneIndexerImpl.java:656)
at org.alfresco.repo.search.impl.lucene.AbstractLuceneIndexerImpl.flushPending(AbstractLuceneIndexerImpl.java:788)
at org.alfresco.repo.search.impl.lucene.ADMLuceneIndexerImpl.doPrepare(ADMLuceneIndexerImpl.java:1337)
at org.alfresco.repo.search.impl.lucene.AbstractLuceneIndexerImpl.prepare(AbstractLuceneIndexerImpl.java:471)
at org.alfresco.repo.search.impl.lucene.AbstractLuceneIndexerAndSearcherFactory.prepare(AbstractLuceneIndexerAndSearcherFactory.java:679)
at org.alfresco.repo.transaction.AlfrescoTransactionSupport$TransactionSynchronizationImpl.beforeCommit(AlfrescoTransactionSupport.java:634)
at org.springframework.transaction.support.TransactionSynchronizationUtils.triggerBeforeCommit(TransactionSynchronizationUtils.java:48)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.triggerBeforeCommit(AbstractPlatformTransactionManager.java:821)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:637)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:624)
at org.springframework.transaction.interceptor.TransactionAspectSupport.commitTransactionAfterReturning(TransactionAspectSupport.java:307)
at org.alfresco.util.transaction.SpringAwareUserTransaction.commit(SpringAwareUserTransaction.java:420)
at org.alfresco.repo.transaction.RetryingTransactionHelper.doInTransaction(RetryingTransactionHelper.java:239)
at org.alfresco.repo.transaction.RetryingTransactionHelper.doInTransaction(RetryingTransactionHelper.java:155)
at org.alfresco.repo.node.index.AbstractReindexComponent.reindex(AbstractReindexComponent.java:220)
at org.alfresco.repo.node.index.IndexRecoveryBootstrapBean.onBootstrap(IndexRecoveryBootstrapBean.java:35)
at org.alfresco.util.AbstractLifecycleBean.onApplicationEvent(AbstractLifecycleBean.java:62)
at org.springframework.context.event.SimpleApplicationEventMulticaster$1.run(SimpleApplicationEventMulticaster.java:77)
at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:49)
at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:75)
at org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:241)
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:349)
at org.springframework.web.context.support.AbstractRefreshableWebApplicationContext.refresh(AbstractRefreshableWebApplicationContext.java:156)
at org.springframework.web.context.ContextLoader.createWebApplicationContext(ContextLoader.java:246)
at org.springframework.web.context.ContextLoader.initWebApplicationContext(ContextLoader.java:184)
at org.springframework.web.context.ContextLoaderListener.contextInitialized(ContextLoaderListener.java:49)
at org.apache.catalina.core.StandardContext.listenerStart(StandardContext.java:3764)
at org.apache.catalina.core.StandardContext.start(StandardContext.java:4216)
at org.apache.catalina.core.ContainerBase.addChildInternal(ContainerBase.java:760)
at org.apache.catalina.core.ContainerBase.addChild(ContainerBase.java:740)
at org.apache.catalina.core.StandardHost.addChild(StandardHost.java:544)
at org.apache.catalina.startup.HostConfig.deployWAR(HostConfig.java:825)
at org.apache.catalina.startup.HostConfig.deployWARs(HostConfig.java:714)
at org.apache.catalina.startup.HostConfig.deployApps(HostConfig.java:490)
at org.apache.catalina.startup.HostConfig.start(HostConfig.java:1138)
at org.apache.catalina.startup.HostConfig.lifecycleEvent(HostConfig.java:311)
at org.apache.catalina.util.LifecycleSupport.fireLifecycleEvent(LifecycleSupport.java:120)
at org.apache.catalina.core.ContainerBase.start(ContainerBase.java:1022)
at org.apache.catalina.core.StandardHost.start(StandardHost.java:736)
at org.apache.catalina.core.ContainerBase.start(ContainerBase.java:1014)
at org.apache.catalina.core.StandardEngine.start(StandardEngine.java:443)
at org.apache.catalina.core.StandardService.start(StandardService.java:448)
at org.apache.catalina.core.StandardServer.start(StandardServer.java:700)
at org.apache.catalina.startup.Catalina.start(Catalina.java:552)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.apache.catalina.startup.Bootstrap.start(Bootstrap.java:295)
at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:433)
Caused by: com.mysql.jdbc.CommunicationsException: Communications link failure due to underlying exception:
** BEGIN NESTED EXCEPTION **
java.net.SocketException
MESSAGE: Broken pipe
STACKTRACE:
java.net.SocketException: Broken pipe
at java.net.SocketOutputStream.socketWrite0(Native Method)
at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:92)
at java.net.SocketOutputStream.write(SocketOutputStream.java:136)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:65)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:123)
at com.mysql.jdbc.MysqlIO.send(MysqlIO.java:2637)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:1554)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:1665)
at com.mysql.jdbc.Connection.execSQL(Connection.java:3124)
at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:1149)
at com.mysql.jdbc.PreparedStatement.executeQuery(PreparedStatement.java:1262)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeQuery(DelegatingPreparedStatement.java:92)
at org.hibernate.jdbc.AbstractBatcher.getResultSet(AbstractBatcher.java:186)
at org.hibernate.loader.Loader.getResultSet(Loader.java:1778)
at org.hibernate.loader.Loader.doQuery(Loader.java:662)
at org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:224)
at org.hibernate.loader.Loader.loadEntity(Loader.java:1851)
at org.hibernate.loader.entity.AbstractEntityLoader.load(AbstractEntityLoader.java:48)
at org.hibernate.loader.entity.AbstractEntityLoader.load(AbstractEntityLoader.java:42)
at org.hibernate.persister.entity.AbstractEntityPersister.load(AbstractEntityPersister.java:2992)
at org.hibernate.event.def.DefaultLoadEventListener.loadFromDatasource(DefaultLoadEventListener.java:395)
at org.hibernate.event.def.DefaultLoadEventListener.doLoad(DefaultLoadEventListener.java:375)
at org.hibernate.event.def.DefaultLoadEventListener.load(DefaultLoadEventListener.java:139)
at org.hibernate.event.def.DefaultLoadEventListener.proxyOrLoad(DefaultLoadEventListener.java:195)
at org.hibernate.event.def.DefaultLoadEventListener.onLoad(DefaultLoadEventListener.java:103)
at org.hibernate.impl.SessionImpl.fireLoad(SessionImpl.java:878)
at org.hibernate.impl.SessionImpl.get(SessionImpl.java:815)
at org.hibernate.impl.SessionImpl.get(SessionImpl.java:808)
at org.springframework.orm.hibernate3.HibernateTemplate$1.doInHibernate(HibernateTemplate.java:465)
at org.springframework.orm.hibernate3.HibernateTemplate.execute(HibernateTemplate.java:367)
at org.springframework.orm.hibernate3.HibernateTemplate.get(HibernateTemplate.java:459)
at org.springframework.orm.hibernate3.HibernateTemplate.get(HibernateTemplate.java:453)
at org.alfresco.repo.node.db.hibernate.HibernateNodeDaoServiceImpl.getNodeStatus(HibernateNodeDaoServiceImpl.java:368)
at org.alfresco.repo.node.db.hibernate.HibernateNodeDaoServiceImpl.getNode(HibernateNodeDaoServiceImpl.java:465)
at sun.reflect.GeneratedMethodAccessor115.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:585)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:281)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:187)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:154)
at org.alfresco.repo.transaction.TransactionalDaoInterceptor.invoke(TransactionalDaoInterceptor.java:68)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:176)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:210)
at $Proxy1.getNode(Unknown Source)
at org.alfresco.repo.node.db.DbNodeServiceImpl.getNodeNotNull(DbNodeServiceImpl.java:137)
at org.alfresco.repo.node.db.DbNodeServiceImpl.getParentAssocs(DbNodeServiceImpl.java:1218)
at org.alfresco.repo.node.AbstractNodeServiceImpl.getParentAssocs(AbstractNodeServiceImpl.java:580)
at sun.reflect.GeneratedMethodAccessor118.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
Any idea will be appriciate,
Itay