Uploaded image for project: 'Bitbucket Data Center'
  1. Bitbucket Data Center
  2. BSERV-4517

Empty activity deletion deadlocks on MySQL

XMLWordPrintable

      Symptoms

      This issue might cause Pull Requests to misbehave as well as a number of performance issues on your instance. As an example of PRs misbehaviour, it's been reported:

      (...) the following error more and more frequently when trying to merge a PR from the Stash UI (happening on multiple repos): "New changes were pushed to master in project/repo while the merge was being performed. Please retry the merge.

      You will find similar stack traces in your logs:

      atlassian.log
      2014-01-24 01:48:07,044 WARN  [rescope-details:thread-2] nj9312 108x272810x0 j74y32 10.1.52.24 SSH - git-receive-pack '/consumer/yp.git' c.a.s.i.p.r.DefaultRescopeProcessor 345: Could not delete empty rescope with ID 1471
org.hibernate.exception.LockAcquisitionException: could not execute statement
	at org.hibernate.dialect.MySQLDialect$1.convert(MySQLDialect.java:411) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at org.hibernate.exception.internal.StandardSQLExceptionConverter.convert(StandardSQLExceptionConverter.java:49) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:125) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:110) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at org.hibernate.engine.jdbc.internal.ResultSetReturnImpl.executeUpdate(ResultSetReturnImpl.java:136) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at org.hibernate.hql.spi.TableBasedDeleteHandlerImpl.execute(TableBasedDeleteHandlerImpl.java:163) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at org.hibernate.hql.internal.ast.exec.MultiTableDeleteExecutor.execute(MultiTableDeleteExecutor.java:63) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at org.hibernate.hql.internal.ast.QueryTranslatorImpl.executeUpdate(QueryTranslatorImpl.java:414) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at org.hibernate.engine.query.spi.HQLQueryPlan.performExecuteUpdate(HQLQueryPlan.java:282) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at org.hibernate.internal.SessionImpl.executeUpdate(SessionImpl.java:1290) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at org.hibernate.internal.QueryImpl.executeUpdate(QueryImpl.java:116) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	at com.atlassian.stash.internal.AbstractHibernateDao.deleteById(AbstractHibernateDao.java:53) ~[stash-dao-impl-2.9.2.jar:na]
	at com.atlassian.stash.internal.AbstractHibernateDao.deleteById(AbstractHibernateDao.java:22) ~[stash-dao-impl-2.9.2.jar:na]
	at com.sun.proxy.$Proxy156.deleteById(Unknown Source) ~[na:na]
	at com.atlassian.stash.internal.pull.rescope.DefaultRescopeProcessor$3.doInTransactionWithoutResult(DefaultRescopeProcessor.java:139) ~[stash-service-impl-2.9.2.jar:na]
	at org.springframework.transaction.support.TransactionCallbackWithoutResult.doInTransaction(TransactionCallbackWithoutResult.java:33) ~[spring-tx-3.2.4.RELEASE.jar:3.2.4.RELEASE]
	at org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:131) ~[spring-tx-3.2.4.RELEASE.jar:3.2.4.RELEASE]
	at com.atlassian.stash.internal.pull.rescope.DefaultRescopeProcessor.deleteActivity(DefaultRescopeProcessor.java:130) [stash-service-impl-2.9.2.jar:na]
	at com.atlassian.stash.internal.pull.rescope.DefaultRescopeProcessor.processAll(DefaultRescopeProcessor.java:100) [stash-service-impl-2.9.2.jar:na]
	at com.atlassian.stash.internal.pull.rescope.DefaultRescopeProcessor.process(DefaultRescopeProcessor.java:77) [stash-service-impl-2.9.2.jar:na]
	at com.atlassian.stash.internal.pull.rescope.DefaultRescopeProcessor$RescopeCalculator.run(DefaultRescopeProcessor.java:219) ~[stash-service-impl-2.9.2.jar:na]
	at com.atlassian.stash.internal.concurrent.StateTransferringExecutor$StateTransferringRunnable.run(StateTransferringExecutor.java:69) ~[stash-platform-2.9.2.jar:na]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) ~[na:1.7.0_25]
	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334) ~[na:1.7.0_25]
	at java.util.concurrent.FutureTask.run(FutureTask.java:166) ~[na:1.7.0_25]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) ~[na:1.7.0_25]
	at java.lang.Thread.run(Thread.java:724) ~[na:1.7.0_25]
	... 10 frames trimmed
Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[na:1.7.0_25]
	at com.mysql.jdbc.Util.handleNewInstance(Util.java:411) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.Util.getInstance(Util.java:386) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1065) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4096) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4028) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2490) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2651) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2734) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2155) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2458) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2375) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2359) ~[mysql-connector-java-5.1.22-bin.jar:na]
	at com.jolbox.bonecp.PreparedStatementHandle.executeUpdate(PreparedStatementHandle.java:203) ~[bonecp-0.7.1.RELEASE.jar:0.7.1.RELEASE]
	at org.hibernate.engine.jdbc.internal.ResultSetReturnImpl.executeUpdate(ResultSetReturnImpl.java:133) ~[hibernate-core-4.2.5.Final.jar:4.2.5.Final]
	... 23 common frames omitted

      Cause

      This issue affects all versions of Stash from 2.5.0 through the just released 2.12.0. It is 100% MySQL-specific, and should not affect customers using any other database.

      Solution

      Upgrade to the indicated fix releases.

      Workaround

      Exactly because of this issue and STASH-4701, we recommend that customers with larger instances (you can consider that 500+ licenses) move off MySQL (you can see the warning we placed on Connecting Stash to MySQL). Therefore, if you can move to an alternative database (for example, PostgreSQL) in case you don't want to upgrade now, that'd also solve this issue for you.

      Leaving as-is will cause you performance issues as well as Pull Requests misbehaviour while merging. For instance:

      (...)the following error more and more frequently when trying to merge a PR from the Stash UI (happening on multiple repos): "New changes were pushed to master in project/repo while the merge was being performed. Please retry the merge."

      The good news is that Stash makes it really easy for you to migrate databases. All you have to do is:

          Form Name

            [BSERV-4517] Empty activity deletion deadlocks on MySQL

            Martin Cleaver added a comment -

            Me two.

            Martin Cleaver added a comment - Me two.

            Martin Cleaver added a comment -

            We see this in Atlassian Stash 3.8.0

            Martin Cleaver added a comment - We see this in Atlassian Stash 3.8.0

            Morgan Tocker added a comment -

            To explain Alexey's SHOW INNODB STATUS output:

            • The first query is trying to INSERT INTO sta_pr_rescope_activity, which requires an EXCLUSIVE lock. It is being blocked by the second query.
            • The second query is not inserting into sta_pr_rescope_activity, but has acquired a SHARED (s) lock on it, preventing the first from getting an exclusive lock.

            Shared locks are not usually required for InnoDB because it supports MVCC (reads can see an earlier version of a row without locks). This is a special circumstance, because InnoDB must get a serializable read of the sta_pr_rescope_activity table in order to delete the same rows from sta_activity on the slave (MySQL uses statement based replication by default.)

            I added a comment on STASH-4701 as well, but this can be addressed largely in MySQL configuration (and part in converting the IN subquery to a JOIN in Stash):

            binlog-format=ROW
# if using 5.6:
# binlog-row-image=minimal
transaction-isolation=READ-COMMITTED
innodb_autoinc_lock_mode=2

            Morgan Tocker added a comment - To explain Alexey's SHOW INNODB STATUS output: The first query is trying to INSERT INTO sta_pr_rescope_activity, which requires an EXCLUSIVE lock. It is being blocked by the second query. The second query is not inserting into sta_pr_rescope_activity, but has acquired a SHARED (s) lock on it, preventing the first from getting an exclusive lock. Shared locks are not usually required for InnoDB because it supports MVCC (reads can see an earlier version of a row without locks). This is a special circumstance, because InnoDB must get a serializable read of the sta_pr_rescope_activity table in order to delete the same rows from sta_activity on the slave (MySQL uses statement based replication by default.) I added a comment on STASH-4701 as well, but this can be addressed largely in MySQL configuration (and part in converting the IN subquery to a JOIN in Stash): binlog-format=ROW # if using 5.6: # binlog-row-image=minimal transaction-isolation=READ-COMMITTED innodb_autoinc_lock_mode=2

            Brent P added a comment -

            Stash 3.2.2 has been released with this fix.

            Brent P added a comment - Stash 3.2.2 has been released with this fix.

            Bryan Turner (Inactive) added a comment -

            All,

            Changes made in bugfix releases for 2.10, 2.11, 2.12 and 3.0 reduced the likelihood of this deadlock, but did not eliminate it. Further changes have been applied in 3.0.7, 3.1.4 and 3.2.1 which will eliminate the deadlock.

            There is still an issue, STASH-4701, with the speed for deleting activities. The fix applied here does not address that speed issue.

            Best regards,
            Bryan Turner
            Atlassian Stash

            Bryan Turner (Inactive) added a comment - All, Changes made in bugfix releases for 2.10, 2.11, 2.12 and 3.0 reduced the likelihood of this deadlock, but did not eliminate it. Further changes have been applied in 3.0.7, 3.1.4 and 3.2.1 which will eliminate the deadlock. There is still an issue, STASH-4701 , with the speed for deleting activities. The fix applied here does not address that speed issue. Best regards, Bryan Turner Atlassian Stash

            Bryan Turner (Inactive) added a comment -

            Alexey,

            Thanks for the updated deadlock details. It shows that it is a different deadlock than we've seen before, but related (and not really surprising).

            Unfortunately, with the current system, it is just not possible to fix. It is a MySQL behavior that's at fault here. Stash is doing perfectly normal things that any reasonable RDBMS would support (and that all of our other supported databases do support without issue). It's going to require some significant changes to Stash to address this any further.

            The best thing for you to do is migrate to another database and stop using MySQL. PostgreSQL (or any other supported database) will be more stable. There is a new setting in Stash you can change that may reduce the deadlocks, but it's going to have some (potentially quite bad) trade-offs. (Sufficiently bad that I'm reluctant to even tell it to you)

            How often is your system showing deadlocks? It should have a theoretical upper bound of ~48 times per day.

            Best regards,
            Bryan Turner
            Atlassian Stash

            Bryan Turner (Inactive) added a comment - Alexey, Thanks for the updated deadlock details. It shows that it is a different deadlock than we've seen before, but related (and not really surprising). Unfortunately, with the current system, it is just not possible to fix. It is a MySQL behavior that's at fault here. Stash is doing perfectly normal things that any reasonable RDBMS would support (and that all of our other supported databases do support without issue). It's going to require some significant changes to Stash to address this any further. The best thing for you to do is migrate to another database and stop using MySQL. PostgreSQL (or any other supported database) will be more stable. There is a new setting in Stash you can change that may reduce the deadlocks, but it's going to have some (potentially quite bad) trade-offs. (Sufficiently bad that I'm reluctant to even tell it to you) How often is your system showing deadlocks? It should have a theoretical upper bound of ~48 times per day. Best regards, Bryan Turner Atlassian Stash

            Alexey Efimov added a comment -

            Ping!

            Alexey Efimov added a comment - Ping!

            Alexey Efimov added a comment - 
            ------------------------
LATEST DETECTED DEADLOCK
------------------------
140410 20:00:02
*** (1) TRANSACTION:
TRANSACTION 260FFDCE, ACTIVE 6 sec inserting
mysql tables in use 1, locked 1
LOCK WAIT 10 lock struct(s), heap size 1248, 5 row lock(s), undo log entries 2
MySQL thread id 190, OS thread handle 0x7faaf2266700, query id 24373011 127.0.0.1 stash update
insert into sta_pr_rescope_activity (from_hash, prev_from_hash, prev_to_hash, to_hash, commits_added, commits_removed, activity_id) values ('1d85987cca552726e9cc6c20c72de2d50eb72383', 'a6f4685b8645ce3c76797299089d45b41c112af2', '96ceef69c58282180f3df05b0b629d593597bac5', '96ceef69c58282180f3df05b0b629d593597bac5', null, null, 733839)
*** (1) WAITING FOR THIS LOCK TO BE GRANTED:
RECORD LOCKS space id 0 page no 32231 n bits 136 index `PRIMARY` of table `stash`.`sta_pr_rescope_activity` trx id 260FFDCE lock_mode X insert intention waiting
*** (2) TRANSACTION:
TRANSACTION 260FF9F8, ACTIVE 50 sec fetching rows, thread declared inside InnoDB 217
mysql tables in use 2, locked 2
2599 lock struct(s), heap size 326072, 1092 row lock(s), undo log entries 540
MySQL thread id 203, OS thread handle 0x7faaf1f19700, query id 24371636 127.0.0.1 stash Sending data
delete from sta_activity where (id) IN (select activity_id from HT_sta_pr_rescope_activity)
*** (2) HOLDS THE LOCK(S):
RECORD LOCKS space id 0 page no 32231 n bits 136 index `PRIMARY` of table `stash`.`sta_pr_rescope_activity` trx id 260FF9F8 lock mode S
*** (2) WAITING FOR THIS LOCK TO BE GRANTED:
RECORD LOCKS space id 0 page no 36069 n bits 152 index `PRIMARY` of table `stash`.`sta_activity` trx id 260FF9F8 lock_mode X locks rec but not gap waiting
*** WE ROLL BACK TRANSACTION (1)
------------

            Alexey Efimov added a comment - ------------------------ LATEST DETECTED DEADLOCK ------------------------ 140410 20:00:02 *** (1) TRANSACTION: TRANSACTION 260FFDCE, ACTIVE 6 sec inserting mysql tables in use 1, locked 1 LOCK WAIT 10 lock struct(s), heap size 1248, 5 row lock(s), undo log entries 2 MySQL thread id 190, OS thread handle 0x7faaf2266700, query id 24373011 127.0.0.1 stash update insert into sta_pr_rescope_activity (from_hash, prev_from_hash, prev_to_hash, to_hash, commits_added, commits_removed, activity_id) values ( '1d85987cca552726e9cc6c20c72de2d50eb72383' , 'a6f4685b8645ce3c76797299089d45b41c112af2' , '96ceef69c58282180f3df05b0b629d593597bac5' , '96ceef69c58282180f3df05b0b629d593597bac5' , null , null , 733839) *** (1) WAITING FOR THIS LOCK TO BE GRANTED: RECORD LOCKS space id 0 page no 32231 n bits 136 index `PRIMARY` of table `stash`.`sta_pr_rescope_activity` trx id 260FFDCE lock_mode X insert intention waiting *** (2) TRANSACTION: TRANSACTION 260FF9F8, ACTIVE 50 sec fetching rows, thread declared inside InnoDB 217 mysql tables in use 2, locked 2 2599 lock struct(s), heap size 326072, 1092 row lock(s), undo log entries 540 MySQL thread id 203, OS thread handle 0x7faaf1f19700, query id 24371636 127.0.0.1 stash Sending data delete from sta_activity where (id) IN (select activity_id from HT_sta_pr_rescope_activity) *** (2) HOLDS THE LOCK(S): RECORD LOCKS space id 0 page no 32231 n bits 136 index `PRIMARY` of table `stash`.`sta_pr_rescope_activity` trx id 260FF9F8 lock mode S *** (2) WAITING FOR THIS LOCK TO BE GRANTED: RECORD LOCKS space id 0 page no 36069 n bits 152 index `PRIMARY` of table `stash`.`sta_activity` trx id 260FF9F8 lock_mode X locks rec but not gap waiting *** WE ROLL BACK TRANSACTION (1) ------------

            Bryan Turner (Inactive) added a comment -

            Alexey,

            From the stack trace this is pretty clearly a separate issue from the other. The property that worked around the other issue will have absolutely no relevance here; this error is not happening on a rescope-details thread.

            Please connect to your database and provide the deadlock details from SHOW ENGINE INNODB STATUS.

            Best regards,
            Bryan Turner
            Atlassian Stash

            Bryan Turner (Inactive) added a comment - Alexey, From the stack trace this is pretty clearly a separate issue from the other. The property that worked around the other issue will have absolutely no relevance here; this error is not happening on a rescope-details thread. Please connect to your database and provide the deadlock details from SHOW ENGINE INNODB STATUS . Best regards, Bryan Turner Atlassian Stash

            Alexey Efimov added a comment -

            In 2.12.1, property still in config:

            2014-04-09 21:31:42,549 DEBUG [http-bio-7990-exec-149] user 1279x6764x19 1fqzwzv 127.0.0.1 "POST /rest/api/latest/projects/PROJECT/repos/repo/pull-requests/13580/merge HTTP/1.0" c.a.s.r.e.ServiceExceptionMapper Mapping S
erviceException to REST response 500
com.atlassian.stash.exception.DataStoreException: A database error has occurred.
        at com.atlassian.stash.internal.aop.ExceptionRewriteAdvice.afterThrowing(ExceptionRewriteAdvice.java:46) ~[stash-platform-2.12.1.jar:na]
        at com.sun.proxy.$Proxy185.merge(Unknown Source) ~[na:na]
        at com.sun.proxy.$Proxy263.merge(Unknown Source) ~[na:na]
        at com.sun.proxy.$Proxy870.merge(Unknown Source) ~[na:na]
        at com.atlassian.stash.internal.rest.pull.PullRequestResource.mergePullRequest(PullRequestResource.java:424) ~[stash-rest-2.12.1.jar:na]
        at com.atlassian.applinks.core.rest.context.ContextFilter.doFilter(ContextFilter.java:25) [applinks-plugin-4.1.1_1393464360000.jar:na]
        at com.atlassian.stash.internal.spring.security.StashAuthenticationFilter.doFilter(StashAuthenticationFilter.java:92) [StashAuthenticationFilter.class:na]
        at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doInsideSpringSecurityChain(BeforeLoginPluginAuthenticationFilter.java:111) [BeforeLoginPluginAuthenticationFilter.class:na]
        at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doFilter(BeforeLoginPluginAuthenticationFilter.java:77) [BeforeLoginPluginAuthenticationFilter.class:na]
        at com.atlassian.security.auth.trustedapps.filter.TrustedApplicationsFilter.doFilter(TrustedApplicationsFilter.java:100) [atlassian-trusted-apps-core-3.0.3.jar:na]
        at com.atlassian.oauth.serviceprovider.internal.servlet.OAuthFilter.doFilter(OAuthFilter.java:61) [atlassian-oauth-service-provider-plugin-1.9.1_1393300222000.jar:na]
        at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doBeforeBeforeLoginFilters(BeforeLoginPluginAuthenticationFilter.java:89) [BeforeLoginPluginAuthenticationFilter.class:na]
        at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doFilter(BeforeLoginPluginAuthenticationFilter.java:75) [BeforeLoginPluginAuthenticationFilter.class:na]
        at com.atlassian.stash.internal.request.DefaultRequestManager.doAsRequest(DefaultRequestManager.java:73) [stash-service-impl-2.12.1.jar:na]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_45]
        at java.lang.Thread.run(Thread.java:744) [na:1.7.0_45]
        ... 214 frames trimmed
Caused by: org.springframework.dao.CannotAcquireLockException: could not execute statement; SQL [n/a]; nested exception is org.hibernate.exception.LockAcquisitionException: could not execute statement
        at org.springframework.orm.hibernate4.SessionFactoryUtils.convertHibernateAccessException(SessionFactoryUtils.java:152) ~[spring-orm-4.0.2.RELEASE.jar:4.0.2.RELEASE]
        at org.springframework.orm.hibernate4.HibernateTransactionManager.convertHibernateAccessException(HibernateTransactionManager.java:681) ~[spring-orm-4.0.2.RELEASE.jar:4.0.2.RELEASE]
        at org.springframework.orm.hibernate4.HibernateTransactionManager.doCommit(HibernateTransactionManager.java:563) ~[spring-orm-4.0.2.RELEASE.jar:4.0.2.RELEASE]
        at org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:757) ~[spring-tx-4.0.2.RELEASE.jar:4.0.2.RELEASE]
        at org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:726) ~[spring-tx-4.0.2.RELEASE.jar:4.0.2.RELEASE]
        ... 16 common frames omitted
Caused by: org.hibernate.exception.LockAcquisitionException: could not execute statement
        at org.hibernate.dialect.MySQLDialect$1.convert(MySQLDialect.java:451) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.exception.internal.StandardSQLExceptionConverter.convert(StandardSQLExceptionConverter.java:49) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:126) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:112) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.engine.jdbc.internal.ResultSetReturnImpl.executeUpdate(ResultSetReturnImpl.java:190) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.persister.entity.AbstractEntityPersister.insert(AbstractEntityPersister.java:3127) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.persister.entity.AbstractEntityPersister.insert(AbstractEntityPersister.java:3581) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.action.internal.EntityInsertAction.execute(EntityInsertAction.java:103) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.engine.spi.ActionQueue.executeActions(ActionQueue.java:461) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.engine.spi.ActionQueue.executeActions(ActionQueue.java:347) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.event.internal.AbstractFlushingEventListener.performExecutions(AbstractFlushingEventListener.java:350) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.event.internal.DefaultFlushEventListener.onFlush(DefaultFlushEventListener.java:56) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.internal.SessionImpl.flush(SessionImpl.java:1222) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.internal.SessionImpl.managedFlush(SessionImpl.java:425) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.engine.transaction.internal.jdbc.JdbcTransaction.beforeTransactionCommit(JdbcTransaction.java:101) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.hibernate.engine.transaction.spi.AbstractTransactionImpl.commit(AbstractTransactionImpl.java:177) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        at org.springframework.orm.hibernate4.HibernateTransactionManager.doCommit(HibernateTransactionManager.java:555) ~[spring-orm-4.0.2.RELEASE.jar:4.0.2.RELEASE]
        ... 18 common frames omitted
Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[na:1.7.0_45]
        at com.mysql.jdbc.Util.handleNewInstance(Util.java:411) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.Util.getInstance(Util.java:386) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1066) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4237) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4169) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2617) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2778) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2834) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2156) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2441) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2366) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2350) ~[mysql-connector-java-5.1.29-bin.jar:na]
        at com.jolbox.bonecp.PreparedStatementHandle.executeUpdate(PreparedStatementHandle.java:203) ~[bonecp-0.7.1.RELEASE.jar:0.7.1.RELEASE]
        at org.hibernate.engine.jdbc.internal.ResultSetReturnImpl.executeUpdate(ResultSetReturnImpl.java:187) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final]
        ... 30 common frames omitted

            Alexey Efimov added a comment - In 2.12.1, property still in config: 2014-04-09 21:31:42,549 DEBUG [http-bio-7990-exec-149] user 1279x6764x19 1fqzwzv 127.0.0.1 "POST / rest /api/latest/projects/PROJECT/repos/repo/pull-requests/13580/merge HTTP/1.0" c.a.s.r.e.ServiceExceptionMapper Mapping S erviceException to REST response 500 com.atlassian.stash.exception.DataStoreException: A database error has occurred. at com.atlassian.stash.internal.aop.ExceptionRewriteAdvice.afterThrowing(ExceptionRewriteAdvice.java:46) ~[stash-platform-2.12.1.jar:na] at com.sun.proxy.$Proxy185.merge(Unknown Source) ~[na:na] at com.sun.proxy.$Proxy263.merge(Unknown Source) ~[na:na] at com.sun.proxy.$Proxy870.merge(Unknown Source) ~[na:na] at com.atlassian.stash.internal. rest .pull.PullRequestResource.mergePullRequest(PullRequestResource.java:424) ~[stash- rest -2.12.1.jar:na] at com.atlassian.applinks.core. rest .context.ContextFilter.doFilter(ContextFilter.java:25) [applinks-plugin-4.1.1_1393464360000.jar:na] at com.atlassian.stash.internal.spring.security.StashAuthenticationFilter.doFilter(StashAuthenticationFilter.java:92) [StashAuthenticationFilter. class: na] at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doInsideSpringSecurityChain(BeforeLoginPluginAuthenticationFilter.java:111) [BeforeLoginPluginAuthenticationFilter. class: na] at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doFilter(BeforeLoginPluginAuthenticationFilter.java:77) [BeforeLoginPluginAuthenticationFilter. class: na] at com.atlassian.security.auth.trustedapps.filter.TrustedApplicationsFilter.doFilter(TrustedApplicationsFilter.java:100) [atlassian-trusted-apps-core-3.0.3.jar:na] at com.atlassian.oauth.serviceprovider.internal.servlet.OAuthFilter.doFilter(OAuthFilter.java:61) [atlassian-oauth-service-provider-plugin-1.9.1_1393300222000.jar:na] at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doBeforeBeforeLoginFilters(BeforeLoginPluginAuthenticationFilter.java:89) [BeforeLoginPluginAuthenticationFilter. class: na] at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doFilter(BeforeLoginPluginAuthenticationFilter.java:75) [BeforeLoginPluginAuthenticationFilter. class: na] at com.atlassian.stash.internal.request.DefaultRequestManager.doAsRequest(DefaultRequestManager.java:73) [stash-service-impl-2.12.1.jar:na] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_45] at java.lang. Thread .run( Thread .java:744) [na:1.7.0_45] ... 214 frames trimmed Caused by: org.springframework.dao.CannotAcquireLockException: could not execute statement; SQL [n/a]; nested exception is org.hibernate.exception.LockAcquisitionException: could not execute statement at org.springframework.orm.hibernate4.SessionFactoryUtils.convertHibernateAccessException(SessionFactoryUtils.java:152) ~[spring-orm-4.0.2.RELEASE.jar:4.0.2.RELEASE] at org.springframework.orm.hibernate4.HibernateTransactionManager.convertHibernateAccessException(HibernateTransactionManager.java:681) ~[spring-orm-4.0.2.RELEASE.jar:4.0.2.RELEASE] at org.springframework.orm.hibernate4.HibernateTransactionManager.doCommit(HibernateTransactionManager.java:563) ~[spring-orm-4.0.2.RELEASE.jar:4.0.2.RELEASE] at org.springframework.transaction.support.AbstractPlatformTransactionManager.processCommit(AbstractPlatformTransactionManager.java:757) ~[spring-tx-4.0.2.RELEASE.jar:4.0.2.RELEASE] at org.springframework.transaction.support.AbstractPlatformTransactionManager.commit(AbstractPlatformTransactionManager.java:726) ~[spring-tx-4.0.2.RELEASE.jar:4.0.2.RELEASE] ... 16 common frames omitted Caused by: org.hibernate.exception.LockAcquisitionException: could not execute statement at org.hibernate.dialect.MySQLDialect$1.convert(MySQLDialect.java:451) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.exception.internal.StandardSQLExceptionConverter.convert(StandardSQLExceptionConverter.java:49) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:126) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:112) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.engine.jdbc.internal.ResultSetReturnImpl.executeUpdate(ResultSetReturnImpl.java:190) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.persister.entity.AbstractEntityPersister.insert(AbstractEntityPersister.java:3127) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.persister.entity.AbstractEntityPersister.insert(AbstractEntityPersister.java:3581) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.action.internal.EntityInsertAction.execute(EntityInsertAction.java:103) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.engine.spi.ActionQueue.executeActions(ActionQueue.java:461) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.engine.spi.ActionQueue.executeActions(ActionQueue.java:347) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.event.internal.AbstractFlushingEventListener.performExecutions(AbstractFlushingEventListener.java:350) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.event.internal.DefaultFlushEventListener.onFlush(DefaultFlushEventListener.java:56) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.internal.SessionImpl.flush(SessionImpl.java:1222) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.internal.SessionImpl.managedFlush(SessionImpl.java:425) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.engine.transaction.internal.jdbc.JdbcTransaction.beforeTransactionCommit(JdbcTransaction.java:101) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.hibernate.engine.transaction.spi.AbstractTransactionImpl.commit(AbstractTransactionImpl.java:177) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] at org.springframework.orm.hibernate4.HibernateTransactionManager.doCommit(HibernateTransactionManager.java:555) ~[spring-orm-4.0.2.RELEASE.jar:4.0.2.RELEASE] ... 18 common frames omitted Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLTransactionRollbackException: Deadlock found when trying to get lock; try restarting transaction at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[na:1.7.0_45] at com.mysql.jdbc.Util.handleNewInstance(Util.java:411) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.Util.getInstance(Util.java:386) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1066) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4237) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4169) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2617) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2778) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2834) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2156) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2441) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2366) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2350) ~[mysql-connector-java-5.1.29-bin.jar:na] at com.jolbox.bonecp.PreparedStatementHandle.executeUpdate(PreparedStatementHandle.java:203) ~[bonecp-0.7.1.RELEASE.jar:0.7.1.RELEASE] at org.hibernate.engine.jdbc.internal.ResultSetReturnImpl.executeUpdate(ResultSetReturnImpl.java:187) ~[hibernate-core-4.3.4.Final.jar:4.3.4.Final] ... 30 common frames omitted

              bturner Bryan Turner (Inactive)
              cofarrell CharlesA
              Affected customers:
              0 This affects my team
              Watchers:
              10 Start watching this issue

                Created:
                Updated:
                Resolved: