Issue Summary
This is reproducible on Data Center: yes
Steps to Reproduce
- Setup Jira 8.x with 100_000_000 comments
- Upgrade to 9.0/9.1.0/9.2.0
Expected Results
Upgrade task should finish in the expected time (~1min per 1M issue/comments/worklogs)
Actual Results
The below exception is thrown in the atlassian-jira.log file:
2022-09-18 10:39:34,319+0000 main ERROR [c.a.upgrade.core.DefaultUpgradeTaskFactoryProcessor] Upgrade task [host,buildNumber=900000] failed 18java.lang.RuntimeException: Error running original upgrade task 19 at com.atlassian.jira.upgrade.AbstractUpgradeTask.runUpgrade(AbstractUpgradeTask.java:63) 20 at com.atlassian.upgrade.core.DefaultUpgradeTaskFactoryProcessor.runOneUpgradeTask(DefaultUpgradeTaskFactoryProcessor.java:109) 21 at com.atlassian.upgrade.core.DefaultUpgradeTaskFactoryProcessor.lambda$performUpgradesUnsafe$13(DefaultUpgradeTaskFactoryProcessor.java:80) 22 at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:174) 23 at java.util.stream.SortedOps$RefSortingSink.end(SortedOps.java:400) 24 at java.util.stream.Sink$ChainedReference.end(Sink.java:258) 25 at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:500) 26 at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:486) 27 at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472) 28 at java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:152) 29 at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) 30 at java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:531) 31 at com.atlassian.upgrade.core.DefaultUpgradeTaskFactoryProcessor.performUpgradesUnsafe(DefaultUpgradeTaskFactoryProcessor.java:81) 32 at com.atlassian.upgrade.core.DefaultUpgradeTaskFactoryProcessor.performUpgrades(DefaultUpgradeTaskFactoryProcessor.java:46) 33 at com.atlassian.upgrade.core.DefaultUpgradeTaskManager.upgradeHostApp(DefaultUpgradeTaskManager.java:41) 34 at com.atlassian.jira.upgrade.LicenseCheckingUpgradeService.executeUpgrades(LicenseCheckingUpgradeService.java:134) 35 at com.atlassian.jira.upgrade.LicenseCheckingUpgradeService.runUpgrades(LicenseCheckingUpgradeService.java:97) 36 at com.atlassian.jira.upgrade.ClusterLockingUpgradeService.runUpgrades(ClusterLockingUpgradeService.java:35) 37 at com.atlassian.jira.upgrade.LoggingUpgradeService.lambda$runUpgradesWithLogging$0(LoggingUpgradeService.java:28) 38 at com.atlassian.jira.upgrade.LoggingUpgradeService.runWithTaskLogging(LoggingUpgradeService.java:43) 39 at com.atlassian.jira.upgrade.LoggingUpgradeService.runUpgradesWithLogging(LoggingUpgradeService.java:28) 40 at com.atlassian.jira.upgrade.IndexingUpgradeService.runUpgrades(IndexingUpgradeService.java:19) 41 at com.atlassian.jira.upgrade.DefaultUpgradeService.runUpgrades(DefaultUpgradeService.java:61) 42 at com.atlassian.jira.upgrade.UpgradeLauncher.upgradeIfNeeded(UpgradeLauncher.java:109) 43 at com.atlassian.jira.upgrade.UpgradeLauncher.start(UpgradeLauncher.java:86) 44 at com.atlassian.jira.startup.ActiveServicesLauncher.start(ActiveServicesLauncher.java:50) 45 at com.atlassian.jira.startup.DefaultJiraLauncher.postDBActivated(DefaultJiraLauncher.java:170) 46 at com.atlassian.jira.startup.DefaultJiraLauncher.lambda$postDbLaunch$2(DefaultJiraLauncher.java:144) 47 at com.atlassian.jira.config.database.DatabaseConfigurationManagerImpl.doNowOrEnqueue(DatabaseConfigurationManagerImpl.java:307) 48 at com.atlassian.jira.config.database.DatabaseConfigurationManagerImpl.doNowOrWhenDatabaseActivated(DatabaseConfigurationManagerImpl.java:202) 49 at com.atlassian.jira.startup.DefaultJiraLauncher.postDbLaunch(DefaultJiraLauncher.java:135) 50 at com.atlassian.jira.startup.DefaultJiraLauncher.lambda$start$0(DefaultJiraLauncher.java:102) 51 at com.atlassian.jira.util.devspeed.JiraDevSpeedTimer.run(JiraDevSpeedTimer.java:31) 52 at com.atlassian.jira.startup.DefaultJiraLauncher.start(DefaultJiraLauncher.java:100) 53 at com.atlassian.jira.startup.LauncherContextListener.initSlowStuff(LauncherContextListener.java:154) 54 at com.atlassian.jira.startup.LauncherContextListener.initSlowStuffInBackground(LauncherContextListener.java:139) 55 at com.atlassian.jira.startup.LauncherContextListener.contextInitialized(LauncherContextListener.java:101) 56 ... 5 filtered 57 at java.util.concurrent.FutureTask.run(FutureTask.java:266) 58 at org.apache.tomcat.util.threads.InlineExecutorService.execute(InlineExecutorService.java:75) 59 at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134) 60 ... 5 filtered 61 at java.util.concurrent.FutureTask.run(FutureTask.java:266) 62 at org.apache.tomcat.util.threads.InlineExecutorService.execute(InlineExecutorService.java:75) 63 at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134) 64 ... 11 filtered 65 at java.lang.reflect.Method.invoke(Method.java:498) 66 ... 2 filtered 67Caused by: com.querydsl.core.QueryException: Caught PSQLException for insert into public.comment_version (comment_id, parent_issue_id, update_time, index_version, deleted) 68select ACTION.id, ACTION.issueid, ?, ?, ? 69from public.jiraaction ACTION 70left join public.comment_version comment_version 71on comment_version.comment_id = ACTION.id 72where comment_version.comment_id is null and ACTION.actiontype = ? 73 ... 2 filtered 74 at com.querydsl.sql.DefaultSQLExceptionTranslator.translate(DefaultSQLExceptionTranslator.java:50) 75 at com.querydsl.sql.Configuration.translate(Configuration.java:459) 76 at com.querydsl.sql.dml.SQLInsertClause.execute(SQLInsertClause.java:445) 77 at com.atlassian.jira.upgrade.tasks.UpgradeTask_Build900000.lambda$populateCommentVersion$4(UpgradeTask_Build900000.java:204) 78 at com.atlassian.jira.database.DefaultQueryDslAccessor$1.lambda$executeQuery$0(DefaultQueryDslAccessor.java:81) 79 at com.atlassian.jira.database.DatabaseAccessorImpl.executeQuery(DatabaseAccessorImpl.java:74) 80 at com.atlassian.jira.database.DefaultQueryDslAccessor$1.executeQuery(DefaultQueryDslAccessor.java:80) 81 at com.atlassian.jira.upgrade.tasks.UpgradeTask_Build900000.populateCommentVersion(UpgradeTask_Build900000.java:198) 82 at com.atlassian.jira.upgrade.tasks.UpgradeTask_Build900000.runMeasured(UpgradeTask_Build900000.java:168) 83 at com.atlassian.jira.upgrade.tasks.UpgradeTask_Build900000.doUpgrade(UpgradeTask_Build900000.java:159) 84 at com.atlassian.jira.upgrade.AbstractUpgradeTask.runUpgrade(AbstractUpgradeTask.java:61) 85 ... 66 more 86Caused by: org.postgresql.util.PSQLException: An I/O error occurred while sending to the backend. 87 at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:349) 88 at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:481) 89 at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:401) 90 at org.postgresql.jdbc.PgPreparedStatement.executeWithFlags(PgPreparedStatement.java:164) 91 at org.postgresql.jdbc.PgPreparedStatement.executeUpdate(PgPreparedStatement.java:130) 92 at org.apache.commons.dbcp2.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:98) 93 at com.atlassian.jira.ofbiz.sql.PreparedStatementWrapper.executeUpdate(PreparedStatementWrapper.java:47) 94 at com.atlassian.jira.diagnostic.connection.DiagnosticPreparedStatement.lambda$executeUpdate$7(DiagnosticPreparedStatement.java:69) 95 at com.atlassian.diagnostics.internal.platform.monitor.db.DefaultDatabaseDiagnosticsCollector.recordExecutionTime(DefaultDatabaseDiagnosticsCollector.java:104) 96 at com.atlassian.jira.diagnostic.connection.DatabaseDiagnosticsCollectorDelegate.recordExecutionTime(DatabaseDiagnosticsCollectorDelegate.java:62) 97 at com.atlassian.jira.diagnostic.connection.DiagnosticPreparedStatement.executeUpdate(DiagnosticPreparedStatement.java:69) 98 at com.querydsl.sql.dml.SQLInsertClause.execute(SQLInsertClause.java:423) 99 ... 74 more 100Caused by: java.net.SocketTimeoutException: Read timed out 101 at java.net.SocketInputStream.socketRead0(Native Method) 102 at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) 103 at java.net.SocketInputStream.read(SocketInputStream.java:171) 104 at java.net.SocketInputStream.read(SocketInputStream.java:141) 105 at sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:464) 106 at sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1346) 107 at sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:73) 108 at sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:962) 109 at org.postgresql.core.VisibleBufferedInputStream.readMore(VisibleBufferedInputStream.java:161) 110 at org.postgresql.core.VisibleBufferedInputStream.ensureBytes(VisibleBufferedInputStream.java:128) 111 at org.postgresql.core.VisibleBufferedInputStream.ensureBytes(VisibleBufferedInputStream.java:113) 112 at org.postgresql.core.VisibleBufferedInputStream.read(VisibleBufferedInputStream.java:73) 113 at org.postgresql.core.PGStream.receiveChar(PGStream.java:443) 114 at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2069) 115 at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:322) 116 ... 86 more 1172022-09-18 10:39:34,321+0000 pool-21-thread-1 WARN [atlassian-diagnostics] 2022-09-18T10:39:34.317Z Component 'Database' alerted 'Database connection leaked' (details: {"activeConnections":1,"idleConnections":21,"maxConnections":40,"connectionAcquiredTimestampInMillis":1663497274119}, trigger: {"pluginKey": "not-detected"}) 1182022-09-18 10:39:34,380+0000 main WARN [c.a.upgrade.core.DefaultUpgradeTaskFactoryProcessor] Failed to complete upgrade task [factory=host,buildNumber=900000]
Workaround
Currently there is no known workaround for this behavior. A workaround will be added here when available