Issue Summary
This is reproducible on Data Center: yes
Steps to Reproduce
- Setup Jira 8.x with 100_000_000 comments
- Upgrade to 9.0/9.1.0/9.2.0
Expected Results
Upgrade task should finish in the expected time (~1min per 1M issue/comments/worklogs)
Actual Results
The below exception is thrown in the atlassian-jira.log file:
2022-09-18 10:39:34,319+0000 main ERROR [c.a.upgrade.core.DefaultUpgradeTaskFactoryProcessor] Upgrade task [host,buildNumber=900000] failed
18java.lang.RuntimeException: Error running original upgrade task
19 at com.atlassian.jira.upgrade.AbstractUpgradeTask.runUpgrade(AbstractUpgradeTask.java:63)
20 at com.atlassian.upgrade.core.DefaultUpgradeTaskFactoryProcessor.runOneUpgradeTask(DefaultUpgradeTaskFactoryProcessor.java:109)
21 at com.atlassian.upgrade.core.DefaultUpgradeTaskFactoryProcessor.lambda$performUpgradesUnsafe$13(DefaultUpgradeTaskFactoryProcessor.java:80)
22 at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:174)
23 at java.util.stream.SortedOps$RefSortingSink.end(SortedOps.java:400)
24 at java.util.stream.Sink$ChainedReference.end(Sink.java:258)
25 at java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:500)
26 at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:486)
27 at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
28 at java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:152)
29 at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
30 at java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:531)
31 at com.atlassian.upgrade.core.DefaultUpgradeTaskFactoryProcessor.performUpgradesUnsafe(DefaultUpgradeTaskFactoryProcessor.java:81)
32 at com.atlassian.upgrade.core.DefaultUpgradeTaskFactoryProcessor.performUpgrades(DefaultUpgradeTaskFactoryProcessor.java:46)
33 at com.atlassian.upgrade.core.DefaultUpgradeTaskManager.upgradeHostApp(DefaultUpgradeTaskManager.java:41)
34 at com.atlassian.jira.upgrade.LicenseCheckingUpgradeService.executeUpgrades(LicenseCheckingUpgradeService.java:134)
35 at com.atlassian.jira.upgrade.LicenseCheckingUpgradeService.runUpgrades(LicenseCheckingUpgradeService.java:97)
36 at com.atlassian.jira.upgrade.ClusterLockingUpgradeService.runUpgrades(ClusterLockingUpgradeService.java:35)
37 at com.atlassian.jira.upgrade.LoggingUpgradeService.lambda$runUpgradesWithLogging$0(LoggingUpgradeService.java:28)
38 at com.atlassian.jira.upgrade.LoggingUpgradeService.runWithTaskLogging(LoggingUpgradeService.java:43)
39 at com.atlassian.jira.upgrade.LoggingUpgradeService.runUpgradesWithLogging(LoggingUpgradeService.java:28)
40 at com.atlassian.jira.upgrade.IndexingUpgradeService.runUpgrades(IndexingUpgradeService.java:19)
41 at com.atlassian.jira.upgrade.DefaultUpgradeService.runUpgrades(DefaultUpgradeService.java:61)
42 at com.atlassian.jira.upgrade.UpgradeLauncher.upgradeIfNeeded(UpgradeLauncher.java:109)
43 at com.atlassian.jira.upgrade.UpgradeLauncher.start(UpgradeLauncher.java:86)
44 at com.atlassian.jira.startup.ActiveServicesLauncher.start(ActiveServicesLauncher.java:50)
45 at com.atlassian.jira.startup.DefaultJiraLauncher.postDBActivated(DefaultJiraLauncher.java:170)
46 at com.atlassian.jira.startup.DefaultJiraLauncher.lambda$postDbLaunch$2(DefaultJiraLauncher.java:144)
47 at com.atlassian.jira.config.database.DatabaseConfigurationManagerImpl.doNowOrEnqueue(DatabaseConfigurationManagerImpl.java:307)
48 at com.atlassian.jira.config.database.DatabaseConfigurationManagerImpl.doNowOrWhenDatabaseActivated(DatabaseConfigurationManagerImpl.java:202)
49 at com.atlassian.jira.startup.DefaultJiraLauncher.postDbLaunch(DefaultJiraLauncher.java:135)
50 at com.atlassian.jira.startup.DefaultJiraLauncher.lambda$start$0(DefaultJiraLauncher.java:102)
51 at com.atlassian.jira.util.devspeed.JiraDevSpeedTimer.run(JiraDevSpeedTimer.java:31)
52 at com.atlassian.jira.startup.DefaultJiraLauncher.start(DefaultJiraLauncher.java:100)
53 at com.atlassian.jira.startup.LauncherContextListener.initSlowStuff(LauncherContextListener.java:154)
54 at com.atlassian.jira.startup.LauncherContextListener.initSlowStuffInBackground(LauncherContextListener.java:139)
55 at com.atlassian.jira.startup.LauncherContextListener.contextInitialized(LauncherContextListener.java:101)
56 ... 5 filtered
57 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
58 at org.apache.tomcat.util.threads.InlineExecutorService.execute(InlineExecutorService.java:75)
59 at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134)
60 ... 5 filtered
61 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
62 at org.apache.tomcat.util.threads.InlineExecutorService.execute(InlineExecutorService.java:75)
63 at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134)
64 ... 11 filtered
65 at java.lang.reflect.Method.invoke(Method.java:498)
66 ... 2 filtered
67Caused by: com.querydsl.core.QueryException: Caught PSQLException for insert into public.comment_version (comment_id, parent_issue_id, update_time, index_version, deleted)
68select ACTION.id, ACTION.issueid, ?, ?, ?
69from public.jiraaction ACTION
70left join public.comment_version comment_version
71on comment_version.comment_id = ACTION.id
72where comment_version.comment_id is null and ACTION.actiontype = ?
73 ... 2 filtered
74 at com.querydsl.sql.DefaultSQLExceptionTranslator.translate(DefaultSQLExceptionTranslator.java:50)
75 at com.querydsl.sql.Configuration.translate(Configuration.java:459)
76 at com.querydsl.sql.dml.SQLInsertClause.execute(SQLInsertClause.java:445)
77 at com.atlassian.jira.upgrade.tasks.UpgradeTask_Build900000.lambda$populateCommentVersion$4(UpgradeTask_Build900000.java:204)
78 at com.atlassian.jira.database.DefaultQueryDslAccessor$1.lambda$executeQuery$0(DefaultQueryDslAccessor.java:81)
79 at com.atlassian.jira.database.DatabaseAccessorImpl.executeQuery(DatabaseAccessorImpl.java:74)
80 at com.atlassian.jira.database.DefaultQueryDslAccessor$1.executeQuery(DefaultQueryDslAccessor.java:80)
81 at com.atlassian.jira.upgrade.tasks.UpgradeTask_Build900000.populateCommentVersion(UpgradeTask_Build900000.java:198)
82 at com.atlassian.jira.upgrade.tasks.UpgradeTask_Build900000.runMeasured(UpgradeTask_Build900000.java:168)
83 at com.atlassian.jira.upgrade.tasks.UpgradeTask_Build900000.doUpgrade(UpgradeTask_Build900000.java:159)
84 at com.atlassian.jira.upgrade.AbstractUpgradeTask.runUpgrade(AbstractUpgradeTask.java:61)
85 ... 66 more
86Caused by: org.postgresql.util.PSQLException: An I/O error occurred while sending to the backend.
87 at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:349)
88 at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:481)
89 at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:401)
90 at org.postgresql.jdbc.PgPreparedStatement.executeWithFlags(PgPreparedStatement.java:164)
91 at org.postgresql.jdbc.PgPreparedStatement.executeUpdate(PgPreparedStatement.java:130)
92 at org.apache.commons.dbcp2.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:98)
93 at com.atlassian.jira.ofbiz.sql.PreparedStatementWrapper.executeUpdate(PreparedStatementWrapper.java:47)
94 at com.atlassian.jira.diagnostic.connection.DiagnosticPreparedStatement.lambda$executeUpdate$7(DiagnosticPreparedStatement.java:69)
95 at com.atlassian.diagnostics.internal.platform.monitor.db.DefaultDatabaseDiagnosticsCollector.recordExecutionTime(DefaultDatabaseDiagnosticsCollector.java:104)
96 at com.atlassian.jira.diagnostic.connection.DatabaseDiagnosticsCollectorDelegate.recordExecutionTime(DatabaseDiagnosticsCollectorDelegate.java:62)
97 at com.atlassian.jira.diagnostic.connection.DiagnosticPreparedStatement.executeUpdate(DiagnosticPreparedStatement.java:69)
98 at com.querydsl.sql.dml.SQLInsertClause.execute(SQLInsertClause.java:423)
99 ... 74 more
100Caused by: java.net.SocketTimeoutException: Read timed out
101 at java.net.SocketInputStream.socketRead0(Native Method)
102 at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
103 at java.net.SocketInputStream.read(SocketInputStream.java:171)
104 at java.net.SocketInputStream.read(SocketInputStream.java:141)
105 at sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:464)
106 at sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1346)
107 at sun.security.ssl.SSLSocketImpl.access$300(SSLSocketImpl.java:73)
108 at sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:962)
109 at org.postgresql.core.VisibleBufferedInputStream.readMore(VisibleBufferedInputStream.java:161)
110 at org.postgresql.core.VisibleBufferedInputStream.ensureBytes(VisibleBufferedInputStream.java:128)
111 at org.postgresql.core.VisibleBufferedInputStream.ensureBytes(VisibleBufferedInputStream.java:113)
112 at org.postgresql.core.VisibleBufferedInputStream.read(VisibleBufferedInputStream.java:73)
113 at org.postgresql.core.PGStream.receiveChar(PGStream.java:443)
114 at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2069)
115 at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:322)
116 ... 86 more
1172022-09-18 10:39:34,321+0000 pool-21-thread-1 WARN [atlassian-diagnostics] 2022-09-18T10:39:34.317Z Component 'Database' alerted 'Database connection leaked' (details: {"activeConnections":1,"idleConnections":21,"maxConnections":40,"connectionAcquiredTimestampInMillis":1663497274119}, trigger: {"pluginKey": "not-detected"})
1182022-09-18 10:39:34,380+0000 main WARN [c.a.upgrade.core.DefaultUpgradeTaskFactoryProcessor] Failed to complete upgrade task [factory=host,buildNumber=900000]
Workaround
Currently there is no known workaround for this behavior. A workaround will be added here when available