Customer Notifications not being sent for all projects - (Caused by deleteTenantJobByJobId failed)

deleteTenantJobByJobId failed for config=CronmanLocalClientConfig{tenantId=TenantId{id=<TenantId>}, asapIssuer=com.atlassian.jira.scheduler.JiraCronmanAsapIssuer$$Lambda$144/1851488762@40a9ec78, timeZone=sun.util.calendar.ZoneInfo[id="America/New_York",offset=-18000000,dstSavings=3600000,useDaylight=true,transitions=235,lastRule=java.util.SimpleTimeZone[id=America/New_York,offset=-18000000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]], bucketId=com.atlassian.jira.SchedulerServiceRegistrar$$Lambda$89/1597494596@4a09f0ed, notificationMethod=SQS, notificationUrl=NotificationUrl{urlString=<urlString>}, wrapPayloadFunction=com.atlassian.scheduler.cronman.service.AmqSqsConfig$$Lambda$3848/1978512511@12632fda, unwrapPayloadFunction=com.atlassian.scheduler.cronman.service.AmqSqsConfig$$Lambda$3849/1200323983@74bc55e9} jobId=sd.custom.notification.batch.send

com.atlassian.scheduler.SchedulerRuntimeException: deleteTenantJobByJobId failed for config=CronmanLocalClientConfig{tenantId=TenantId{id=<TenantId>}, asapIssuer=com.atlassian.jira.scheduler.JiraCronmanAsapIssuer$$Lambda$144/1851488762@40a9ec78, timeZone=sun.util.calendar.ZoneInfo[id="America/New_York",offset=-18000000,dstSavings=3600000,useDaylight=true,transitions=235,lastRule=java.util.SimpleTimeZone[id=America/New_York,offset=-18000000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]], bucketId=com.atlassian.jira.SchedulerServiceRegistrar$$Lambda$89/1597494596@4a09f0ed, notificationMethod=SQS, notificationUrl=NotificationUrl{urlString=<urlString>}, wrapPayloadFunction=com.atlassian.scheduler.cronman.service.AmqSqsConfig$$Lambda$3848/1978512511@12632fda, unwrapPayloadFunction=com.atlassian.scheduler.cronman.service.AmqSqsConfig$$Lambda$3849/1200323983@74bc55e9} jobId=sd.custom.notification.batch.send at com.atlassian.scheduler.cronman.service.CompositeSchedulerService.unscheduleJob(CompositeSchedulerService.java:253) at com.atlassian.scheduler.core.DelegatingSchedulerService.unscheduleJob(DelegatingSchedulerService.java:80) at sun.reflect.GeneratedMethodAccessor214.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.atlassian.plugin.util.ContextClassLoaderSettingInvocationHandler.invoke(ContextClassLoaderSettingInvocationHandler.java:26) at com.sun.proxy.$Proxy457.unscheduleJob(Unknown Source) at sun.reflect.GeneratedMethodAccessor214.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:302) at org.eclipse.gemini.blueprint.service.importer.support.internal.aop.ServiceInvoker.doInvoke(ServiceInvoker.java:56) at org.eclipse.gemini.blueprint.service.importer.support.internal.aop.ServiceInvoker.invoke(ServiceInvoker.java:60) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.support.DelegatingIntroductionInterceptor.doProceed(DelegatingIntroductionInterceptor.java:133) at org.springframework.aop.support.DelegatingIntroductionInterceptor.invoke(DelegatingIntroductionInterceptor.java:121) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.eclipse.gemini.blueprint.service.util.internal.aop.ServiceTCCLInterceptor$1.run(ServiceTCCLInterceptor.java:61) at org.eclipse.gemini.blueprint.util.internal.PrivilegedUtils.executeWithCustomTCCL(PrivilegedUtils.java:130) at org.eclipse.gemini.blueprint.service.util.internal.aop.ServiceTCCLInterceptor.invokePrivileged(ServiceTCCLInterceptor.java:58) at org.eclipse.gemini.blueprint.service.util.internal.aop.ServiceTCCLInterceptor.invoke(ServiceTCCLInterceptor.java:50) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.eclipse.gemini.blueprint.service.importer.support.LocalBundleContextAdvice.invoke(LocalBundleContextAdvice.java:57) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.support.DelegatingIntroductionInterceptor.doProceed(DelegatingIntroductionInterceptor.java:133) at org.springframework.aop.support.DelegatingIntroductionInterceptor.invoke(DelegatingIntroductionInterceptor.java:121) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:208) at com.sun.proxy.$Proxy4510.unscheduleJob(Unknown Source) at com.atlassian.servicedesk.plugins.notifications.internal.scheduler.NotificationBatchJobRunner.unscheduleBatchSending(NotificationBatchJobRunner.java:93) at com.atlassian.servicedesk.plugins.notifications.internal.scheduler.NotificationBatchJobRunner.sendNotifications(NotificationBatchJobRunner.java:88) at com.atlassian.servicedesk.plugins.notifications.internal.scheduler.NotificationBatchJobRunner.runWithClusterLock(NotificationBatchJobRunner.java:107) at com.atlassian.servicedesk.plugins.notifications.internal.scheduler.NotificationBatchJobRunner.processMessage(NotificationBatchJobRunner.java:81) at com.atlassian.messagequeue.internal.core.NestedMessageConsumer.consume(NestedMessageConsumer.java:116) at com.atlassian.messagequeue.internal.sqs.SQSMessageConsumer.processReceiveMessageResult(SQSMessageConsumer.java:110) at com.atlassian.messagequeue.internal.sqs.SQSMessageConsumer.run(SQSMessageConsumer.java:62) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Caused by: com.atlassian.scheduler.cronman.client.CronmanLocalClientException: deleteTenantJobByJobId failed for config=CronmanLocalClientConfig{tenantId=TenantId{id=6753a4af-f227-446d-9ae6-d99c3eaf95c9}, asapIssuer=com.atlassian.jira.scheduler.JiraCronmanAsapIssuer$$Lambda$144/1851488762@40a9ec78, timeZone=sun.util.calendar.ZoneInfo[id="America/New_York",offset=-18000000,dstSavings=3600000,useDaylight=true,transitions=235,lastRule=java.util.SimpleTimeZone[id=America/New_York,offset=-18000000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]], bucketId=com.atlassian.jira.SchedulerServiceRegistrar$$Lambda$89/1597494596@4a09f0ed, notificationMethod=SQS, notificationUrl=NotificationUrl{urlString=https://sqs.us-west-2.amazonaws.com/234617365107/rps-prod-west2-jira-prod-us-9-amq}, wrapPayloadFunction=com.atlassian.scheduler.cronman.service.AmqSqsConfig$$Lambda$3848/1978512511@12632fda, unwrapPayloadFunction=com.atlassian.scheduler.cronman.service.AmqSqsConfig$$Lambda$3849/1200323983@74bc55e9} jobId=sd.custom.notification.batch.send at com.atlassian.scheduler.cronman.client.CronmanLocalClient.getException(CronmanLocalClient.java:231) at com.atlassian.scheduler.cronman.client.CronmanLocalClient.delete(CronmanLocalClient.java:160) at com.atlassian.scheduler.cronman.service.CronmanJobScheduler.unscheduleJob(CronmanJobScheduler.java:256) at com.atlassian.scheduler.cronman.service.CompositeSchedulerService.unscheduleJob(CompositeSchedulerService.java:250) ... 38 more at com.atlassian.scheduler.cronman.rest.client.CronmanClient.callUnverified(CronmanClient.java:282) at com.atlassian.scheduler.cronman.rest.client.CronmanClient.deleteTenantJobByJobId(CronmanClient.java:225) at com.atlassian.scheduler.cronman.client.CronmanLocalClient.delete(CronmanLocalClient.java:156) ... 40 more Caused by: java.net.SocketTimeoutException: timeout at okio.Okio$4.newTimeoutException(Okio.java:227) at okio.AsyncTimeout.exit(AsyncTimeout.java:284) at okio.AsyncTimeout$2.read(AsyncTimeout.java:240) at okio.RealBufferedSource.indexOf(RealBufferedSource.java:325) at okio.RealBufferedSource.indexOf(RealBufferedSource.java:314) at okhttp3.internal.http1.Http1Codec.readResponse(Http1Codec.java:191) at okhttp3.internal.http1.Http1Codec.readResponseHeaders(Http1Codec.java:132) at okhttp3.internal.http.CallServerInterceptor.intercept(CallServerInterceptor.java:54) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:45) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:179) at okhttp3.RealCall.execute(RealCall.java:63) at com.atlassian.scheduler.cronman.rest.client.CronmanClient.callUnverified(CronmanClient.java:275) ... 42 more Caused by: java.net.SocketException: Socket closed at java.net.SocketInputStream.read(SocketInputStream.java:204) at java.net.SocketInputStream.read(SocketInputStream.java:141) at sun.security.ssl.InputRecord.readFully(InputRecord.java:465) at sun.security.ssl.InputRecord.read(InputRecord.java:503) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973) at sun.security.ssl.AppInputStream.read(AppInputStream.java:105) at okio.Okio$2.read(Okio.java:138) at okio.AsyncTimeout$2.read(AsyncTimeout.java:236) ... 63 more

SELECT * FROM clusteredjob WHERE job_runner_key='sd.custom.notification.batch.job.runner'