Details
-
Suggestion
-
Resolution: Obsolete
-
None
Description
Occasionally there may be transient problems communicating with the AWS. This causes the state of the Instance to be changed into UNKNOWN, which automatically brings down the instance according to Bamboo.
We should be more tolerant of transient errors. We should mark UNKNOWN as a temporary state and only switching it to TERMINATED once a certain threshold of errors have been received.
An example is this error appearing on J2BAC after running for around 9 days. New agents can be restarted without any dramas.
com.xerox.amazonws.ec2.EC2Exception: Problem parsing returned message.
at com.xerox.amazonws.ec2.Jec2.makeRequestInt(Jec2.java:1682)
at com.xerox.amazonws.ec2.Jec2.describeInstances(Jec2.java:678)
at com.atlassian.aws.ec2.RemoteEC2InstanceImpl.describeInstance(RemoteEC2InstanceImpl.java:272)
at com.atlassian.aws.ec2.EC2InstanceState$2.supervise(EC2InstanceState.java:77)
at com.atlassian.aws.ec2.RemoteEC2InstanceImpl.backgroundSupervise(RemoteEC2InstanceImpl.java:413)
at com.atlassian.aws.ec2.RemoteEC2InstanceImpl.access$300(RemoteEC2InstanceImpl.java:25)
at com.atlassian.aws.ec2.RemoteEC2InstanceImpl$2.run(RemoteEC2InstanceImpl.java:125)
at com.atlassian.aws.ec2.RemoteEC2InstanceImpl$CatchingRunnableDecorator.run(RemoteEC2InstanceImpl.java:96)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:417)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:280)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:135)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$101(ScheduledThreadPoolExecutor.java:65)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.runPeriodic(ScheduledThreadPoolExecutor.java:142)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:166)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:650)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:675)
at java.lang.Thread.run(Thread.java:595)
Caused by: javax.xml.bind.UnmarshalException: unexpected element (uri:"", local:"html"). Expected elements are <Unknown macro: {http}ActivateDesktopProduct>,<
Unknown macro: {http}ActivateDesktopProductResponse>,<
Unknown macro: {http}ActivateDesktopProductResult>,<
Unknown macro: {http}ActivateHostedProduct>,<
Unknown macro: {http}ActivateHostedProductResponse>,<
Unknown macro: {http}ActivateHostedProductResult>,<
Unknown macro: {http}AddGrant>,<
Unknown macro: {http}AddGrantResponse>,<
Unknown macro: {http}AllocateAddress>,<
Unknown macro: {http}AllocateAddressResponse>,<
Unknown macro: {http}AssociateAddress>,<
Unknown macro: {http}AssociateAddressResponse>,<
Unknown macro: {http}AttachVolume>,<
Unknown macro: {http}AttachVolumeResponse>,<
Unknown macro: {http}AttributedValue>,<
Unknown macro: {http}AuthorizeSecurityGroupIngress>,<
Unknown macro: {http}AuthorizeSecurityGroupIngressResponse>,<
Unknown macro: {http}ChangeMessageVisibility>,<
Unknown macro: {http}ChangeMessageVisibilityResponse>,<
Unknown macro: {http}ConfirmProductInstance>,<
Unknown macro: {http}ConfirmProductInstanceResponse>,<
Unknown macro: {http}CreateKeyPair>,<
Unknown macro: {http}CreateKeyPairResponse>,<
Unknown macro: {http}CreateQueue>,<
Unknown macro: {http}CreateQueueResponse>,<
Unknown macro: {http}CreateSecurityGroup>,<
Unknown macro: {http}CreateSecurityGroupResponse>,<
Unknown macro: {http}CreateSnapshot>,<
Unknown macro: {http}CreateSnapshotResponse>,<
Unknown macro: {http}CreateVolume>,<
Unknown macro: {http}CreateVolumeResponse>,<
Unknown macro: {http}DeleteKeyPair>,<
Unknown macro: {http}DeleteKeyPairResponse>,<
Unknown macro: {http}DeleteMessage>,<
Unknown macro: {http}DeleteMessageResponse>,<
Unknown macro: {http}DeleteQueue>,<
Unknown macro: {http}DeleteQueueResponse>,<
Unknown macro: {http}DeleteSecurityGroup>,<
Unknown macro: {http}DeleteSecurityGroupResponse>,<
Unknown macro: {http}DeleteSnapshot>,<
Unknown macro: {http}DeleteSnapshotResponse>,<
Unknown macro: {http}DeleteVolume>,<
Unknown macro: {http}DeleteVolumeResponse>,<
Unknown macro: {http}DeregisterImage>,<
Unknown macro: {http}DeregisterImageResponse>,<
Unknown macro: {http}DescribeAddresses>,<
Unknown macro: {http}DescribeAddressesResponse>,<
Unknown macro: {http}DescribeAvailabilityZones>,<
Unknown macro: {http}DescribeAvailabilityZonesResponse>,<
Unknown macro: {http}DescribeImageAttribute>,<
Unknown macro: {http}DescribeImageAttributeResponse>,<
Unknown macro: {http}DescribeImages>,<
Unknown macro: {http}DescribeImagesResponse>,<
Unknown macro: {http}DescribeInstances>,<
Unknown macro: {http}DescribeInstancesResponse>,<
Unknown macro: {http}DescribeKeyPairs>,<
Unknown macro: {http}DescribeKeyPairsResponse>,<
Unknown macro: {http}DescribeSecurityGroups>,<
Unknown macro: {http}DescribeSecurityGroupsResponse>,<
Unknown macro: {http}DescribeSnapshots>,<
Unknown macro: {http}DescribeSnapshotsResponse>,<
Unknown macro: {http}DescribeVolumes>,<
Unknown macro: {http}DescribeVolumesResponse>,<
Unknown macro: {http}DetachVolume>,<
Unknown macro: {http}DetachVolumeResponse>,<
Unknown macro: {http}DisassociateAddress>,<
Unknown macro: {http}DisassociateAddressResponse>,<
Unknown macro: {http}Error>,<
Unknown macro: {http}ErrorResponse>,<
Unknown macro: {http}GetActiveSubscriptionsByPid>,<
Unknown macro: {http}GetActiveSubscriptionsByPidResponse>,<
Unknown macro: {http}GetActiveSubscriptionsByPidResult>,<
Unknown macro: {http}GetConsoleOutput>,<
Unknown macro: {http}GetConsoleOutputResponse>,<
Unknown macro: {http}GetQueueAttributes>,<
Unknown macro: {http}GetQueueAttributesResponse>,<
Unknown macro: {http}GetVisibilityTimeout>,<
Unknown macro: {http}GetVisibilityTimeoutResponse>,<
Unknown macro: {http}ListGrants>,<
Unknown macro: {http}ListGrantsResponse>,<
Unknown macro: {http}ListQueues>,<
Unknown macro: {http}ListQueuesResponse>,<
Unknown macro: {http}Message>,<
Unknown macro: {http}MessageBody>,<
Unknown macro: {http}MessageId>,<
Unknown macro: {http}ModifyImageAttribute>,<
Unknown macro: {http}ModifyImageAttributeResponse>,<
Unknown macro: {http}PeekMessage>,<
Unknown macro: {http}PeekMessageResponse>,<
Unknown macro: {http}RebootInstances>,<
Unknown macro: {http}RebootInstancesResponse>,<
Unknown macro: {http}ReceiveMessage>,<
Unknown macro: {http}ReceiveMessageResponse>,<
Unknown macro: {http}RefreshUserToken>,<
Unknown macro: {http}RefreshUserTokenResponse>,<
Unknown macro: {http}RefreshUserTokenResult>,<
Unknown macro: {http}RegisterImage>,<
Unknown macro: {http}RegisterImageResponse>,<
Unknown macro: {http}ReleaseAddress>,<
Unknown macro: {http}ReleaseAddressResponse>,<
Unknown macro: {http}RemoveGrant>,<
Unknown macro: {http}RemoveGrantResponse>,<
Unknown macro: {http}ResetImageAttribute>,<
Unknown macro: {http}ResetImageAttributeResponse>,<{}Response>,<
Unknown macro: {http}ResponseMetadata>,<
Unknown macro: {http}ResponseStatus>,<
Unknown macro: {http}RevokeSecurityGroupIngress>,<
Unknown macro: {http}RevokeSecurityGroupIngressResponse>,<
Unknown macro: {http}RunInstances>,<
Unknown macro: {http}RunInstancesResponse>,<
Unknown macro: {http}SendMessage>,<
Unknown macro: {http}SendMessageResponse>,<
Unknown macro: {http}SetQueueAttributes>,<
Unknown macro: {http}SetQueueAttributesResponse>,<
Unknown macro: {http}SetVisibilityTimeout>,<
Unknown macro: {http}SetVisibilityTimeoutResponse>,<
Unknown macro: {http}TerminateInstances>,<
Unknown macro: {http}TerminateInstancesResponse>,<
Unknown macro: {http}VerifyProductSubscriptionByPid>,<
Unknown macro: {http}VerifyProductSubscriptionByPidResponse>,<
Unknown macro: {http}VerifyProductSubscriptionByPidResult>,<
Unknown macro: {http}VerifyProductSubscriptionByTokens>,<
Unknown macro: {http}VerifyProductSubscriptionByTokensResponse>,<
Unknown macro: {http}VerifyProductSubscriptionByTokensResult>
at com.sun.xml.bind.v2.runtime.unmarshaller.UnmarshallingContext.handleEvent(UnmarshallingContext.java:556)
at com.sun.xml.bind.v2.runtime.unmarshaller.Loader.reportError(Loader.java:199)
at com.sun.xml.bind.v2.runtime.unmarshaller.Loader.reportError(Loader.java:194)
at com.sun.xml.bind.v2.runtime.unmarshaller.Loader.reportUnexpectedChildElement(Loader.java:71)
at com.sun.xml.bind.v2.runtime.unmarshaller.UnmarshallingContext$DefaultRootLoader.childElement(UnmarshallingContext.java:962)
at com.sun.xml.bind.v2.runtime.unmarshaller.UnmarshallingContext._startElement(UnmarshallingContext.java:399)
at com.sun.xml.bind.v2.runtime.unmarshaller.UnmarshallingContext.startElement(UnmarshallingContext.java:380)
at com.sun.xml.bind.v2.runtime.unmarshaller.SAXConnector.startElement(SAXConnector.java:101)
at org.apache.xerces.parsers.AbstractSAXParser.startElement(Unknown Source)
at org.apache.xerces.impl.XMLNSDocumentScannerImpl.scanStartElement(Unknown Source)
at org.apache.xerces.impl.XMLNSDocumentScannerImpl$NSContentDispatcher.scanRootElementHook(Unknown Source)
at org.apache.xerces.impl.XMLDocumentFragmentScannerImpl$FragmentContentDispatcher.dispatch(Unknown Source)
at org.apache.xerces.impl.XMLDocumentFragmentScannerImpl.scanDocument(Unknown Source)
at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
at org.apache.xerces.parsers.XML11Configuration.parse(Unknown Source)
at org.apache.xerces.parsers.XMLParser.parse(Unknown Source)
at org.apache.xerces.parsers.AbstractSAXParser.parse(Unknown Source)
at org.apache.xerces.jaxp.SAXParserImpl$JAXPSAXParser.parse(Unknown Source)
at com.sun.xml.bind.v2.runtime.unmarshaller.UnmarshallerImpl.unmarshal0(UnmarshallerImpl.java:195)
at com.sun.xml.bind.v2.runtime.unmarshaller.UnmarshallerImpl.unmarshal(UnmarshallerImpl.java:168)
at javax.xml.bind.helpers.AbstractUnmarshallerImpl.unmarshal(AbstractUnmarshallerImpl.java:137)
at javax.xml.bind.helpers.AbstractUnmarshallerImpl.unmarshal(AbstractUnmarshallerImpl.java:184)
at com.xerox.amazonws.common.JAXBuddy.deserializeXMLStream(JAXBuddy.java:80)
at com.xerox.amazonws.common.AWSQueryConnection.createException(AWSQueryConnection.java:449)
at com.xerox.amazonws.common.AWSQueryConnection.makeRequest(AWSQueryConnection.java:358)
at com.xerox.amazonws.ec2.Jec2.makeRequestInt(Jec2.java:1678)