Stash fails to connect to User Directory behind a reverse proxy

2014-10-24 22:18:39,154 ERROR [http-nio-8443-exec-10] superadmin @1VRW1K7x1338x222x0 1e52nv5 127.0.0.1 "POST /plugins/servlet/embedded-crowd/configure/crowd/ HTTP/1.1" c.a.c.e.a.ConfigurationController Configuration test failed for user directory: [ KP Crowd Server], type: [ CROWD ]
com.atlassian.crowd.exception.runtime.OperationFailedException: The following URL does not specify a valid Crowd User Management REST service: https://<reverse_proxy_address>/crowd/rest/usermanagement/1/search?entity-type=user&start-index=0&max-results=1&expand=user
	at com.atlassian.crowd.embedded.core.CrowdDirectoryServiceImpl.testConnection(CrowdDirectoryServiceImpl.java:78) ~[CrowdDirectoryServiceImpl.class:na]
	at com.atlassian.stash.internal.crowd.CustomizedCrowdDirectoryService.testConnection(CustomizedCrowdDirectoryService.java:43) ~[CustomizedCrowdDirectoryService.class:na]
	at com.atlassian.crowd.embedded.admin.ConfigurationController.onSubmit(ConfigurationController.java:95) ~[embedded-crowd-admin-plugin-1.8-m2_1413897170000.jar:na]
	at com.atlassian.applinks.core.rest.context.ContextFilter.doFilter(ContextFilter.java:25) [applinks-plugin-4.3.0_1413897172000.jar:na]
	at com.atlassian.stash.internal.spring.security.StashAuthenticationFilter.doFilter(StashAuthenticationFilter.java:86) [StashAuthenticationFilter.class:na]
	at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doInsideSpringSecurityChain(BeforeLoginPluginAuthenticationFilter.java:111) [BeforeLoginPluginAuthenticationFilter.class:na]
	at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doFilter(BeforeLoginPluginAuthenticationFilter.java:77) [BeforeLoginPluginAuthenticationFilter.class:na]
	at com.atlassian.security.auth.trustedapps.filter.TrustedApplicationsFilter.doFilter(TrustedApplicationsFilter.java:100) [TrustedApplicationsFilter.class:na]
	at com.atlassian.oauth.serviceprovider.internal.servlet.OAuthFilter.doFilter(OAuthFilter.java:69) [atlassian-oauth-service-provider-plugin-1.9.2_1413897172000.jar:na]
	at com.atlassian.analytics.client.filter.DefaultAnalyticsFilter.doFilter(DefaultAnalyticsFilter.java:33) [analytics-client-3.48_1413897174000.jar:na]
	at com.atlassian.core.filters.AbstractHttpFilter.doFilter(AbstractHttpFilter.java:31) [AbstractHttpFilter.class:na]
	at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doBeforeBeforeLoginFilters(BeforeLoginPluginAuthenticationFilter.java:89) [BeforeLoginPluginAuthenticationFilter.class:na]
	at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doFilter(BeforeLoginPluginAuthenticationFilter.java:75) [BeforeLoginPluginAuthenticationFilter.class:na]
	at com.atlassian.stash.internal.request.DefaultRequestManager.doAsRequest(DefaultRequestManager.java:85) [DefaultRequestManager.class:na]
	at com.hazelcast.web.WebFilter.doFilter(WebFilter.java:463) [WebFilter.class:3.3.2-atlassian-3]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_25]
	at java.lang.Thread.run(Thread.java:745) [na:1.8.0_25]
	... 226 frames trimmed
Caused by: com.atlassian.crowd.exception.InvalidCrowdServiceException: The following URL does not specify a valid Crowd User Management REST service: https://<reverse_proxy_address>/crowd/rest/usermanagement/1/search?entity-type=user&start-index=0&max-results=1&expand=user
	at com.atlassian.crowd.integration.rest.service.RestExecutor$MethodExecutor.executeCrowdServiceMethod(RestExecutor.java:628) ~[RestExecutor$MethodExecutor.class:na]
	at com.atlassian.crowd.integration.rest.service.RestExecutor$MethodExecutor.andReceive(RestExecutor.java:464) ~[RestExecutor$MethodExecutor.class:na]
	at com.atlassian.crowd.integration.rest.service.RestCrowdClient.searchUsers(RestCrowdClient.java:553) ~[RestCrowdClient.class:na]
	at com.atlassian.crowd.integration.rest.service.RestCrowdClient.testConnection(RestCrowdClient.java:541) ~[RestCrowdClient.class:na]
	at com.atlassian.crowd.directory.RemoteCrowdDirectory.testConnection(RemoteCrowdDirectory.java:830) ~[RemoteCrowdDirectory.class:na]
	at com.atlassian.crowd.embedded.core.CrowdDirectoryServiceImpl.testConnection(CrowdDirectoryServiceImpl.java:69) ~[CrowdDirectoryServiceImpl.class:na]
	... 17 common frames omitted

2014-11-13 11:00:47,638 WARN  [http-nio-7990-exec-7] @P1PMTOx660x10717x0 10.0.3.88,10.0.0.123 "GET /rest/api/1.0/users/usert@somain.com/repos HTTP/1.1" c.a.s.i.s.s.PluginAuthenticationProvider Could not authenticate user@domain.com; authentication by com.atlassian.stash.stash-authentication:crowdHttpAuthHandler failed
com.atlassian.stash.user.AuthenticationSystemException: The remote authentication server is not available. Please try again later.
	at com.atlassian.stash.internal.crowd.RiotPolice.authenticate(RiotPolice.java:113) ~[RiotPolice.class:na]
	at com.atlassian.stash.internal.user.DefaultUserService.authenticate(DefaultUserService.java:94) ~[DefaultUserService.class:na]
	at com.atlassian.stash.internal.auth.EmbeddedCrowdHttpAuthenticationHandler.authenticate(EmbeddedCrowdHttpAuthenticationHandler.java:40) ~[EmbeddedCrowdHttpAuthenticationHandler.class:na]
	at com.atlassian.stash.internal.spring.security.PluginAuthenticationProvider$1.perform(PluginAuthenticationProvider.java:96) ~[PluginAuthenticationProvider$1.class:na]
	at com.atlassian.stash.internal.spring.security.PluginAuthenticationProvider$1.perform(PluginAuthenticationProvider.java:93) ~[PluginAuthenticationProvider$1.class:na]
	at com.atlassian.stash.internal.auth.DefaultCaptchaService.authenticateWithCaptcha(DefaultCaptchaService.java:71) ~[DefaultCaptchaService.class:na]
	at com.atlassian.stash.internal.spring.security.PluginAuthenticationProvider.attemptAuthentication(PluginAuthenticationProvider.java:113) [PluginAuthenticationProvider.class:na]
	at com.atlassian.stash.internal.spring.security.PluginAuthenticationProvider.authenticate(PluginAuthenticationProvider.java:60) [PluginAuthenticationProvider.class:na]
	at com.atlassian.stash.internal.spring.security.StashAuthenticationFilter.doFilter(StashAuthenticationFilter.java:100) [StashAuthenticationFilter.class:na]
	at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doInsideSpringSecurityChain(BeforeLoginPluginAuthenticationFilter.java:111) [BeforeLoginPluginAuthenticationFilter.class:na]
	at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doFilter(BeforeLoginPluginAuthenticationFilter.java:77) [BeforeLoginPluginAuthenticationFilter.class:na]
	at com.atlassian.security.auth.trustedapps.filter.TrustedApplicationsFilter.doFilter(TrustedApplicationsFilter.java:100) [TrustedApplicationsFilter.class:na]
	at com.atlassian.oauth.serviceprovider.internal.servlet.OAuthFilter.doFilter(OAuthFilter.java:69) [atlassian-oauth-service-provider-plugin-1.9.2_1415217634000.jar:na]
	at com.atlassian.security.auth.trustedapps.filter.TrustedApplicationsFilter.doFilter(TrustedApplicationsFilter.java:100) [TrustedApplicationsFilter.class:na]
	at com.atlassian.oauth.serviceprovider.internal.servlet.OAuthFilter.doFilter(OAuthFilter.java:69) [atlassian-oauth-service-provider-plugin-1.9.2_1415217634000.jar:na]
	at com.atlassian.analytics.client.filter.DefaultAnalyticsFilter.doFilter(DefaultAnalyticsFilter.java:32) [analytics-client-3.53_1414682400000.jar:na]
	at com.atlassian.analytics.client.filter.AbstractHttpFilter.doFilter(AbstractHttpFilter.java:32) [analytics-client-3.53_1414682400000.jar:na]
	at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doBeforeBeforeLoginFilters(BeforeLoginPluginAuthenticationFilter.java:89) [BeforeLoginPluginAuthenticationFilter.class:na]
	at com.atlassian.stash.internal.web.auth.BeforeLoginPluginAuthenticationFilter.doFilter(BeforeLoginPluginAuthenticationFilter.java:75) [BeforeLoginPluginAuthenticationFilter.class:na]
	at com.atlassian.stash.internal.request.DefaultRequestManager.doAsRequest(DefaultRequestManager.java:85) [DefaultRequestManager.class:na]
	at com.atlassian.stash.internal.hazelcast.ConfigurableWebFilter.doFilter(ConfigurableWebFilter.java:35) [ConfigurableWebFilter.class:na]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [na:1.7.0_60]
	at java.lang.Thread.run(Thread.java:745) [na:1.7.0_60]
	... 226 frames trimmed
Caused by: com.atlassian.crowd.exception.runtime.OperationFailedException: null
	at com.atlassian.crowd.embedded.core.CrowdServiceImpl.convertOperationFailedException(CrowdServiceImpl.java:915) ~[CrowdServiceImpl.class:na]
	at com.atlassian.crowd.embedded.core.CrowdServiceImpl.authenticate(CrowdServiceImpl.java:80) ~[CrowdServiceImpl.class:na]
	at com.atlassian.stash.internal.crowd.RiotPolice.authenticate(RiotPolice.java:98) ~[RiotPolice.class:na]
	... 23 common frames omitted
Caused by: java.net.SocketException: Connection reset
	at java.net.SocketInputStream.read(SocketInputStream.java:196) ~[na:1.7.0_60]
	at java.net.SocketInputStream.read(SocketInputStream.java:122) ~[na:1.7.0_60]
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442) ~[na:1.7.0_60]
	at sun.security.ssl.InputRecord.read(InputRecord.java:480) ~[na:1.7.0_60]
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927) ~[na:1.7.0_60]
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884) ~[na:1.7.0_60]
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:102) ~[na:1.7.0_60]
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136) ~[SessionInputBufferImpl.class:4.3.2]
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152) ~[SessionInputBufferImpl.class:4.3.2]
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270) ~[SessionInputBufferImpl.class:4.3.2]
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140) ~[DefaultHttpResponseParser.class:4.3.5]
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) ~[DefaultHttpResponseParser.class:4.3.5]
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260) ~[AbstractMessageParser.class:4.3.2]
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161) ~[DefaultBHttpClientConnection.class:4.3.2]
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:153) ~[CPoolProxy.class:4.3.5]
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271) ~[HttpRequestExecutor.class:4.3.2]
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) ~[HttpRequestExecutor.class:4.3.2]
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:254) ~[MainClientExec.class:4.3.5]
	at org.apache.http.impl.client.cache.CachingExec.callBackend(CachingExec.java:575) ~[CachingExec.class:4.3.5]
	at org.apache.http.impl.client.cache.CachingExec.execute(CachingExec.java:256) ~[CachingExec.class:4.3.5]
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:195) ~[ProtocolExec.class:4.3.5]
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:86) ~[RetryExec.class:4.3.5]
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108) ~[RedirectExec.class:4.3.5]
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184) ~[InternalHttpClient.class:4.3.5]
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:72) ~[CloseableHttpClient.class:4.3.5]
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) ~[CloseableHttpClient.class:4.3.5]
	at com.atlassian.crowd.integration.rest.service.RestExecutor$MethodExecutor.rewriteAndExecute(RestExecutor.java:607) ~[RestExecutor$MethodExecutor.class:na]
	at com.atlassian.crowd.integration.rest.service.RestExecutor$MethodExecutor.executeCrowdServiceMethod(RestExecutor.java:622) ~[RestExecutor$MethodExecutor.class:na]
	at com.atlassian.crowd.integration.rest.service.RestExecutor$MethodExecutor.andReceive(RestExecutor.java:464) ~[RestExecutor$MethodExecutor.class:na]
	at com.atlassian.crowd.integration.rest.service.RestCrowdClient.authenticateUser(RestCrowdClient.java:139) ~[RestCrowdClient.class:na]
	at com.atlassian.crowd.directory.RemoteCrowdDirectory.authenticate(RemoteCrowdDirectory.java:194) ~[RemoteCrowdDirectory.class:na]
	at com.atlassian.crowd.directory.DbCachingRemoteDirectory.authenticateAndUpdateInternalUser(DbCachingRemoteDirectory.java:295) ~[DbCachingRemoteDirectory.class:na]
	at com.atlassian.crowd.directory.DbCachingRemoteDirectory.authenticate(DbCachingRemoteDirectory.java:200) ~[DbCachingRemoteDirectory.class:na]
	at com.atlassian.crowd.manager.directory.DirectoryManagerGeneric.authenticateUser(DirectoryManagerGeneric.java:283) ~[DirectoryManagerGeneric.class:na]
	at com.atlassian.stash.internal.crowd.CustomizedDirectoryManager.authenticateUser(CustomizedDirectoryManager.java:53) ~[CustomizedDirectoryManager.class:na]
	at com.atlassian.crowd.manager.application.ApplicationServiceGeneric.authenticateUser(ApplicationServiceGeneric.java:196) ~[ApplicationServiceGeneric.class:na]
	at com.atlassian.stash.internal.crowd.CustomizedApplicationService.authenticateUser(CustomizedApplicationService.java:44) ~[CustomizedApplicationService.class:na]
	at com.atlassian.crowd.embedded.core.CrowdServiceImpl.authenticate(CrowdServiceImpl.java:68) ~[CrowdServiceImpl.class:na]
	... 24 common frames omitted