From c47c3b2f9e83b3a68cb45a962caac3da46a0ddc0 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sun, 25 Oct 2020 02:09:56 +0100 Subject: [PATCH] Make api_request to CHP's REST API more reliable --- jupyterhub/proxy.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/jupyterhub/proxy.py b/jupyterhub/proxy.py index 0666bcda..e8a8c3e9 100644 --- a/jupyterhub/proxy.py +++ b/jupyterhub/proxy.py @@ -39,10 +39,16 @@ from traitlets import observe from traitlets import Unicode from traitlets.config import LoggingConfigurable +try: + from tornado.curl_httpclient import CurlError +except ImportError: + CurlError = NotImplementedError + from . import utils from .metrics import CHECK_ROUTES_DURATION_SECONDS from .metrics import PROXY_POLL_DURATION_SECONDS from .objects import Server +from .utils import exponential_backoff from .utils import make_ssl_context from .utils import url_path_join from jupyterhub.traitlets import Command @@ -768,10 +774,25 @@ class ConfigurableHTTPProxy(Proxy): method=method, headers={'Authorization': 'token {}'.format(self.auth_token)}, body=body, + connect_timeout=1, # default: 20s + request_timeout=5, # default: 20s ) - async with self.semaphore: - result = await client.fetch(req) - return result + + async def _wait_for_api_request(): + try: + async with self.semaphore: + return await client.fetch(req) + except (TimeoutError, CurlError) as error: + self.log.debug("api_request to proxy failed: {0}".format(error)) + # A falsy return value triggers exponential_backoff to retry + return False + + result = await exponential_backoff( + _wait_for_api_request, + "api_request to proxy failed to complete in 20 seconds", + timeout=20, + ) + return result async def add_route(self, routespec, target, data): body = data or {}