diff --git a/jupyterhub/spawner.py b/jupyterhub/spawner.py index e5c1c7e1..1e5111c1 100644 --- a/jupyterhub/spawner.py +++ b/jupyterhub/spawner.py @@ -16,7 +16,7 @@ from subprocess import Popen from tempfile import mkdtemp from tornado import gen -from tornado.ioloop import PeriodicCallback +from tornado.ioloop import PeriodicCallback, IOLoop from traitlets.config import LoggingConfigurable from traitlets import ( @@ -25,7 +25,7 @@ from traitlets import ( ) from .traitlets import Command, ByteSpecification -from .utils import random_port, url_path_join +from .utils import random_port, url_path_join, DT_MIN, DT_MAX, DT_SCALE class Spawner(LoggingConfigurable): @@ -628,17 +628,21 @@ class Spawner(LoggingConfigurable): self.log.exception("Unhandled error in poll callback for %s", self) return status - death_interval = Float(0.1) + death_interval = Float(DT_MIN) @gen.coroutine def wait_for_death(self, timeout=10): """Wait for the single-user server to die, up to timeout seconds""" - for i in range(int(timeout / self.death_interval)): + loop = IOLoop.current() + tic = loop.time() + dt = self.death_interval + while dt > 0: status = yield self.poll() if status is not None: break else: - yield gen.sleep(self.death_interval) + yield gen.sleep(dt) + dt = min(dt * DT_SCALE, DT_MAX, timeout - (loop.time() - tic)) def _try_setcwd(path): diff --git a/jupyterhub/utils.py b/jupyterhub/utils.py index 1ac76a2e..07ab161f 100644 --- a/jupyterhub/utils.py +++ b/jupyterhub/utils.py @@ -48,6 +48,12 @@ def can_connect(ip, port): else: return True +# exponential falloff factors: +# start at 100ms, falloff by 2x +# never longer than 5s +DT_MIN = 0.1 +DT_SCALE = 2 +DT_MAX = 5 @gen.coroutine def wait_for_server(ip, port, timeout=10): @@ -56,11 +62,13 @@ def wait_for_server(ip, port, timeout=10): ip = '127.0.0.1' loop = ioloop.IOLoop.current() tic = loop.time() - while loop.time() - tic < timeout: + dt = DT_MIN + while dt > 0: if can_connect(ip, port): return else: - yield gen.sleep(0.1) + yield gen.sleep(dt) + dt = min(dt * DT_SCALE, DT_MAX, timeout - (loop.time() - tic)) raise TimeoutError( "Server at {ip}:{port} didn't respond in {timeout} seconds".format(**locals()) ) @@ -75,7 +83,8 @@ def wait_for_http_server(url, timeout=10): loop = ioloop.IOLoop.current() tic = loop.time() client = AsyncHTTPClient() - while loop.time() - tic < timeout: + dt = DT_MIN + while dt > 0: try: r = yield client.fetch(url, follow_redirects=False) except HTTPError as e: @@ -86,16 +95,17 @@ def wait_for_http_server(url, timeout=10): # but 502 or other proxy error is conceivable app_log.warning( "Server at %s responded with error: %s", url, e.code) - yield gen.sleep(0.1) + yield gen.sleep(dt) else: app_log.debug("Server at %s responded with %s", url, e.code) return e.response except (OSError, socket.error) as e: if e.errno not in {errno.ECONNABORTED, errno.ECONNREFUSED, errno.ECONNRESET}: app_log.warning("Failed to connect to %s (%s)", url, e) - yield gen.sleep(0.1) + yield gen.sleep(dt) else: return r + dt = min(dt * DT_SCALE, DT_MAX, timeout - (loop.time() - tic)) raise TimeoutError( "Server at {url} didn't respond in {timeout} seconds".format(**locals())