hazza/jupyterhub
1
0
Fork 0
mirror of https://github.com/jupyterhub/jupyterhub.git synced 2025-10-17 23:13:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

parallelize startup poll

Browse Source 
puts each check for a running spawner in a coroutine and runs them all concurrently.

Note: this will only improve performance when a large number of Spawners are running and `yield spawner.poll()` takes a nontrivial amount of time.
This is because these are coroutines, not threads. If instantiating Spawners themselves takes a long time, performance will not be affected.
This commit is contained in:
Min RK
2018-01-31 11:23:04 +01:00
parent dde7b5ea68
commit 6d6041a3c1
1 changed files with 36 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
jupyterhub/app.py
View File

@@ -1204,8 +1204,6 @@ class JupyterHub(Application):
def init_spawners(self):
db = self.db
user_summaries = ['']
def _user_summary(user):
parts = ['{0: >8}'.format(user.name)]
if user.admin:
@@ -1225,39 +1223,49 @@ class JupyterHub(Application):
yield self.proxy.delete_user(user, server_name)
yield user.stop(server_name)
@gen.coroutine
def check_spawner(user, name, spawner):
status = 0
if spawner.server:
try:
status = yield spawner.poll()
except Exception:
self.log.exception("Failed to poll spawner for %s, assuming the spawner is not running.",
spawner._log_name)
status = -1
if status is None:
self.log.info("%s still running", user.name)
spawner.add_poll_callback(user_stopped, user, name)
spawner.start_polling()
else:
# user not running. This is expected if server is None,
# but indicates the user's server died while the Hub wasn't running
# if spawner.server is defined.
if spawner.server:
self.log.warning("%s appears to have stopped while the Hub was down", spawner._log_name)
# remove server entry from db
db.delete(spawner.orm_spawner.server)
spawner.server = None
else:
self.log.debug("%s not running", spawner._log_name)
# parallelize checks for running Spawners
check_futures = []
for orm_user in db.query(orm.User):
self.users[orm_user.id] = user = User(orm_user, self.tornado_settings)
self.log.debug("Loading state for %s from db", user.name)
for name, spawner in user.spawners.items():
status = 0
if spawner.server:
try:
status = yield spawner.poll()
except Exception:
self.log.exception("Failed to poll spawner for %s, assuming the spawner is not running.",
spawner._log_name)
status = -1
f = check_spawner(user, name, spawner)
check_futures.append(f)
if status is None:
self.log.info("%s still running", user.name)
spawner.add_poll_callback(user_stopped, user, name)
spawner.start_polling()
else:
# user not running. This is expected if server is None,
# but indicates the user's server died while the Hub wasn't running
# if spawner.server is defined.
if spawner.server:
self.log.warning("%s appears to have stopped while the Hub was down", spawner._log_name)
# remove server entry from db
db.delete(spawner.orm_spawner.server)
spawner.server = None
else:
self.log.debug("%s not running", spawner._log_name)
db.commit()
# await checks after submitting them all
for f in check_futures:
yield f
user_summaries.append(_user_summary(user))
user_summaries = map(_user_summary, self.users.values())
self.log.debug("Loaded users: %s", '\n'.join(user_summaries))
self.log.debug("Loaded users:\n%s", '\n'.join(user_summaries))
db.commit()
def init_oauth(self):