diff --git a/ci/init-db.sh b/ci/init-db.sh index a26c30ab..0d3d3223 100755 --- a/ci/init-db.sh +++ b/ci/init-db.sh @@ -21,7 +21,7 @@ fi # Configure a set of databases in the database server for upgrade tests # this list must be in sync with versions in test_db.py:test_upgrade set -x -for SUFFIX in '' _upgrade_110 _upgrade_122 _upgrade_130 _upgrade_150 _upgrade_211; do +for SUFFIX in '' _upgrade_110 _upgrade_122 _upgrade_130 _upgrade_150 _upgrade_211 _upgrade_311; do $SQL_CLIENT "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true $SQL_CLIENT "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE_DATABASE_ARGS:-};" done diff --git a/docs/source/_static/rest-api.yml b/docs/source/_static/rest-api.yml index b611f870..48e49d49 100644 --- a/docs/source/_static/rest-api.yml +++ b/docs/source/_static/rest-api.yml @@ -1498,6 +1498,9 @@ components: read:groups: Read group models. read:groups:name: Read group names. delete:groups: Delete groups. + admin:services: + Create, read, update, delete services, not including services + defined from config files. list:services: List services, including at least their names. read:services: Read service models. read:services:name: Read service names. diff --git a/docs/source/howto/rest.md b/docs/source/howto/rest.md index 886acdc5..503b4ba2 100644 --- a/docs/source/howto/rest.md +++ b/docs/source/howto/rest.md @@ -24,6 +24,7 @@ such as: - Checking which users are active - Adding or removing users +- Adding or removing services - Stopping or starting single user notebook servers - Authenticating services - Communicating with an individual Jupyter server's REST API diff --git a/docs/source/reference/services.md b/docs/source/reference/services.md index 81ffaf1d..f3d4ed38 100644 --- a/docs/source/reference/services.md +++ b/docs/source/reference/services.md @@ -174,6 +174,47 @@ c.JupyterHub.services = [ In this case, the `url` field will be passed along to the Service as `JUPYTERHUB_SERVICE_URL`. +## Adding or removing services at runtime + +Only externally-managed services can be added at runtime by using JupyterHub’s REST API. + +### Add a new service + +To add a new service, send a POST request to this endpoint + +``` +POST /hub/api/services/:servicename +``` + +**Required scope: `admin:services`** + +**Payload**: The payload should contain the definition of the service to be created. The endpoint supports the same properties as externally-managed services defined in the config file. + +**Possible responses** + +- `201 Created`: The service and related objects are created (and started in case of a Hub-managed one) successfully. +- `400 Bad Request`: The payload is invalid or JupyterHub can not create the service. +- `409 Conflict`: The service with the same name already exists. + +### Remove an existing service + +To remove an existing service, send a DELETE request to this endpoint + +``` +DELETE /hub/api/services/:servicename +``` + +**Required scope: `admin:services`** + +**Payload**: `None` + +**Possible responses** + +- `200 OK`: The service and related objects are removed (and stopped in case of a Hub-managed one) successfully. +- `400 Bad Request`: JupyterHub can not remove the service. +- `404 Not Found`: The requested service does not exist. +- `405 Not Allowed`: The requested service is created from the config file, it can not be removed at runtime. + ## Writing your own Services When writing your own services, you have a few decisions to make (in addition diff --git a/jupyterhub/alembic/versions/3c2384c5aae1_add_from_config_column_to_the_services_.py b/jupyterhub/alembic/versions/3c2384c5aae1_add_from_config_column_to_the_services_.py new file mode 100644 index 00000000..26b832a3 --- /dev/null +++ b/jupyterhub/alembic/versions/3c2384c5aae1_add_from_config_column_to_the_services_.py @@ -0,0 +1,51 @@ +"""Add from_config column to the services table + +Revision ID: 3c2384c5aae1 +Revises: 0eee8c825d24 +Create Date: 2023-02-27 16:22:26.196231 +""" + +# revision identifiers, used by Alembic. +revision = '3c2384c5aae1' +down_revision = '0eee8c825d24' +branch_labels = None +depends_on = None + +import sqlalchemy as sa +from alembic import op + +from jupyterhub.orm import JSONDict, JSONList + +COL_DATA = [ + {'name': 'url', 'type': sa.Unicode(length=2047)}, + {'name': 'oauth_client_allowed_scopes', 'type': JSONDict()}, + {'name': 'info', 'type': JSONDict()}, + {'name': 'display', 'type': sa.Boolean}, + {'name': 'oauth_no_confirm', 'type': sa.Boolean}, + {'name': 'command', 'type': JSONList()}, + {'name': 'cwd', 'type': sa.Unicode(length=2047)}, + {'name': 'environment', 'type': JSONDict()}, + {'name': 'user', 'type': sa.Unicode(255)}, +] + + +def upgrade(): + engine = op.get_bind().engine + tables = sa.inspect(engine).get_table_names() + if 'services' in tables: + op.add_column( + 'services', + sa.Column('from_config', sa.Boolean, default=True), + ) + op.execute('UPDATE services SET from_config = true') + for item in COL_DATA: + op.add_column( + 'services', + sa.Column(item['name'], item['type'], nullable=True), + ) + + +def downgrade(): + op.drop_column('services', sa.Column('from_config')) + for item in COL_DATA: + op.drop_column('services', sa.Column(item['name'])) diff --git a/jupyterhub/alembic/versions/651f5419b74d_api_token_scopes.py b/jupyterhub/alembic/versions/651f5419b74d_api_token_scopes.py index 483a91c5..6be31b5f 100644 --- a/jupyterhub/alembic/versions/651f5419b74d_api_token_scopes.py +++ b/jupyterhub/alembic/versions/651f5419b74d_api_token_scopes.py @@ -13,16 +13,37 @@ depends_on = None import sqlalchemy as sa from alembic import op -from sqlalchemy import Column, ForeignKey, Table +from sqlalchemy import Column, ForeignKey, Table, text from sqlalchemy.orm import relationship from sqlalchemy.orm.session import Session -from jupyterhub import orm, roles, scopes +from jupyterhub import orm, roles + + +def access_scopes(oauth_client: orm.OAuthClient, db: Session): + """Return scope(s) required to access an oauth client + This is a clone of `scopes.access_scopes` without using + the `orm.Service` + """ + scopes = set() + if oauth_client.identifier == "jupyterhub": + return frozenset() + spawner = oauth_client.spawner + if spawner: + scopes.add(f"access:servers!server={spawner.user.name}/{spawner.name}") + else: + statement = "SELECT * FROM services WHERE oauth_client_id = :identifier" + service = db.execute( + text(statement), {"identifier": oauth_client.identifier} + ).fetchall() + if len(service) > 0: + scopes.add(f"access:services!service={service[0].name}") + + return frozenset(scopes) def upgrade(): c = op.get_bind() - tables = sa.inspect(c.engine).get_table_names() # oauth codes are short lived, no need to upgrade them @@ -100,7 +121,7 @@ def upgrade(): db = Session(bind=c) for oauth_client in db.query(orm.OAuthClient): allowed_scopes = set(roles.roles_to_scopes(oauth_client.allowed_roles)) - allowed_scopes.update(scopes.access_scopes(oauth_client)) + allowed_scopes.update(access_scopes(oauth_client, db)) oauth_client.allowed_scopes = sorted(allowed_scopes) db.commit() # drop token-role relationship diff --git a/jupyterhub/apihandlers/base.py b/jupyterhub/apihandlers/base.py index f174f0dd..f8bbd569 100644 --- a/jupyterhub/apihandlers/base.py +++ b/jupyterhub/apihandlers/base.py @@ -395,6 +395,23 @@ class APIHandler(BaseHandler): _group_model_types = {'name': str, 'users': list, 'roles': list} + _service_model_types = { + 'name': str, + 'admin': bool, + 'url': str, + 'oauth_client_allowed_scopes': list, + 'api_token': str, + 'info': dict, + 'display': bool, + 'oauth_no_confirm': bool, + 'command': list, + 'cwd': str, + 'environment': dict, + 'user': str, + 'oauth_client_id': str, + 'oauth_redirect_uri': str, + } + def _check_model(self, model, model_types, name): """Check a model provided by a REST API request @@ -433,6 +450,15 @@ class APIHandler(BaseHandler): 400, ("group names must be str, not %r", type(groupname)) ) + def _check_service_model(self, model): + """Check a request-provided service model from a REST API""" + self._check_model(model, self._service_model_types, 'service') + service_name = model.get('name') + if not isinstance(service_name, str): + raise web.HTTPError( + 400, ("Service name must be str, not %r", type(service_name)) + ) + def get_api_pagination(self): default_limit = self.settings["api_page_default_limit"] max_limit = self.settings["api_page_max_limit"] diff --git a/jupyterhub/apihandlers/services.py b/jupyterhub/apihandlers/services.py index 43e69aa7..b9d23d84 100644 --- a/jupyterhub/apihandlers/services.py +++ b/jupyterhub/apihandlers/services.py @@ -5,8 +5,14 @@ Currently GET-only, no actions can be taken to modify services. # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import json +from typing import Optional, Tuple -from ..scopes import Scope, needs_scope +from tornado import web + +from .. import orm +from ..roles import get_default_roles +from ..scopes import Scope, _check_token_scopes, needs_scope +from ..services.service import Service from .base import APIHandler @@ -25,9 +31,171 @@ class ServiceListAPIHandler(APIHandler): class ServiceAPIHandler(APIHandler): @needs_scope('read:services', 'read:services:name', 'read:roles:services') def get(self, service_name): + if service_name not in self.services: + raise web.HTTPError(404, f"No such service: {service_name}") service = self.services[service_name] self.write(json.dumps(self.service_model(service))) + def _check_service_scopes(self, spec: dict): + user = self.current_user + requested_scopes = [] + if spec.get('admin'): + default_roles = get_default_roles() + admin_scopes = [ + role['scopes'] for role in default_roles if role['name'] == 'admin' + ] + requested_scopes.extend(admin_scopes[0]) + + requested_client_scope = spec.get('oauth_client_allowed_scopes') + if requested_client_scope is not None: + requested_scopes.extend(requested_client_scope) + if len(requested_scopes) > 0: + try: + _check_token_scopes(requested_scopes, user, None) + except ValueError as e: + raise web.HTTPError(400, str(e)) + + async def add_service(self, spec: dict) -> Service: + """Add a new service and related objects to the database + + Args: + spec (dict): The service specification + + Raises: + web.HTTPError: Raise if the service is not created + + Returns: + Service: Returns the service instance. + + """ + + self._check_service_model(spec) + self._check_service_scopes(spec) + + service_name = spec["name"] + managed = bool(spec.get('command')) + if managed: + msg = f"Can not create managed service {service_name} at runtime" + self.log.error(msg, exc_info=True) + raise web.HTTPError(400, msg) + try: + new_service = self.service_from_spec(spec) + except Exception: + msg = f"Failed to create service {service_name}" + self.log.error(msg, exc_info=True) + raise web.HTTPError(400, msg) + + if new_service is None: + raise web.HTTPError(400, f"Failed to create service {service_name}") + + if new_service.api_token: + # Add api token to database + await self.app._add_tokens( + {new_service.api_token: new_service.name}, kind='service' + ) + if new_service.url: + # Start polling for external service + service_status = await self.app.start_service(service_name, new_service) + if not service_status: + self.log.error( + 'Failed to start service %s', + service_name, + exc_info=True, + ) + + if new_service.oauth_no_confirm: + oauth_no_confirm_list = self.settings.get('oauth_no_confirm_list') + msg = f"Allowing service {new_service.name} to complete OAuth without confirmation on an authorization web page" + self.log.warning(msg) + oauth_no_confirm_list.add(new_service.oauth_client_id) + + return new_service + + @needs_scope('admin:services') + async def post(self, service_name: str): + data = self.get_json_body() + service, _ = self.find_service(service_name) + + if service is not None: + raise web.HTTPError(409, f"Service {service_name} already exists") + + if not data or not isinstance(data, dict): + raise web.HTTPError(400, "Invalid service data") + + data['name'] = service_name + new_service = await self.add_service(data) + self.write(json.dumps(self.service_model(new_service))) + self.set_status(201) + + @needs_scope('admin:services') + async def delete(self, service_name: str): + service, orm_service = self.find_service(service_name) + + if service is None: + raise web.HTTPError(404, f"Service {service_name} does not exist") + + if service.from_config: + raise web.HTTPError( + 405, f"Service {service_name} is not modifiable at runtime" + ) + try: + await self.remove_service(service, orm_service) + self.services.pop(service_name) + except Exception: + msg = f"Failed to remove service {service_name}" + self.log.error(msg, exc_info=True) + raise web.HTTPError(400, msg) + + self.set_status(200) + + async def remove_service(self, service: Service, orm_service: orm.Service) -> None: + """Remove a service and all related objects from the database. + + Args: + service (Service): the service object to be removed + + orm_service (orm.Service): The `orm.Service` object linked + with `service` + """ + if service.managed: + await service.stop() + + if service.oauth_client: + self.oauth_provider.remove_client(service.oauth_client_id) + + if orm_service._server_id is not None: + orm_server = ( + self.db.query(orm.Server).filter_by(id=orm_service._server_id).first() + ) + if orm_server is not None: + self.db.delete(orm_server) + + if service.oauth_no_confirm: + oauth_no_confirm_list = self.settings.get('oauth_no_confirm_list') + oauth_no_confirm_list.discard(service.oauth_client_id) + + self.db.delete(orm_service) + self.db.commit() + + def service_from_spec(self, spec) -> Optional[Service]: + """Create service from api request""" + service = self.app.service_from_spec(spec, from_config=False) + self.db.commit() + return service + + def find_service( + self, name: str + ) -> Tuple[Optional[Service], Optional[orm.Service]]: + """Get a service by name + return None if no such service + """ + orm_service = orm.Service.find(db=self.db, name=name) + if orm_service is not None: + service = self.services.get(name) + return service, orm_service + + return (None, None) + default_handlers = [ (r"/api/services", ServiceListAPIHandler), diff --git a/jupyterhub/app.py b/jupyterhub/app.py index c993a6f0..430564bf 100644 --- a/jupyterhub/app.py +++ b/jupyterhub/app.py @@ -21,6 +21,7 @@ from functools import partial from getpass import getuser from operator import itemgetter from textwrap import dedent +from typing import Optional from urllib.parse import unquote, urlparse, urlunparse if sys.version_info[:2] < (3, 3): @@ -2323,7 +2324,7 @@ class JupyterHub(Application): if not self.authenticator.validate_username(name): raise ValueError("Token user name %r is not valid" % name) if kind == 'service': - if not any(service["name"] == name for service in self.services): + if not any(service_name == name for service_name in self._service_map): self.log.warning( f"service {name} not in services, creating implicitly. It is recommended to register services using services list." ) @@ -2386,8 +2387,20 @@ class JupyterHub(Application): ) pc.start() - def init_services(self): - self._service_map.clear() + def service_from_orm( + self, + orm_service: orm.Service, + ) -> Service: + """Create the service instance and related objects from + ORM data. + + Args: + orm_service (orm.Service): The `orm.Service` object + + Returns: + Service: the created service + """ + if self.domain: domain = 'services.' + self.domain parsed = urlparse(self.subdomain_host) @@ -2395,118 +2408,208 @@ class JupyterHub(Application): else: domain = host = '' - for spec in self.services: - if 'name' not in spec: - raise ValueError('service spec must have a name: %r' % spec) - name = spec['name'] - # get/create orm - orm_service = orm.Service.find(self.db, name=name) - if orm_service is None: - # not found, create a new one - orm_service = orm.Service(name=name) - self.db.add(orm_service) - if spec.get('admin', False): - self.log.warning( - f"Service {name} sets `admin: True`, which is deprecated in JupyterHub 2.0." - " You can assign now assign roles via `JupyterHub.load_roles` configuration." - " If you specify services in the admin role configuration, " - "the Service admin flag will be ignored." + name = orm_service.name + service = Service( + parent=self, + app=self, + base_url=self.base_url, + db=self.db, + orm=orm_service, + roles=orm_service.roles, + domain=domain, + host=host, + hub=self.hub, + ) + traits = service.traits(input=True) + for key, trait in traits.items(): + if not trait.metadata.get("in_db", True): + continue + orm_value = getattr(orm_service, key) + if orm_value is not None: + setattr(service, key, orm_value) + + if orm_service.oauth_client is not None: + service.oauth_client_id = orm_service.oauth_client.identifier + service.oauth_redirect_uri = orm_service.oauth_client.redirect_uri + + self._service_map[name] = service + + return service + + def service_from_spec( + self, + spec: Dict, + from_config=True, + ) -> Optional[Service]: + """Create the service instance and related objects from + config data. + + Args: + spec (Dict): The spec of service, defined in the config file. + from_config (bool, optional): `True` if the service will be created + from the config file, `False` if it is created from REST API. + Defaults to `True`. + + Returns: + Optional[Service]: The created service + """ + + if self.domain: + domain = 'services.' + self.domain + parsed = urlparse(self.subdomain_host) + host = f'{parsed.scheme}://services.{parsed.netloc}' + else: + domain = host = '' + + if 'name' not in spec: + raise ValueError('service spec must have a name: %r' % spec) + + name = spec['name'] + # get/create orm + orm_service = orm.Service.find(self.db, name=name) + if orm_service is None: + # not found, create a new one + orm_service = orm.Service(name=name, from_config=from_config) + self.db.add(orm_service) + if spec.get('admin', False): + self.log.warning( + f"Service {name} sets `admin: True`, which is deprecated in JupyterHub 2.0." + " You can assign now assign roles via `JupyterHub.load_roles` configuration." + " If you specify services in the admin role configuration, " + "the Service admin flag will be ignored." + ) + roles.update_roles(self.db, entity=orm_service, roles=['admin']) + else: + # Do nothing if the config file tries to modify a API-base service + # or vice versa. + if orm_service.from_config != from_config: + if from_config: + self.log.error( + f"The service {name} from the config file is trying to modify a runtime-created service with the same name" ) - roles.update_roles(self.db, entity=orm_service, roles=['admin']) - orm_service.admin = spec.get('admin', False) - self.db.commit() - service = Service( - parent=self, - app=self, - base_url=self.base_url, - db=self.db, - orm=orm_service, - roles=orm_service.roles, - domain=domain, - host=host, - hub=self.hub, + else: + self.log.error( + f"The runtime-created service {name} is trying to modify a config-based service with the same name" + ) + return + orm_service.admin = spec.get('admin', False) + + self.db.commit() + service = Service( + parent=self, + app=self, + base_url=self.base_url, + db=self.db, + orm=orm_service, + roles=orm_service.roles, + domain=domain, + host=host, + hub=self.hub, + ) + + traits = service.traits(input=True) + for key, value in spec.items(): + trait = traits.get(key) + if trait is None: + raise AttributeError("No such service field: %s" % key) + setattr(service, key, value) + # also set the value on the orm object + # unless it's marked as not in the db + # (e.g. on the oauth object) + if trait.metadata.get("in_db", True): + setattr(orm_service, key, value) + + if service.api_token: + self.service_tokens[service.api_token] = service.name + elif service.managed: + # generate new token + # TODO: revoke old tokens? + service.api_token = service.orm.new_api_token(note="generated at startup") + + if service.url: + parsed = urlparse(service.url) + if parsed.scheme not in {"http", "https"}: + raise ValueError( + f"Unsupported scheme in URL for service {name}: {service.url}. Must be http[s]" + ) + + port = None + if parsed.port is not None: + port = parsed.port + elif parsed.scheme == 'http': + port = 80 + elif parsed.scheme == 'https': + port = 443 + + server = service.orm.server = orm.Server( + proto=parsed.scheme, + ip=parsed.hostname, + port=port, + cookie_name=service.oauth_client_id, + base_url=service.prefix, ) + self.db.add(server) + else: + service.orm.server = None - traits = service.traits(input=True) - for key, value in spec.items(): - if key not in traits: - raise AttributeError("No such service field: %s" % key) - setattr(service, key, value) - - if service.api_token: - self.service_tokens[service.api_token] = service.name - elif service.managed: - # generate new token - # TODO: revoke old tokens? - service.api_token = service.orm.new_api_token( - note="generated at startup" - ) - - if service.url: - parsed = urlparse(service.url) - if parsed.port is not None: - port = parsed.port - elif parsed.scheme == 'http': - port = 80 - elif parsed.scheme == 'https': - port = 443 - server = service.orm.server = orm.Server( - proto=parsed.scheme, - ip=parsed.hostname, - port=port, - cookie_name=service.oauth_client_id, - base_url=service.prefix, - ) - self.db.add(server) - - else: - service.orm.server = None - - if service.oauth_available: - allowed_scopes = set() - if service.oauth_client_allowed_scopes: - allowed_scopes.update(service.oauth_client_allowed_scopes) - if service.oauth_roles: - if not allowed_scopes: - # DEPRECATED? It's still convenient and valid, - # e.g. 'admin' - allowed_roles = list( - self.db.query(orm.Role).filter( - orm.Role.name.in_(service.oauth_roles) - ) + if service.oauth_available: + allowed_scopes = set() + if service.oauth_client_allowed_scopes: + allowed_scopes.update(service.oauth_client_allowed_scopes) + if service.oauth_roles: + if not allowed_scopes: + # DEPRECATED? It's still convenient and valid, + # e.g. 'admin' + allowed_roles = list( + self.db.query(orm.Role).filter( + orm.Role.name.in_(service.oauth_roles) ) - allowed_scopes.update(roles.roles_to_scopes(allowed_roles)) - else: - self.log.warning( - f"Ignoring oauth_roles for {service.name}: {service.oauth_roles}," - f" using oauth_client_allowed_scopes={allowed_scopes}." - ) - oauth_client = self.oauth_provider.add_client( - client_id=service.oauth_client_id, - client_secret=service.api_token, - redirect_uri=service.oauth_redirect_uri, - description="JupyterHub service %s" % service.name, - ) - service.orm.oauth_client = oauth_client - # add access-scopes, derived from OAuthClient itself - allowed_scopes.update(scopes.access_scopes(oauth_client)) - oauth_client.allowed_scopes = sorted(allowed_scopes) + ) + allowed_scopes.update(roles.roles_to_scopes(allowed_roles)) + else: + self.log.warning( + f"Ignoring oauth_roles for {service.name}: {service.oauth_roles}," + f" using oauth_client_allowed_scopes={allowed_scopes}." + ) + oauth_client = self.oauth_provider.add_client( + client_id=service.oauth_client_id, + client_secret=service.api_token, + redirect_uri=service.oauth_redirect_uri, + description="JupyterHub service %s" % service.name, + ) + service.orm.oauth_client = oauth_client + # add access-scopes, derived from OAuthClient itself + allowed_scopes.update(scopes.access_scopes(oauth_client)) + oauth_client.allowed_scopes = sorted(allowed_scopes) + else: + if service.oauth_client: + self.db.delete(service.oauth_client) + + self._service_map[name] = service + + return service + + def init_services(self): + self._service_map.clear() + for spec in self.services: + self.service_from_spec(spec, from_config=True) + + for service_orm in self.db.query(orm.Service): + if service_orm.from_config: + # delete config-based services from db + # that are not in current config file: + if service_orm.name not in self._service_map: + self.db.delete(service_orm) else: - if service.oauth_client: - self.db.delete(service.oauth_client) + self.service_from_orm(service_orm) - self._service_map[name] = service - - # delete services from db not in service config: - for service in self.db.query(orm.Service): - if service.name not in self._service_map: - self.db.delete(service) self.db.commit() async def check_services_health(self): """Check connectivity of all services""" for name, service in self._service_map.items(): if not service.url: + # no URL to check, nothing to do continue try: await Server.from_orm(service.orm.server).wait_up(timeout=1, http=True) @@ -2687,7 +2790,6 @@ class JupyterHub(Application): for user in self.users.values(): for spawner in user.spawners.values(): oauth_client_ids.add(spawner.oauth_client_id) - for i, oauth_client in enumerate(self.db.query(orm.OAuthClient)): if oauth_client.identifier not in oauth_client_ids: self.log.warning("Deleting OAuth client %s", oauth_client.identifier) @@ -3129,6 +3231,72 @@ class JupyterHub(Application): await self.proxy.check_routes(self.users, self._service_map, routes) + async def start_service( + self, + service_name: str, + service: Service, + ssl_context: Optional[ssl.SSLContext] = None, + ) -> bool: + """Start a managed service or poll for external service + + Args: + service_name (str): Name of the service. + service (Service): The service object. + + Returns: + boolean: Returns `True` if the service is started successfully, + returns `False` otherwise. + """ + if ssl_context is None: + ssl_context = make_ssl_context( + self.internal_ssl_key, + self.internal_ssl_cert, + cafile=self.internal_ssl_ca, + purpose=ssl.Purpose.CLIENT_AUTH, + ) + + msg = f'{service_name} at {service.url}' if service.url else service_name + if service.managed: + self.log.info("Starting managed service %s", msg) + try: + await service.start() + except Exception as e: + self.log.critical( + "Failed to start service %s", service_name, exc_info=True + ) + return False + else: + self.log.info("Adding external service %s", msg) + + if service.url: + tries = 10 if service.managed else 1 + for i in range(tries): + try: + await Server.from_orm(service.orm.server).wait_up( + http=True, timeout=1, ssl_context=ssl_context + ) + except AnyTimeoutError: + if service.managed: + status = await service.spawner.poll() + if status is not None: + self.log.error( + "Service %s exited with status %s", + service_name, + status, + ) + return False + else: + return True + else: + self.log.error( + "Cannot connect to %s service %s at %s. Is it running?", + service.kind, + service_name, + service.url, + ) + return False + return True + async def start(self): """Start the whole thing""" self.io_loop = loop = IOLoop.current() @@ -3214,55 +3382,29 @@ class JupyterHub(Application): # start the service(s) for service_name, service in self._service_map.items(): - msg = f'{service_name} at {service.url}' if service.url else service_name - if service.managed: - self.log.info("Starting managed service %s", msg) - try: - await service.start() - except Exception as e: - self.log.critical( - "Failed to start service %s", service_name, exc_info=True - ) + service_ready = await self.start_service(service_name, service, ssl_context) + if not service_ready: + if service.from_config: + # Stop the application if a config-based service failed to start. self.exit(1) - else: - self.log.info("Adding external service %s", msg) - - if service.url: - tries = 10 if service.managed else 1 - for i in range(tries): - try: - await Server.from_orm(service.orm.server).wait_up( - http=True, timeout=1, ssl_context=ssl_context - ) - except AnyTimeoutError: - if service.managed: - status = await service.spawner.poll() - if status is not None: - self.log.error( - "Service %s exited with status %s", - service_name, - status, - ) - break - else: - break else: + # Only warn for database-based service, so that admin can connect + # to hub to remove the service. self.log.error( - "Cannot connect to %s service %s at %s. Is it running?", - service.kind, + "Failed to reach externally managed service %s", service_name, - service.url, + exc_info=True, ) await self.proxy.check_routes(self.users, self._service_map) - if self.service_check_interval and any( - s.url for s in self._service_map.values() - ): - pc = PeriodicCallback( + # Check services health + self._check_services_health_callback = None + if self.service_check_interval: + self._check_services_health_callback = PeriodicCallback( self.check_services_health, 1e3 * self.service_check_interval ) - pc.start() + self._check_services_health_callback.start() if self.last_activity_interval: pc = PeriodicCallback( diff --git a/jupyterhub/oauth/provider.py b/jupyterhub/oauth/provider.py index 9f45a07e..4a7533ac 100644 --- a/jupyterhub/oauth/provider.py +++ b/jupyterhub/oauth/provider.py @@ -668,6 +668,18 @@ class JupyterHubOAuthServer(WebApplicationServer): self.db.commit() return orm_client + def remove_client(self, client_id): + """Remove a client by its id if it is existed.""" + orm_client = ( + self.db.query(orm.OAuthClient).filter_by(identifier=client_id).one_or_none() + ) + if orm_client is not None: + self.db.delete(orm_client) + self.db.commit() + app_log.info("Removed client %s", client_id) + else: + app_log.warning("No such client %s", client_id) + def fetch_by_client_id(self, client_id): """Find a client by its id""" client = self.db.query(orm.OAuthClient).filter_by(identifier=client_id).first() diff --git a/jupyterhub/orm.py b/jupyterhub/orm.py index c0fb80ba..71cc2f76 100644 --- a/jupyterhub/orm.py +++ b/jupyterhub/orm.py @@ -394,6 +394,26 @@ class Service(Base): 'Role', secondary='service_role_map', back_populates='services' ) + url = Column(Unicode(2047), nullable=True) + + oauth_client_allowed_scopes = Column(JSONList, nullable=True) + + info = Column(JSONDict, nullable=True) + + display = Column(Boolean, nullable=True) + + oauth_no_confirm = Column(Boolean, nullable=True) + + command = Column(JSONList, nullable=True) + + cwd = Column(Unicode(4095), nullable=True) + + environment = Column(JSONDict, nullable=True) + + user = Column(Unicode(255), nullable=True) + + from_config = Column(Boolean, default=True) + api_tokens = relationship( "APIToken", back_populates="service", cascade="all, delete-orphan" ) @@ -416,6 +436,7 @@ class Service(Base): ondelete='SET NULL', ), ) + oauth_client = relationship( 'OAuthClient', back_populates="service", diff --git a/jupyterhub/roles.py b/jupyterhub/roles.py index a53e5a53..3ae2d008 100644 --- a/jupyterhub/roles.py +++ b/jupyterhub/roles.py @@ -34,6 +34,7 @@ def get_default_roles(): 'admin-ui', 'admin:users', 'admin:servers', + 'admin:services', 'tokens', 'admin:groups', 'list:services', diff --git a/jupyterhub/scopes.py b/jupyterhub/scopes.py index 00170133..d7b91f16 100644 --- a/jupyterhub/scopes.py +++ b/jupyterhub/scopes.py @@ -123,6 +123,10 @@ scope_definitions = { 'delete:groups': { 'description': "Delete groups.", }, + 'admin:services': { + 'description': 'Create, read, update, delete services, not including services defined from config files.', + 'subscopes': ['list:services', 'read:services', 'read:roles:services'], + }, 'list:services': { 'description': 'List services, including at least their names.', 'subscopes': ['read:services:name'], @@ -435,7 +439,7 @@ def _expand_self_scope(username): @lru_cache(maxsize=65535) def _expand_scope(scope): - """Returns a scope and all all subscopes + """Returns a scope and all subscopes Arguments: scope (str): the scope to expand diff --git a/jupyterhub/services/service.py b/jupyterhub/services/service.py index 2f6f8a27..24c3f37c 100644 --- a/jupyterhub/services/service.py +++ b/jupyterhub/services/service.py @@ -180,9 +180,12 @@ class Service(LoggingConfigurable): - user: str The name of a system user to become. If unspecified, run as the same user as the Hub. + """ - # inputs: + # traits tagged with `input=True` are accepted as input from configuration / API + # input traits are also persisted to the db UNLESS they are also tagged with `in_db=False` + name = Unicode( help="""The name of the service. @@ -205,7 +208,7 @@ class Service(LoggingConfigurable): DEPRECATED in 3.0: use oauth_client_allowed_scopes """ - ).tag(input=True) + ).tag(input=True, in_db=False) oauth_client_allowed_scopes = List( help="""OAuth allowed scopes. @@ -225,7 +228,7 @@ class Service(LoggingConfigurable): If unspecified, an API token will be generated for managed services. """ - ).tag(input=True) + ).tag(input=True, in_db=False) info = Dict( help="""Provide a place to include miscellaneous information about the service, @@ -310,7 +313,7 @@ class Service(LoggingConfigurable): You shouldn't generally need to change this. Default: `service-` """ - ).tag(input=True) + ).tag(input=True, in_db=False) @default('oauth_client_id') def _default_client_id(self): @@ -331,7 +334,7 @@ class Service(LoggingConfigurable): You shouldn't generally need to change this. Default: `/services/:name/oauth_callback` """ - ).tag(input=True) + ).tag(input=True, in_db=False) @default('oauth_redirect_uri') def _default_redirect_uri(self): @@ -371,6 +374,11 @@ class Service(LoggingConfigurable): else: return self.server.base_url + @property + def from_config(self): + """Is the service defined from config file?""" + return self.orm.from_config + def __repr__(self): return "<{cls}(name={name}{managed})>".format( cls=self.__class__.__name__, diff --git a/jupyterhub/tests/conftest.py b/jupyterhub/tests/conftest.py index e9b2260e..98ec9fb8 100644 --- a/jupyterhub/tests/conftest.py +++ b/jupyterhub/tests/conftest.py @@ -167,6 +167,8 @@ async def cleanup_after(request, io_loop): app = MockHub.instance() if app.db_file.closed: return + + # cleanup users for uid, user in list(app.users.items()): for name, spawner in list(user.spawners.items()): if spawner.active: @@ -181,6 +183,16 @@ async def cleanup_after(request, io_loop): # delete groups for group in app.db.query(orm.Group): app.db.delete(group) + + # clear services + for name, service in app._service_map.items(): + if service.managed: + service.stop() + for orm_service in app.db.query(orm.Service): + if orm_service.oauth_client: + app.oauth_provider.remove_client(orm_service.oauth_client_id) + app.db.delete(orm_service) + app._service_map.clear() app.db.commit() @@ -262,10 +274,7 @@ class MockServiceSpawner(jupyterhub.services.service._ServiceSpawner): poll_interval = 1 -_mock_service_counter = 0 - - -async def _mockservice(request, app, external=False, url=False): +async def _mockservice(request, app, name, external=False, url=False): """ Add a service to the application @@ -281,9 +290,6 @@ async def _mockservice(request, app, external=False, url=False): If True, register the service at a URL (as opposed to headless, API-only). """ - global _mock_service_counter - _mock_service_counter += 1 - name = 'mock-service-%i' % _mock_service_counter spec = {'name': name, 'command': mockservice_cmd, 'admin': True} if url: if app.internal_ssl: @@ -330,22 +336,33 @@ async def _mockservice(request, app, external=False, url=False): return service +_service_name_counter = 0 + + @fixture -async def mockservice(request, app): +def service_name(): + global _service_name_counter + _service_name_counter += 1 + name = f'test-service-{_service_name_counter}' + return name + + +@fixture +async def mockservice(request, app, service_name): """Mock a service with no external service url""" - yield await _mockservice(request, app, url=False) + yield await _mockservice(request, app, name=service_name, url=False) @fixture -async def mockservice_external(request, app): +async def mockservice_external(request, app, service_name): """Mock an externally managed service (don't start anything)""" - yield await _mockservice(request, app, external=True, url=False) + yield await _mockservice(request, app, name=service_name, external=True, url=False) @fixture -async def mockservice_url(request, app): +async def mockservice_url(request, app, service_name): """Mock a service with its own url to test external services""" - yield await _mockservice(request, app, url=True) + yield await _mockservice(request, app, name=service_name, url=True) @fixture @@ -534,3 +551,17 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): queries = db_counts[nodeid] if queries: terminalreporter.line(f"{queries:<6} {nodeid}") + + +@fixture +def service_data(service_name): + """Data used to create service at runtime""" + return { + "name": service_name, + "oauth_client_id": f"service-{service_name}", + "api_token": f"api_token-{service_name}", + "oauth_redirect_uri": "http://127.0.0.1:5555/oauth_callback-from-api", + "oauth_no_confirm": True, + "oauth_client_allowed_scopes": ["inherit"], + "info": {'foo': 'bar'}, + } diff --git a/jupyterhub/tests/test_api.py b/jupyterhub/tests/test_api.py index b7002e79..3a01dc7a 100644 --- a/jupyterhub/tests/test_api.py +++ b/jupyterhub/tests/test_api.py @@ -4,10 +4,12 @@ import json import re import sys import uuid +from copy import deepcopy from datetime import datetime, timedelta from unittest import mock from urllib.parse import quote, urlparse +import pytest from pytest import fixture, mark from tornado.httputil import url_concat @@ -2062,7 +2064,7 @@ async def test_get_services(app, mockservice_url): async def test_get_service(app, mockservice_url): mockservice = mockservice_url db = app.db - r = await api_request(app, 'services/%s' % mockservice.name) + r = await api_request(app, f"services/{mockservice.name}") r.raise_for_status() assert r.status_code == 200 @@ -2081,19 +2083,271 @@ async def test_get_service(app, mockservice_url): } r = await api_request( app, - 'services/%s' % mockservice.name, + f"services/{mockservice.name}", headers={'Authorization': 'token %s' % mockservice.api_token}, ) r.raise_for_status() + r = await api_request( - app, 'services/%s' % mockservice.name, headers=auth_header(db, 'user') + app, f"services/{mockservice.name}", headers=auth_header(db, 'user') ) assert r.status_code == 403 + r = await api_request(app, "services/nosuchservice") + assert r.status_code == 404 + + +@pytest.fixture +def service_admin_user(create_user_with_scopes): + return create_user_with_scopes('admin:services') + + +@mark.services +async def test_create_service(app, service_admin_user, service_name, service_data): + db = app.db + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, service_admin_user.name), + data=json.dumps(service_data), + method='post', + ) + + r.raise_for_status() + assert r.status_code == 201 + assert r.json()['name'] == service_name + orm_service = orm.Service.find(db, service_name) + assert orm_service is not None + + oath_client = ( + db.query(orm.OAuthClient) + .filter_by(identifier=service_data['oauth_client_id']) + .first() + ) + assert oath_client.redirect_uri == service_data['oauth_redirect_uri'] + + assert service_name in app._service_map + assert ( + app._service_map[service_name].oauth_no_confirm + == service_data['oauth_no_confirm'] + ) + + +@mark.services +async def test_create_service_no_role(app, service_name, service_data): + db = app.db + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, 'user'), + data=json.dumps(service_data), + method='post', + ) + + assert r.status_code == 403 + + +@mark.services +async def test_create_service_conflict( + app, service_admin_user, mockservice, service_data, service_name +): + db = app.db + app.services = [{'name': service_name}] + app.init_services() + + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, service_admin_user.name), + data=json.dumps(service_data), + method='post', + ) + + assert r.status_code == 409 + + +@mark.services +async def test_create_service_duplication( + app, service_admin_user, service_name, service_data +): + db = app.db + + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, service_admin_user.name), + data=json.dumps(service_data), + method='post', + ) + assert r.status_code == 201 + + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, service_admin_user.name), + data=json.dumps(service_data), + method='post', + ) + assert r.status_code == 409 + + +@mark.services +async def test_create_managed_service( + app, service_admin_user, service_name, service_data +): + db = app.db + managed_service_data = deepcopy(service_data) + managed_service_data['command'] = ['foo'] + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, service_admin_user.name), + data=json.dumps(managed_service_data), + method='post', + ) + + assert r.status_code == 400 + assert 'Can not create managed service' in r.json()['message'] + orm_service = orm.Service.find(db, service_name) + assert orm_service is None + + +@mark.services +async def test_create_admin_service(app, admin_user, service_name, service_data): + db = app.db + managed_service_data = deepcopy(service_data) + managed_service_data['admin'] = True + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, admin_user.name), + data=json.dumps(managed_service_data), + method='post', + ) + + assert r.status_code == 201 + orm_service = orm.Service.find(db, service_name) + assert orm_service is not None + + +@mark.services +async def test_create_admin_service_without_admin_right( + app, service_admin_user, service_data, service_name +): + db = app.db + managed_service_data = deepcopy(service_data) + managed_service_data['admin'] = True + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, service_admin_user.name), + data=json.dumps(managed_service_data), + method='post', + ) + + assert r.status_code == 400 + assert 'Not assigning requested scopes' in r.json()['message'] + orm_service = orm.Service.find(db, service_name) + assert orm_service is None + + +@mark.services +async def test_create_service_with_scope( + app, create_user_with_scopes, service_name, service_data +): + db = app.db + managed_service_data = deepcopy(service_data) + managed_service_data['oauth_client_allowed_scopes'] = ["admin:users"] + managed_service_data['oauth_client_id'] = "service-client-with-scope" + user_with_scope = create_user_with_scopes('admin:services', 'admin:users') + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, user_with_scope.name), + data=json.dumps(managed_service_data), + method='post', + ) + + assert r.status_code == 201 + orm_service = orm.Service.find(db, service_name) + assert orm_service is not None + + +@mark.services +async def test_create_service_without_requested_scope( + app, + service_admin_user, + service_data, + service_name, +): + db = app.db + managed_service_data = deepcopy(service_data) + managed_service_data['oauth_client_allowed_scopes'] = ["admin:users"] + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, service_admin_user.name), + data=json.dumps(managed_service_data), + method='post', + ) + + assert r.status_code == 400 + assert 'Not assigning requested scopes' in r.json()['message'] + orm_service = orm.Service.find(db, service_name) + assert orm_service is None + + +@mark.services +async def test_delete_service(app, service_admin_user, service_name, service_data): + db = app.db + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, service_admin_user.name), + data=json.dumps(service_data), + method='post', + ) + assert r.status_code == 201 + + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, service_admin_user.name), + method='delete', + ) + assert r.status_code == 200 + + orm_service = orm.Service.find(db, service_name) + assert orm_service is None + + oath_client = ( + db.query(orm.OAuthClient) + .filter_by(identifier=service_data['oauth_client_id']) + .first() + ) + assert oath_client is None + + assert service_name not in app._service_map + + r = await api_request(app, f"services/{service_name}", method="delete") + assert r.status_code == 404 + + +@mark.services +async def test_delete_service_from_config(app, service_admin_user, mockservice): + db = app.db + service_name = mockservice.name + r = await api_request( + app, + f'services/{service_name}', + headers=auth_header(db, service_admin_user.name), + method='delete', + ) + assert r.status_code == 405 + assert r.json()['message'] == f'Service {service_name} is not modifiable at runtime' + async def test_root_api(app): - base_url = app.hub.url - url = ujoin(base_url, 'api') kwargs = {} if app.internal_ssl: kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key) diff --git a/jupyterhub/tests/test_app.py b/jupyterhub/tests/test_app.py index f1967dce..4bc4316f 100644 --- a/jupyterhub/tests/test_app.py +++ b/jupyterhub/tests/test_app.py @@ -220,7 +220,7 @@ def test_cookie_secret_env(tmpdir, request): assert not os.path.exists(hub.cookie_secret_file) -def test_cookie_secret_string_(): +def test_cookie_secret_string(): cfg = Config() cfg.JupyterHub.cookie_secret = "not hex" @@ -270,18 +270,41 @@ async def test_load_groups(tmpdir, request): ) -async def test_resume_spawners(tmpdir, request): - if not os.getenv('JUPYTERHUB_TEST_DB_URL'): - p = patch.dict( - os.environ, - { - 'JUPYTERHUB_TEST_DB_URL': 'sqlite:///%s' - % tmpdir.join('jupyterhub.sqlite') - }, - ) - p.start() - request.addfinalizer(p.stop) +@pytest.fixture +def persist_db(tmpdir): + """ensure db will persist (overrides default sqlite://:memory:)""" + if os.getenv('JUPYTERHUB_TEST_DB_URL'): + # already using a db, no need + yield + return + with patch.dict( + os.environ, + {'JUPYTERHUB_TEST_DB_URL': f"sqlite:///{tmpdir.join('jupyterhub.sqlite')}"}, + ): + yield + +@pytest.fixture +def new_hub(request, tmpdir, persist_db): + """Fixture to launch a new hub for testing""" + + async def new_hub(): + kwargs = {} + ssl_enabled = getattr(request.module, "ssl_enabled", False) + if ssl_enabled: + kwargs['internal_certs_location'] = str(tmpdir) + app = MockHub(test_clean_db=False, **kwargs) + app.config.ConfigurableHTTPProxy.should_start = False + app.config.ConfigurableHTTPProxy.auth_token = 'unused' + request.addfinalizer(app.stop) + await app.initialize([]) + + return app + + return new_hub + + +async def test_resume_spawners(tmpdir, request, new_hub): async def new_hub(): kwargs = {} ssl_enabled = getattr(request.module, "ssl_enabled", False) @@ -473,3 +496,42 @@ async def test_user_creation(tmpdir, request): "in-group", "in-role", } + + +async def test_recreate_service_from_database( + request, new_hub, service_name, service_data +): + # create a hub and add a service (not from config) + app = await new_hub() + app.service_from_spec(service_data, from_config=False) + app.stop() + + # new hub, should load service from db + app = await new_hub() + assert service_name in app._service_map + + # verify keys + service = app._service_map[service_name] + for key, value in service_data.items(): + if key in {'api_token'}: + # skip some keys + continue + assert getattr(service, key) == value + + assert ( + service_data['oauth_client_id'] in app.tornado_settings['oauth_no_confirm_list'] + ) + oauth_client = ( + app.db.query(orm.OAuthClient) + .filter_by(identifier=service_data['oauth_client_id']) + .first() + ) + assert oauth_client.redirect_uri == service_data['oauth_redirect_uri'] + + # delete service from db, start one more + app.db.delete(service.orm) + app.db.commit() + + # start one more, service should be gone + app = await new_hub() + assert service_name not in app._service_map diff --git a/jupyterhub/tests/test_db.py b/jupyterhub/tests/test_db.py index f8c379bc..d9941ea1 100644 --- a/jupyterhub/tests/test_db.py +++ b/jupyterhub/tests/test_db.py @@ -44,7 +44,9 @@ def generate_old_db(env_dir, hub_version, db_url): # changes to this version list must also be reflected # in ci/init-db.sh -@pytest.mark.parametrize('hub_version', ["1.1.0", "1.2.2", "1.3.0", "1.5.0", "2.1.1"]) +@pytest.mark.parametrize( + 'hub_version', ['1.1.0', '1.2.2', '1.3.0', '1.5.0', '2.1.1', '3.1.1'] +) async def test_upgrade(tmpdir, hub_version): db_url = os.getenv('JUPYTERHUB_TEST_DB_URL') if db_url: diff --git a/jupyterhub/tests/test_roles.py b/jupyterhub/tests/test_roles.py index 7bd90d63..f5e905fc 100644 --- a/jupyterhub/tests/test_roles.py +++ b/jupyterhub/tests/test_roles.py @@ -236,6 +236,16 @@ def test_orm_roles_delete_cascade(db): ['tokens!group=hobbits'], {'tokens!group=hobbits', 'read:tokens!group=hobbits'}, ), + ( + ['admin:services'], + { + 'read:roles:services', + 'read:services:name', + 'admin:services', + 'list:services', + 'read:services', + }, + ), ], ) def test_get_expanded_scopes(db, scopes, expected_scopes):