hazza/jupyterhub
1
0
Fork 0
mirror of https://github.com/jupyterhub/jupyterhub.git synced 2025-10-18 15:33:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4381 from trungleduc/service-api

Browse Source 
Add REST API for managing services at runtime
This commit is contained in:
Min RK
2023-08-09 11:41:18 +02:00
committed by GitHub
parent d268633a2c 7dbb4ce1ff
commit f215324c44
19 changed files with 1044 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ci/init-db.sh
View File

@@ -21,7 +21,7 @@ fi
# Configure a set of databases in the database server for upgrade tests
# this list must be in sync with versions in test_db.py:test_upgrade
set -x
for SUFFIX in '' _upgrade_110 _upgrade_122 _upgrade_130 _upgrade_150 _upgrade_211; do
for SUFFIX in '' _upgrade_110 _upgrade_122 _upgrade_130 _upgrade_150 _upgrade_211 _upgrade_311; do
$SQL_CLIENT "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true
$SQL_CLIENT "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE_DATABASE_ARGS:-};"
done

3
docs/source/_static/rest-api.yml
View File

@@ -1498,6 +1498,9 @@ components:
read:groups: Read group models.
read:groups:name: Read group names.
delete:groups: Delete groups.
admin:services:
Create, read, update, delete services, not including services
defined from config files.
list:services: List services, including at least their names.
read:services: Read service models.
read:services:name: Read service names.

1
docs/source/howto/rest.md
View File

@@ -24,6 +24,7 @@ such as:
- Checking which users are active
- Adding or removing users
- Adding or removing services
- Stopping or starting single user notebook servers
- Authenticating services
- Communicating with an individual Jupyter server's REST API

41
docs/source/reference/services.md
View File

@@ -174,6 +174,47 @@ c.JupyterHub.services = [
In this case, the `url` field will be passed along to the Service as
`JUPYTERHUB_SERVICE_URL`.
## Adding or removing services at runtime
Only externally-managed services can be added at runtime by using JupyterHub’s REST API.
### Add a new service
To add a new service, send a POST request to this endpoint
```
POST /hub/api/services/:servicename
```
**Required scope: `admin:services`**
**Payload**: The payload should contain the definition of the service to be created. The endpoint supports the same properties as externally-managed services defined in the config file.
**Possible responses**
- `201 Created`: The service and related objects are created (and started in case of a Hub-managed one) successfully.
- `400 Bad Request`: The payload is invalid or JupyterHub can not create the service.
- `409 Conflict`: The service with the same name already exists.
### Remove an existing service
To remove an existing service, send a DELETE request to this endpoint
```
DELETE /hub/api/services/:servicename
```
**Required scope: `admin:services`**
**Payload**: `None`
**Possible responses**
- `200 OK`: The service and related objects are removed (and stopped in case of a Hub-managed one) successfully.
- `400 Bad Request`: JupyterHub can not remove the service.
- `404 Not Found`: The requested service does not exist.
- `405 Not Allowed`: The requested service is created from the config file, it can not be removed at runtime.
## Writing your own Services
When writing your own services, you have a few decisions to make (in addition

51
jupyterhub/alembic/versions/3c2384c5aae1_add_from_config_column_to_the_services_.py Normal file
View File

@@ -0,0 +1,51 @@
"""Add from_config column to the services table
Revision ID: 3c2384c5aae1
Revises: 0eee8c825d24
Create Date: 2023-02-27 16:22:26.196231
"""
# revision identifiers, used by Alembic.
revision = '3c2384c5aae1'
down_revision = '0eee8c825d24'
branch_labels = None
depends_on = None
import sqlalchemy as sa
from alembic import op
from jupyterhub.orm import JSONDict, JSONList
COL_DATA = [
{'name': 'url', 'type': sa.Unicode(length=2047)},
{'name': 'oauth_client_allowed_scopes', 'type': JSONDict()},
{'name': 'info', 'type': JSONDict()},
{'name': 'display', 'type': sa.Boolean},
{'name': 'oauth_no_confirm', 'type': sa.Boolean},
{'name': 'command', 'type': JSONList()},
{'name': 'cwd', 'type': sa.Unicode(length=2047)},
{'name': 'environment', 'type': JSONDict()},
{'name': 'user', 'type': sa.Unicode(255)},
]
def upgrade():
engine = op.get_bind().engine
tables = sa.inspect(engine).get_table_names()
if 'services' in tables:
op.add_column(
'services',
sa.Column('from_config', sa.Boolean, default=True),
)
op.execute('UPDATE services SET from_config = true')
for item in COL_DATA:
op.add_column(
'services',
sa.Column(item['name'], item['type'], nullable=True),
)
def downgrade():
op.drop_column('services', sa.Column('from_config'))
for item in COL_DATA:
op.drop_column('services', sa.Column(item['name']))

29
jupyterhub/alembic/versions/651f5419b74d_api_token_scopes.py
View File

@@ -13,16 +13,37 @@ depends_on = None
import sqlalchemy as sa
from alembic import op
from sqlalchemy import Column, ForeignKey, Table
from sqlalchemy import Column, ForeignKey, Table, text
from sqlalchemy.orm import relationship
from sqlalchemy.orm.session import Session
from jupyterhub import orm, roles, scopes
from jupyterhub import orm, roles
def access_scopes(oauth_client: orm.OAuthClient, db: Session):
"""Return scope(s) required to access an oauth client
This is a clone of `scopes.access_scopes` without using
the `orm.Service`
"""
scopes = set()
if oauth_client.identifier == "jupyterhub":
return frozenset()
spawner = oauth_client.spawner
if spawner:
scopes.add(f"access:servers!server={spawner.user.name}/{spawner.name}")
else:
statement = "SELECT * FROM services WHERE oauth_client_id = :identifier"
service = db.execute(
text(statement), {"identifier": oauth_client.identifier}
).fetchall()
if len(service) > 0:
scopes.add(f"access:services!service={service[0].name}")
return frozenset(scopes)
def upgrade():
c = op.get_bind()
tables = sa.inspect(c.engine).get_table_names()
# oauth codes are short lived, no need to upgrade them
@@ -100,7 +121,7 @@ def upgrade():
db = Session(bind=c)
for oauth_client in db.query(orm.OAuthClient):
allowed_scopes = set(roles.roles_to_scopes(oauth_client.allowed_roles))
allowed_scopes.update(scopes.access_scopes(oauth_client))
allowed_scopes.update(access_scopes(oauth_client, db))
oauth_client.allowed_scopes = sorted(allowed_scopes)
db.commit()
# drop token-role relationship

26
jupyterhub/apihandlers/base.py
View File

@@ -395,6 +395,23 @@ class APIHandler(BaseHandler):
_group_model_types = {'name': str, 'users': list, 'roles': list}
_service_model_types = {
'name': str,
'admin': bool,
'url': str,
'oauth_client_allowed_scopes': list,
'api_token': str,
'info': dict,
'display': bool,
'oauth_no_confirm': bool,
'command': list,
'cwd': str,
'environment': dict,
'user': str,
'oauth_client_id': str,
'oauth_redirect_uri': str,
}
def _check_model(self, model, model_types, name):
"""Check a model provided by a REST API request
@@ -433,6 +450,15 @@ class APIHandler(BaseHandler):
400, ("group names must be str, not %r", type(groupname))
)
def _check_service_model(self, model):
"""Check a request-provided service model from a REST API"""
self._check_model(model, self._service_model_types, 'service')
service_name = model.get('name')
if not isinstance(service_name, str):
raise web.HTTPError(
400, ("Service name must be str, not %r", type(service_name))
)
def get_api_pagination(self):
default_limit = self.settings["api_page_default_limit"]
max_limit = self.settings["api_page_max_limit"]

170
jupyterhub/apihandlers/services.py
View File

@@ -5,8 +5,14 @@ Currently GET-only, no actions can be taken to modify services.
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import json
from typing import Optional, Tuple
from ..scopes import Scope, needs_scope
from tornado import web
from .. import orm
from ..roles import get_default_roles
from ..scopes import Scope, _check_token_scopes, needs_scope
from ..services.service import Service
from .base import APIHandler
@@ -25,9 +31,171 @@ class ServiceListAPIHandler(APIHandler):
class ServiceAPIHandler(APIHandler):
@needs_scope('read:services', 'read:services:name', 'read:roles:services')
def get(self, service_name):
if service_name not in self.services:
raise web.HTTPError(404, f"No such service: {service_name}")
service = self.services[service_name]
self.write(json.dumps(self.service_model(service)))
def _check_service_scopes(self, spec: dict):
user = self.current_user
requested_scopes = []
if spec.get('admin'):
default_roles = get_default_roles()
admin_scopes = [
role['scopes'] for role in default_roles if role['name'] == 'admin'
]
requested_scopes.extend(admin_scopes[0])
requested_client_scope = spec.get('oauth_client_allowed_scopes')
if requested_client_scope is not None:
requested_scopes.extend(requested_client_scope)
if len(requested_scopes) > 0:
try:
_check_token_scopes(requested_scopes, user, None)
except ValueError as e:
raise web.HTTPError(400, str(e))
async def add_service(self, spec: dict) -> Service:
"""Add a new service and related objects to the database
Args:
spec (dict): The service specification
Raises:
web.HTTPError: Raise if the service is not created
Returns:
Service: Returns the service instance.
"""
self._check_service_model(spec)
self._check_service_scopes(spec)
service_name = spec["name"]
managed = bool(spec.get('command'))
if managed:
msg = f"Can not create managed service {service_name} at runtime"
self.log.error(msg, exc_info=True)
raise web.HTTPError(400, msg)
try:
new_service = self.service_from_spec(spec)
except Exception:
msg = f"Failed to create service {service_name}"
self.log.error(msg, exc_info=True)
raise web.HTTPError(400, msg)
if new_service is None:
raise web.HTTPError(400, f"Failed to create service {service_name}")
if new_service.api_token:
# Add api token to database
await self.app._add_tokens(
{new_service.api_token: new_service.name}, kind='service'
)
if new_service.url:
# Start polling for external service
service_status = await self.app.start_service(service_name, new_service)
if not service_status:
self.log.error(
'Failed to start service %s',
service_name,
exc_info=True,
)
if new_service.oauth_no_confirm:
oauth_no_confirm_list = self.settings.get('oauth_no_confirm_list')
msg = f"Allowing service {new_service.name} to complete OAuth without confirmation on an authorization web page"
self.log.warning(msg)
oauth_no_confirm_list.add(new_service.oauth_client_id)
return new_service
@needs_scope('admin:services')
async def post(self, service_name: str):
data = self.get_json_body()
service, _ = self.find_service(service_name)
if service is not None:
raise web.HTTPError(409, f"Service {service_name} already exists")
if not data or not isinstance(data, dict):
raise web.HTTPError(400, "Invalid service data")
data['name'] = service_name
new_service = await self.add_service(data)
self.write(json.dumps(self.service_model(new_service)))
self.set_status(201)
@needs_scope('admin:services')
async def delete(self, service_name: str):
service, orm_service = self.find_service(service_name)
if service is None:
raise web.HTTPError(404, f"Service {service_name} does not exist")
if service.from_config:
raise web.HTTPError(
405, f"Service {service_name} is not modifiable at runtime"
)
try:
await self.remove_service(service, orm_service)
self.services.pop(service_name)
except Exception:
msg = f"Failed to remove service {service_name}"
self.log.error(msg, exc_info=True)
raise web.HTTPError(400, msg)
self.set_status(200)
async def remove_service(self, service: Service, orm_service: orm.Service) -> None:
"""Remove a service and all related objects from the database.
Args:
service (Service): the service object to be removed
orm_service (orm.Service): The `orm.Service` object linked
with `service`
"""
if service.managed:
await service.stop()
if service.oauth_client:
self.oauth_provider.remove_client(service.oauth_client_id)
if orm_service._server_id is not None:
orm_server = (
self.db.query(orm.Server).filter_by(id=orm_service._server_id).first()
)
if orm_server is not None:
self.db.delete(orm_server)
if service.oauth_no_confirm:
oauth_no_confirm_list = self.settings.get('oauth_no_confirm_list')
oauth_no_confirm_list.discard(service.oauth_client_id)
self.db.delete(orm_service)
self.db.commit()
def service_from_spec(self, spec) -> Optional[Service]:
"""Create service from api request"""
service = self.app.service_from_spec(spec, from_config=False)
self.db.commit()
return service
def find_service(
self, name: str
) -> Tuple[Optional[Service], Optional[orm.Service]]:
"""Get a service by name
return None if no such service
"""
orm_service = orm.Service.find(db=self.db, name=name)
if orm_service is not None:
service = self.services.get(name)
return service, orm_service
return (None, None)
default_handlers = [
(r"/api/services", ServiceListAPIHandler),

250
jupyterhub/app.py
View File

@@ -21,6 +21,7 @@ from functools import partial
from getpass import getuser
from operator import itemgetter
from textwrap import dedent
from typing import Optional
from urllib.parse import unquote, urlparse, urlunparse
if sys.version_info[:2] < (3, 3):
@@ -2323,7 +2324,7 @@ class JupyterHub(Application):
if not self.authenticator.validate_username(name):
raise ValueError("Token user name %r is not valid" % name)
if kind == 'service':
if not any(service["name"] == name for service in self.services):
if not any(service_name == name for service_name in self._service_map):
self.log.warning(
f"service {name} not in services, creating implicitly. It is recommended to register services using services list."
)
@@ -2386,8 +2387,73 @@ class JupyterHub(Application):
)
pc.start()
def init_services(self):
self._service_map.clear()
def service_from_orm(
self,
orm_service: orm.Service,
) -> Service:
"""Create the service instance and related objects from
ORM data.
Args:
orm_service (orm.Service): The `orm.Service` object
Returns:
Service: the created service
"""
if self.domain:
domain = 'services.' + self.domain
parsed = urlparse(self.subdomain_host)
host = f'{parsed.scheme}://services.{parsed.netloc}'
else:
domain = host = ''
name = orm_service.name
service = Service(
parent=self,
app=self,
base_url=self.base_url,
db=self.db,
orm=orm_service,
roles=orm_service.roles,
domain=domain,
host=host,
hub=self.hub,
)
traits = service.traits(input=True)
for key, trait in traits.items():
if not trait.metadata.get("in_db", True):
continue
orm_value = getattr(orm_service, key)
if orm_value is not None:
setattr(service, key, orm_value)
if orm_service.oauth_client is not None:
service.oauth_client_id = orm_service.oauth_client.identifier
service.oauth_redirect_uri = orm_service.oauth_client.redirect_uri
self._service_map[name] = service
return service
def service_from_spec(
self,
spec: Dict,
from_config=True,
) -> Optional[Service]:
"""Create the service instance and related objects from
config data.
Args:
spec (Dict): The spec of service, defined in the config file.
from_config (bool, optional): `True` if the service will be created
from the config file, `False` if it is created from REST API.
Defaults to `True`.
Returns:
Optional[Service]: The created service
"""
if self.domain:
domain = 'services.' + self.domain
parsed = urlparse(self.subdomain_host)
@@ -2395,15 +2461,15 @@ class JupyterHub(Application):
else:
domain = host = ''
for spec in self.services:
if 'name' not in spec:
raise ValueError('service spec must have a name: %r' % spec)
name = spec['name']
# get/create orm
orm_service = orm.Service.find(self.db, name=name)
if orm_service is None:
# not found, create a new one
orm_service = orm.Service(name=name)
orm_service = orm.Service(name=name, from_config=from_config)
self.db.add(orm_service)
if spec.get('admin', False):
self.log.warning(
@@ -2413,7 +2479,21 @@ class JupyterHub(Application):
"the Service admin flag will be ignored."
)
roles.update_roles(self.db, entity=orm_service, roles=['admin'])
else:
# Do nothing if the config file tries to modify a API-base service
# or vice versa.
if orm_service.from_config != from_config:
if from_config:
self.log.error(
f"The service {name} from the config file is trying to modify a runtime-created service with the same name"
)
else:
self.log.error(
f"The runtime-created service {name} is trying to modify a config-based service with the same name"
)
return
orm_service.admin = spec.get('admin', False)
self.db.commit()
service = Service(
parent=self,
@@ -2429,27 +2509,38 @@ class JupyterHub(Application):
traits = service.traits(input=True)
for key, value in spec.items():
if key not in traits:
trait = traits.get(key)
if trait is None:
raise AttributeError("No such service field: %s" % key)
setattr(service, key, value)
# also set the value on the orm object
# unless it's marked as not in the db
# (e.g. on the oauth object)
if trait.metadata.get("in_db", True):
setattr(orm_service, key, value)
if service.api_token:
self.service_tokens[service.api_token] = service.name
elif service.managed:
# generate new token
# TODO: revoke old tokens?
service.api_token = service.orm.new_api_token(
note="generated at startup"
)
service.api_token = service.orm.new_api_token(note="generated at startup")
if service.url:
parsed = urlparse(service.url)
if parsed.scheme not in {"http", "https"}:
raise ValueError(
f"Unsupported scheme in URL for service {name}: {service.url}. Must be http[s]"
)
port = None
if parsed.port is not None:
port = parsed.port
elif parsed.scheme == 'http':
port = 80
elif parsed.scheme == 'https':
port = 443
server = service.orm.server = orm.Server(
proto=parsed.scheme,
ip=parsed.hostname,
@@ -2458,7 +2549,6 @@ class JupyterHub(Application):
base_url=service.prefix,
)
self.db.add(server)
else:
service.orm.server = None
@@ -2497,16 +2587,29 @@ class JupyterHub(Application):
self._service_map[name] = service
# delete services from db not in service config:
for service in self.db.query(orm.Service):
if service.name not in self._service_map:
self.db.delete(service)
return service
def init_services(self):
self._service_map.clear()
for spec in self.services:
self.service_from_spec(spec, from_config=True)
for service_orm in self.db.query(orm.Service):
if service_orm.from_config:
# delete config-based services from db
# that are not in current config file:
if service_orm.name not in self._service_map:
self.db.delete(service_orm)
else:
self.service_from_orm(service_orm)
self.db.commit()
async def check_services_health(self):
"""Check connectivity of all services"""
for name, service in self._service_map.items():
if not service.url:
# no URL to check, nothing to do
continue
try:
await Server.from_orm(service.orm.server).wait_up(timeout=1, http=True)
@@ -2687,7 +2790,6 @@ class JupyterHub(Application):
for user in self.users.values():
for spawner in user.spawners.values():
oauth_client_ids.add(spawner.oauth_client_id)
for i, oauth_client in enumerate(self.db.query(orm.OAuthClient)):
if oauth_client.identifier not in oauth_client_ids:
self.log.warning("Deleting OAuth client %s", oauth_client.identifier)
@@ -3129,6 +3231,72 @@ class JupyterHub(Application):
await self.proxy.check_routes(self.users, self._service_map, routes)
async def start_service(
self,
service_name: str,
service: Service,
ssl_context: Optional[ssl.SSLContext] = None,
) -> bool:
"""Start a managed service or poll for external service
Args:
service_name (str): Name of the service.
service (Service): The service object.
Returns:
boolean: Returns `True` if the service is started successfully,
returns `False` otherwise.
"""
if ssl_context is None:
ssl_context = make_ssl_context(
self.internal_ssl_key,
self.internal_ssl_cert,
cafile=self.internal_ssl_ca,
purpose=ssl.Purpose.CLIENT_AUTH,
)
msg = f'{service_name} at {service.url}' if service.url else service_name
if service.managed:
self.log.info("Starting managed service %s", msg)
try:
await service.start()
except Exception as e:
self.log.critical(
"Failed to start service %s", service_name, exc_info=True
)
return False
else:
self.log.info("Adding external service %s", msg)
if service.url:
tries = 10 if service.managed else 1
for i in range(tries):
try:
await Server.from_orm(service.orm.server).wait_up(
http=True, timeout=1, ssl_context=ssl_context
)
except AnyTimeoutError:
if service.managed:
status = await service.spawner.poll()
if status is not None:
self.log.error(
"Service %s exited with status %s",
service_name,
status,
)
return False
else:
return True
else:
self.log.error(
"Cannot connect to %s service %s at %s. Is it running?",
service.kind,
service_name,
service.url,
)
return False
return True
async def start(self):
"""Start the whole thing"""
self.io_loop = loop = IOLoop.current()
@@ -3214,55 +3382,29 @@ class JupyterHub(Application):
# start the service(s)
for service_name, service in self._service_map.items():
msg = f'{service_name} at {service.url}' if service.url else service_name
if service.managed:
self.log.info("Starting managed service %s", msg)
try:
await service.start()
except Exception as e:
self.log.critical(
"Failed to start service %s", service_name, exc_info=True
)
service_ready = await self.start_service(service_name, service, ssl_context)
if not service_ready:
if service.from_config:
# Stop the application if a config-based service failed to start.
self.exit(1)
else:
self.log.info("Adding external service %s", msg)
if service.url:
tries = 10 if service.managed else 1
for i in range(tries):
try:
await Server.from_orm(service.orm.server).wait_up(
http=True, timeout=1, ssl_context=ssl_context
)
except AnyTimeoutError:
if service.managed:
status = await service.spawner.poll()
if status is not None:
# Only warn for database-based service, so that admin can connect
# to hub to remove the service.
self.log.error(
"Service %s exited with status %s",
"Failed to reach externally managed service %s",
service_name,
status,
)
break
else:
break
else:
self.log.error(
"Cannot connect to %s service %s at %s. Is it running?",
service.kind,
service_name,
service.url,
exc_info=True,
)
await self.proxy.check_routes(self.users, self._service_map)
if self.service_check_interval and any(
s.url for s in self._service_map.values()
):
pc = PeriodicCallback(
# Check services health
self._check_services_health_callback = None
if self.service_check_interval:
self._check_services_health_callback = PeriodicCallback(
self.check_services_health, 1e3 * self.service_check_interval
)
pc.start()
self._check_services_health_callback.start()
if self.last_activity_interval:
pc = PeriodicCallback(

12
jupyterhub/oauth/provider.py
View File

@@ -668,6 +668,18 @@ class JupyterHubOAuthServer(WebApplicationServer):
self.db.commit()
return orm_client
def remove_client(self, client_id):
"""Remove a client by its id if it is existed."""
orm_client = (
self.db.query(orm.OAuthClient).filter_by(identifier=client_id).one_or_none()
)
if orm_client is not None:
self.db.delete(orm_client)
self.db.commit()
app_log.info("Removed client %s", client_id)
else:
app_log.warning("No such client %s", client_id)
def fetch_by_client_id(self, client_id):
"""Find a client by its id"""
client = self.db.query(orm.OAuthClient).filter_by(identifier=client_id).first()

21
jupyterhub/orm.py
View File

@@ -394,6 +394,26 @@ class Service(Base):
'Role', secondary='service_role_map', back_populates='services'
)
url = Column(Unicode(2047), nullable=True)
oauth_client_allowed_scopes = Column(JSONList, nullable=True)
info = Column(JSONDict, nullable=True)
display = Column(Boolean, nullable=True)
oauth_no_confirm = Column(Boolean, nullable=True)
command = Column(JSONList, nullable=True)
cwd = Column(Unicode(4095), nullable=True)
environment = Column(JSONDict, nullable=True)
user = Column(Unicode(255), nullable=True)
from_config = Column(Boolean, default=True)
api_tokens = relationship(
"APIToken", back_populates="service", cascade="all, delete-orphan"
)
@@ -416,6 +436,7 @@ class Service(Base):
ondelete='SET NULL',
),
)
oauth_client = relationship(
'OAuthClient',
back_populates="service",

1
jupyterhub/roles.py
View File

@@ -34,6 +34,7 @@ def get_default_roles():
'admin-ui',
'admin:users',
'admin:servers',
'admin:services',
'tokens',
'admin:groups',
'list:services',

6
jupyterhub/scopes.py
View File

@@ -123,6 +123,10 @@ scope_definitions = {
'delete:groups': {
'description': "Delete groups.",
},
'admin:services': {
'description': 'Create, read, update, delete services, not including services defined from config files.',
'subscopes': ['list:services', 'read:services', 'read:roles:services'],
},
'list:services': {
'description': 'List services, including at least their names.',
'subscopes': ['read:services:name'],
@@ -435,7 +439,7 @@ def _expand_self_scope(username):
@lru_cache(maxsize=65535)
def _expand_scope(scope):
"""Returns a scope and all all subscopes
"""Returns a scope and all subscopes
Arguments:
scope (str): the scope to expand

18
jupyterhub/services/service.py
View File

@@ -180,9 +180,12 @@ class Service(LoggingConfigurable):
- user: str
The name of a system user to become.
If unspecified, run as the same user as the Hub.
"""
# inputs:
# traits tagged with `input=True` are accepted as input from configuration / API
# input traits are also persisted to the db UNLESS they are also tagged with `in_db=False`
name = Unicode(
help="""The name of the service.
@@ -205,7 +208,7 @@ class Service(LoggingConfigurable):
DEPRECATED in 3.0: use oauth_client_allowed_scopes
"""
).tag(input=True)
).tag(input=True, in_db=False)
oauth_client_allowed_scopes = List(
help="""OAuth allowed scopes.
@@ -225,7 +228,7 @@ class Service(LoggingConfigurable):
If unspecified, an API token will be generated for managed services.
"""
).tag(input=True)
).tag(input=True, in_db=False)
info = Dict(
help="""Provide a place to include miscellaneous information about the service,
@@ -310,7 +313,7 @@ class Service(LoggingConfigurable):
You shouldn't generally need to change this.
Default: `service-<name>`
"""
).tag(input=True)
).tag(input=True, in_db=False)
@default('oauth_client_id')
def _default_client_id(self):
@@ -331,7 +334,7 @@ class Service(LoggingConfigurable):
You shouldn't generally need to change this.
Default: `/services/:name/oauth_callback`
"""
).tag(input=True)
).tag(input=True, in_db=False)
@default('oauth_redirect_uri')
def _default_redirect_uri(self):
@@ -371,6 +374,11 @@ class Service(LoggingConfigurable):
else:
return self.server.base_url
@property
def from_config(self):
"""Is the service defined from config file?"""
return self.orm.from_config
def __repr__(self):
return "<{cls}(name={name}{managed})>".format(
cls=self.__class__.__name__,

57
jupyterhub/tests/conftest.py
View File

@@ -167,6 +167,8 @@ async def cleanup_after(request, io_loop):
app = MockHub.instance()
if app.db_file.closed:
return
# cleanup users
for uid, user in list(app.users.items()):
for name, spawner in list(user.spawners.items()):
if spawner.active:
@@ -181,6 +183,16 @@ async def cleanup_after(request, io_loop):
# delete groups
for group in app.db.query(orm.Group):
app.db.delete(group)
# clear services
for name, service in app._service_map.items():
if service.managed:
service.stop()
for orm_service in app.db.query(orm.Service):
if orm_service.oauth_client:
app.oauth_provider.remove_client(orm_service.oauth_client_id)
app.db.delete(orm_service)
app._service_map.clear()
app.db.commit()
@@ -262,10 +274,7 @@ class MockServiceSpawner(jupyterhub.services.service._ServiceSpawner):
poll_interval = 1
_mock_service_counter = 0
async def _mockservice(request, app, external=False, url=False):
async def _mockservice(request, app, name, external=False, url=False):
"""
Add a service to the application
@@ -281,9 +290,6 @@ async def _mockservice(request, app, external=False, url=False):
If True, register the service at a URL
(as opposed to headless, API-only).
"""
global _mock_service_counter
_mock_service_counter += 1
name = 'mock-service-%i' % _mock_service_counter
spec = {'name': name, 'command': mockservice_cmd, 'admin': True}
if url:
if app.internal_ssl:
@@ -330,22 +336,33 @@ async def _mockservice(request, app, external=False, url=False):
return service
_service_name_counter = 0
@fixture
async def mockservice(request, app):
def service_name():
global _service_name_counter
_service_name_counter += 1
name = f'test-service-{_service_name_counter}'
return name
@fixture
async def mockservice(request, app, service_name):
"""Mock a service with no external service url"""
yield await _mockservice(request, app, url=False)
yield await _mockservice(request, app, name=service_name, url=False)
@fixture
async def mockservice_external(request, app):
async def mockservice_external(request, app, service_name):
"""Mock an externally managed service (don't start anything)"""
yield await _mockservice(request, app, external=True, url=False)
yield await _mockservice(request, app, name=service_name, external=True, url=False)
@fixture
async def mockservice_url(request, app):
async def mockservice_url(request, app, service_name):
"""Mock a service with its own url to test external services"""
yield await _mockservice(request, app, url=True)
yield await _mockservice(request, app, name=service_name, url=True)
@fixture
@@ -534,3 +551,17 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
queries = db_counts[nodeid]
if queries:
terminalreporter.line(f"{queries:<6} {nodeid}")
@fixture
def service_data(service_name):
"""Data used to create service at runtime"""
return {
"name": service_name,
"oauth_client_id": f"service-{service_name}",
"api_token": f"api_token-{service_name}",
"oauth_redirect_uri": "http://127.0.0.1:5555/oauth_callback-from-api",
"oauth_no_confirm": True,
"oauth_client_allowed_scopes": ["inherit"],
"info": {'foo': 'bar'},
}

264
jupyterhub/tests/test_api.py
View File

@@ -4,10 +4,12 @@ import json
import re
import sys
import uuid
from copy import deepcopy
from datetime import datetime, timedelta
from unittest import mock
from urllib.parse import quote, urlparse
import pytest
from pytest import fixture, mark
from tornado.httputil import url_concat
@@ -2062,7 +2064,7 @@ async def test_get_services(app, mockservice_url):
async def test_get_service(app, mockservice_url):
mockservice = mockservice_url
db = app.db
r = await api_request(app, 'services/%s' % mockservice.name)
r = await api_request(app, f"services/{mockservice.name}")
r.raise_for_status()
assert r.status_code == 200
@@ -2081,19 +2083,271 @@ async def test_get_service(app, mockservice_url):
}
r = await api_request(
app,
'services/%s' % mockservice.name,
f"services/{mockservice.name}",
headers={'Authorization': 'token %s' % mockservice.api_token},
)
r.raise_for_status()
r = await api_request(
app, 'services/%s' % mockservice.name, headers=auth_header(db, 'user')
app, f"services/{mockservice.name}", headers=auth_header(db, 'user')
)
assert r.status_code == 403
r = await api_request(app, "services/nosuchservice")
assert r.status_code == 404
@pytest.fixture
def service_admin_user(create_user_with_scopes):
return create_user_with_scopes('admin:services')
@mark.services
async def test_create_service(app, service_admin_user, service_name, service_data):
db = app.db
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, service_admin_user.name),
data=json.dumps(service_data),
method='post',
)
r.raise_for_status()
assert r.status_code == 201
assert r.json()['name'] == service_name
orm_service = orm.Service.find(db, service_name)
assert orm_service is not None
oath_client = (
db.query(orm.OAuthClient)
.filter_by(identifier=service_data['oauth_client_id'])
.first()
)
assert oath_client.redirect_uri == service_data['oauth_redirect_uri']
assert service_name in app._service_map
assert (
app._service_map[service_name].oauth_no_confirm
== service_data['oauth_no_confirm']
)
@mark.services
async def test_create_service_no_role(app, service_name, service_data):
db = app.db
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, 'user'),
data=json.dumps(service_data),
method='post',
)
assert r.status_code == 403
@mark.services
async def test_create_service_conflict(
app, service_admin_user, mockservice, service_data, service_name
):
db = app.db
app.services = [{'name': service_name}]
app.init_services()
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, service_admin_user.name),
data=json.dumps(service_data),
method='post',
)
assert r.status_code == 409
@mark.services
async def test_create_service_duplication(
app, service_admin_user, service_name, service_data
):
db = app.db
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, service_admin_user.name),
data=json.dumps(service_data),
method='post',
)
assert r.status_code == 201
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, service_admin_user.name),
data=json.dumps(service_data),
method='post',
)
assert r.status_code == 409
@mark.services
async def test_create_managed_service(
app, service_admin_user, service_name, service_data
):
db = app.db
managed_service_data = deepcopy(service_data)
managed_service_data['command'] = ['foo']
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, service_admin_user.name),
data=json.dumps(managed_service_data),
method='post',
)
assert r.status_code == 400
assert 'Can not create managed service' in r.json()['message']
orm_service = orm.Service.find(db, service_name)
assert orm_service is None
@mark.services
async def test_create_admin_service(app, admin_user, service_name, service_data):
db = app.db
managed_service_data = deepcopy(service_data)
managed_service_data['admin'] = True
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, admin_user.name),
data=json.dumps(managed_service_data),
method='post',
)
assert r.status_code == 201
orm_service = orm.Service.find(db, service_name)
assert orm_service is not None
@mark.services
async def test_create_admin_service_without_admin_right(
app, service_admin_user, service_data, service_name
):
db = app.db
managed_service_data = deepcopy(service_data)
managed_service_data['admin'] = True
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, service_admin_user.name),
data=json.dumps(managed_service_data),
method='post',
)
assert r.status_code == 400
assert 'Not assigning requested scopes' in r.json()['message']
orm_service = orm.Service.find(db, service_name)
assert orm_service is None
@mark.services
async def test_create_service_with_scope(
app, create_user_with_scopes, service_name, service_data
):
db = app.db
managed_service_data = deepcopy(service_data)
managed_service_data['oauth_client_allowed_scopes'] = ["admin:users"]
managed_service_data['oauth_client_id'] = "service-client-with-scope"
user_with_scope = create_user_with_scopes('admin:services', 'admin:users')
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, user_with_scope.name),
data=json.dumps(managed_service_data),
method='post',
)
assert r.status_code == 201
orm_service = orm.Service.find(db, service_name)
assert orm_service is not None
@mark.services
async def test_create_service_without_requested_scope(
app,
service_admin_user,
service_data,
service_name,
):
db = app.db
managed_service_data = deepcopy(service_data)
managed_service_data['oauth_client_allowed_scopes'] = ["admin:users"]
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, service_admin_user.name),
data=json.dumps(managed_service_data),
method='post',
)
assert r.status_code == 400
assert 'Not assigning requested scopes' in r.json()['message']
orm_service = orm.Service.find(db, service_name)
assert orm_service is None
@mark.services
async def test_delete_service(app, service_admin_user, service_name, service_data):
db = app.db
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, service_admin_user.name),
data=json.dumps(service_data),
method='post',
)
assert r.status_code == 201
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, service_admin_user.name),
method='delete',
)
assert r.status_code == 200
orm_service = orm.Service.find(db, service_name)
assert orm_service is None
oath_client = (
db.query(orm.OAuthClient)
.filter_by(identifier=service_data['oauth_client_id'])
.first()
)
assert oath_client is None
assert service_name not in app._service_map
r = await api_request(app, f"services/{service_name}", method="delete")
assert r.status_code == 404
@mark.services
async def test_delete_service_from_config(app, service_admin_user, mockservice):
db = app.db
service_name = mockservice.name
r = await api_request(
app,
f'services/{service_name}',
headers=auth_header(db, service_admin_user.name),
method='delete',
)
assert r.status_code == 405
assert r.json()['message'] == f'Service {service_name} is not modifiable at runtime'
async def test_root_api(app):
base_url = app.hub.url
url = ujoin(base_url, 'api')
kwargs = {}
if app.internal_ssl:
kwargs['cert'] = (app.internal_ssl_cert, app.internal_ssl_key)

84
jupyterhub/tests/test_app.py
View File

@@ -220,7 +220,7 @@ def test_cookie_secret_env(tmpdir, request):
assert not os.path.exists(hub.cookie_secret_file)
def test_cookie_secret_string_():
def test_cookie_secret_string():
cfg = Config()
cfg.JupyterHub.cookie_secret = "not hex"
@@ -270,18 +270,41 @@ async def test_load_groups(tmpdir, request):
)
async def test_resume_spawners(tmpdir, request):
if not os.getenv('JUPYTERHUB_TEST_DB_URL'):
p = patch.dict(
@pytest.fixture
def persist_db(tmpdir):
"""ensure db will persist (overrides default sqlite://:memory:)"""
if os.getenv('JUPYTERHUB_TEST_DB_URL'):
# already using a db, no need
yield
return
with patch.dict(
os.environ,
{
'JUPYTERHUB_TEST_DB_URL': 'sqlite:///%s'
% tmpdir.join('jupyterhub.sqlite')
},
)
p.start()
request.addfinalizer(p.stop)
{'JUPYTERHUB_TEST_DB_URL': f"sqlite:///{tmpdir.join('jupyterhub.sqlite')}"},
):
yield
@pytest.fixture
def new_hub(request, tmpdir, persist_db):
"""Fixture to launch a new hub for testing"""
async def new_hub():
kwargs = {}
ssl_enabled = getattr(request.module, "ssl_enabled", False)
if ssl_enabled:
kwargs['internal_certs_location'] = str(tmpdir)
app = MockHub(test_clean_db=False, **kwargs)
app.config.ConfigurableHTTPProxy.should_start = False
app.config.ConfigurableHTTPProxy.auth_token = 'unused'
request.addfinalizer(app.stop)
await app.initialize([])
return app
return new_hub
async def test_resume_spawners(tmpdir, request, new_hub):
async def new_hub():
kwargs = {}
ssl_enabled = getattr(request.module, "ssl_enabled", False)
@@ -473,3 +496,42 @@ async def test_user_creation(tmpdir, request):
"in-group",
"in-role",
}
async def test_recreate_service_from_database(
request, new_hub, service_name, service_data
):
# create a hub and add a service (not from config)
app = await new_hub()
app.service_from_spec(service_data, from_config=False)
app.stop()
# new hub, should load service from db
app = await new_hub()
assert service_name in app._service_map
# verify keys
service = app._service_map[service_name]
for key, value in service_data.items():
if key in {'api_token'}:
# skip some keys
continue
assert getattr(service, key) == value
assert (
service_data['oauth_client_id'] in app.tornado_settings['oauth_no_confirm_list']
)
oauth_client = (
app.db.query(orm.OAuthClient)
.filter_by(identifier=service_data['oauth_client_id'])
.first()
)
assert oauth_client.redirect_uri == service_data['oauth_redirect_uri']
# delete service from db, start one more
app.db.delete(service.orm)
app.db.commit()
# start one more, service should be gone
app = await new_hub()
assert service_name not in app._service_map

4
jupyterhub/tests/test_db.py
View File

@@ -44,7 +44,9 @@ def generate_old_db(env_dir, hub_version, db_url):
# changes to this version list must also be reflected
# in ci/init-db.sh
@pytest.mark.parametrize('hub_version', ["1.1.0", "1.2.2", "1.3.0", "1.5.0", "2.1.1"])
@pytest.mark.parametrize(
'hub_version', ['1.1.0', '1.2.2', '1.3.0', '1.5.0', '2.1.1', '3.1.1']
)
async def test_upgrade(tmpdir, hub_version):
db_url = os.getenv('JUPYTERHUB_TEST_DB_URL')
if db_url:

10
jupyterhub/tests/test_roles.py
View File

@@ -236,6 +236,16 @@ def test_orm_roles_delete_cascade(db):
['tokens!group=hobbits'],
{'tokens!group=hobbits', 'read:tokens!group=hobbits'},
),
(
['admin:services'],
{
'read:roles:services',
'read:services:name',
'admin:services',
'list:services',
'read:services',
},
),
],
)
def test_get_expanded_scopes(db, scopes, expected_scopes):