hazza/docker-stacks
1
0
Fork 0
mirror of https://github.com/jupyter/docker-stacks.git synced 2025-10-10 19:42:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into asalikhov/automatic_conda_versioning

Browse Source
This commit is contained in:
Ayaz Salikhov
2021-06-29 02:28:53 +03:00
parent f1010ec6f1 d75d70f3e6
commit 3c5e7c9cb4
39 changed files with 504 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.flake8
View File

@@ -1,5 +1,4 @@
[flake8]
ignore = W605,W503,W504,H306,H238,H301,H202
max-line-length = 120
per-file-ignores =
test/test_packages.py:E501
max-line-length = 88
select = C,E,F,W,B,B950
extend-ignore = E203, E501

11
.github/dependabot.yml vendored Normal file
View File

@@ -0,0 +1,11 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "daily"

11
.github/workflows/docker.yml vendored
View File

@@ -59,14 +59,13 @@ jobs:
run: make -C main hook-all
- name: Push Wiki to GitHub
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main'
# Pass GITHUB_REPOSITORY directly to avoid conflict with GitHub Actions built-in env var
run: make -C main git-commit GITHUB_REPOSITORY='${{github.repository}}.wiki'
env:
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
LOCAL_PATH: ../wiki
uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release
with:
commit_message: "[ci skip] Automated publish for ${{github.sha}}"
repository: wiki/
- name: Login to Docker Hub
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main'
uses: docker/login-action@v1
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # dependabot updates to latest release
with:
username: ${{secrets.DOCKERHUB_USERNAME}}
password: ${{secrets.DOCKERHUB_TOKEN}}

8
.github/workflows/sphinx.yml vendored
View File

@@ -42,8 +42,6 @@ jobs:
sphinx-intl update -p ./_build/gettext -l en
- name: Push Strings to Master
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main'
run: make git-commit
env:
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
GITHUB_REPOSITORY: ${{github.repository}}
LOCAL_PATH: ./docs/locale/en
uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release
with:
commit_message: "[ci skip] Automated publish for ${{github.sha}}"

35
.pre-commit-config.yaml
View File

@@ -3,47 +3,58 @@ ci:
skip: [hadolint-docker]
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.0.1
# Autoformat: Python code
- repo: https://github.com/ambv/black
rev: 21.6b0
hooks:
- id: check-yaml
files: .*\.(yaml|yml)$
- id: black
args: [--target-version=py39]
# Autoformat: YAML, JSON, Markdown, etc.
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v2.3.2
hooks:
- id: prettier
# Lint: Dockerfile
- repo: https://github.com/hadolint/hadolint.git
rev: v2.5.0
hooks:
- id: hadolint-docker
# FIXME: remove after https://github.com/hadolint/hadolint/issues/628 is resolved
entry: hadolint/hadolint:v2.5.0 hadolint
# Lint: YAML
- repo: https://github.com/adrienverge/yamllint.git
rev: v1.26.1
hooks:
- id: yamllint
args: ["-d {extends: relaxed, rules: {line-length: disable}}", "-s"]
files: \.(yaml|yml)$
# Lint: Bash scripts
- repo: https://github.com/openstack-dev/bashate.git
rev: 2.0.0
hooks:
- id: bashate
args: ["--ignore=E006"]
# Lint: Shell scripts
- repo: https://github.com/shellcheck-py/shellcheck-py
rev: v0.7.2.1
hooks:
- id: shellcheck
args: ["-x"]
# Lint: Python
- repo: https://github.com/PyCQA/flake8
rev: 3.9.2
hooks:
- id: flake8
- repo: https://github.com/pre-commit/mirrors-autopep8
rev: v1.5.7
hooks:
- id: autopep8
# Lint: Markdown
- repo: https://github.com/igorshubovych/markdownlint-cli
rev: v0.27.1
hooks:
- id: markdownlint
args: ["--fix"]
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v2.3.1
hooks:
- id: prettier

16
Makefile
View File

@@ -62,22 +62,6 @@ dev-env: ## install libraries required to build docs and run tests
docs: ## build HTML documentation
sphinx-build docs/ docs/_build/
git-commit: LOCAL_PATH?=.
git-commit: GITHUB_SHA?=$(shell git rev-parse HEAD)
git-commit: GITHUB_REPOSITORY?=jupyter/docker-stacks
git-commit: GITHUB_TOKEN?=
git-commit: ## commit outstading git changes and push to remote
@git config --global user.name "GitHub Actions"
@git config --global user.email "actions@users.noreply.github.com"
@echo "Publishing outstanding changes in $(LOCAL_PATH) to $(GITHUB_REPOSITORY)"
@cd $(LOCAL_PATH) && \
git remote add publisher https://$(GITHUB_TOKEN)@github.com/$(GITHUB_REPOSITORY).git && \
git checkout master && \
git add -A -- . && \
git commit -m "[ci skip] Automated publish for $(GITHUB_SHA)" || exit 0
@cd $(LOCAL_PATH) && git push -u publisher master
hook/%: WIKI_PATH?=../wiki
hook/%: ## run post-build hooks for an image
python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \

10
all-spark-notebook/test/test_spark_notebooks.py
View File

@@ -22,10 +22,12 @@ def test_nbconvert(container, test_file):
output_dir = "/tmp"
timeout_ms = 600
LOGGER.info(f"Test that {test_file} notebook can be executed ...")
command = "jupyter nbconvert --to markdown " + \
f"--ExecutePreprocessor.timeout={timeout_ms} " + \
f"--output-dir {output_dir} " + \
f"--execute {cont_data_dir}/{test_file}.ipynb"
command = (
"jupyter nbconvert --to markdown "
+ f"--ExecutePreprocessor.timeout={timeout_ms} "
+ f"--output-dir {output_dir} "
+ f"--execute {cont_data_dir}/{test_file}.ipynb"
)
c = container.run(
volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
tty=True,

41
base-notebook/jupyter_notebook_config.py
View File

@@ -8,7 +8,7 @@ import errno
import stat
c = get_config() # noqa: F821
c.NotebookApp.ip = '0.0.0.0'
c.NotebookApp.ip = "0.0.0.0"
c.NotebookApp.port = 8888
c.NotebookApp.open_browser = False
@@ -16,9 +16,9 @@ c.NotebookApp.open_browser = False
c.FileContentsManager.delete_to_trash = False
# Generate a self-signed certificate
if 'GEN_CERT' in os.environ:
if "GEN_CERT" in os.environ:
dir_name = jupyter_data_dir()
pem_file = os.path.join(dir_name, 'notebook.pem')
pem_file = os.path.join(dir_name, "notebook.pem")
try:
os.makedirs(dir_name)
except OSError as exc: # Python >2.5
@@ -28,28 +28,37 @@ if 'GEN_CERT' in os.environ:
raise
# Generate an openssl.cnf file to set the distinguished name
cnf_file = os.path.join(os.getenv('CONDA_DIR', '/usr/lib'), 'ssl', 'openssl.cnf')
cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf")
if not os.path.isfile(cnf_file):
with open(cnf_file, 'w') as fh:
fh.write('''\
with open(cnf_file, "w") as fh:
fh.write(
"""\
[req]
distinguished_name = req_distinguished_name
[req_distinguished_name]
''')
"""
)
# Generate a certificate if one doesn't exist on disk
subprocess.check_call(['openssl', 'req', '-new',
'-newkey', 'rsa:2048',
'-days', '365',
'-nodes', '-x509',
'-subj', '/C=XX/ST=XX/L=XX/O=generated/CN=generated',
'-keyout', pem_file,
'-out', pem_file])
subprocess.check_call(
[
"openssl",
"req",
"-new",
"-newkey=rsa:2048",
"-days=365",
"-nodes",
"-x509",
"-subj=/C=XX/ST=XX/L=XX/O=generated/CN=generated",
f"-keyout={pem_file}",
f"-out={pem_file}",
]
)
# Restrict access to the file
os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR)
c.NotebookApp.certfile = pem_file
# Change default umask for all subprocesses of the notebook server if set in
# the environment
if 'NB_UMASK' in os.environ:
os.umask(int(os.environ['NB_UMASK'], 8))
if "NB_UMASK" in os.environ:
os.umask(int(os.environ["NB_UMASK"], 8))

114
base-notebook/test/test_container_options.py
View File

@@ -11,59 +11,55 @@ LOGGER = logging.getLogger(__name__)
def test_cli_args(container, http_client):
"""Container should respect notebook server command line args
(e.g., disabling token security)"""
c = container.run(
command=["start-notebook.sh", "--NotebookApp.token=''"]
)
resp = http_client.get('http://localhost:8888')
c = container.run(command=["start-notebook.sh", "--NotebookApp.token=''"])
resp = http_client.get("http://localhost:8888")
resp.raise_for_status()
logs = c.logs(stdout=True).decode('utf-8')
logs = c.logs(stdout=True).decode("utf-8")
LOGGER.debug(logs)
assert 'login_submit' not in resp.text
assert "login_submit" not in resp.text
@pytest.mark.filterwarnings('ignore:Unverified HTTPS request')
@pytest.mark.filterwarnings("ignore:Unverified HTTPS request")
def test_unsigned_ssl(container, http_client):
"""Container should generate a self-signed SSL certificate
and notebook server should use it to enable HTTPS.
"""
container.run(
environment=['GEN_CERT=yes']
)
container.run(environment=["GEN_CERT=yes"])
# NOTE: The requests.Session backing the http_client fixture does not retry
# properly while the server is booting up. An SSL handshake error seems to
# abort the retry logic. Forcing a long sleep for the moment until I have
# time to dig more.
time.sleep(5)
resp = http_client.get('https://localhost:8888', verify=False)
resp = http_client.get("https://localhost:8888", verify=False)
resp.raise_for_status()
assert 'login_submit' in resp.text
assert "login_submit" in resp.text
def test_uid_change(container):
"""Container should change the UID of the default user."""
c = container.run(
tty=True,
user='root',
environment=['NB_UID=1010'],
command=['start.sh', 'bash', '-c', 'id && touch /opt/conda/test-file']
user="root",
environment=["NB_UID=1010"],
command=["start.sh", "bash", "-c", "id && touch /opt/conda/test-file"],
)
# usermod is slow so give it some time
c.wait(timeout=120)
assert 'uid=1010(jovyan)' in c.logs(stdout=True).decode('utf-8')
assert "uid=1010(jovyan)" in c.logs(stdout=True).decode("utf-8")
def test_gid_change(container):
"""Container should change the GID of the default user."""
c = container.run(
tty=True,
user='root',
environment=['NB_GID=110'],
command=['start.sh', 'id']
user="root",
environment=["NB_GID=110"],
command=["start.sh", "id"],
)
c.wait(timeout=10)
logs = c.logs(stdout=True).decode('utf-8')
assert 'gid=110(jovyan)' in logs
assert 'groups=110(jovyan),100(users)' in logs
logs = c.logs(stdout=True).decode("utf-8")
assert "gid=110(jovyan)" in logs
assert "groups=110(jovyan),100(users)" in logs
def test_nb_user_change(container):
@@ -72,11 +68,8 @@ def test_nb_user_change(container):
running_container = container.run(
tty=True,
user="root",
environment=[
f"NB_USER={nb_user}",
"CHOWN_HOME=yes"
],
command=['start.sh', 'bash', '-c', 'sleep infinity']
environment=[f"NB_USER={nb_user}", "CHOWN_HOME=yes"],
command=["start.sh", "bash", "-c", "sleep infinity"],
)
# Give the chown time to complete. Use sleep, not wait, because the
@@ -98,25 +91,27 @@ def test_nb_user_change(container):
expected_output = f"{nb_user} users"
cmd = running_container.exec_run(command, workdir=f"/home/{nb_user}")
output = cmd.output.decode("utf-8").strip("\n")
assert output == expected_output, f"Bad owner for the {nb_user} home folder {output}, expected {expected_output}"
assert (
output == expected_output
), f"Bad owner for the {nb_user} home folder {output}, expected {expected_output}"
def test_chown_extra(container):
"""Container should change the UID/GID of CHOWN_EXTRA."""
c = container.run(
tty=True,
user='root',
user="root",
environment=[
'NB_UID=1010',
'NB_GID=101',
'CHOWN_EXTRA=/opt/conda',
'CHOWN_EXTRA_OPTS=-R'
"NB_UID=1010",
"NB_GID=101",
"CHOWN_EXTRA=/opt/conda",
"CHOWN_EXTRA_OPTS=-R",
],
command=['start.sh', 'bash', '-c', 'stat -c \'%n:%u:%g\' /opt/conda/LICENSE.txt']
command=["start.sh", "bash", "-c", "stat -c '%n:%u:%g' /opt/conda/LICENSE.txt"],
)
# chown is slow so give it some time
c.wait(timeout=120)
assert '/opt/conda/LICENSE.txt:1010:101' in c.logs(stdout=True).decode('utf-8')
assert "/opt/conda/LICENSE.txt:1010:101" in c.logs(stdout=True).decode("utf-8")
def test_chown_home(container):
@@ -124,53 +119,59 @@ def test_chown_home(container):
group to the current value of NB_UID and NB_GID."""
c = container.run(
tty=True,
user='root',
environment=[
'CHOWN_HOME=yes',
'CHOWN_HOME_OPTS=-R'
user="root",
environment=["CHOWN_HOME=yes", "CHOWN_HOME_OPTS=-R"],
command=[
"start.sh",
"bash",
"-c",
"chown root:root /home/jovyan && ls -alsh /home",
],
command=['start.sh', 'bash', '-c', 'chown root:root /home/jovyan && ls -alsh /home']
)
c.wait(timeout=120)
assert "Changing ownership of /home/jovyan to 1000:100 with options '-R'" in c.logs(stdout=True).decode('utf-8')
assert "Changing ownership of /home/jovyan to 1000:100 with options '-R'" in c.logs(
stdout=True
).decode("utf-8")
def test_sudo(container):
"""Container should grant passwordless sudo to the default user."""
c = container.run(
tty=True,
user='root',
environment=['GRANT_SUDO=yes'],
command=['start.sh', 'sudo', 'id']
user="root",
environment=["GRANT_SUDO=yes"],
command=["start.sh", "sudo", "id"],
)
rv = c.wait(timeout=10)
assert rv == 0 or rv["StatusCode"] == 0
assert 'uid=0(root)' in c.logs(stdout=True).decode('utf-8')
assert "uid=0(root)" in c.logs(stdout=True).decode("utf-8")
def test_sudo_path(container):
"""Container should include /opt/conda/bin in the sudo secure_path."""
c = container.run(
tty=True,
user='root',
environment=['GRANT_SUDO=yes'],
command=['start.sh', 'sudo', 'which', 'jupyter']
user="root",
environment=["GRANT_SUDO=yes"],
command=["start.sh", "sudo", "which", "jupyter"],
)
rv = c.wait(timeout=10)
assert rv == 0 or rv["StatusCode"] == 0
assert c.logs(stdout=True).decode('utf-8').rstrip().endswith('/opt/conda/bin/jupyter')
logs = c.logs(stdout=True).decode("utf-8")
assert logs.rstrip().endswith("/opt/conda/bin/jupyter")
def test_sudo_path_without_grant(container):
"""Container should include /opt/conda/bin in the sudo secure_path."""
c = container.run(
tty=True,
user='root',
command=['start.sh', 'which', 'jupyter']
user="root",
command=["start.sh", "which", "jupyter"],
)
rv = c.wait(timeout=10)
assert rv == 0 or rv["StatusCode"] == 0
assert c.logs(stdout=True).decode('utf-8').rstrip().endswith('/opt/conda/bin/jupyter')
logs = c.logs(stdout=True).decode("utf-8")
assert logs.rstrip().endswith("/opt/conda/bin/jupyter")
def test_group_add(container, tmpdir):
@@ -178,10 +179,11 @@ def test_group_add(container, tmpdir):
group.
"""
c = container.run(
user='1010:1010',
group_add=['users'],
command=['start.sh', 'id']
user="1010:1010",
group_add=["users"],
command=["start.sh", "id"],
)
rv = c.wait(timeout=5)
assert rv == 0 or rv["StatusCode"] == 0
assert 'uid=1010 gid=1010 groups=1010,100(users)' in c.logs(stdout=True).decode('utf-8')
logs = c.logs(stdout=True).decode("utf-8")
assert "uid=1010 gid=1010 groups=1010,100(users)" in logs

2
base-notebook/test/test_package_managers.py
View File

@@ -24,7 +24,7 @@ def test_package_manager(container, package_manager, version_arg):
)
c = container.run(
tty=True,
command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"]
command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"],
)
rv = c.wait(timeout=5)
logs = c.logs(stdout=True).decode("utf-8")

2
base-notebook/test/test_pandoc.py
View File

@@ -10,7 +10,7 @@ def test_pandoc(container):
"""Pandoc shall be able to convert MD to HTML."""
c = container.run(
tty=True,
command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc']
command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc'],
)
c.wait(timeout=10)
logs = c.logs(stdout=True).decode("utf-8")

5
base-notebook/test/test_python.py
View File

@@ -10,7 +10,10 @@ LOGGER = logging.getLogger(__name__)
def test_python_version(container, python_next_version="3.10"):
"""Check that python version is lower than the next version"""
LOGGER.info(f"Checking that python version is lower than {python_next_version}")
c = container.run(tty=True, command=["start.sh"])
c = container.run(
tty=True,
command=["start.sh"],
)
cmd = c.exec_run("python --version")
output = cmd.output.decode("utf-8")
actual_python_version = version.parse(output.split()[1])

11
base-notebook/test/test_start_container.py
View File

@@ -19,7 +19,11 @@ def test_start_notebook(container, http_client, env, expected_server):
LOGGER.info(
f"Test that the start-notebook launches the {expected_server} server from the env {env} ..."
)
c = container.run(tty=True, environment=env, command=["start-notebook.sh"])
c = container.run(
tty=True,
environment=env,
command=["start-notebook.sh"],
)
resp = http_client.get("http://localhost:8888")
logs = c.logs(stdout=True).decode("utf-8")
LOGGER.debug(logs)
@@ -40,7 +44,10 @@ def test_tini_entrypoint(container, pid=1, command="tini"):
https://superuser.com/questions/632979/if-i-know-the-pid-number-of-a-process-how-can-i-get-its-name
"""
LOGGER.info(f"Test that {command} is launched as PID {pid} ...")
c = container.run(tty=True, command=["start.sh"])
c = container.run(
tty=True,
command=["start.sh"],
)
# Select the PID 1 and get the corresponding command
cmd = c.exec_run(f"ps -p {pid} -o comm=")
output = cmd.output.decode("utf-8").strip("\n")

23
conftest.py
View File

@@ -14,26 +14,26 @@ from requests.adapters import HTTPAdapter
LOGGER = logging.getLogger(__name__)
@pytest.fixture(scope='session')
@pytest.fixture(scope="session")
def http_client():
"""Requests session with retries and backoff."""
s = requests.Session()
retries = Retry(total=5, backoff_factor=1)
s.mount('http://', HTTPAdapter(max_retries=retries))
s.mount('https://', HTTPAdapter(max_retries=retries))
s.mount("http://", HTTPAdapter(max_retries=retries))
s.mount("https://", HTTPAdapter(max_retries=retries))
return s
@pytest.fixture(scope='session')
@pytest.fixture(scope="session")
def docker_client():
"""Docker client configured based on the host environment"""
return docker.from_env()
@pytest.fixture(scope='session')
@pytest.fixture(scope="session")
def image_name():
"""Image name to test"""
return os.getenv('TEST_IMAGE')
return os.getenv("TEST_IMAGE")
class TrackedContainer:
@@ -78,7 +78,10 @@ class TrackedContainer:
all_kwargs.update(self.kwargs)
all_kwargs.update(kwargs)
LOGGER.info(f"Running {self.image_name} with args {all_kwargs} ...")
self.container = self.docker_client.containers.run(self.image_name, **all_kwargs)
self.container = self.docker_client.containers.run(
self.image_name,
**all_kwargs,
)
return self.container
def remove(self):
@@ -87,7 +90,7 @@ class TrackedContainer:
self.container.remove(force=True)
@pytest.fixture(scope='function')
@pytest.fixture(scope="function")
def container(docker_client, image_name):
"""Notebook container with initial configuration appropriate for testing
(e.g., HTTP port exposed to the host for HTTP calls).
@@ -98,9 +101,7 @@ def container(docker_client, image_name):
docker_client,
image_name,
detach=True,
ports={
'8888/tcp': 8888
}
ports={"8888/tcp": 8888},
)
yield container
container.remove()

2
datascience-notebook/test/test_julia.py
View File

@@ -10,7 +10,7 @@ def test_julia(container):
LOGGER.info("Test that julia is correctly installed ...")
running_container = container.run(
tty=True,
command=["start.sh", "bash", "-c", "sleep infinity"]
command=["start.sh", "bash", "-c", "sleep infinity"],
)
command = "julia --version"
cmd = running_container.exec_run(command)

92
docs/conf.py
View File

@@ -25,40 +25,37 @@
# If your documentation needs a minimal Sphinx version, state it here.
#
needs_sphinx = '2.1'
needs_sphinx = "2.1"
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'myst_parser',
'sphinx_copybutton'
]
extensions = ["myst_parser", "sphinx_copybutton"]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
source_suffix = {
'.rst': 'restructuredtext',
'.md': 'markdown',
".rst": "restructuredtext",
".md": "markdown",
}
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = 'docker-stacks'
copyright = '2018- Project Jupyter'
author = 'Project Jupyter'
project = "docker-stacks"
copyright = "2018- Project Jupyter"
author = "Project Jupyter"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = 'latest'
version = "latest"
# The full version, including alpha/beta/rc tags.
release = 'latest'
release = "latest"
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@@ -70,10 +67,10 @@ language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
@@ -84,27 +81,27 @@ todo_include_todos = False
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
html_theme = "alabaster"
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
html_theme_options = {
'description': "Jupyter Docker Stacks",
'fixed_sidebar': False,
'show_relbars': True,
'github_user': 'jupyter',
'github_repo': 'docker-stacks',
'github_type': 'star',
'logo': 'jupyter-logo.svg',
'logo_text_align': 'left'
"description": "Jupyter Docker Stacks",
"fixed_sidebar": False,
"show_relbars": True,
"github_user": "jupyter",
"github_repo": "docker-stacks",
"github_type": "star",
"logo": "jupyter-logo.svg",
"logo_text_align": "left",
}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = ["_static"]
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
@@ -115,18 +112,18 @@ html_static_path = ['_static']
# 'searchbox.html']``.
#
html_sidebars = {
'**': [
'about.html',
'navigation.html',
'relations.html',
'searchbox.html',
"**": [
"about.html",
"navigation.html",
"relations.html",
"searchbox.html",
]
}
# -- Options for HTMLHelp output ------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'docker-stacksdoc'
htmlhelp_basename = "docker-stacksdoc"
# -- Options for LaTeX output ---------------------------------------------
@@ -135,15 +132,12 @@ latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
@@ -153,8 +147,13 @@ latex_elements = {
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'docker-stacks.tex', 'docker-stacks Documentation',
'Project Jupyter', 'manual'),
(
master_doc,
"docker-stacks.tex",
"docker-stacks Documentation",
"Project Jupyter",
"manual",
),
]
@@ -162,10 +161,7 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'docker-stacks', 'docker-stacks Documentation',
[author], 1)
]
man_pages = [(master_doc, "docker-stacks", "docker-stacks Documentation", [author], 1)]
# -- Options for Texinfo output -------------------------------------------
@@ -174,9 +170,15 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'docker-stacks', 'docker-stacks Documentation',
author, 'docker-stacks', 'One line description of project.',
'Miscellaneous'),
(
master_doc,
"docker-stacks",
"docker-stacks Documentation",
author,
"docker-stacks",
"One line description of project.",
"Miscellaneous",
),
]
# -- Extension configuration ----------------------------------------------
@@ -187,4 +189,4 @@ linkcheck_anchors = False
# -- Translation ----------------------------------------------------------
gettext_uuid = True
locale_dirs = ['locale/']
locale_dirs = ["locale/"]

2
minimal-notebook/test/test_inkscape.py
View File

@@ -11,7 +11,7 @@ def test_inkscape(container):
LOGGER.info("Test that inkscape is working by printing its version ...")
c = container.run(
tty=True,
command=["start.sh", "bash", "-c", "inkscape --version"]
command=["start.sh", "bash", "-c", "inkscape --version"],
)
c.wait(timeout=10)
logs = c.logs(stdout=True).decode("utf-8")

4
minimal-notebook/test/test_nbconvert.py
View File

@@ -24,7 +24,9 @@ def test_nbconvert(container, test_file, output_format):
host_data_dir = os.path.join(THIS_DIR, "data")
cont_data_dir = "/home/jovyan/data"
output_dir = "/tmp"
LOGGER.info(f"Test that the example notebook {test_file} can be converted to {output_format} ...")
LOGGER.info(
f"Test that the example notebook {test_file} can be converted to {output_format} ..."
)
command = f"jupyter nbconvert {cont_data_dir}/{test_file}.ipynb --output-dir {output_dir} --to {output_format}"
c = container.run(
volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},

18
pyspark-notebook/test/test_spark.py
View File

@@ -10,21 +10,9 @@ def test_spark_shell(container):
"""Checking if Spark (spark-shell) is running properly"""
c = container.run(
tty=True,
command=['start.sh', 'bash', '-c', 'spark-shell <<< "1+1"']
command=["start.sh", "bash", "-c", 'spark-shell <<< "1+1"'],
)
c.wait(timeout=60)
logs = c.logs(stdout=True).decode('utf-8')
logs = c.logs(stdout=True).decode("utf-8")
LOGGER.debug(logs)
assert 'res0: Int = 2' in logs, "spark-shell does not work"
def test_pyspark(container):
"""PySpark should be in the Python path"""
c = container.run(
tty=True,
command=['start.sh', 'python', '-c', 'import pyspark']
)
rv = c.wait(timeout=30)
logs = c.logs(stdout=True).decode('utf-8')
LOGGER.debug(logs)
assert rv == 0 or rv["StatusCode"] == 0, "pyspark not in PYTHONPATH"
assert "res0: Int = 2" in logs, "spark-shell does not work"

4
pyspark-notebook/test/units/unit_spark.py Normal file
View File

@@ -0,0 +1,4 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import pyspark # noqa: F401

7
scipy-notebook/test/data/matplotlib_1.py
View File

@@ -14,8 +14,11 @@ s = 1 + np.sin(2 * np.pi * t)
fig, ax = plt.subplots()
ax.plot(t, s)
ax.set(xlabel='time (s)', ylabel='voltage (mV)',
title='About as simple as it gets, folks')
ax.set(
xlabel="time (s)",
ylabel="voltage (mV)",
title="About as simple as it gets, folks",
)
ax.grid()
# Note that the test can be run headless by checking if an image is produced
file_path = os.path.join("/tmp", "test.png")

18
scipy-notebook/test/data/matplotlib_fonts_1.py
View File

@@ -3,16 +3,22 @@ import matplotlib
import matplotlib.pyplot as plt
import os
matplotlib.rcParams['pgf.texsystem'] = 'pdflatex'
matplotlib.rcParams.update({'font.family': 'serif', 'font.size': 18,
'axes.labelsize': 20, 'axes.titlesize': 24,
'figure.titlesize': 28})
matplotlib.rcParams['text.usetex'] = True
matplotlib.rcParams["pgf.texsystem"] = "pdflatex"
matplotlib.rcParams.update(
{
"font.family": "serif",
"font.size": 18,
"axes.labelsize": 20,
"axes.titlesize": 24,
"figure.titlesize": 28,
}
)
matplotlib.rcParams["text.usetex"] = True
fig, ax = plt.subplots(1, 1)
x = [1, 2]
y = [1, 2]
ax.plot(x, y, label='a label')
ax.plot(x, y, label="a label")
ax.legend(fontsize=15)
file_path = os.path.join("/tmp", "test_fonts.png")

2
scipy-notebook/test/test_extensions.py
View File

@@ -27,7 +27,7 @@ def test_check_extension(container, extension):
LOGGER.info(f"Checking the extension: {extension} ...")
c = container.run(
tty=True,
command=["start.sh", "jupyter", "labextension", "check", extension]
command=["start.sh", "jupyter", "labextension", "check", extension],
)
rv = c.wait(timeout=10)
logs = c.logs(stdout=True).decode("utf-8")

14
scipy-notebook/test/test_matplotlib.py
View File

@@ -13,9 +13,17 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__))
@pytest.mark.parametrize(
"test_file,expected_file,description",
[
("matplotlib_1.py", "test.png", "Test that matplotlib is able to plot a graph and write it as an image ..."),
("matplotlib_fonts_1.py", "test_fonts.png", "Test cm-super latex labels in matplotlib ...")
]
(
"matplotlib_1.py",
"test.png",
"Test that matplotlib is able to plot a graph and write it as an image ...",
),
(
"matplotlib_fonts_1.py",
"test_fonts.png",
"Test cm-super latex labels in matplotlib ...",
),
],
)
def test_matplotlib(container, test_file, expected_file, description):
"""Various tests performed on matplotlib

32
scipy-notebook/test/test_pandas.py
View File

@@ -1,32 +0,0 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import logging
import pytest
LOGGER = logging.getLogger(__name__)
@pytest.mark.parametrize(
"name,command_list",
[
(
"Sum series",
[
"import pandas as pd",
"import numpy as np",
"np.random.seed(0)",
"print(pd.Series(np.random.randint(0, 7, size=10)).sum())"
]
),
],
)
def test_pandas(container, name, command_list):
"""Basic pandas tests"""
LOGGER.info(f"Testing pandas: {name} ...")
command = ';'.join(command_list)
c = container.run(tty=True, command=["start.sh", "python", "-c", command])
rv = c.wait(timeout=30)
logs = c.logs(stdout=True).decode("utf-8")
LOGGER.debug(logs)
assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"

9
scipy-notebook/test/units/unit_pandas.py Normal file
View File

@@ -0,0 +1,9 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import numpy as np
import pandas as pd
np.random.seed(0)
print(pd.Series(np.random.randint(0, 7, size=10)).sum())

33
tagging/create_manifests.py
View File

@@ -19,7 +19,12 @@ BUILD_TIMESTAMP = datetime.datetime.utcnow().isoformat()[:-7] + "Z"
MARKDOWN_LINE_BREAK = "<br />"
def append_build_history_line(short_image_name: str, owner: str, wiki_path: str, all_tags: List[str]) -> None:
def append_build_history_line(
short_image_name: str,
owner: str,
wiki_path: str,
all_tags: List[str],
) -> None:
logger.info("Appending build history line")
date_column = f"`{BUILD_TIMESTAMP}`"
@@ -28,11 +33,13 @@ def append_build_history_line(short_image_name: str, owner: str, wiki_path: str,
)
commit_hash = GitHelper.commit_hash()
commit_hash_tag = GitHelper.commit_hash_tag()
links_column = MARKDOWN_LINE_BREAK.join([
links_column = MARKDOWN_LINE_BREAK.join(
[
f"[Git diff](https://github.com/jupyter/docker-stacks/commit/{commit_hash})",
f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/{short_image_name}/Dockerfile)",
f"[Build manifest](./{short_image_name}-{commit_hash_tag})"
])
f"[Build manifest](./{short_image_name}-{commit_hash_tag})",
]
)
build_history_line = "|".join([date_column, image_column, links_column]) + "|"
home_wiki_file = os.path.join(wiki_path, "Home.md")
@@ -49,16 +56,20 @@ def create_manifest_file(
owner: str,
wiki_path: str,
manifests: List[ManifestInterface],
container
container,
) -> None:
manifest_names = [manifest.__name__ for manifest in manifests]
logger.info(f"Using manifests: {manifest_names}")
commit_hash_tag = GitHelper.commit_hash_tag()
manifest_file = os.path.join(wiki_path, f"manifests/{short_image_name}-{commit_hash_tag}.md")
manifest_file = os.path.join(
wiki_path,
f"manifests/{short_image_name}-{commit_hash_tag}.md",
)
markdown_pieces = [ManifestHeader.create_header(short_image_name, owner, BUILD_TIMESTAMP)] + \
[manifest.markdown_piece(container) for manifest in manifests]
markdown_pieces = [
ManifestHeader.create_header(short_image_name, owner, BUILD_TIMESTAMP)
] + [manifest.markdown_piece(container) for manifest in manifests]
markdown_content = "\n\n".join(markdown_pieces) + "\n"
with open(manifest_file, "w") as f:
@@ -81,7 +92,11 @@ if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("--short-image-name", required=True, help="Short image name to apply tags for")
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name to apply tags for",
)
arg_parser.add_argument("--owner", required=True, help="Owner of the image")
arg_parser.add_argument("--wiki-path", required=True, help="Path to the wiki pages")
args = arg_parser.parse_args()

11
tagging/docker_runner.py
View File

@@ -8,7 +8,12 @@ logger = logging.getLogger(__name__)
class DockerRunner:
def __init__(self, image_name: str, docker_client=docker.from_env(), command: str = "sleep infinity"):
def __init__(
self,
image_name: str,
docker_client=docker.from_env(),
command: str = "sleep infinity",
):
self.container = None
self.image_name = image_name
self.command = command
@@ -17,7 +22,9 @@ class DockerRunner:
def __enter__(self):
logger.info(f"Creating container for image {self.image_name} ...")
self.container = self.docker_client.containers.run(
image=self.image_name, command=self.command, detach=True,
image=self.image_name,
command=self.command,
detach=True,
)
logger.info(f"Container {self.container.name} created")
return self.container

4
tagging/get_taggers_and_manifests.py
View File

@@ -6,7 +6,9 @@ from .manifests import ManifestInterface
from .taggers import TaggerInterface
def get_taggers_and_manifests(short_image_name: str) -> Tuple[List[TaggerInterface], List[ManifestInterface]]:
def get_taggers_and_manifests(
short_image_name: str,
) -> Tuple[List[TaggerInterface], List[ManifestInterface]]:
taggers: List[TaggerInterface] = []
manifests: List[ManifestInterface] = []
while short_image_name is not None:

64
tagging/images_hierarchy.py
View File

@@ -2,15 +2,29 @@
# Distributed under the terms of the Modified BSD License.
from dataclasses import dataclass, field
from typing import Optional, List
from .taggers import TaggerInterface, \
SHATagger, \
UbuntuVersionTagger, PythonVersionTagger, \
JupyterNotebookVersionTagger, JupyterLabVersionTagger, JupyterHubVersionTagger, \
RVersionTagger, TensorflowVersionTagger, JuliaVersionTagger, \
SparkVersionTagger, HadoopVersionTagger, JavaVersionTagger
from .manifests import ManifestInterface, \
CondaEnvironmentManifest, AptPackagesManifest, \
RPackagesManifest, JuliaPackagesManifest, SparkInfoManifest
from .taggers import (
TaggerInterface,
SHATagger,
UbuntuVersionTagger,
PythonVersionTagger,
JupyterNotebookVersionTagger,
JupyterLabVersionTagger,
JupyterHubVersionTagger,
RVersionTagger,
TensorflowVersionTagger,
JuliaVersionTagger,
SparkVersionTagger,
HadoopVersionTagger,
JavaVersionTagger,
)
from .manifests import (
ManifestInterface,
CondaEnvironmentManifest,
AptPackagesManifest,
RPackagesManifest,
JuliaPackagesManifest,
SparkInfoManifest,
)
@dataclass
@@ -25,41 +39,37 @@ ALL_IMAGES = {
parent_image=None,
taggers=[
SHATagger,
UbuntuVersionTagger, PythonVersionTagger,
JupyterNotebookVersionTagger, JupyterLabVersionTagger, JupyterHubVersionTagger
UbuntuVersionTagger,
PythonVersionTagger,
JupyterNotebookVersionTagger,
JupyterLabVersionTagger,
JupyterHubVersionTagger,
],
manifests=[
CondaEnvironmentManifest, AptPackagesManifest
]
),
"minimal-notebook": ImageDescription(
parent_image="base-notebook"
),
"scipy-notebook": ImageDescription(
parent_image="minimal-notebook"
manifests=[CondaEnvironmentManifest, AptPackagesManifest],
),
"minimal-notebook": ImageDescription(parent_image="base-notebook"),
"scipy-notebook": ImageDescription(parent_image="minimal-notebook"),
"r-notebook": ImageDescription(
parent_image="minimal-notebook",
taggers=[RVersionTagger],
manifests=[RPackagesManifest]
manifests=[RPackagesManifest],
),
"tensorflow-notebook": ImageDescription(
parent_image="scipy-notebook",
taggers=[TensorflowVersionTagger]
parent_image="scipy-notebook", taggers=[TensorflowVersionTagger]
),
"datascience-notebook": ImageDescription(
parent_image="scipy-notebook",
taggers=[RVersionTagger, JuliaVersionTagger],
manifests=[RPackagesManifest, JuliaPackagesManifest]
manifests=[RPackagesManifest, JuliaPackagesManifest],
),
"pyspark-notebook": ImageDescription(
parent_image="scipy-notebook",
taggers=[SparkVersionTagger, HadoopVersionTagger, JavaVersionTagger],
manifests=[SparkInfoManifest]
manifests=[SparkInfoManifest],
),
"all-spark-notebook": ImageDescription(
parent_image="pyspark-notebook",
taggers=[RVersionTagger],
manifests=[RPackagesManifest]
)
manifests=[RPackagesManifest],
),
}

68
tagging/manifests.py
View File

@@ -10,24 +10,30 @@ logger = logging.getLogger(__name__)
def quoted_output(container, cmd: str) -> str:
return "\n".join([
return "\n".join(
[
"```",
DockerRunner.run_simple_command(container, cmd, print_result=False),
"```"
])
"```",
]
)
class ManifestHeader:
"""ManifestHeader doesn't fall under common interface and we run it separately"""
@staticmethod
def create_header(short_image_name: str, owner: str, build_timestamp: str) -> str:
commit_hash = GitHelper.commit_hash()
commit_hash_tag = GitHelper.commit_hash_tag()
commit_message = GitHelper.commit_message()
image_size = docker["images", f"{owner}/{short_image_name}:latest", "--format", "{{.Size}}"]().rstrip()
image_size = docker[
"images", f"{owner}/{short_image_name}:latest", "--format", "{{.Size}}"
]().rstrip()
return "\n".join([
return "\n".join(
[
f"# Build manifest for image: {short_image_name}:{commit_hash_tag}",
"",
"## Build Info",
@@ -39,12 +45,14 @@ class ManifestHeader:
"* Git commit message:",
"```",
f"{commit_message}",
"```"
])
"```",
]
)
class ManifestInterface:
"""Common interface for all manifests"""
@staticmethod
def markdown_piece(container) -> str:
raise NotImplementedError
@@ -53,56 +61,72 @@ class ManifestInterface:
class CondaEnvironmentManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
return "\n".join(
[
"## Python Packages",
"",
quoted_output(container, "python --version"),
"",
quoted_output(container, "conda info"),
"",
quoted_output(container, "conda list")
])
quoted_output(container, "conda list"),
]
)
class AptPackagesManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
return "\n".join(
[
"## Apt Packages",
"",
quoted_output(container, "apt list --installed")
])
quoted_output(container, "apt list --installed"),
]
)
class RPackagesManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
return "\n".join(
[
"## R Packages",
"",
quoted_output(container, "R --version"),
"",
quoted_output(container, "R --silent -e 'installed.packages(.Library)[, c(1,3)]'")
])
quoted_output(
container,
"R --silent -e 'installed.packages(.Library)[, c(1,3)]'",
),
]
)
class JuliaPackagesManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
return "\n".join(
[
"## Julia Packages",
"",
quoted_output(container, "julia -E 'using InteractiveUtils; versioninfo()'"),
quoted_output(
container,
"julia -E 'using InteractiveUtils; versioninfo()'",
),
"",
quoted_output(container, "julia -E 'import Pkg; Pkg.status()'")
])
quoted_output(container, "julia -E 'import Pkg; Pkg.status()'"),
]
)
class SparkInfoManifest(ManifestInterface):
@staticmethod
def markdown_piece(container) -> str:
return "\n".join([
return "\n".join(
[
"## Apache Spark",
"",
quoted_output(container, "/usr/local/spark/bin/spark-submit --version"),
])
]
)

10
tagging/tag_image.py
View File

@@ -21,7 +21,9 @@ def tag_image(short_image_name: str, owner: str) -> None:
for tagger in taggers:
tagger_name = tagger.__name__
tag_value = tagger.tag_value(container)
logger.info(f"Applying tag tagger_name: {tagger_name} tag_value: {tag_value}")
logger.info(
f"Applying tag tagger_name: {tagger_name} tag_value: {tag_value}"
)
docker["tag", image, f"{owner}/{short_image_name}:{tag_value}"]()
@@ -29,7 +31,11 @@ if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("--short-image-name", required=True, help="Short image name to apply tags for")
arg_parser.add_argument(
"--short-image-name",
required=True,
help="Short image name to apply tags for",
)
arg_parser.add_argument("--owner", required=True, help="Owner of the image")
args = arg_parser.parse_args()

14
tagging/taggers.py
View File

@@ -16,11 +16,11 @@ def _get_env_variable(container, variable: str) -> str:
env = DockerRunner.run_simple_command(
container,
cmd="env",
print_result=False
print_result=False,
).split()
for env_entry in env:
if env_entry.startswith(variable):
return env_entry[len(variable) + 1:]
return env_entry[len(variable) + 1 :]
raise KeyError(variable)
@@ -29,15 +29,16 @@ def _get_pip_package_version(container, package: str) -> str:
package_info = DockerRunner.run_simple_command(
container,
cmd=f"pip show {package}",
print_result=False
print_result=False,
)
version_line = package_info.split("\n")[1]
assert version_line.startswith(VERSION_PREFIX)
return version_line[len(VERSION_PREFIX):]
return version_line[len(VERSION_PREFIX) :]
class TaggerInterface:
"""Common interface for all taggers"""
@staticmethod
def tag_value(container) -> str:
raise NotImplementedError
@@ -52,7 +53,10 @@ class SHATagger(TaggerInterface):
class UbuntuVersionTagger(TaggerInterface):
@staticmethod
def tag_value(container) -> str:
os_release = DockerRunner.run_simple_command(container, "cat /etc/os-release").split("\n")
os_release = DockerRunner.run_simple_command(
container,
"cat /etc/os-release",
).split("\n")
for line in os_release:
if line.startswith("VERSION_ID"):
return "ubuntu-" + line.split("=")[1].strip('"')

30
tensorflow-notebook/test/test_tensorflow.py
View File

@@ -1,30 +0,0 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import logging
import pytest
LOGGER = logging.getLogger(__name__)
@pytest.mark.parametrize(
"name,command",
[
(
"Hello world",
"import tensorflow as tf;print(tf.constant('Hello, TensorFlow'))",
),
(
"Sum",
"import tensorflow as tf;print(tf.reduce_sum(tf.random.normal([1000, 1000])))",
),
],
)
def test_tensorflow(container, name, command):
"""Basic tensorflow tests"""
LOGGER.info(f"Testing tensorflow: {name} ...")
c = container.run(tty=True, command=["start.sh", "python", "-c", command])
rv = c.wait(timeout=30)
logs = c.logs(stdout=True).decode("utf-8")
LOGGER.debug(logs)
assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"

7
tensorflow-notebook/test/units/unit_tensorflow.py Normal file
View File

@@ -0,0 +1,7 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import tensorflow as tf
print(tf.constant("Hello, TensorFlow"))
print(tf.reduce_sum(tf.random.normal([1000, 1000])))

15
test/README.md Normal file
View File

@@ -0,0 +1,15 @@
# Docker stacks testing
We test our images using `pytest` module.
`conftest.py` and `pytest.ini` in the root of our repository define the environment in which tests are run.
More info on pytest can be found [here](https://docs.pytest.org/en/latest/contents.html).
There are two kinds of tests we use:
- General tests - these are located in [this](https://github.com/jupyter/docker-stacks/blob/master/test) folder
- Image specific tests - for example, [base-notebook/test](https://github.com/jupyter/docker-stacks/blob/master/base-notebook/test) folder
We also have a way to easily run arbitrary python files in a container.
This is useful for running unit tests of packages we use, so we put these files in `{image}/test/units` folder.
An example of such a test is [unit_pandas.py](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/test/units/unit_pandas.py).

21
test/helpers.py
View File

@@ -34,8 +34,7 @@ LOGGER = logging.getLogger(__name__)
class CondaPackageHelper:
"""Conda package helper permitting to get information about packages
"""
"""Conda package helper permitting to get information about packages"""
def __init__(self, container):
# if isinstance(container, TrackedContainer):
@@ -51,7 +50,7 @@ class CondaPackageHelper:
LOGGER.info(f"Starting container {container.image_name} ...")
return container.run(
tty=True,
command=["start.sh", "bash", "-c", "sleep infinity"]
command=["start.sh", "bash", "-c", "sleep infinity"],
)
@staticmethod
@@ -76,7 +75,9 @@ class CondaPackageHelper:
if self.specs is None:
LOGGER.info("Grabing the list of specifications ...")
self.specs = CondaPackageHelper._packages_from_json(
self._execute_command(CondaPackageHelper._conda_export_command(from_history=True))
self._execute_command(
CondaPackageHelper._conda_export_command(from_history=True)
)
)
return self.specs
@@ -112,9 +113,7 @@ class CondaPackageHelper:
def available_packages(self):
"""Return the available packages"""
if self.available is None:
LOGGER.info(
"Grabing the list of available packages (can take a while) ..."
)
LOGGER.info("Grabing the list of available packages (can take a while) ...")
# Keeping command line output since `conda search --outdated --json` is way too long ...
self.available = CondaPackageHelper._extract_available(
self._execute_command(["conda", "search", "--outdated"])
@@ -145,10 +144,11 @@ class CondaPackageHelper:
continue
current = min(inst_vs, key=CondaPackageHelper.semantic_cmp)
newest = avail_vs[-1]
if avail_vs and current != newest:
if (
CondaPackageHelper.semantic_cmp(current) <
CondaPackageHelper.semantic_cmp(newest)
avail_vs
and current != newest
and CondaPackageHelper.semantic_cmp(current)
< CondaPackageHelper.semantic_cmp(newest)
):
self.comparison.append(
{"Package": pkg, "Current": current, "Newest": newest}
@@ -162,6 +162,7 @@ class CondaPackageHelper:
def mysplit(string):
def version_substrs(x):
return re.findall(r"([A-z]+|\d+)", x)
return list(chain(map(version_substrs, string.split("."))))
def str_ord(string):

7
test/test_packages.py
View File

@@ -68,7 +68,7 @@ EXCLUDED_PACKAGES = [
"protobuf",
"r-irkernel",
"unixodbc",
"bzip2"
"bzip2",
]
@@ -133,8 +133,9 @@ def _import_packages(package_helper, filtered_packages, check_function, max_fail
for package in filtered_packages:
LOGGER.info(f"Trying to import {package}")
try:
assert check_function(package_helper, package) == 0, \
f"Package [{package}] import failed"
assert (
check_function(package_helper, package) == 0
), f"Package [{package}] import failed"
except AssertionError as err:
failures[package] = err
if len(failures) > max_failures:

35
test/test_units.py Normal file
View File

@@ -0,0 +1,35 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import logging
import os
LOGGER = logging.getLogger(__name__)
THIS_DIR = os.path.dirname(os.path.realpath(__file__))
def test_units(container):
"""Various units tests
Add a py file in the {image}/test/units dir and it will be automatically tested
"""
short_image_name = container.image_name[container.image_name.rfind("/") + 1 :]
host_data_dir = os.path.join(THIS_DIR, f"../{short_image_name}/test/units")
LOGGER.info(f"Searching for units tests in {host_data_dir}")
cont_data_dir = "/home/jovyan/data"
if not os.path.exists(host_data_dir):
LOGGER.info(f"Not found unit tests for image: {container.image_name}")
return
for test_file in os.listdir(host_data_dir):
LOGGER.info(f"Running unit test: {test_file}")
c = container.run(
volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
tty=True,
command=["start.sh", "python", f"{cont_data_dir}/{test_file}"],
)
rv = c.wait(timeout=30)
logs = c.logs(stdout=True).decode("utf-8")
LOGGER.debug(logs)
assert rv == 0 or rv["StatusCode"] == 0