diff --git a/.github/actions/download-manifests/action.yml b/.github/actions/download-manifests/action.yml index 79ac0682..5692f01e 100644 --- a/.github/actions/download-manifests/action.yml +++ b/.github/actions/download-manifests/action.yml @@ -18,6 +18,16 @@ inputs: runs: using: composite steps: + - name: Download artifact 📥 + uses: actions/download-artifact@v3 + with: + name: docker-stacks-foundation-aarch64-history_line + path: ${{ inputs.histLineDir }} + - name: Download artifact 📥 + uses: actions/download-artifact@v3 + with: + name: docker-stacks-foundation-x86_64-history_line + path: ${{ inputs.histLineDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 with: @@ -94,6 +104,16 @@ runs: name: all-spark-notebook-x86_64-history_line path: ${{ inputs.histLineDir }} + - name: Download artifact 📥 + uses: actions/download-artifact@v3 + with: + name: docker-stacks-foundation-aarch64-manifest + path: ${{ inputs.manifestDir }} + - name: Download artifact 📥 + uses: actions/download-artifact@v3 + with: + name: docker-stacks-foundation-x86_64-manifest + path: ${{ inputs.manifestDir }} - name: Download artifact 📥 uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 498d6273..d5736e0d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -23,6 +23,7 @@ on: - "all-spark-notebook/**" - "base-notebook/**" - "datascience-notebook/**" + - "docker-stacks-foundation/**" - "minimal-notebook/**" - "pyspark-notebook/**" - "r-notebook/**" @@ -49,6 +50,7 @@ on: - "all-spark-notebook/**" - "base-notebook/**" - "datascience-notebook/**" + - "docker-stacks-foundation/**" - "minimal-notebook/**" - "pyspark-notebook/**" - "r-notebook/**" @@ -67,18 +69,36 @@ concurrency: cancel-in-progress: true jobs: - aarch64-base: + aarch64-foundation: uses: ./.github/workflows/docker-build-test-upload.yml with: parentImage: "" + image: docker-stacks-foundation + platform: aarch64 + runsOn: ARM64 + + x86_64-foundation: + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: "" + image: docker-stacks-foundation + platform: x86_64 + runsOn: ubuntu-latest + + aarch64-base: + needs: [aarch64-foundation] + uses: ./.github/workflows/docker-build-test-upload.yml + with: + parentImage: docker-stacks-foundation image: base-notebook platform: aarch64 runsOn: ARM64 x86_64-base: + needs: [x86_64-foundation] uses: ./.github/workflows/docker-build-test-upload.yml with: - parentImage: "" + parentImage: docker-stacks-foundation image: base-notebook platform: x86_64 runsOn: ubuntu-latest @@ -206,6 +226,7 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} needs: [ + aarch64-foundation, aarch64-base, aarch64-minimal, aarch64-scipy, @@ -218,6 +239,7 @@ jobs: matrix: image: [ + docker-stacks-foundation, base-notebook, minimal-notebook, scipy-notebook, @@ -237,6 +259,7 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} needs: [ + x86_64-foundation, x86_64-base, x86_64-minimal, x86_64-scipy, @@ -250,6 +273,7 @@ jobs: matrix: image: [ + docker-stacks-foundation, base-notebook, minimal-notebook, scipy-notebook, @@ -273,6 +297,7 @@ jobs: matrix: image: [ + docker-stacks-foundation, base-notebook, minimal-notebook, scipy-notebook, diff --git a/.github/workflows/hub-overview.yml b/.github/workflows/hub-overview.yml index ec0fd6c9..171c3857 100644 --- a/.github/workflows/hub-overview.yml +++ b/.github/workflows/hub-overview.yml @@ -13,6 +13,7 @@ on: - "all-spark-notebook/README.md" - "base-notebook/README.md" - "datascience-notebook/README.md" + - "docker-stacks-foundation/README.md" - "minimal-notebook/README.md" - "pyspark-notebook/README.md" - "r-notebook/README.md" @@ -27,8 +28,10 @@ jobs: strategy: matrix: include: + - image: docker-stacks-foundation + description: "Small base image on which Jupyter applications can be built from https://github.com/jupyter/docker-stacks" - image: base-notebook - description: "Small base image for Jupyter Notebook stacks from https://github.com/jupyter/docker-stacks" + description: "Base image for Jupyter Notebook stacks from https://github.com/jupyter/docker-stacks" - image: minimal-notebook description: "Minimal Jupyter Notebook Python Stack from https://github.com/jupyter/docker-stacks" - image: scipy-notebook diff --git a/Makefile b/Makefile index 5669d9d1..2c4e593b 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ OWNER?=jupyter # Need to list the images in build dependency order # All of the images ALL_IMAGES:= \ + docker-stacks-foundation \ base-notebook \ minimal-notebook \ r-notebook \ @@ -19,6 +20,7 @@ ALL_IMAGES:= \ all-spark-notebook AARCH64_IMAGES:= \ + docker-stacks-foundation \ base-notebook \ minimal-notebook \ r-notebook \ diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 9a3a836b..f5390d39 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -1,16 +1,10 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. - -# Ubuntu 22.04 (jammy) -# https://hub.docker.com/_/ubuntu/tags?page=1&name=jammy -ARG ROOT_CONTAINER=ubuntu:22.04 - -FROM $ROOT_CONTAINER +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/docker-stacks-foundation +FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " -ARG NB_USER="jovyan" -ARG NB_UID="1000" -ARG NB_GID="100" # Fix: https://github.com/hadolint/hadolint/wiki/DL4006 # Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 @@ -20,115 +14,31 @@ USER root # Install all OS dependencies for notebook server that starts but lacks all # features (e.g., download as all possible file formats) -ENV DEBIAN_FRONTEND noninteractive RUN apt-get update --yes && \ - # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as - # the ubuntu base image is rebuilt too seldom sometimes (less than once a month) - apt-get upgrade --yes && \ apt-get install --yes --no-install-recommends \ - # - bzip2 is necessary to extract the micromamba executable. - bzip2 \ - ca-certificates \ fonts-liberation \ - locales \ # - pandoc is used to convert notebooks to html files # it's not present in aarch64 ubuntu image, so we install it here pandoc \ # - run-one - a wrapper script that runs no more # than one unique instance of some command with a unique set of arguments, # we use `run-one-constantly` to support `RESTARTABLE` option - run-one \ - sudo \ - # - tini is installed as a helpful container entrypoint that reaps zombie - # processes and such of the actual executable we want to start, see - # https://github.com/krallin/tini#why-tini for details. - tini \ - wget && \ - apt-get clean && rm -rf /var/lib/apt/lists/* && \ - echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ - locale-gen - -# Configure environment -ENV CONDA_DIR=/opt/conda \ - SHELL=/bin/bash \ - NB_USER="${NB_USER}" \ - NB_UID=${NB_UID} \ - NB_GID=${NB_GID} \ - LC_ALL=en_US.UTF-8 \ - LANG=en_US.UTF-8 \ - LANGUAGE=en_US.UTF-8 -ENV PATH="${CONDA_DIR}/bin:${PATH}" \ - HOME="/home/${NB_USER}" - -# Copy a script that we will use to correct permissions after running certain commands -COPY fix-permissions /usr/local/bin/fix-permissions -RUN chmod a+rx /usr/local/bin/fix-permissions - -# Enable prompt color in the skeleton .bashrc before creating the default NB_USER -# hadolint ignore=SC2016 -RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ - # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 - echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc - -# Create NB_USER with name jovyan user with UID=1000 and in the 'users' group -# and make sure these dirs are writable by the `users` group. -RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ - sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ - sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ - useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \ - mkdir -p "${CONDA_DIR}" && \ - chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ - chmod g+w /etc/passwd && \ - fix-permissions "${HOME}" && \ - fix-permissions "${CONDA_DIR}" + run-one && \ + apt-get clean && rm -rf /var/lib/apt/lists/* USER ${NB_UID} -# Pin python version here, or set it to "default" -ARG PYTHON_VERSION=3.10 - -# Setup work directory for backward-compatibility -RUN mkdir "/home/${NB_USER}/work" && \ - fix-permissions "/home/${NB_USER}" - -# Download and install Micromamba, and initialize Conda prefix. -# -# Similar projects using Micromamba: -# - Micromamba-Docker: -# - repo2docker: -# Install Python, Mamba, Jupyter Notebook, Lab, and Hub +# Install Jupyter Notebook, Lab, and Hub # Generate a notebook server config -# Cleanup temporary files and remove Micromamba +# Cleanup temporary files # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change -COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" WORKDIR /tmp -RUN set -x && \ - arch=$(uname -m) && \ - if [ "${arch}" = "x86_64" ]; then \ - # Should be simpler, see - arch="64"; \ - fi && \ - wget -qO /tmp/micromamba.tar.bz2 \ - "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ - tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ - rm /tmp/micromamba.tar.bz2 && \ - PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ - if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ - # Install the packages - ./micromamba install \ - --root-prefix="${CONDA_DIR}" \ - --prefix="${CONDA_DIR}" \ - --yes \ - "${PYTHON_SPECIFIER}" \ - 'mamba' \ - 'notebook' \ - 'jupyterhub' \ - 'jupyterlab' && \ - rm micromamba && \ - # Pin major.minor version of python - mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ +RUN mamba install --quiet --yes \ + 'notebook' \ + 'jupyterhub' \ + 'jupyterlab' && \ jupyter notebook --generate-config && \ mamba clean --all -f -y && \ npm cache clean --force && \ @@ -140,11 +50,10 @@ RUN set -x && \ EXPOSE 8888 # Configure container startup -ENTRYPOINT ["tini", "-g", "--"] CMD ["start-notebook.sh"] # Copy local files as late as possible to avoid cache busting -COPY start.sh start-notebook.sh start-singleuser.sh /usr/local/bin/ +COPY start-notebook.sh start-singleuser.sh /usr/local/bin/ # Currently need to have both jupyter_notebook_config and jupyter_server_config to support classic and lab COPY jupyter_server_config.py /etc/jupyter/ diff --git a/docker-stacks-foundation/.dockerignore b/docker-stacks-foundation/.dockerignore new file mode 100644 index 00000000..9dea340f --- /dev/null +++ b/docker-stacks-foundation/.dockerignore @@ -0,0 +1,2 @@ +# Documentation +README.md diff --git a/docker-stacks-foundation/Dockerfile b/docker-stacks-foundation/Dockerfile new file mode 100644 index 00000000..9d048a3e --- /dev/null +++ b/docker-stacks-foundation/Dockerfile @@ -0,0 +1,135 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Ubuntu 22.04 (jammy) +# https://hub.docker.com/_/ubuntu/tags?page=1&name=jammy +ARG ROOT_CONTAINER=ubuntu:22.04 + +FROM $ROOT_CONTAINER + +LABEL maintainer="Jupyter Project " +ARG NB_USER="jovyan" +ARG NB_UID="1000" +ARG NB_GID="100" + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# Install all OS dependencies for notebook server that starts but lacks all +# features (e.g., download as all possible file formats) +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update --yes && \ + # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as + # the ubuntu base image is rebuilt too seldom sometimes (less than once a month) + apt-get upgrade --yes && \ + apt-get install --yes --no-install-recommends \ + # - bzip2 is necessary to extract the micromamba executable. + bzip2 \ + ca-certificates \ + locales \ + sudo \ + # - tini is installed as a helpful container entrypoint that reaps zombie + # processes and such of the actual executable we want to start, see + # https://github.com/krallin/tini#why-tini for details. + tini \ + wget && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen + +# Configure environment +ENV CONDA_DIR=/opt/conda \ + SHELL=/bin/bash \ + NB_USER="${NB_USER}" \ + NB_UID=${NB_UID} \ + NB_GID=${NB_GID} \ + LC_ALL=en_US.UTF-8 \ + LANG=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 +ENV PATH="${CONDA_DIR}/bin:${PATH}" \ + HOME="/home/${NB_USER}" + +# Copy a script that we will use to correct permissions after running certain commands +COPY fix-permissions /usr/local/bin/fix-permissions +RUN chmod a+rx /usr/local/bin/fix-permissions + +# Enable prompt color in the skeleton .bashrc before creating the default NB_USER +# hadolint ignore=SC2016 +RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ + # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 + echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc + +# Create NB_USER with name jovyan user with UID=1000 and in the 'users' group +# and make sure these dirs are writable by the `users` group. +RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ + sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ + sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ + useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \ + mkdir -p "${CONDA_DIR}" && \ + chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ + chmod g+w /etc/passwd && \ + fix-permissions "${HOME}" && \ + fix-permissions "${CONDA_DIR}" + +USER ${NB_UID} + +# Pin python version here, or set it to "default" +ARG PYTHON_VERSION=3.10 + +# Setup work directory for backward-compatibility +RUN mkdir "/home/${NB_USER}/work" && \ + fix-permissions "/home/${NB_USER}" + +# Download and install Micromamba, and initialize Conda prefix. +# +# Similar projects using Micromamba: +# - Micromamba-Docker: +# - repo2docker: +# Install Python, Mamba and jupyter_core +# Cleanup temporary files and remove Micromamba +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" +WORKDIR /tmp +RUN set -x && \ + arch=$(uname -m) && \ + if [ "${arch}" = "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + wget -qO /tmp/micromamba.tar.bz2 \ + "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ + tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ + rm /tmp/micromamba.tar.bz2 && \ + PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ + if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ + # Install the packages + ./micromamba install \ + --root-prefix="${CONDA_DIR}" \ + --prefix="${CONDA_DIR}" \ + --yes \ + "${PYTHON_SPECIFIER}" \ + 'mamba' \ + 'jupyter_core' && \ + rm micromamba && \ + # Pin major.minor version of python + mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +# Configure container startup +ENTRYPOINT ["tini", "-g", "--"] +CMD ["start.sh"] + +# Copy local files as late as possible to avoid cache busting +COPY start.sh /usr/local/bin/ + +# Switch back to jovyan to avoid accidental container runs as root +USER ${NB_UID} + +WORKDIR "${HOME}" diff --git a/docker-stacks-foundation/README.md b/docker-stacks-foundation/README.md new file mode 100644 index 00000000..122c9aa1 --- /dev/null +++ b/docker-stacks-foundation/README.md @@ -0,0 +1,12 @@ +# Base Jupyter Stack + +[![docker pulls](https://img.shields.io/docker/pulls/jupyter/docker-stacks-foundation.svg)](https://hub.docker.com/r/jupyter/docker-stacks-foundation/) +[![docker stars](https://img.shields.io/docker/stars/jupyter/docker-stacks-foundation.svg)](https://hub.docker.com/r/jupyter/docker-stacks-foundation/) +[![image size](https://img.shields.io/docker/image-size/jupyter/docker-stacks-foundation/latest)](https://hub.docker.com/r/jupyter/docker-stacks-foundation/ "jupyter/docker-stacks-foundation image size") + +GitHub Actions in the project builds and pushes this image to Docker Hub. + +Please visit the project documentation site for help to use and contribute to this image and others. + +- [Jupyter Docker Stacks on ReadTheDocs](https://jupyter-docker-stacks.readthedocs.io/en/latest/index.html) +- [Selecting an Image :: Core Stacks :: jupyter/docker-stacks-foundation](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-docker-stacks-foundation) diff --git a/base-notebook/fix-permissions b/docker-stacks-foundation/fix-permissions similarity index 100% rename from base-notebook/fix-permissions rename to docker-stacks-foundation/fix-permissions diff --git a/base-notebook/initial-condarc b/docker-stacks-foundation/initial-condarc similarity index 100% rename from base-notebook/initial-condarc rename to docker-stacks-foundation/initial-condarc diff --git a/base-notebook/start.sh b/docker-stacks-foundation/start.sh similarity index 100% rename from base-notebook/start.sh rename to docker-stacks-foundation/start.sh diff --git a/docs/contributing/tests.md b/docs/contributing/tests.md index 2c5578a6..bccec310 100644 --- a/docs/contributing/tests.md +++ b/docs/contributing/tests.md @@ -11,10 +11,10 @@ We use `pytest` module to run tests on the image. `conftest.py` and `pytest.ini` in the `tests` folder define the environment in which tests are run. More info on `pytest` can be found [here](https://docs.pytest.org/en/latest/contents.html). -The actual image-specific test files are located in folders like `tests/-notebook/`. +The actual image-specific test files are located in folders like `tests//` (e.g., `tests/docker-stacks-foundation/`, `tests/minimal-notebook/`, etc.). ```{note} -If your test is located in `tests/-notebook/`, it will be run against `jupyter/-notebook` image and against all the [images inherited from this image](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships. +If your test is located in `tests//`, it will be run against `jupyter/` image and against all the [images inherited from this image](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships. ``` Many tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/reference/fixtures.html) @@ -23,7 +23,7 @@ defined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/main/ ## Unit tests If you want to run a python script in one of our images, you could add a unit test. -You can do this by creating a `tests/-notebook/units/` directory, if it doesn't already exist and put your file there. +You can do this by creating a `tests//units/` directory, if it doesn't already exist and put your file there. Files in this folder will be executed in the container when tests are run. You could see an [example for the TensorFlow package here](https://github.com/jupyter/docker-stacks/blob/HEAD/tests/tensorflow-notebook/units/unit_tensorflow.py). @@ -31,13 +31,13 @@ You could see an [example for the TensorFlow package here](https://github.com/ju Please follow the process below to add new tests: -1. Add your test code to one of the modules in `tests/-notebook/` directory or create a new module. +1. Add your test code to one of the modules in `tests//` directory or create a new module. 2. Build one or more images you intend to test and run the tests locally. If you use `make`, call: ```bash - make build/-notebook - make test/-notebook + make build/ + make test/ ``` 3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) diff --git a/docs/images/inherit.svg b/docs/images/inherit.svg index 6073a953..db85c7fc 100644 --- a/docs/images/inherit.svg +++ b/docs/images/inherit.svg @@ -1,6 +1,6 @@ - + @@ -12,51 +12,56 @@ - - + + + ubuntu (LTS with point release) - base-notebook + docker-stacks-foundation - minimal-notebook + base-notebook - scipy-notebook - - r-notebook + minimal-notebook - tensorflow-notebook - (only x86_64) + scipy-notebook - datascience-notebook - - pyspark-notebook + r-notebook + + tensorflow-notebook + (only x86_64) + + datascience-notebook - all-spark-notebook + pyspark-notebook + + all-spark-notebook - - - - + + - - - - - - - + + + + + + + + + + + diff --git a/docs/maintaining/tasks.md b/docs/maintaining/tasks.md index fa853a83..0e614a6e 100644 --- a/docs/maintaining/tasks.md +++ b/docs/maintaining/tasks.md @@ -26,9 +26,9 @@ When a new `Python` version is released, we wait for two things: ## Updating the Ubuntu Base Image -`base-notebook` is based on the LTS Ubuntu docker image. +`docker-stacks-foundation` is based on the LTS Ubuntu docker image. We wait for the first point release of the new LTS Ubuntu before updating the version. -Other images are directly or indirectly inherited from `base-notebook`. +Other images are directly or indirectly inherited from `docker-stacks-foundation`. We rebuild our images automatically each week, which means they frequently receive the updates. When there's a security fix in the Ubuntu base image, it's a good idea to manually trigger images rebuild [from the GitHub actions workflow UI](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml). diff --git a/docs/using/common.md b/docs/using/common.md index 6239168b..1f5f0e18 100644 --- a/docs/using/common.md +++ b/docs/using/common.md @@ -1,6 +1,6 @@ # Common Features -By default, a container launched from any Jupyter Docker Stacks image runs a Jupyter Server with JupyterLab frontend. +Except `jupyter/docker-stacks-foundation`, a container launched from any Jupyter Docker Stacks image runs a Jupyter Server with JupyterLab frontend. The container does so by executing a `start-notebook.sh` script. This script configures the internal container environment and then runs `jupyter lab`, passing any command-line arguments received. diff --git a/docs/using/selecting.md b/docs/using/selecting.md index 0df65d2d..b0536f3d 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -16,27 +16,46 @@ This section provides details about the first. The Jupyter team maintains a set of Docker image definitions in the GitHub repository. The following sections describe these images, including their contents, relationships, and versioning strategy. +### jupyter/docker-stacks-foundation + +[Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/docker-stacks-foundation) | +[Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/docker-stacks-foundation/Dockerfile) | +[Docker Hub image tags](https://hub.docker.com/r/jupyter/docker-stacks-foundation/tags/) + +`jupyter/docker-stacks-foundation` is a small image supporting a majority of [options common across all core stacks](common.md). +It is the basis for all other stacks on which Jupyter-related applications can be built +(e.g., kernel-based containers, [nbclient](https://github.com/jupyter/nbclient) applications, etc.). +As such, it does not contain application-level software like Jupyter Notebook server, Jupyter Lab or Jupyter Hub. + +It contains: + +- Package managers + - [conda](https://github.com/conda/conda): "cross-platform, language-agnostic binary package manager". + - [mamba](https://github.com/mamba-org/mamba): "reimplementation of the conda package manager in C++". We use this package manager by default when installing packages. +- Unprivileged user `jovyan` (`uid=1000`, configurable, [see options in the common features section](./common.md) of this documentation) in group `users` (`gid=100`) + with ownership over the `/home/jovyan` and `/opt/conda` paths +- `tini` as the container entrypoint +- A `start.sh` script as the default command - useful for running alternative commands in the container as applications are added (e.g. `ipython`, `jupyter kernelgateway`, `jupyter lab`) +- Options for a passwordless sudo +- No preinstalled scientific computing packages + ### jupyter/base-notebook [Source on GitHub](https://github.com/jupyter/docker-stacks/tree/main/base-notebook) | [Dockerfile commit history](https://github.com/jupyter/docker-stacks/commits/main/base-notebook/Dockerfile) | [Docker Hub image tags](https://hub.docker.com/r/jupyter/base-notebook/tags/) -`jupyter/base-notebook` is a small image supporting the [options common across all core stacks](common.md). -It is the basis for all other stacks and contains: +`jupyter/base-notebook` adds base Jupyter server applications like Notebook, Jupyter Lab and Jupyter Hub +and serves as the basis for all other stacks besides `jupyter/docker-stacks-foundation`. +It contains: + +- Everything in `jupyter/docker-stacks-foundation` - Minimally-functional Jupyter Notebook server (e.g., no LaTeX support for saving notebooks as PDFs) -- [Miniforge](https://github.com/conda-forge/miniforge) Python 3.x in `/opt/conda` with two package managers - - [conda](https://github.com/conda/conda): "cross-platform, language-agnostic binary package manager". - - [mamba](https://github.com/mamba-org/mamba): "reimplementation of the conda package manager in C++". We use this package manager by default when installing packages. - `notebook`, `jupyterhub` and `jupyterlab` packages -- No preinstalled scientific computing packages -- Unprivileged user `jovyan` (`uid=1000`, configurable, [see options in the common features section](./common.md) of this documentation) in group `users` (`gid=100`) - with ownership over the `/home/jovyan` and `/opt/conda` paths -- `tini` as the container entrypoint and a `start-notebook.sh` script as the default command +- A `start-notebook.sh` script as the default command - A `start-singleuser.sh` script useful for launching containers in JupyterHub -- A `start.sh` script useful for running alternative commands in the container (e.g. `ipython`, `jupyter kernelgateway`, `jupyter lab`) -- Options for a self-signed HTTPS certificate and passwordless sudo +- Options for a self-signed HTTPS certificate ### jupyter/minimal-notebook @@ -191,7 +210,7 @@ The following diagram depicts the build dependency tree of the core images. (i.e Any given image inherits the complete content of all ancestor images pointing to it. [![Image inheritance -diagram](../images/inherit.svg)](http://interactive.blockdiag.com/?compression=deflate&src=eJyFzrEKwjAQxvG9T3FkskM3KUrRJ3DTUShJe9XQ9C4kKbWK7266CCmCW_jnd_Apw03fanmDVwbQYidHE-qOKXj9RDjAvsrihxjVSGG80uZ0OcOkwx0sawrg0KD0mAsojqDiqyAOqJj7Kp4lYRGDJj1Ik6B1W5xvtJ0TlZbFiIDk2XWGp2-PA5nMDI9dWZfbXPy-bGWQsSI1-HeJ-7PCzt5K1ydq3RYnjSnW8v0BwS-D-w) +diagram](../images/inherit.svg)](http://interactive.blockdiag.com/image?compression=deflate&encoding=base64&src=eJyFj0FqxDAMRfc5hchqsvCuDFNCe4Lu2mVhUBKlNXGkYMtkMqV3r70pOLRk-_T-56tz0k-DxQ_4qgAGGjE6vY7CGuyd4Ake2yod6thF1vjOp5e3V1itfsIilhU8OcJATQ3mGYbURd4ExX4KZpTIA6oVbnP1P7ec61KDYVHqRKYsFyAbs2U7oyukPcte6O2yFVZJslMrcRA_Oll_eXpM2G1wu5yv54em_juZFmOixD0dLvEHK5YtLOinwtqz7KFzZm9-_wDZDphP) ### Builds diff --git a/tagging/images_hierarchy.py b/tagging/images_hierarchy.py index 145b329a..452d2a68 100644 --- a/tagging/images_hierarchy.py +++ b/tagging/images_hierarchy.py @@ -38,7 +38,7 @@ class ImageDescription: ALL_IMAGES = { - "base-notebook": ImageDescription( + "docker-stacks-foundation": ImageDescription( parent_image=None, taggers=[ SHATagger(), @@ -46,11 +46,16 @@ ALL_IMAGES = { UbuntuVersionTagger(), PythonMajorMinorVersionTagger(), PythonVersionTagger(), + ], + manifests=[CondaEnvironmentManifest(), AptPackagesManifest()], + ), + "base-notebook": ImageDescription( + parent_image="docker-stacks-foundation", + taggers=[ JupyterNotebookVersionTagger(), JupyterLabVersionTagger(), JupyterHubVersionTagger(), ], - manifests=[CondaEnvironmentManifest(), AptPackagesManifest()], ), "minimal-notebook": ImageDescription(parent_image="base-notebook"), "scipy-notebook": ImageDescription(parent_image="minimal-notebook"), diff --git a/tests/base-notebook/test_container_options.py b/tests/base-notebook/test_container_options.py index bf2a36df..bb3129e6 100644 --- a/tests/base-notebook/test_container_options.py +++ b/tests/base-notebook/test_container_options.py @@ -1,7 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. import logging -import pathlib import time import pytest # type: ignore @@ -30,6 +29,31 @@ def test_cli_args(container: TrackedContainer, http_client: requests.Session) -> assert "login_submit" not in resp.text +def test_nb_user_change(container: TrackedContainer) -> None: + """Container should change the username (`NB_USER`) of the default user.""" + nb_user = "nayvoj" + running_container = container.run_detached( + tty=True, + user="root", + environment=[f"NB_USER={nb_user}", "CHOWN_HOME=yes"], + command=["start.sh", "bash", "-c", "sleep infinity"], + ) + + # Give the chown time to complete. + # Use sleep, not wait, because the container sleeps forever. + time.sleep(1) + LOGGER.info( + f"Checking if home folder of {nb_user} contains the hidden '.jupyter' folder with appropriate permissions ..." + ) + command = f'stat -c "%F %U %G" /home/{nb_user}/.jupyter' + expected_output = f"directory {nb_user} users" + cmd = running_container.exec_run(command, workdir=f"/home/{nb_user}") + output = cmd.output.decode("utf-8").strip("\n") + assert ( + output == expected_output + ), f"Hidden folder .jupyter was not copied properly to {nb_user} home folder. stat: {output}, expected {expected_output}" + + @pytest.mark.filterwarnings("ignore:Unverified HTTPS request") def test_unsigned_ssl( container: TrackedContainer, http_client: requests.Session @@ -54,282 +78,3 @@ def test_unsigned_ssl( assert "ERROR" not in logs warnings = TrackedContainer.get_warnings(logs) assert not warnings - - -def test_uid_change(container: TrackedContainer) -> None: - """Container should change the UID of the default user.""" - logs = container.run_and_wait( - timeout=120, # usermod is slow so give it some time - tty=True, - user="root", - environment=["NB_UID=1010"], - command=["start.sh", "bash", "-c", "id && touch /opt/conda/test-file"], - ) - assert "uid=1010(jovyan)" in logs - - -def test_gid_change(container: TrackedContainer) -> None: - """Container should change the GID of the default user.""" - logs = container.run_and_wait( - timeout=10, - tty=True, - user="root", - environment=["NB_GID=110"], - command=["start.sh", "id"], - ) - assert "gid=110(jovyan)" in logs - assert "groups=110(jovyan),100(users)" in logs - - -def test_nb_user_change(container: TrackedContainer) -> None: - """Container should change the username (`NB_USER`) of the default user.""" - nb_user = "nayvoj" - running_container = container.run_detached( - tty=True, - user="root", - environment=[f"NB_USER={nb_user}", "CHOWN_HOME=yes"], - command=["start.sh", "bash", "-c", "sleep infinity"], - ) - - # Give the chown time to complete. - # Use sleep, not wait, because the container sleeps forever. - time.sleep(1) - LOGGER.info(f"Checking if the user is changed to {nb_user} by the start script ...") - output = running_container.logs().decode("utf-8") - assert "ERROR" not in output - assert "WARNING" not in output - assert ( - f"username: jovyan -> {nb_user}" in output - ), f"User is not changed to {nb_user}" - - LOGGER.info(f"Checking {nb_user} id ...") - command = "id" - expected_output = f"uid=1000({nb_user}) gid=100(users) groups=100(users)" - cmd = running_container.exec_run(command, user=nb_user, workdir=f"/home/{nb_user}") - output = cmd.output.decode("utf-8").strip("\n") - assert output == expected_output, f"Bad user {output}, expected {expected_output}" - - LOGGER.info(f"Checking if {nb_user} owns his home folder ...") - command = f'stat -c "%U %G" /home/{nb_user}/' - expected_output = f"{nb_user} users" - cmd = running_container.exec_run(command, workdir=f"/home/{nb_user}") - output = cmd.output.decode("utf-8").strip("\n") - assert ( - output == expected_output - ), f"Bad owner for the {nb_user} home folder {output}, expected {expected_output}" - - LOGGER.info( - f"Checking if home folder of {nb_user} contains the hidden '.jupyter' folder with appropriate permissions ..." - ) - command = f'stat -c "%F %U %G" /home/{nb_user}/.jupyter' - expected_output = f"directory {nb_user} users" - cmd = running_container.exec_run(command, workdir=f"/home/{nb_user}") - output = cmd.output.decode("utf-8").strip("\n") - assert ( - output == expected_output - ), f"Hidden folder .jupyter was not copied properly to {nb_user} home folder. stat: {output}, expected {expected_output}" - - -def test_chown_extra(container: TrackedContainer) -> None: - """Container should change the UID/GID of a comma separated - CHOWN_EXTRA list of folders.""" - logs = container.run_and_wait( - timeout=120, # chown is slow so give it some time - tty=True, - user="root", - environment=[ - "NB_UID=1010", - "NB_GID=101", - "CHOWN_EXTRA=/home/jovyan,/opt/conda/bin", - "CHOWN_EXTRA_OPTS=-R", - ], - command=[ - "start.sh", - "bash", - "-c", - "stat -c '%n:%u:%g' /home/jovyan/.bashrc /opt/conda/bin/jupyter", - ], - ) - assert "/home/jovyan/.bashrc:1010:101" in logs - assert "/opt/conda/bin/jupyter:1010:101" in logs - - -def test_chown_home(container: TrackedContainer) -> None: - """Container should change the NB_USER home directory owner and - group to the current value of NB_UID and NB_GID.""" - logs = container.run_and_wait( - timeout=120, # chown is slow so give it some time - tty=True, - user="root", - environment=[ - "CHOWN_HOME=yes", - "CHOWN_HOME_OPTS=-R", - "NB_USER=kitten", - "NB_UID=1010", - "NB_GID=101", - ], - command=["start.sh", "bash", "-c", "stat -c '%n:%u:%g' /home/kitten/.bashrc"], - ) - assert "/home/kitten/.bashrc:1010:101" in logs - - -def test_sudo(container: TrackedContainer) -> None: - """Container should grant passwordless sudo to the default user.""" - logs = container.run_and_wait( - timeout=10, - tty=True, - user="root", - environment=["GRANT_SUDO=yes"], - command=["start.sh", "sudo", "id"], - ) - assert "uid=0(root)" in logs - - -def test_sudo_path(container: TrackedContainer) -> None: - """Container should include /opt/conda/bin in the sudo secure_path.""" - logs = container.run_and_wait( - timeout=10, - tty=True, - user="root", - environment=["GRANT_SUDO=yes"], - command=["start.sh", "sudo", "which", "jupyter"], - ) - assert logs.rstrip().endswith("/opt/conda/bin/jupyter") - - -def test_sudo_path_without_grant(container: TrackedContainer) -> None: - """Container should include /opt/conda/bin in the sudo secure_path.""" - logs = container.run_and_wait( - timeout=10, - tty=True, - user="root", - command=["start.sh", "which", "jupyter"], - ) - assert logs.rstrip().endswith("/opt/conda/bin/jupyter") - - -def test_group_add(container: TrackedContainer) -> None: - """Container should run with the specified uid, gid, and secondary - group. It won't be possible to modify /etc/passwd since gid is nonzero, so - additionally verify that setting gid=0 is suggested in a warning. - """ - logs = container.run_and_wait( - timeout=5, - no_warnings=False, - user="1010:1010", - group_add=["users"], # Ensures write access to /home/jovyan - command=["start.sh", "id"], - ) - warnings = TrackedContainer.get_warnings(logs) - assert len(warnings) == 1 - assert "Try setting gid=0" in warnings[0] - assert "uid=1010 gid=1010 groups=1010,100(users)" in logs - - -def test_set_uid(container: TrackedContainer) -> None: - """Container should run with the specified uid and NB_USER. - The /home/jovyan directory will not be writable since it's owned by 1000:users. - Additionally verify that "--group-add=users" is suggested in a warning to restore - write access. - """ - logs = container.run_and_wait( - timeout=5, - no_warnings=False, - user="1010", - command=["start.sh", "id"], - ) - assert "uid=1010(jovyan) gid=0(root)" in logs - warnings = TrackedContainer.get_warnings(logs) - assert len(warnings) == 1 - assert "--group-add=users" in warnings[0] - - -def test_set_uid_and_nb_user(container: TrackedContainer) -> None: - """Container should run with the specified uid and NB_USER.""" - logs = container.run_and_wait( - timeout=5, - no_warnings=False, - user="1010", - environment=["NB_USER=kitten"], - group_add=["users"], # Ensures write access to /home/jovyan - command=["start.sh", "id"], - ) - assert "uid=1010(kitten) gid=0(root)" in logs - warnings = TrackedContainer.get_warnings(logs) - assert len(warnings) == 1 - assert "user is kitten but home is /home/jovyan" in warnings[0] - - -def test_container_not_delete_bind_mount( - container: TrackedContainer, tmp_path: pathlib.Path -) -> None: - """Container should not delete host system files when using the (docker) - -v bind mount flag and mapping to /home/jovyan. - """ - d = tmp_path / "data" - d.mkdir() - p = d / "foo.txt" - p.write_text("some-content") - - container.run_and_wait( - timeout=5, - tty=True, - user="root", - working_dir="/home/", - environment=[ - "NB_USER=user", - "CHOWN_HOME=yes", - ], - volumes={d: {"bind": "/home/jovyan/data", "mode": "rw"}}, - command=["start.sh", "ls"], - ) - assert p.read_text() == "some-content" - assert len(list(tmp_path.iterdir())) == 1 - - -@pytest.mark.parametrize("enable_root", [False, True]) -def test_jupyter_env_vars_to_unset_as_root( - container: TrackedContainer, enable_root: bool -) -> None: - """Environment variables names listed in JUPYTER_ENV_VARS_TO_UNSET - should be unset in the final environment.""" - root_args = {"user": "root"} if enable_root else {} - logs = container.run_and_wait( - timeout=10, - tty=True, - environment=[ - "JUPYTER_ENV_VARS_TO_UNSET=SECRET_ANIMAL,UNUSED_ENV,SECRET_FRUIT", - "FRUIT=bananas", - "SECRET_ANIMAL=cats", - "SECRET_FRUIT=mango", - ], - command=[ - "start.sh", - "bash", - "-c", - "echo I like $FRUIT and ${SECRET_FRUIT:-stuff}, and love ${SECRET_ANIMAL:-to keep secrets}!", - ], - **root_args, # type: ignore - ) - assert "I like bananas and stuff, and love to keep secrets!" in logs - - -def test_secure_path(container: TrackedContainer, tmp_path: pathlib.Path) -> None: - """Make sure that the sudo command has conda's python (not system's) on path. - See . - """ - d = tmp_path / "data" - d.mkdir() - p = d / "wrong_python.sh" - p.write_text('#!/bin/bash\necho "Wrong python executable invoked!"') - p.chmod(0o755) - - logs = container.run_and_wait( - timeout=5, - tty=True, - user="root", - volumes={p: {"bind": "/usr/bin/python", "mode": "ro"}}, - command=["start.sh", "python", "--version"], - ) - assert "Wrong python" not in logs - assert "Python" in logs diff --git a/tests/base-notebook/test_npm_package_manager.py b/tests/base-notebook/test_npm_package_manager.py new file mode 100644 index 00000000..4d94dfbb --- /dev/null +++ b/tests/base-notebook/test_npm_package_manager.py @@ -0,0 +1,14 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +import logging + +from tests.conftest import TrackedContainer +from tests.package_helper import run_package_manager + +LOGGER = logging.getLogger(__name__) + + +def test_npm_package_manager(container: TrackedContainer) -> None: + """Test that npm is installed and runs.""" + run_package_manager(container, "npm", "--version") diff --git a/tests/base-notebook/test_outdated.py b/tests/docker-stacks-foundation/test_outdated.py similarity index 100% rename from tests/base-notebook/test_outdated.py rename to tests/docker-stacks-foundation/test_outdated.py diff --git a/tests/base-notebook/test_package_managers.py b/tests/docker-stacks-foundation/test_package_managers.py similarity index 59% rename from tests/base-notebook/test_package_managers.py rename to tests/docker-stacks-foundation/test_package_managers.py index d831b1e6..9e8dd5b9 100644 --- a/tests/base-notebook/test_package_managers.py +++ b/tests/docker-stacks-foundation/test_package_managers.py @@ -6,6 +6,7 @@ import logging import pytest # type: ignore from tests.conftest import TrackedContainer +from tests.package_helper import run_package_manager LOGGER = logging.getLogger(__name__) @@ -16,19 +17,11 @@ LOGGER = logging.getLogger(__name__) ("apt", "--version"), ("conda", "--version"), ("mamba", "--version"), - ("npm", "--version"), ("pip", "--version"), ], ) def test_package_manager( - container: TrackedContainer, package_manager: str, version_arg: tuple[str, ...] + container: TrackedContainer, package_manager: str, version_arg: str ) -> None: - """Test the notebook start-notebook script""" - LOGGER.info( - f"Test that the package manager {package_manager} is working properly ..." - ) - container.run_and_wait( - timeout=5, - tty=True, - command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"], - ) + """Test that package managers are installed and run.""" + run_package_manager(container, package_manager, version_arg) diff --git a/tests/base-notebook/test_python.py b/tests/docker-stacks-foundation/test_python.py similarity index 100% rename from tests/base-notebook/test_python.py rename to tests/docker-stacks-foundation/test_python.py diff --git a/tests/base-notebook/test_units.py b/tests/docker-stacks-foundation/test_units.py similarity index 100% rename from tests/base-notebook/test_units.py rename to tests/docker-stacks-foundation/test_units.py diff --git a/tests/docker-stacks-foundation/test_user_options.py b/tests/docker-stacks-foundation/test_user_options.py new file mode 100644 index 00000000..cfb344cf --- /dev/null +++ b/tests/docker-stacks-foundation/test_user_options.py @@ -0,0 +1,290 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import logging +import pathlib +import time + +import pytest # type: ignore + +from tests.conftest import TrackedContainer + +LOGGER = logging.getLogger(__name__) + + +def test_uid_change(container: TrackedContainer) -> None: + """Container should change the UID of the default user.""" + logs = container.run_and_wait( + timeout=120, # usermod is slow so give it some time + tty=True, + user="root", + environment=["NB_UID=1010"], + command=["start.sh", "bash", "-c", "id && touch /opt/conda/test-file"], + ) + assert "uid=1010(jovyan)" in logs + + +def test_gid_change(container: TrackedContainer) -> None: + """Container should change the GID of the default user.""" + logs = container.run_and_wait( + timeout=10, + tty=True, + user="root", + environment=["NB_GID=110"], + command=["start.sh", "id"], + ) + assert "gid=110(jovyan)" in logs + assert "groups=110(jovyan),100(users)" in logs + + +def test_nb_user_change(container: TrackedContainer) -> None: + """Container should change the username (`NB_USER`) of the default user.""" + nb_user = "nayvoj" + running_container = container.run_detached( + tty=True, + user="root", + environment=[f"NB_USER={nb_user}", "CHOWN_HOME=yes"], + command=["start.sh", "bash", "-c", "sleep infinity"], + ) + + # Give the chown time to complete. + # Use sleep, not wait, because the container sleeps forever. + time.sleep(1) + LOGGER.info(f"Checking if the user is changed to {nb_user} by the start script ...") + output = running_container.logs().decode("utf-8") + assert "ERROR" not in output + assert "WARNING" not in output + assert ( + f"username: jovyan -> {nb_user}" in output + ), f"User is not changed to {nb_user}" + + LOGGER.info(f"Checking {nb_user} id ...") + command = "id" + expected_output = f"uid=1000({nb_user}) gid=100(users) groups=100(users)" + cmd = running_container.exec_run(command, user=nb_user, workdir=f"/home/{nb_user}") + output = cmd.output.decode("utf-8").strip("\n") + assert output == expected_output, f"Bad user {output}, expected {expected_output}" + + LOGGER.info(f"Checking if {nb_user} owns his home folder ...") + command = f'stat -c "%U %G" /home/{nb_user}/' + expected_output = f"{nb_user} users" + cmd = running_container.exec_run(command, workdir=f"/home/{nb_user}") + output = cmd.output.decode("utf-8").strip("\n") + assert ( + output == expected_output + ), f"Bad owner for the {nb_user} home folder {output}, expected {expected_output}" + + LOGGER.info( + f"Checking if home folder of {nb_user} contains the 'work' folder with appropriate permissions ..." + ) + command = f'stat -c "%F %U %G" /home/{nb_user}/work' + expected_output = f"directory {nb_user} users" + cmd = running_container.exec_run(command, workdir=f"/home/{nb_user}") + output = cmd.output.decode("utf-8").strip("\n") + assert ( + output == expected_output + ), f"Folder work was not copied properly to {nb_user} home folder. stat: {output}, expected {expected_output}" + + +def test_chown_extra(container: TrackedContainer) -> None: + """Container should change the UID/GID of a comma separated + CHOWN_EXTRA list of folders.""" + logs = container.run_and_wait( + timeout=120, # chown is slow so give it some time + tty=True, + user="root", + environment=[ + "NB_UID=1010", + "NB_GID=101", + "CHOWN_EXTRA=/home/jovyan,/opt/conda/bin", + "CHOWN_EXTRA_OPTS=-R", + ], + command=[ + "start.sh", + "bash", + "-c", + "stat -c '%n:%u:%g' /home/jovyan/.bashrc /opt/conda/bin/jupyter", + ], + ) + assert "/home/jovyan/.bashrc:1010:101" in logs + assert "/opt/conda/bin/jupyter:1010:101" in logs + + +def test_chown_home(container: TrackedContainer) -> None: + """Container should change the NB_USER home directory owner and + group to the current value of NB_UID and NB_GID.""" + logs = container.run_and_wait( + timeout=120, # chown is slow so give it some time + tty=True, + user="root", + environment=[ + "CHOWN_HOME=yes", + "CHOWN_HOME_OPTS=-R", + "NB_USER=kitten", + "NB_UID=1010", + "NB_GID=101", + ], + command=["start.sh", "bash", "-c", "stat -c '%n:%u:%g' /home/kitten/.bashrc"], + ) + assert "/home/kitten/.bashrc:1010:101" in logs + + +def test_sudo(container: TrackedContainer) -> None: + """Container should grant passwordless sudo to the default user.""" + logs = container.run_and_wait( + timeout=10, + tty=True, + user="root", + environment=["GRANT_SUDO=yes"], + command=["start.sh", "sudo", "id"], + ) + assert "uid=0(root)" in logs + + +def test_sudo_path(container: TrackedContainer) -> None: + """Container should include /opt/conda/bin in the sudo secure_path.""" + logs = container.run_and_wait( + timeout=10, + tty=True, + user="root", + environment=["GRANT_SUDO=yes"], + command=["start.sh", "sudo", "which", "jupyter"], + ) + assert logs.rstrip().endswith("/opt/conda/bin/jupyter") + + +def test_sudo_path_without_grant(container: TrackedContainer) -> None: + """Container should include /opt/conda/bin in the sudo secure_path.""" + logs = container.run_and_wait( + timeout=10, + tty=True, + user="root", + command=["start.sh", "which", "jupyter"], + ) + assert logs.rstrip().endswith("/opt/conda/bin/jupyter") + + +def test_group_add(container: TrackedContainer) -> None: + """Container should run with the specified uid, gid, and secondary + group. It won't be possible to modify /etc/passwd since gid is nonzero, so + additionally verify that setting gid=0 is suggested in a warning. + """ + logs = container.run_and_wait( + timeout=5, + no_warnings=False, + user="1010:1010", + group_add=["users"], # Ensures write access to /home/jovyan + command=["start.sh", "id"], + ) + warnings = TrackedContainer.get_warnings(logs) + assert len(warnings) == 1 + assert "Try setting gid=0" in warnings[0] + assert "uid=1010 gid=1010 groups=1010,100(users)" in logs + + +def test_set_uid(container: TrackedContainer) -> None: + """Container should run with the specified uid and NB_USER. + The /home/jovyan directory will not be writable since it's owned by 1000:users. + Additionally verify that "--group-add=users" is suggested in a warning to restore + write access. + """ + logs = container.run_and_wait( + timeout=5, + no_warnings=False, + user="1010", + command=["start.sh", "id"], + ) + assert "uid=1010(jovyan) gid=0(root)" in logs + warnings = TrackedContainer.get_warnings(logs) + assert len(warnings) == 1 + assert "--group-add=users" in warnings[0] + + +def test_set_uid_and_nb_user(container: TrackedContainer) -> None: + """Container should run with the specified uid and NB_USER.""" + logs = container.run_and_wait( + timeout=5, + no_warnings=False, + user="1010", + environment=["NB_USER=kitten"], + group_add=["users"], # Ensures write access to /home/jovyan + command=["start.sh", "id"], + ) + assert "uid=1010(kitten) gid=0(root)" in logs + warnings = TrackedContainer.get_warnings(logs) + assert len(warnings) == 1 + assert "user is kitten but home is /home/jovyan" in warnings[0] + + +def test_container_not_delete_bind_mount( + container: TrackedContainer, tmp_path: pathlib.Path +) -> None: + """Container should not delete host system files when using the (docker) + -v bind mount flag and mapping to /home/jovyan. + """ + d = tmp_path / "data" + d.mkdir() + p = d / "foo.txt" + p.write_text("some-content") + + container.run_and_wait( + timeout=5, + tty=True, + user="root", + working_dir="/home/", + environment=[ + "NB_USER=user", + "CHOWN_HOME=yes", + ], + volumes={d: {"bind": "/home/jovyan/data", "mode": "rw"}}, + command=["start.sh", "ls"], + ) + assert p.read_text() == "some-content" + assert len(list(tmp_path.iterdir())) == 1 + + +@pytest.mark.parametrize("enable_root", [False, True]) +def test_jupyter_env_vars_to_unset_as_root( + container: TrackedContainer, enable_root: bool +) -> None: + """Environment variables names listed in JUPYTER_ENV_VARS_TO_UNSET + should be unset in the final environment.""" + root_args = {"user": "root"} if enable_root else {} + logs = container.run_and_wait( + timeout=10, + tty=True, + environment=[ + "JUPYTER_ENV_VARS_TO_UNSET=SECRET_ANIMAL,UNUSED_ENV,SECRET_FRUIT", + "FRUIT=bananas", + "SECRET_ANIMAL=cats", + "SECRET_FRUIT=mango", + ], + command=[ + "start.sh", + "bash", + "-c", + "echo I like $FRUIT and ${SECRET_FRUIT:-stuff}, and love ${SECRET_ANIMAL:-to keep secrets}!", + ], + **root_args, # type: ignore + ) + assert "I like bananas and stuff, and love to keep secrets!" in logs + + +def test_secure_path(container: TrackedContainer, tmp_path: pathlib.Path) -> None: + """Make sure that the sudo command has conda's python (not system's) on path. + See . + """ + d = tmp_path / "data" + d.mkdir() + p = d / "wrong_python.sh" + p.write_text('#!/bin/bash\necho "Wrong python executable invoked!"') + p.chmod(0o755) + + logs = container.run_and_wait( + timeout=5, + tty=True, + user="root", + volumes={p: {"bind": "/usr/bin/python", "mode": "ro"}}, + command=["start.sh", "python", "--version"], + ) + assert "Wrong python" not in logs + assert "Python" in logs diff --git a/tests/images_hierarchy.py b/tests/images_hierarchy.py index 744f9c2f..c210a1a8 100644 --- a/tests/images_hierarchy.py +++ b/tests/images_hierarchy.py @@ -8,7 +8,8 @@ THIS_DIR = Path(__file__).parent.resolve() # Please, take a look at the hierarchy of the images here: # https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#image-relationships ALL_IMAGES = { - "base-notebook": None, + "docker-stacks-foundation": None, + "base-notebook": "docker-stacks-foundation", "minimal-notebook": "base-notebook", "scipy-notebook": "minimal-notebook", "r-notebook": "minimal-notebook", diff --git a/tests/package_helper.py b/tests/package_helper.py index 7524fe0a..c9fa09f2 100644 --- a/tests/package_helper.py +++ b/tests/package_helper.py @@ -37,6 +37,21 @@ from tests.conftest import TrackedContainer LOGGER = logging.getLogger(__name__) +def run_package_manager( + container: TrackedContainer, package_manager: str, version_arg: str +) -> None: + """Runs the given package manager with its version argument.""" + + LOGGER.info( + f"Test that the package manager {package_manager} is working properly ..." + ) + container.run_and_wait( + timeout=5, + tty=True, + command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"], + ) + + class CondaPackageHelper: """Conda package helper permitting to get information about packages"""