From 434ebfaa13f025e73f3e55ed4af326135813dc44 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 2 Feb 2022 16:19:21 +0100 Subject: [PATCH 01/36] Use micromamba to bootstrap mamba --- base-notebook/Dockerfile | 86 +++++++++++++++------------------------- 1 file changed, 32 insertions(+), 54 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index c8d69034..41eb5fca 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -67,13 +67,24 @@ RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashr # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc +# Download and install micromamba, and initialize Conda prefix +# hadolint ignore=DL3003 +RUN set -x && \ + cd /usr/local && \ + arch=$(uname -m) && \ + if [ "${arch}" == "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + wget -qO- "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" | tar -xvj bin/micromamba && \ + micromamba create --prefix="${CONDA_DIR}" + # Create NB_USER with name jovyan user with UID=1000 and in the 'users' group # and make sure these dirs are writable by the `users` group. RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \ - mkdir -p "${CONDA_DIR}" && \ chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ chmod g+w /etc/passwd && \ fix-permissions "${HOME}" && \ @@ -86,66 +97,33 @@ ARG PYTHON_VERSION=default RUN mkdir "/home/${NB_USER}/work" && \ fix-permissions "/home/${NB_USER}" -# Install conda as jovyan and check the sha256 sum provided on the download site -WORKDIR /tmp - -# CONDA_MIRROR is a mirror prefix to speed up downloading -# For example, people from mainland China could set it as -# https://mirrors.tuna.tsinghua.edu.cn/github-release/conda-forge/miniforge/LatestRelease -ARG CONDA_MIRROR=https://github.com/conda-forge/miniforge/releases/latest/download - -# ---- Miniforge installer ---- -# Check https://github.com/conda-forge/miniforge/releases -# Package Manager and Python implementation to use (https://github.com/conda-forge/miniforge) -# We're using Mambaforge installer, possible options: -# - conda only: either Miniforge3 to use Python or Miniforge-pypy3 to use PyPy -# - conda + mamba: either Mambaforge to use Python or Mambaforge-pypy3 to use PyPy -# Installation: conda, mamba, pip -RUN set -x && \ - # Miniforge installer - miniforge_arch=$(uname -m) && \ - miniforge_installer="Mambaforge-Linux-${miniforge_arch}.sh" && \ - wget --quiet "${CONDA_MIRROR}/${miniforge_installer}" && \ - /bin/bash "${miniforge_installer}" -f -b -p "${CONDA_DIR}" && \ - rm "${miniforge_installer}" && \ - # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html - conda config --system --set auto_update_conda false && \ - conda config --system --set show_channel_urls true && \ - if [[ "${PYTHON_VERSION}" != "default" ]]; then mamba install --quiet --yes python="${PYTHON_VERSION}"; fi && \ - # Pin major.minor version of python - mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ - # Using conda to update all packages: https://github.com/mamba-org/mamba/issues/1092 - conda update --all --quiet --yes && \ - conda clean --all -f -y && \ - rm -rf "/home/${NB_USER}/.cache/yarn" && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -# Using fixed version of mamba in arm, because the latest one has problems with arm under qemu -# See: https://github.com/jupyter/docker-stacks/issues/1539 -RUN set -x && \ - arch=$(uname -m) && \ - if [ "${arch}" == "aarch64" ]; then \ - mamba install --quiet --yes \ - 'mamba<0.18' && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}"; \ - fi; - -# Install Jupyter Notebook, Lab, and Hub +# Configure the environment +# Install Python, Mamba, Jupyter Notebook, Lab, and Hub # Generate a notebook server config # Cleanup temporary files # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change -RUN mamba install --quiet --yes \ - 'notebook' \ - 'jupyterhub' \ - 'jupyterlab' && \ +RUN set -x && \ + PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ + if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ + # TODO: use `micromamba config` or report issue. + echo "show_channel_urls: true" >> "${CONDA_DIR}/.condarc" && \ + echo "channels:" >> "${CONDA_DIR}/.condarc" && \ + echo " - conda-forge" >> "${CONDA_DIR}/.condarc" && \ + micromamba install --prefix="${CONDA_DIR}" --quiet --yes \ + "${PYTHON_SPECIFIER}" \ + 'mamba' \ + 'notebook' \ + 'jupyterhub' \ + 'jupyterlab' && \ + # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html + conda config --system --set auto_update_conda false && \ + # Pin major.minor version of python + mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ + jupyter notebook --generate-config && \ mamba clean --all -f -y && \ npm cache clean --force && \ - jupyter notebook --generate-config && \ jupyter lab clean && \ rm -rf "/home/${NB_USER}/.cache/yarn" && \ fix-permissions "${CONDA_DIR}" && \ From 1284cfa438bdf409560d8269687628138b286d35 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 2 Feb 2022 18:05:35 +0100 Subject: [PATCH 02/36] Pin Python=3.9 --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 41eb5fca..1755877d 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -91,7 +91,7 @@ RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ fix-permissions "${CONDA_DIR}" USER ${NB_UID} -ARG PYTHON_VERSION=default +ARG PYTHON_VERSION=3.9 # Setup work directory for backward-compatibility RUN mkdir "/home/${NB_USER}/work" && \ From 3dd8f6e2f44f1c92f3e74bcf4a56312abb2a8cf8 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 2 Feb 2022 18:12:40 +0100 Subject: [PATCH 03/36] Add links for Micromamba --- base-notebook/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 1755877d..f0e3a6b2 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -67,7 +67,11 @@ RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashr # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc -# Download and install micromamba, and initialize Conda prefix +# Download and install Micromamba, and initialize Conda prefix. +# +# Similar projects using Micromamba: +# - Micromamba-Docker: +# - repo2docker: # hadolint ignore=DL3003 RUN set -x && \ cd /usr/local && \ From 5da971792541e73e515ed45fc821362b082555b0 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 2 Feb 2022 19:31:50 +0100 Subject: [PATCH 04/36] Make bzip2 dependency explicit --- base-notebook/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index f0e3a6b2..c99779c0 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -40,7 +40,9 @@ RUN apt-get update --yes && \ # processes and such of the actual executable we want to start, see # https://github.com/krallin/tini#why-tini for details. tini \ - wget && \ + wget \ + # - bzip2 is necessary to extract the micromamba executable. + bzip2 && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ locale-gen From a36293a1013f298d186216d77497676f019f88a5 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 2 Feb 2022 21:15:17 +0100 Subject: [PATCH 05/36] Turn on wget debug --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index c99779c0..795d7020 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -82,7 +82,7 @@ RUN set -x && \ # Should be simpler, see arch="64"; \ fi && \ - wget -qO- "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" | tar -xvj bin/micromamba && \ + wget --debug -qO- "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" | tar -xvj bin/micromamba && \ micromamba create --prefix="${CONDA_DIR}" # Create NB_USER with name jovyan user with UID=1000 and in the 'users' group From a392d30cb06c893e899142dd81e020e88d726712 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 2 Feb 2022 21:27:51 +0100 Subject: [PATCH 06/36] Remove micromamba after use --- base-notebook/Dockerfile | 72 +++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 795d7020..7188933f 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -69,60 +69,62 @@ RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashr # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc +# Create NB_USER with name jovyan user with UID=1000 and in the 'users' group +# and make sure these dirs are writable by the `users` group. +# Setup work directory for backward-compatibility +RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ + sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ + sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ + useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \ + mkdir -p "${CONDA_DIR}" && \ + chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ + chmod g+w /etc/passwd && \ + mkdir "/home/${NB_USER}/work" && \ + fix-permissions "${HOME}" + +USER ${NB_UID} +ARG PYTHON_VERSION=3.9 + # Download and install Micromamba, and initialize Conda prefix. # -# Similar projects using Micromamba: -# - Micromamba-Docker: -# - repo2docker: -# hadolint ignore=DL3003 +# Similar projects using Micromamba: +# - Micromamba-Docker: +# - repo2docker: +# Configure the environment +# Install Python, Mamba, Jupyter Notebook, Lab, and Hub +# Generate a notebook server config +# Cleanup temporary files and remove Micromamba +# Correct permissions +# Do all this in a single RUN command to avoid duplicating all of the +# files across image layers when the permissions change +WORKDIR /tmp RUN set -x && \ - cd /usr/local && \ arch=$(uname -m) && \ if [ "${arch}" == "x86_64" ]; then \ # Should be simpler, see arch="64"; \ fi && \ - wget --debug -qO- "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" | tar -xvj bin/micromamba && \ - micromamba create --prefix="${CONDA_DIR}" - -# Create NB_USER with name jovyan user with UID=1000 and in the 'users' group -# and make sure these dirs are writable by the `users` group. -RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ - sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ - sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ - useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \ - chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ - chmod g+w /etc/passwd && \ - fix-permissions "${HOME}" && \ - fix-permissions "${CONDA_DIR}" - -USER ${NB_UID} -ARG PYTHON_VERSION=3.9 - -# Setup work directory for backward-compatibility -RUN mkdir "/home/${NB_USER}/work" && \ - fix-permissions "/home/${NB_USER}" - -# Configure the environment -# Install Python, Mamba, Jupyter Notebook, Lab, and Hub -# Generate a notebook server config -# Cleanup temporary files -# Correct permissions -# Do all this in a single RUN command to avoid duplicating all of the -# files across image layers when the permissions change -RUN set -x && \ + wget -qO- "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" \ + | tar -xvj bin/micromamba && \ + # bin/micromamba create --prefix="${CONDA_DIR}" && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ # TODO: use `micromamba config` or report issue. echo "show_channel_urls: true" >> "${CONDA_DIR}/.condarc" && \ echo "channels:" >> "${CONDA_DIR}/.condarc" && \ echo " - conda-forge" >> "${CONDA_DIR}/.condarc" && \ - micromamba install --prefix="${CONDA_DIR}" --quiet --yes \ + bin/micromamba install \ + --root-prefix="${CONDA_DIR}" \ + --prefix="${CONDA_DIR}" \ + --quiet --yes \ "${PYTHON_SPECIFIER}" \ 'mamba' \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ + # There should be just a single binary inside bin/ + rm bin/micromamba && \ + rmdir bin && \ # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html conda config --system --set auto_update_conda false && \ # Pin major.minor version of python From c09a07da459bd970bedf5260802481c159c79463 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 2 Feb 2022 22:29:51 +0100 Subject: [PATCH 07/36] Switch from wget to curl --- base-notebook/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 7188933f..caf17056 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -41,6 +41,7 @@ RUN apt-get update --yes && \ # https://github.com/krallin/tini#why-tini for details. tini \ wget \ + curl \ # - bzip2 is necessary to extract the micromamba executable. bzip2 && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ @@ -104,7 +105,7 @@ RUN set -x && \ # Should be simpler, see arch="64"; \ fi && \ - wget -qO- "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" \ + curl -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" \ | tar -xvj bin/micromamba && \ # bin/micromamba create --prefix="${CONDA_DIR}" && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ From 3d81383c6b9b950d0cdef5ac6fed834c811b754b Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 2 Feb 2022 22:56:46 +0100 Subject: [PATCH 08/36] Add lots of debugging --- base-notebook/Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index caf17056..c7b42d49 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -99,14 +99,18 @@ ARG PYTHON_VERSION=3.9 # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change WORKDIR /tmp +# hadolint ignore=DL4001 RUN set -x && \ arch=$(uname -m) && \ if [ "${arch}" == "x86_64" ]; then \ # Should be simpler, see arch="64"; \ fi && \ - curl -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" \ - | tar -xvj bin/micromamba && \ + curl --verbose -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" > micromamba.tar.bz2 && \ + wget --debug -qO micromamba2.tar.bz2 "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ + ls -al && \ + md5sum ./* && \ + tar -xvjf micromamba.tar.bz2 bin/micromamba && \ # bin/micromamba create --prefix="${CONDA_DIR}" && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ From 7ecf4c18d1ed51324123d82a6a1a6a67f3279e21 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 2 Feb 2022 23:18:48 +0100 Subject: [PATCH 09/36] More diagnostics --- base-notebook/Dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index c7b42d49..41e22700 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -99,17 +99,24 @@ ARG PYTHON_VERSION=3.9 # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change WORKDIR /tmp -# hadolint ignore=DL4001 +# hadolint ignore=DL4001,DL3047 RUN set -x && \ arch=$(uname -m) && \ if [ "${arch}" == "x86_64" ]; then \ # Should be simpler, see arch="64"; \ fi && \ - curl --verbose -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" > micromamba.tar.bz2 && \ + uname -a && \ + echo "${arch}" && \ + wget --version && \ + curl --version && \ wget --debug -qO micromamba2.tar.bz2 "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ ls -al && \ md5sum ./* && \ + head -c 100 micromamba2.tar.bz2 && \ + curl --verbose -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" > micromamba.tar.bz2 && \ + ls -al && \ + md5sum ./* && \ tar -xvjf micromamba.tar.bz2 bin/micromamba && \ # bin/micromamba create --prefix="${CONDA_DIR}" && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ From f731fc9f9354c60dd2e2bc7e8c3c54faec4bedae Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 3 Feb 2022 20:06:51 +0100 Subject: [PATCH 10/36] Get it running, I hope --- base-notebook/Dockerfile | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 41e22700..4e5564fe 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -5,6 +5,23 @@ # https://hub.docker.com/_/ubuntu/?tab=tags&name=focal ARG ROOT_CONTAINER=ubuntu:focal +## BEGIN MICROMAMBA WORKAROUND +# For some reason, downloading micromamba with wget or curl leads to a segfault. +# Download them in a separate stage with alpine. +# Context: +FROM alpine as download_micromamba_stage +WORKDIR /tmp +RUN set -x && \ + arch=$(uname -m) && \ + if [ "${arch}" = "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + echo $arch && \ + apk add --no-cache curl==7.80.0-r0 && \ + curl -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" > micromamba.tar.bz2 +## END MICROMAMBA WORKAROUND + FROM $ROOT_CONTAINER LABEL maintainer="Jupyter Project " @@ -99,24 +116,9 @@ ARG PYTHON_VERSION=3.9 # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change WORKDIR /tmp +COPY --chown="${NB_UID}:${NB_GID}" --from=download_micromamba_stage /tmp/micromamba.tar.bz2 /tmp # hadolint ignore=DL4001,DL3047 RUN set -x && \ - arch=$(uname -m) && \ - if [ "${arch}" == "x86_64" ]; then \ - # Should be simpler, see - arch="64"; \ - fi && \ - uname -a && \ - echo "${arch}" && \ - wget --version && \ - curl --version && \ - wget --debug -qO micromamba2.tar.bz2 "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ - ls -al && \ - md5sum ./* && \ - head -c 100 micromamba2.tar.bz2 && \ - curl --verbose -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" > micromamba.tar.bz2 && \ - ls -al && \ - md5sum ./* && \ tar -xvjf micromamba.tar.bz2 bin/micromamba && \ # bin/micromamba create --prefix="${CONDA_DIR}" && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ @@ -134,6 +136,7 @@ RUN set -x && \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ + rm micromamba.tar.bz2 && \ # There should be just a single binary inside bin/ rm bin/micromamba && \ rmdir bin && \ From fe247bf7a579ea271d7fff973160a18966ee4f2a Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 3 Feb 2022 21:55:09 +0100 Subject: [PATCH 11/36] Cleanup of #1594 To be merged into maresb:micromamba after we see if CI passes --- base-notebook/Dockerfile | 28 ++++++++++------------------ base-notebook/initial-condarc | 6 ++++++ 2 files changed, 16 insertions(+), 18 deletions(-) create mode 100644 base-notebook/initial-condarc diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 4e5564fe..8f05ccd7 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -10,7 +10,7 @@ ARG ROOT_CONTAINER=ubuntu:focal # Download them in a separate stage with alpine. # Context: FROM alpine as download_micromamba_stage -WORKDIR /tmp +# hadolint ignore=DL3018 RUN set -x && \ arch=$(uname -m) && \ if [ "${arch}" = "x86_64" ]; then \ @@ -18,8 +18,9 @@ RUN set -x && \ arch="64"; \ fi && \ echo $arch && \ - apk add --no-cache curl==7.80.0-r0 && \ - curl -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" > micromamba.tar.bz2 + apk add --no-cache curl && \ + curl -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" \ + -o /tmp/micromamba.tar.bz2 ## END MICROMAMBA WORKAROUND FROM $ROOT_CONTAINER @@ -58,7 +59,6 @@ RUN apt-get update --yes && \ # https://github.com/krallin/tini#why-tini for details. tini \ wget \ - curl \ # - bzip2 is necessary to extract the micromamba executable. bzip2 && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ @@ -115,19 +115,14 @@ ARG PYTHON_VERSION=3.9 # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change -WORKDIR /tmp COPY --chown="${NB_UID}:${NB_GID}" --from=download_micromamba_stage /tmp/micromamba.tar.bz2 /tmp -# hadolint ignore=DL4001,DL3047 +COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" RUN set -x && \ - tar -xvjf micromamba.tar.bz2 bin/micromamba && \ - # bin/micromamba create --prefix="${CONDA_DIR}" && \ + tar -xvjf /tmp/micromamba.tar.bz2 \ + --directory=/tmp --strip-components=1 bin/micromamba && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ - # TODO: use `micromamba config` or report issue. - echo "show_channel_urls: true" >> "${CONDA_DIR}/.condarc" && \ - echo "channels:" >> "${CONDA_DIR}/.condarc" && \ - echo " - conda-forge" >> "${CONDA_DIR}/.condarc" && \ - bin/micromamba install \ + /tmp/micromamba install \ --root-prefix="${CONDA_DIR}" \ --prefix="${CONDA_DIR}" \ --quiet --yes \ @@ -136,12 +131,9 @@ RUN set -x && \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ - rm micromamba.tar.bz2 && \ + rm /tmp/micromamba.tar.bz2 && \ # There should be just a single binary inside bin/ - rm bin/micromamba && \ - rmdir bin && \ - # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html - conda config --system --set auto_update_conda false && \ + rm /tmp/micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ jupyter notebook --generate-config && \ diff --git a/base-notebook/initial-condarc b/base-notebook/initial-condarc new file mode 100644 index 00000000..383aad3c --- /dev/null +++ b/base-notebook/initial-condarc @@ -0,0 +1,6 @@ +# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html + +auto_update_conda: false +show_channel_urls: true +channels: + - conda-forge From 293266e9be7fba77f1c4ceeaac024ba99f700c60 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 3 Feb 2022 22:03:34 +0100 Subject: [PATCH 12/36] Minimize extraneous changes --- base-notebook/Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 8f05ccd7..770b2229 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -98,11 +98,16 @@ RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ chmod g+w /etc/passwd && \ mkdir "/home/${NB_USER}/work" && \ - fix-permissions "${HOME}" + fix-permissions "${HOME}" && \ + fix-permissions "${CONDA_DIR}" USER ${NB_UID} ARG PYTHON_VERSION=3.9 +# Setup work directory for backward-compatibility +RUN mkdir "/home/${NB_USER}/work" && \ + fix-permissions "/home/${NB_USER}" + # Download and install Micromamba, and initialize Conda prefix. # # Similar projects using Micromamba: From 2e237cd3009101a63474db419694ca128473bdea Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 3 Feb 2022 22:04:43 +0100 Subject: [PATCH 13/36] Remove incorrectly added comment --- base-notebook/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 770b2229..10d70665 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -89,7 +89,6 @@ RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashr # Create NB_USER with name jovyan user with UID=1000 and in the 'users' group # and make sure these dirs are writable by the `users` group. -# Setup work directory for backward-compatibility RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ From 93df1105892f7ade3660675b48066053ae1309c6 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 3 Feb 2022 22:05:50 +0100 Subject: [PATCH 14/36] Remove obsolete comment --- base-notebook/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 10d70665..b41d2987 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -112,7 +112,6 @@ RUN mkdir "/home/${NB_USER}/work" && \ # Similar projects using Micromamba: # - Micromamba-Docker: # - repo2docker: -# Configure the environment # Install Python, Mamba, Jupyter Notebook, Lab, and Hub # Generate a notebook server config # Cleanup temporary files and remove Micromamba From 2bf1b25198ae18800f8a09a298e1d3ebfa92bfcc Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 3 Feb 2022 23:44:00 +0100 Subject: [PATCH 15/36] Delete extraneous mkdir --- base-notebook/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index b41d2987..82e42b9a 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -96,7 +96,6 @@ RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ mkdir -p "${CONDA_DIR}" && \ chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ chmod g+w /etc/passwd && \ - mkdir "/home/${NB_USER}/work" && \ fix-permissions "${HOME}" && \ fix-permissions "${CONDA_DIR}" From 4d45138b7de7789341a3ffbc93ea500f13c0569d Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Fri, 4 Feb 2022 12:27:43 +0300 Subject: [PATCH 16/36] Update base-notebook/Dockerfile --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 82e42b9a..2aae5326 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -118,7 +118,7 @@ RUN mkdir "/home/${NB_USER}/work" && \ # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change COPY --chown="${NB_UID}:${NB_GID}" --from=download_micromamba_stage /tmp/micromamba.tar.bz2 /tmp -COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" +COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" RUN set -x && \ tar -xvjf /tmp/micromamba.tar.bz2 \ --directory=/tmp --strip-components=1 bin/micromamba && \ From 11a28bafbd449d073c9c35c709348efdb78dbc66 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 4 Feb 2022 10:42:54 +0100 Subject: [PATCH 17/36] Use WORKDIR /tmp --- base-notebook/Dockerfile | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 2aae5326..30ff55d7 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -119,12 +119,20 @@ RUN mkdir "/home/${NB_USER}/work" && \ # files across image layers when the permissions change COPY --chown="${NB_UID}:${NB_GID}" --from=download_micromamba_stage /tmp/micromamba.tar.bz2 /tmp COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" +WORKDIR /tmp RUN set -x && \ - tar -xvjf /tmp/micromamba.tar.bz2 \ - --directory=/tmp --strip-components=1 bin/micromamba && \ + tar -xvjf micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ - /tmp/micromamba install \ + MAMBA_SPECIFIER="mamba" && \ + # Using fixed version of mamba in arm, because the latest one has problems with arm under qemu + # See: https://github.com/jupyter/docker-stacks/issues/1539 + arch=$(uname -m) && \ + if [ "${arch}" == "aarch64" ]; then \ + MAMBA_SPECIFIER='mamba<0.18'; \ + fi && \ + # Install the packages + ./micromamba install \ --root-prefix="${CONDA_DIR}" \ --prefix="${CONDA_DIR}" \ --quiet --yes \ @@ -133,9 +141,8 @@ RUN set -x && \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ - rm /tmp/micromamba.tar.bz2 && \ - # There should be just a single binary inside bin/ - rm /tmp/micromamba && \ + rm micromamba.tar.bz2 && \ + rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ jupyter notebook --generate-config && \ From 00218f25ec79a8b426e6a13af56448f0d0f86834 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Fri, 4 Feb 2022 12:45:32 +0300 Subject: [PATCH 18/36] rm micromamba.tar.bz2 earlier --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 30ff55d7..e9a87b34 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -122,6 +122,7 @@ COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" WORKDIR /tmp RUN set -x && \ tar -xvjf micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ + rm micromamba.tar.bz2 && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ MAMBA_SPECIFIER="mamba" && \ @@ -141,7 +142,6 @@ RUN set -x && \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ - rm micromamba.tar.bz2 && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ From acb73743a0b5c8c16e5316e46cc3e8909ebf5f07 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 4 Feb 2022 16:21:58 +0100 Subject: [PATCH 19/36] Downgrade micromamba version --- base-notebook/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index e9a87b34..f9ac17e3 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -10,6 +10,10 @@ ARG ROOT_CONTAINER=ubuntu:focal # Download them in a separate stage with alpine. # Context: FROM alpine as download_micromamba_stage + +# ARG MICROMAMBA_VERSION=latest +ARG MICROMAMBA_VERSION=0.17.0 + # hadolint ignore=DL3018 RUN set -x && \ arch=$(uname -m) && \ @@ -19,7 +23,7 @@ RUN set -x && \ fi && \ echo $arch && \ apk add --no-cache curl && \ - curl -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" \ + curl -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/${MICROMAMBA_VERSION}" \ -o /tmp/micromamba.tar.bz2 ## END MICROMAMBA WORKAROUND From 0e66bfdb05cce6529a75195eeeaf5b682c62e7de Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 4 Feb 2022 21:35:44 +0100 Subject: [PATCH 20/36] Upgrade micromamba again --- base-notebook/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index f9ac17e3..ab18de9a 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -11,8 +11,8 @@ ARG ROOT_CONTAINER=ubuntu:focal # Context: FROM alpine as download_micromamba_stage -# ARG MICROMAMBA_VERSION=latest -ARG MICROMAMBA_VERSION=0.17.0 +ARG MICROMAMBA_VERSION=latest +# ARG MICROMAMBA_VERSION=0.17.0 # hadolint ignore=DL3018 RUN set -x && \ From b69c06f3db4de49d07b52e19cb1c469e76ed7a87 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 6 Feb 2022 12:39:13 +0100 Subject: [PATCH 21/36] Bump to latest mamba --- base-notebook/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index ab18de9a..e3f77ac1 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -134,7 +134,8 @@ RUN set -x && \ # See: https://github.com/jupyter/docker-stacks/issues/1539 arch=$(uname -m) && \ if [ "${arch}" == "aarch64" ]; then \ - MAMBA_SPECIFIER='mamba<0.18'; \ + # MAMBA_SPECIFIER='mamba<0.18'; \ + MAMBA_SPECIFIER='mamba'; \ fi && \ # Install the packages ./micromamba install \ From 3319daec1d06da8748edda76560e76359529caff Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 10 Feb 2022 09:30:01 +0100 Subject: [PATCH 22/36] Bump Python to 3.10 Conda-forge now has a TF build for 3.10. --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index e3f77ac1..ac073b32 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -104,7 +104,7 @@ RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ fix-permissions "${CONDA_DIR}" USER ${NB_UID} -ARG PYTHON_VERSION=3.9 +ARG PYTHON_VERSION=3.10 # Setup work directory for backward-compatibility RUN mkdir "/home/${NB_USER}/work" && \ From da7e1c8bce7f3ea811bfa1b6d57c81ae52da0791 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 13 Feb 2022 12:59:54 +0100 Subject: [PATCH 23/36] Enable verbose for micromamba --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index ac073b32..ea04cfcb 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -141,7 +141,7 @@ RUN set -x && \ ./micromamba install \ --root-prefix="${CONDA_DIR}" \ --prefix="${CONDA_DIR}" \ - --quiet --yes \ + -vvv --yes \ "${PYTHON_SPECIFIER}" \ 'mamba' \ 'notebook' \ From 1922be6ef8174d4483c7823592eaa4dd9f24bf41 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 1 Apr 2022 09:47:00 +0200 Subject: [PATCH 24/36] Remove the hacks --- base-notebook/Dockerfile | 43 ++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index ea04cfcb..cb3b811e 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -5,28 +5,6 @@ # https://hub.docker.com/_/ubuntu/?tab=tags&name=focal ARG ROOT_CONTAINER=ubuntu:focal -## BEGIN MICROMAMBA WORKAROUND -# For some reason, downloading micromamba with wget or curl leads to a segfault. -# Download them in a separate stage with alpine. -# Context: -FROM alpine as download_micromamba_stage - -ARG MICROMAMBA_VERSION=latest -# ARG MICROMAMBA_VERSION=0.17.0 - -# hadolint ignore=DL3018 -RUN set -x && \ - arch=$(uname -m) && \ - if [ "${arch}" = "x86_64" ]; then \ - # Should be simpler, see - arch="64"; \ - fi && \ - echo $arch && \ - apk add --no-cache curl && \ - curl -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/${MICROMAMBA_VERSION}" \ - -o /tmp/micromamba.tar.bz2 -## END MICROMAMBA WORKAROUND - FROM $ROOT_CONTAINER LABEL maintainer="Jupyter Project " @@ -105,6 +83,7 @@ RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ USER ${NB_UID} ARG PYTHON_VERSION=3.10 +ARG MICROMAMBA_VERSION=latest # Setup work directory for backward-compatibility RUN mkdir "/home/${NB_USER}/work" && \ @@ -121,18 +100,26 @@ RUN mkdir "/home/${NB_USER}/work" && \ # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change -COPY --chown="${NB_UID}:${NB_GID}" --from=download_micromamba_stage /tmp/micromamba.tar.bz2 /tmp COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" WORKDIR /tmp RUN set -x && \ - tar -xvjf micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ - rm micromamba.tar.bz2 && \ + arch=$(uname -m) && \ + if [ "${arch}" = "x86_64" ]; then \ + # Should be simpler, see + arch="64"; \ + fi && \ + echo $arch && \ + # curl -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/${MICROMAMBA_VERSION}" \ + # -o /tmp/micromamba.tar.bz2 && \ + wget -qO /tmp/micromamba.tar.bz2 \ + "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/${MICROMAMBA_VERSION}" && \ + tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ + rm /tmp/micromamba.tar.bz2 && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ MAMBA_SPECIFIER="mamba" && \ # Using fixed version of mamba in arm, because the latest one has problems with arm under qemu # See: https://github.com/jupyter/docker-stacks/issues/1539 - arch=$(uname -m) && \ if [ "${arch}" == "aarch64" ]; then \ # MAMBA_SPECIFIER='mamba<0.18'; \ MAMBA_SPECIFIER='mamba'; \ @@ -141,9 +128,9 @@ RUN set -x && \ ./micromamba install \ --root-prefix="${CONDA_DIR}" \ --prefix="${CONDA_DIR}" \ - -vvv --yes \ + --yes \ "${PYTHON_SPECIFIER}" \ - 'mamba' \ + "${MAMBA_SPECIFIER}" \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ From 796ea4bf68e8319f44162353aa5b82596d04c9f3 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Fri, 1 Apr 2022 12:30:21 +0100 Subject: [PATCH 25/36] Update base-notebook/Dockerfile --- base-notebook/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index cb3b811e..96f55849 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -108,9 +108,6 @@ RUN set -x && \ # Should be simpler, see arch="64"; \ fi && \ - echo $arch && \ - # curl -L "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/${MICROMAMBA_VERSION}" \ - # -o /tmp/micromamba.tar.bz2 && \ wget -qO /tmp/micromamba.tar.bz2 \ "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/${MICROMAMBA_VERSION}" && \ tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ From 27040a5d0ad21e1d8de213b1ffe2a2dd69c71a63 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 1 Apr 2022 20:55:59 +0200 Subject: [PATCH 26/36] Pin Python to 3.9 --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 96f55849..5cf8b2eb 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -82,7 +82,7 @@ RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ fix-permissions "${CONDA_DIR}" USER ${NB_UID} -ARG PYTHON_VERSION=3.10 +ARG PYTHON_VERSION=3.9 ARG MICROMAMBA_VERSION=latest # Setup work directory for backward-compatibility From bfd803941b7da5f494be92f31d05712f8496b00a Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 3 Apr 2022 16:41:07 +0200 Subject: [PATCH 27/36] Allow pinning of MAMBA_VERSION --- base-notebook/Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 4507a64e..86a00a69 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -83,8 +83,11 @@ RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ fix-permissions "${CONDA_DIR}" USER ${NB_UID} + +# Pin versions here, or set them to "default" ARG PYTHON_VERSION=3.9 -ARG MICROMAMBA_VERSION=latest +ARG MAMBA_VERSION=default +ARG MICROMAMBA_VERSION=default # Setup work directory for backward-compatibility RUN mkdir "/home/${NB_USER}/work" && \ @@ -109,13 +112,15 @@ RUN set -x && \ # Should be simpler, see arch="64"; \ fi && \ + if [[ "${MICROMAMBA_VERSION}" == "default" ]]; then MICROMAMBA_VERSION="latest"; fi && \ wget -qO /tmp/micromamba.tar.bz2 \ "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/${MICROMAMBA_VERSION}" && \ tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ rm /tmp/micromamba.tar.bz2 && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ - MAMBA_SPECIFIER="mamba" && \ + MAMBA_SPECIFIER="mamba=${MAMBA_VERSION}" && \ + if [[ "${MAMBA_VERSION}" == "default" ]]; then MAMBA_SPECIFIER="mamba"; fi && \ # Using fixed version of mamba in arm, because the latest one has problems with arm under qemu # See: https://github.com/jupyter/docker-stacks/issues/1539 if [ "${arch}" == "aarch64" ]; then \ From 3f67c20598c72568d43df42ec43d32f08ad4c907 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Mon, 4 Apr 2022 12:24:18 +0200 Subject: [PATCH 28/36] Enable verbose mode on mamba install --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 86a00a69..a2ee0e89 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -131,7 +131,7 @@ RUN set -x && \ ./micromamba install \ --root-prefix="${CONDA_DIR}" \ --prefix="${CONDA_DIR}" \ - --yes \ + -vvv --yes \ "${PYTHON_SPECIFIER}" \ "${MAMBA_SPECIFIER}" \ 'notebook' \ From d10583c5641983114899fe6ab65fdd7612c84b13 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Mon, 4 Apr 2022 19:44:46 +0200 Subject: [PATCH 29/36] Slightly reduce micromamba verbosity --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index a2ee0e89..3d6daab2 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -131,7 +131,7 @@ RUN set -x && \ ./micromamba install \ --root-prefix="${CONDA_DIR}" \ --prefix="${CONDA_DIR}" \ - -vvv --yes \ + -vv --yes \ "${PYTHON_SPECIFIER}" \ "${MAMBA_SPECIFIER}" \ 'notebook' \ From 2d22fe1642def444973906afb393bc7e72762e89 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Tue, 5 Apr 2022 22:42:56 +0200 Subject: [PATCH 30/36] Try 1=download_threads=extract_threads --- base-notebook/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 3d6daab2..3890e9a2 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -127,6 +127,8 @@ RUN set -x && \ # MAMBA_SPECIFIER='mamba<0.18'; \ MAMBA_SPECIFIER='mamba'; \ fi && \ + ./micromamba config set download_threads 1 && \ + ./micromamba config set extract_threads 1 && \ # Install the packages ./micromamba install \ --root-prefix="${CONDA_DIR}" \ @@ -137,6 +139,8 @@ RUN set -x && \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ + ./micromamba config remove-key download_threads && \ + ./micromamba config remove-key extract_threads && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ From 0b772947582d824b3164173c6e2f18d92f0a3327 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 6 Apr 2022 11:03:54 +0200 Subject: [PATCH 31/36] Set lock_timeout=300 --- base-notebook/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 3890e9a2..872aba96 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -127,8 +127,7 @@ RUN set -x && \ # MAMBA_SPECIFIER='mamba<0.18'; \ MAMBA_SPECIFIER='mamba'; \ fi && \ - ./micromamba config set download_threads 1 && \ - ./micromamba config set extract_threads 1 && \ + ./micromamba config set lock_timeout 300 && \ # Install the packages ./micromamba install \ --root-prefix="${CONDA_DIR}" \ @@ -139,8 +138,7 @@ RUN set -x && \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ - ./micromamba config remove-key download_threads && \ - ./micromamba config remove-key extract_threads && \ + ./micromamba config remove-key lock_timeout && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ From 88fdffe4e62e18b37672999e7ee498bb1aefa5a1 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 29 Apr 2022 20:11:13 +0200 Subject: [PATCH 32/36] Workaround for hanging via extract_threads=1 --- base-notebook/Dockerfile | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 872aba96..0ec3308e 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -121,24 +121,21 @@ RUN set -x && \ if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ MAMBA_SPECIFIER="mamba=${MAMBA_VERSION}" && \ if [[ "${MAMBA_VERSION}" == "default" ]]; then MAMBA_SPECIFIER="mamba"; fi && \ - # Using fixed version of mamba in arm, because the latest one has problems with arm under qemu - # See: https://github.com/jupyter/docker-stacks/issues/1539 if [ "${arch}" == "aarch64" ]; then \ - # MAMBA_SPECIFIER='mamba<0.18'; \ - MAMBA_SPECIFIER='mamba'; \ + # Prevent libmamba from sporadically hanging on arm64 under QEMU + # + ./micromamba config set extract_threads 1; \ fi && \ - ./micromamba config set lock_timeout 300 && \ # Install the packages ./micromamba install \ --root-prefix="${CONDA_DIR}" \ --prefix="${CONDA_DIR}" \ - -vv --yes \ + --yes \ "${PYTHON_SPECIFIER}" \ "${MAMBA_SPECIFIER}" \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ - ./micromamba config remove-key lock_timeout && \ rm micromamba && \ # Pin major.minor version of python mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ From c9e507c94b07ae86ebcd14a3845099a2e5700175 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 29 Apr 2022 23:33:54 +0200 Subject: [PATCH 33/36] Don't exclude Mamba from tests --- tests/base-notebook/test_packages.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/base-notebook/test_packages.py b/tests/base-notebook/test_packages.py index dea67b80..accef2ea 100644 --- a/tests/base-notebook/test_packages.py +++ b/tests/base-notebook/test_packages.py @@ -68,11 +68,6 @@ EXCLUDED_PACKAGES = [ "ca-certificates", "conda-forge::blas[build=openblas]", "hdf5", - # TODO(asalikhov) - # When we remove a workaround for arm regarding mamba, we can - # test installation of mamba as well and remove this exception. - # See: - "mamba[version='<0.18']", "openssl", "protobuf", "python", From 0b66a720876bd1868ed9d3c72224131b4fcc6c63 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 30 Apr 2022 20:25:21 +0300 Subject: [PATCH 34/36] Keep apt list sorted --- base-notebook/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index b474ce56..d43d830b 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -26,6 +26,8 @@ RUN apt-get update --yes && \ # the ubuntu base image is rebuilt too seldom sometimes (less than once a month) apt-get upgrade --yes && \ apt-get install --yes --no-install-recommends \ + # - bzip2 is necessary to extract the micromamba executable. + bzip2 \ ca-certificates \ fonts-liberation \ locales \ @@ -41,9 +43,7 @@ RUN apt-get update --yes && \ # processes and such of the actual executable we want to start, see # https://github.com/krallin/tini#why-tini for details. tini \ - wget \ - # - bzip2 is necessary to extract the micromamba executable. - bzip2 && \ + wget && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ locale-gen From f6704028bc07d61523eb60d1461f9d99f6f2962c Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 30 Apr 2022 20:41:20 +0300 Subject: [PATCH 35/36] Remove mamba and micromamba explicit versions --- base-notebook/Dockerfile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index d43d830b..72b90a95 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -86,8 +86,6 @@ USER ${NB_UID} # Pin versions here, or set them to "default" ARG PYTHON_VERSION=3.9 -ARG MAMBA_VERSION=default -ARG MICROMAMBA_VERSION=default # Setup work directory for backward-compatibility RUN mkdir "/home/${NB_USER}/work" && \ @@ -112,15 +110,11 @@ RUN set -x && \ # Should be simpler, see arch="64"; \ fi && \ - if [[ "${MICROMAMBA_VERSION}" == "default" ]]; then MICROMAMBA_VERSION="latest"; fi && \ wget -qO /tmp/micromamba.tar.bz2 \ - "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/${MICROMAMBA_VERSION}" && \ + "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ rm /tmp/micromamba.tar.bz2 && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ - if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ - MAMBA_SPECIFIER="mamba=${MAMBA_VERSION}" && \ - if [[ "${MAMBA_VERSION}" == "default" ]]; then MAMBA_SPECIFIER="mamba"; fi && \ if [ "${arch}" == "aarch64" ]; then \ # Prevent libmamba from sporadically hanging on arm64 under QEMU # @@ -132,7 +126,7 @@ RUN set -x && \ --prefix="${CONDA_DIR}" \ --yes \ "${PYTHON_SPECIFIER}" \ - "${MAMBA_SPECIFIER}" \ + 'mamba' \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ From a8579ea3329a9f9509acb369c0eef28a779e3b53 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 30 Apr 2022 20:43:47 +0300 Subject: [PATCH 36/36] Fix choosing python version --- base-notebook/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 72b90a95..376bb5fb 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -84,7 +84,7 @@ RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ USER ${NB_UID} -# Pin versions here, or set them to "default" +# Pin python version here, or set it to "default" ARG PYTHON_VERSION=3.9 # Setup work directory for backward-compatibility @@ -115,6 +115,7 @@ RUN set -x && \ tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ rm /tmp/micromamba.tar.bz2 && \ PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ + if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ if [ "${arch}" == "aarch64" ]; then \ # Prevent libmamba from sporadically hanging on arm64 under QEMU #