From 96ab5502e1e582063d543b4bc4e2adeb1bd7f837 Mon Sep 17 00:00:00 2001 From: romainx Date: Fri, 13 Nov 2020 17:55:31 +0100 Subject: [PATCH 1/9] Miniconda -> Miniforge --- base-notebook/Dockerfile | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 8d2a3bb9..629764ee 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -19,16 +19,16 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Miniconda installation +# Miniforge installation # Default values can be overridden at build time # (ARGS are in lower case to distinguish them from ENV) -# Check https://repo.anaconda.com/miniconda/ -# Miniconda archive to install -ARG miniconda_version="4.8.3" -# Archive MD5 checksum -ARG miniconda_checksum="d63adf39f2c220950a063e0529d4ff74" -# Conda version that can be different from the archive +# Check https://github.com/conda-forge/miniforge/releases +# Conda version ARG conda_version="4.9.0" +# Miniforge archive to install +ARG miniforge_version="${conda_version}-3" +# Miniforge installer +ARG miniforge_installer="Miniforge3-${miniforge_version}-Linux-x86_64.sh" # Install all OS dependencies for notebook server that starts but lacks all # features (e.g., download as all possible file formats) @@ -57,7 +57,8 @@ ENV CONDA_DIR=/opt/conda \ LANG=en_US.UTF-8 \ LANGUAGE=en_US.UTF-8 ENV PATH=$CONDA_DIR/bin:$PATH \ - HOME=/home/$NB_USER + HOME=/home/$NB_USER \ + CONDA_VERSION="${conda_version}" # Copy a script that we will use to correct permissions after running certain commands COPY fix-permissions /usr/local/bin/fix-permissions @@ -82,25 +83,22 @@ RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ fix-permissions $CONDA_DIR USER $NB_UID -WORKDIR $HOME ARG PYTHON_VERSION=default # Setup work directory for backward-compatibility -RUN mkdir /home/$NB_USER/work && \ - fix-permissions /home/$NB_USER - -# Install conda as jovyan and check the md5 sum provided on the download site -ENV MINICONDA_VERSION="${miniconda_version}" \ - CONDA_VERSION="${conda_version}" +RUN mkdir "/home/$NB_USER/work" && \ + fix-permissions "/home/$NB_USER" +# Install conda as jovyan and check the sha256 sum provided on the download site WORKDIR /tmp -RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \ - echo "${miniconda_checksum} *Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \ - /bin/bash Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ - rm Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \ + +RUN wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${miniforge_version}/${miniforge_installer}" && \ + wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${miniforge_version}/${miniforge_installer}.sha256" && \ + sha256sum -c "${miniforge_installer}.sha256" && \ + /bin/bash "${miniforge_installer}" -f -b -p $CONDA_DIR && \ + rm "${miniforge_installer}" "${miniforge_installer}.sha256" && \ # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \ - conda config --system --prepend channels conda-forge && \ conda config --system --set auto_update_conda false && \ conda config --system --set show_channel_urls true && \ conda config --system --set channel_priority strict && \ From 9194225482c9a99e6ad8553b14f4ebbf4a0430a7 Mon Sep 17 00:00:00 2001 From: romainx Date: Fri, 13 Nov 2020 18:32:36 +0100 Subject: [PATCH 2/9] Exclude bzip2 from python import tests --- test/test_packages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_packages.py b/test/test_packages.py index 46a6ecb3..6c44a66f 100644 --- a/test/test_packages.py +++ b/test/test_packages.py @@ -68,6 +68,7 @@ EXCLUDED_PACKAGES = [ "protobuf", "r-irkernel", "unixodbc", + "bzip2" ] From 8763e8c628b3ce79dea1b431ed5d6ca6514c287a Mon Sep 17 00:00:00 2001 From: romainx Date: Fri, 13 Nov 2020 19:29:02 +0100 Subject: [PATCH 3/9] Doc update --- docs/using/selecting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using/selecting.md b/docs/using/selecting.md index 9d0d254b..13d5a300 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -28,7 +28,7 @@ and versioning strategy. [options common across all core stacks](common.md). It is the basis for all other stacks. - Minimally-functional Jupyter Notebook server (e.g., no LaTeX support for saving notebooks as PDFs) -- [Miniconda](https://conda.io/miniconda.html) Python 3.x in `/opt/conda` +- [Miniforge](https://github.com/conda-forge/miniforge) Python 3.x in `/opt/conda` - No preinstalled scientific computing packages - Unprivileged user `jovyan` (`uid=1000`, configurable, see options) in group `users` (`gid=100`) with ownership over the `/home/jovyan` and `/opt/conda` paths From d7b51cf6b82b2957802040eef49b09989dcf20bf Mon Sep 17 00:00:00 2001 From: romainx Date: Fri, 13 Nov 2020 21:40:45 +0100 Subject: [PATCH 4/9] Add checksum in the dockerfile instead of downloading it --- base-notebook/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 629764ee..fcc98f38 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -27,8 +27,9 @@ USER root ARG conda_version="4.9.0" # Miniforge archive to install ARG miniforge_version="${conda_version}-3" -# Miniforge installer +# Miniforge installer and its checksum ARG miniforge_installer="Miniforge3-${miniforge_version}-Linux-x86_64.sh" +ARG miniforge_checksum="29f0eb17dd02aceb0dfd4dad2654e974b1699baed06ee6d350b0ab4a2ccf3d02" # Install all OS dependencies for notebook server that starts but lacks all # features (e.g., download as all possible file formats) @@ -93,10 +94,9 @@ RUN mkdir "/home/$NB_USER/work" && \ WORKDIR /tmp RUN wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${miniforge_version}/${miniforge_installer}" && \ - wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${miniforge_version}/${miniforge_installer}.sha256" && \ - sha256sum -c "${miniforge_installer}.sha256" && \ + echo "${miniforge_checksum} *${miniforge_installer}" | sha256sum --check && \ /bin/bash "${miniforge_installer}" -f -b -p $CONDA_DIR && \ - rm "${miniforge_installer}" "${miniforge_installer}.sha256" && \ + rm "${miniforge_installer}" && \ # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \ conda config --system --set auto_update_conda false && \ From f9167c26196f3a4458a257df631a685779d05e41 Mon Sep 17 00:00:00 2001 From: romainx Date: Sat, 14 Nov 2020 11:56:53 +0100 Subject: [PATCH 5/9] strict channel priority is no more necessary --- base-notebook/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index fcc98f38..9b91f7df 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -101,7 +101,6 @@ RUN wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${m echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \ conda config --system --set auto_update_conda false && \ conda config --system --set show_channel_urls true && \ - conda config --system --set channel_priority strict && \ if [ ! $PYTHON_VERSION = 'default' ]; then conda install --yes python=$PYTHON_VERSION; fi && \ conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed 's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \ conda install --quiet --yes "conda=${CONDA_VERSION}" && \ From 7d550a0a835bb4d5d0ecd8f0bd41417aedffb7f3 Mon Sep 17 00:00:00 2001 From: romainx Date: Sat, 14 Nov 2020 11:58:07 +0100 Subject: [PATCH 6/9] Document how to use defaults channels to install packages --- docs/using/common.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/using/common.md b/docs/using/common.md index 8ad83c9c..6012fcaa 100644 --- a/docs/using/common.md +++ b/docs/using/common.md @@ -113,3 +113,17 @@ pip install some-package conda install some-package ``` +### Using alternative channels + +Conda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel. +However, alternative channels can be used either one shot by overwriting the default channel in the installation command or by configuring `conda` to use different channels. +The examples below shows how to use the [anaconda default channels](https://repo.anaconda.com/pkgs/main) instead of `conda-forge` to install packages. + +```bash +# using defaults channels to install a package +conda install --channel defaults humanize +# configure conda to add default channels at the top of the list +conda config --system --prepend channels defaults +# install a package +conda install humanize +``` \ No newline at end of file From c6764656cc1fc10a5a6927316fdb57b1b667421d Mon Sep 17 00:00:00 2001 From: romainx Date: Sat, 14 Nov 2020 12:03:13 +0100 Subject: [PATCH 7/9] typo --- docs/using/common.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using/common.md b/docs/using/common.md index 6012fcaa..09b7fa9d 100644 --- a/docs/using/common.md +++ b/docs/using/common.md @@ -117,7 +117,7 @@ conda install some-package Conda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel. However, alternative channels can be used either one shot by overwriting the default channel in the installation command or by configuring `conda` to use different channels. -The examples below shows how to use the [anaconda default channels](https://repo.anaconda.com/pkgs/main) instead of `conda-forge` to install packages. +The examples below show how to use the [anaconda default channels](https://repo.anaconda.com/pkgs/main) instead of `conda-forge` to install packages. ```bash # using defaults channels to install a package From 1d2581b4f831d64138c94d92e1247ee2348b170b Mon Sep 17 00:00:00 2001 From: romainx Date: Sat, 14 Nov 2020 18:49:08 +0100 Subject: [PATCH 8/9] Split installer definition in several args --- base-notebook/Dockerfile | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 9b91f7df..a61feb57 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -19,16 +19,25 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Miniforge installation +# ---- Miniforge installer ---- # Default values can be overridden at build time # (ARGS are in lower case to distinguish them from ENV) # Check https://github.com/conda-forge/miniforge/releases # Conda version ARG conda_version="4.9.0" +# Miniforge installer patch version +ARG miniforge_patch_number="3" +# Miniforge installer architecture +ARG miniforge_arch="x86_64" +# Python implementation to use +# can be either Miniforge3 to use Python or Miniforge-pypy3 to use PyPy +ARG miniforge_python="Miniforge3" + # Miniforge archive to install -ARG miniforge_version="${conda_version}-3" -# Miniforge installer and its checksum -ARG miniforge_installer="Miniforge3-${miniforge_version}-Linux-x86_64.sh" +ARG miniforge_version="${conda_version}-${miniforge_patch_number}" +# Miniforge installer +ARG miniforge_installer="${miniforge_python}-${miniforge_version}-Linux-${miniforge_arch}.sh" +# Miniforge checksum ARG miniforge_checksum="29f0eb17dd02aceb0dfd4dad2654e974b1699baed06ee6d350b0ab4a2ccf3d02" # Install all OS dependencies for notebook server that starts but lacks all From 5f140e6972a2c6ec9f0c924546e50ac1c6e05e34 Mon Sep 17 00:00:00 2001 From: romainx Date: Wed, 25 Nov 2020 21:35:05 +0100 Subject: [PATCH 9/9] Bump Miniforge patch number to the latest released Fix the CondaPackageHelper to handle cases when conda spec is expressed with two `=` signs instead of a single one. --- base-notebook/Dockerfile | 4 ++-- test/helpers.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index a61feb57..b383b6a1 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -26,7 +26,7 @@ USER root # Conda version ARG conda_version="4.9.0" # Miniforge installer patch version -ARG miniforge_patch_number="3" +ARG miniforge_patch_number="4" # Miniforge installer architecture ARG miniforge_arch="x86_64" # Python implementation to use @@ -38,7 +38,7 @@ ARG miniforge_version="${conda_version}-${miniforge_patch_number}" # Miniforge installer ARG miniforge_installer="${miniforge_python}-${miniforge_version}-Linux-${miniforge_arch}.sh" # Miniforge checksum -ARG miniforge_checksum="29f0eb17dd02aceb0dfd4dad2654e974b1699baed06ee6d350b0ab4a2ccf3d02" +ARG miniforge_checksum="dae28a05f0fcfed0b47c66468e8434ab42cb1ff90de96540a506949cdecd2b5a" # Install all OS dependencies for notebook server that starts but lacks all # features (e.g., download as all possible file formats) diff --git a/test/helpers.py b/test/helpers.py index 01d46bb6..867b19a9 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -93,11 +93,11 @@ class CondaPackageHelper: # Since we only manage packages installed through conda here dependencies = filter(lambda x: isinstance(x, str), dependencies) packages_dict = dict() - for split in map(lambda x: x.split("=", 1), dependencies): + for split in map(lambda x: re.split("=?=", x), dependencies): # default values package = split[0] version = set() - # cheking if it's a proper version by testing if the first char is a digit + # checking if it's a proper version by testing if the first char is a digit if len(split) > 1: if split[1][0].isdigit(): # package + version case