diff --git a/Makefile b/Makefile index 3ff3928d..2c4e6160 100644 --- a/Makefile +++ b/Makefile @@ -112,9 +112,9 @@ build-all-multi: $(foreach I, $(MULTI_IMAGES), build-multi/$(I)) $(foreach I, $( -check-outdated/%: ## check the outdated conda packages in a stack and produce a report (experimental) +check-outdated/%: ## check the outdated mamba/conda packages in a stack and produce a report (experimental) @TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py -check-outdated-all: $(foreach I, $(ALL_IMAGES), check-outdated/$(I)) ## check all the stacks for outdated conda packages +check-outdated-all: $(foreach I, $(ALL_IMAGES), check-outdated/$(I)) ## check all the stacks for outdated packages diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 36b8f56e..358f3e78 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -95,10 +95,10 @@ RUN set -x && \ # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html conda config --system --set auto_update_conda false && \ conda config --system --set show_channel_urls true && \ - if [[ "${PYTHON_VERSION}" != "default" ]]; then conda install --yes python="${PYTHON_VERSION}"; fi && \ - conda list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ - conda update --all --quiet --yes && \ - conda clean --all -f -y && \ + if [[ "${PYTHON_VERSION}" != "default" ]]; then mamba install --quiet --yes python="${PYTHON_VERSION}"; fi && \ + mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ + mamba update --all --quiet --yes && \ + mamba clean --all -f -y && \ rm -rf "/home/${NB_USER}/.cache/yarn" && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" @@ -109,11 +109,11 @@ RUN set -x && \ # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change -RUN conda install --quiet --yes \ +RUN mamba install --quiet --yes \ 'notebook' \ 'jupyterhub' \ 'jupyterlab' && \ - conda clean --all -f -y && \ + mamba clean --all -f -y && \ npm cache clean --force && \ jupyter notebook --generate-config && \ jupyter lab clean && \ diff --git a/binder/README.ipynb b/binder/README.ipynb index 31685ec3..0ee8c1c4 100644 --- a/binder/README.ipynb +++ b/binder/README.ipynb @@ -66,14 +66,14 @@ "metadata": {}, "outputs": [], "source": [ - "!which conda" + "!which mamba" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The user has read/write access to the root conda environment." + "The user has read/write access to the root mamba environment." ] }, { @@ -89,7 +89,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The following packages are conda-installed in the base image to support [Jupyter Notebook](https://github.com/jupyter/notebook), [JupyterLab](https://github.com/jupyterlab/jupyterlab), and their use in [JupyterHub](https://github.com/jupyterhub/jupyterhub) environments (e.g., [MyBinder](https://mybinder.org/))." + "The following packages are mamba-installed in the base image to support [Jupyter Notebook](https://github.com/jupyter/notebook), [JupyterLab](https://github.com/jupyterlab/jupyterlab), and their use in [JupyterHub](https://github.com/jupyterhub/jupyterhub) environments (e.g., [MyBinder](https://mybinder.org/))." ] }, { @@ -98,7 +98,7 @@ "metadata": {}, "outputs": [], "source": [ - "!conda list" + "!mamba list" ] }, { diff --git a/datascience-notebook/Dockerfile b/datascience-notebook/Dockerfile index 734fcae0..cf7770f7 100644 --- a/datascience-notebook/Dockerfile +++ b/datascience-notebook/Dockerfile @@ -54,7 +54,7 @@ RUN mkdir /etc/julia && \ USER ${NB_UID} # R packages including IRKernel which gets installed globally. -RUN conda install --quiet --yes \ +RUN mamba install --quiet --yes \ 'r-base' \ 'r-caret' \ 'r-crayon' \ @@ -75,7 +75,7 @@ RUN conda install --quiet --yes \ 'r-tidyverse' \ 'rpy2' \ 'unixodbc' && \ - conda clean --all -f -y && \ + mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/docs/contributing/stacks.md b/docs/contributing/stacks.md index 5614365f..9332da6b 100644 --- a/docs/contributing/stacks.md +++ b/docs/contributing/stacks.md @@ -18,7 +18,7 @@ Feel free to follow it or pave your own path using alternative services and buil First, install [cookiecutter](https://github.com/cookiecutter/cookiecutter) using pip or conda: ```bash -pip install cookiecutter # or conda install cookiecutter +pip install cookiecutter # or mamba install cookiecutter ``` Run the cookiecutter command pointing to the [jupyter/cookiecutter-docker-stacks](https://github.com/jupyter/cookiecutter-docker-stacks) project on GitHub. diff --git a/docs/using/common.md b/docs/using/common.md index f63d992f..b42ecb87 100644 --- a/docs/using/common.md +++ b/docs/using/common.md @@ -161,6 +161,11 @@ You can use either `pip`, `conda` or `mamba` to install new packages without any ```bash # install a package into the default (python 3.x) environment and cleanup after the installation +mamba install --quiet --yes some-package && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + pip install --quiet --no-cache-dir some-package && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" @@ -169,27 +174,22 @@ conda install --quiet --yes some-package && \ conda clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" - -mamba install --quiet --yes some-package && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" ``` ### Using alternative channels Conda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel. -However, alternative channels can be used either one shot by overwriting the default channel in the installation command or by configuring `conda` to use different channels. +However, alternative channels can be used either one shot by overwriting the default channel in the installation command or by configuring `mamba` to use different channels. The examples below show how to use the [anaconda default channels](https://repo.anaconda.com/pkgs/main) instead of `conda-forge` to install packages. ```bash # using defaults channels to install a package -conda install --channel defaults humanize +mamba install --channel defaults humanize # configure conda to add default channels at the top of the list conda config --system --prepend channels defaults # install a package -conda install --quiet --yes humanize && \ - conda clean --all -f -y && \ +mamba install --quiet --yes humanize && \ + mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" ``` diff --git a/docs/using/recipes.md b/docs/using/recipes.md index 31da8c44..e230c2b6 100644 --- a/docs/using/recipes.md +++ b/docs/using/recipes.md @@ -21,7 +21,7 @@ docker run -it -e GRANT_SUDO=yes --user root jupyter/minimal-notebook **You should only enable `sudo` if you trust the user and/or if the container is running on an isolated host.** See [Docker security documentation](https://docs.docker.com/engine/security/userns-remap/) for more information about running containers as `root`. -## Using `pip install` or `conda install` in a Child Docker image +## Using `mamba install` or `pip install` in a Child Docker image Create a new Dockerfile like the one shown below. @@ -61,8 +61,8 @@ For conda, the Dockerfile is similar: FROM jupyter/datascience-notebook:33add21fab64 # Install from requirements.txt file COPY --chown=${NB_UID}:${NB_GID} requirements.txt /tmp/ -RUN conda install --yes --file /tmp/requirements.txt && \ - conda clean --all -f -y && \ +RUN mamba install --yes --file /tmp/requirements.txt && \ + mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" ``` @@ -81,8 +81,8 @@ FROM jupyter/scipy-notebook:latest # Create a Python 2.x environment using conda including at least the ipython kernel # and the kernda utility. Add any additional packages you want available for use # in a Python 2 notebook to the first line here (e.g., pandas, matplotlib, etc.) -RUN conda create --quiet --yes -p "${CONDA_DIR}/envs/python2" python=2.7 ipython ipykernel kernda && \ - conda clean --all -f -y +RUN mamba create --quiet --yes -p "${CONDA_DIR}/envs/python2" python=2.7 ipython ipykernel kernda && \ + mamba clean --all -f -y USER root @@ -109,17 +109,17 @@ FROM jupyter/minimal-notebook:latest ARG conda_env=python36 ARG py_ver=3.6 -# you can add additional libraries you want conda to install by listing them below the first line and ending with "&& \" -RUN conda create --quiet --yes -p "${CONDA_DIR}/envs/${conda_env}" python=${py_ver} ipython ipykernel && \ - conda clean --all -f -y +# you can add additional libraries you want mamba to install by listing them below the first line and ending with "&& \" +RUN mamba create --quiet --yes -p "${CONDA_DIR}/envs/${conda_env}" python=${py_ver} ipython ipykernel && \ + mamba clean --all -f -y # alternatively, you can comment out the lines above and uncomment those below # if you'd prefer to use a YAML file present in the docker build context # COPY --chown=${NB_UID}:${NB_GID} environment.yml "/home/${NB_USER}/tmp/" # RUN cd "/home/${NB_USER}/tmp/" && \ -# conda env create -p "${CONDA_DIR}/envs/${conda_env}" -f environment.yml && \ -# conda clean --all -f -y +# mamba env create -p "${CONDA_DIR}/envs/${conda_env}" -f environment.yml && \ +# mamba clean --all -f -y # create Python 3.x environment and link it to jupyter @@ -195,8 +195,8 @@ notebooks, with no conversion, adding javascript Reveal.js: ```bash # Add Live slideshows with RISE -RUN conda install --quiet --yes -c damianavila82 rise && \ - conda clean --all -f -y && \ +RUN mamba install --quiet --yes -c damianavila82 rise && \ + mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" ``` @@ -206,12 +206,12 @@ Credit: [Paolo D.](https://github.com/pdonorio) based on ## xgboost -You need to install conda's gcc for Python xgboost to work properly. +You need to install conda-forge's gcc for Python xgboost to work properly. Otherwise, you'll get an exception about libgomp.so.1 missing GOMP_4.0. ```bash -conda install --quiet --yes gcc && \ - conda clean --all -f -y && \ +mamba install --quiet --yes gcc && \ + mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" @@ -521,29 +521,6 @@ RUN pip install --quiet --no-cache-dir jupyter_contrib_nbextensions && \ Ref: -## Enable auto-sklearn notebooks - -Using `auto-sklearn` requires `swig`, which the other notebook images lack, so it cant be experimented with. -Also, there is no Conda package for `auto-sklearn`. - -```dockerfile -ARG BASE_CONTAINER=jupyter/scipy-notebook -FROM jupyter/scipy-notebook:latest - -USER root - -# autosklearn requires swig, which no other image has -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends swig && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -USER ${NB_UID} - -RUN pip install --quiet --no-cache-dir auto-sklearn && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" -``` - ## Enable Delta Lake in Spark notebooks Please note that the [Delta Lake](https://delta.io/) packages are only available for Spark version > `3.0`. diff --git a/docs/using/selecting.md b/docs/using/selecting.md index 32507846..2ea012c1 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -28,7 +28,7 @@ It is the basis for all other stacks. - Minimally-functional Jupyter Notebook server (e.g., no LaTeX support for saving notebooks as PDFs) - [Miniforge](https://github.com/conda-forge/miniforge) Python 3.x in `/opt/conda` with two package managers - [conda](https://github.com/conda/conda): "cross-platform, language-agnostic binary package manager". - - [mamba](https://github.com/mamba-org/mamba): "reimplementation of the conda package manager in C++". + - [mamba](https://github.com/mamba-org/mamba): "reimplementation of the conda package manager in C++". We use this package manager by default when installing packages. - `notebook`, `jupyterhub` and `jupyterlab` packages - No preinstalled scientific computing packages - Unprivileged user `jovyan` (`uid=1000`, configurable, see options) in group `users` (`gid=100`) diff --git a/examples/docker-compose/notebook/Dockerfile b/examples/docker-compose/notebook/Dockerfile index f89cb0bf..ccee9eaf 100644 --- a/examples/docker-compose/notebook/Dockerfile +++ b/examples/docker-compose/notebook/Dockerfile @@ -12,5 +12,5 @@ USER root USER ${NB_UID} # Switch back to jovyan to avoid accidental container runs as root -# Add permanent pip/conda installs, data files, other user libs here +# Add permanent mamba/pip/conda installs, data files, other user libs here # e.g., RUN pip install --quiet --no-cache-dir flake8 diff --git a/examples/make-deploy/Dockerfile b/examples/make-deploy/Dockerfile index f89cb0bf..ccee9eaf 100644 --- a/examples/make-deploy/Dockerfile +++ b/examples/make-deploy/Dockerfile @@ -12,5 +12,5 @@ USER root USER ${NB_UID} # Switch back to jovyan to avoid accidental container runs as root -# Add permanent pip/conda installs, data files, other user libs here +# Add permanent mamba/pip/conda installs, data files, other user libs here # e.g., RUN pip install --quiet --no-cache-dir flake8 diff --git a/examples/source-to-image/README.md b/examples/source-to-image/README.md index 33efd904..80bf932e 100644 --- a/examples/source-to-image/README.md +++ b/examples/source-to-image/README.md @@ -99,8 +99,8 @@ The next steps are: ```bash if [ -f "/home/${NB_USER}/environment.yml" ]; then - conda env update --name root --file "/home/${NB_USER}/environment.yml" - conda clean --all -f -y + mamba env update --name root --file "/home/${NB_USER}/environment.yml" + mamba clean --all -f -y else if [ -f "/home/${NB_USER}/requirements.txt" ]; then pip --no-cache-dir install -r "/home/${NB_USER}/requirements.txt" diff --git a/examples/source-to-image/assemble b/examples/source-to-image/assemble index d03b1092..56f23aa9 100755 --- a/examples/source-to-image/assemble +++ b/examples/source-to-image/assemble @@ -25,8 +25,8 @@ rm -rf /tmp/src # use 'pip' instead. if [ -f "/home/${NB_USER}/environment.yml" ]; then - conda env update --name root --file "/home/${NB_USER}/environment.yml" - conda clean --all -f -y + mamba env update --name root --file "/home/${NB_USER}/environment.yml" + mamba clean --all -f -y else if [ -f "/home/${NB_USER}/requirements.txt" ]; then pip --no-cache-dir install -r "/home/${NB_USER}/requirements.txt" diff --git a/r-notebook/Dockerfile b/r-notebook/Dockerfile index 8f07d406..b0110fc6 100644 --- a/r-notebook/Dockerfile +++ b/r-notebook/Dockerfile @@ -25,11 +25,12 @@ RUN ln -s /bin/tar /bin/gtar USER ${NB_UID} # R packages including IRKernel which gets installed globally. -RUN conda install --quiet --yes \ +RUN mamba install --quiet --yes \ 'r-base' \ 'r-caret' \ 'r-crayon' \ 'r-devtools' \ + 'r-e1071' # dependency of the caret R package \ 'r-forecast' \ 'r-hexbin' \ 'r-htmltools' \ @@ -45,12 +46,6 @@ RUN conda install --quiet --yes \ 'r-tidymodels' \ 'r-tidyverse' \ 'unixodbc' && \ - conda clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -# Install e1071 R package (dependency of the caret R package) -RUN conda install --quiet --yes 'r-e1071' && \ - conda clean --all -f -y && \ + mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/scipy-notebook/Dockerfile b/scipy-notebook/Dockerfile index d6df8e20..27d104cc 100644 --- a/scipy-notebook/Dockerfile +++ b/scipy-notebook/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update --yes && \ USER ${NB_UID} # Install Python 3 packages -RUN conda install --quiet --yes \ +RUN mamba install --quiet --yes \ 'altair' \ 'beautifulsoup4' \ 'bokeh' \ @@ -46,7 +46,7 @@ RUN conda install --quiet --yes \ 'sympy' \ 'widgetsnbextension'\ 'xlrd' && \ - conda clean --all -f -y && \ + mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/tagging/manifests.py b/tagging/manifests.py index a7d1edc3..2c262fc1 100644 --- a/tagging/manifests.py +++ b/tagging/manifests.py @@ -67,9 +67,9 @@ class CondaEnvironmentManifest(ManifestInterface): "", quoted_output(container, "python --version"), "", - quoted_output(container, "conda info"), + quoted_output(container, "mamba info"), "", - quoted_output(container, "conda list"), + quoted_output(container, "mamba list"), ] ) diff --git a/test/helpers.py b/test/helpers.py index 98b8d75c..0e1dc25f 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -55,8 +55,8 @@ class CondaPackageHelper: @staticmethod def _conda_export_command(from_history=False): - """Return the conda export command with or without history""" - cmd = ["conda", "env", "export", "-n", "base", "--json", "--no-builds"] + """Return the mamba export command with or without history""" + cmd = ["mamba", "env", "export", "-n", "base", "--json", "--no-builds"] if from_history: cmd.append("--from-history") return cmd @@ -92,7 +92,7 @@ class CondaPackageHelper: # dependencies = filter(lambda x: isinstance(x, str), json.loads(env_export).get("dependencies")) dependencies = json.loads(env_export).get("dependencies") # Filtering packages installed through pip in this case it's a dict {'pip': ['toree==0.3.0']} - # Since we only manage packages installed through conda here + # Since we only manage packages installed through mamba here dependencies = filter(lambda x: isinstance(x, str), dependencies) packages_dict = dict() for split in map(lambda x: re.split("=?=", x), dependencies): @@ -114,9 +114,9 @@ class CondaPackageHelper: """Return the available packages""" if self.available is None: LOGGER.info("Grabing the list of available packages (can take a while) ...") - # Keeping command line output since `conda search --outdated --json` is way too long ... + # Keeping command line output since `mamba search --outdated --json` is way too long ... self.available = CondaPackageHelper._extract_available( - self._execute_command(["conda", "search", "--outdated"]) + self._execute_command(["mamba", "search", "--outdated"]) ) return self.available diff --git a/test/test_packages.py b/test/test_packages.py index 01240aff..92d51ee2 100644 --- a/test/test_packages.py +++ b/test/test_packages.py @@ -12,7 +12,7 @@ The goal is to detect import errors that can be caused by incompatibilities betw - #1012: issue importing `sympy` - #966: isssue importing `pyarrow` -This module checks dynamically, through the `CondaPackageHelper`, only the specified packages i.e. packages requested by `conda install` in the `Dockerfile`s. +This module checks dynamically, through the `CondaPackageHelper`, only the specified packages i.e. packages requested by `mamba install` in the `Dockerfile`s. This means that it does not check dependencies. This choice is a tradeoff to cover the main requirements while achieving reasonable test duration. However it could be easily changed (or completed) to cover also dependencies `package_helper.installed_packages()` instead of `package_helper.specified_packages()`. @@ -149,7 +149,7 @@ def _import_packages(package_helper, filtered_packages, check_function, max_fail @pytest.fixture(scope="function") def r_packages(packages): """Return an iterable of R packages""" - # package[2:] is to remove the leading "r-" appended by conda on R packages + # package[2:] is to remove the leading "r-" appended on R packages return map( lambda package: package_map(package[2:]), filter(r_package_predicate, packages) )