From c8f5fbd61f26193c01d749abb4cfdf45a6cfa76d Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Oct 2024 23:15:14 +0000 Subject: [PATCH 01/11] Update gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 14241e2c..b0f3c848 100644 --- a/.gitignore +++ b/.gitignore @@ -106,8 +106,10 @@ ipython_config.py #pdm.lock # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it # in version control. -# https://pdm.fming.dev/#use-with-ide +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control .pdm.toml +.pdm-python +.pdm-build/ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ From a1265be650bff7d1d38f0f2f68c10e32c369aedf Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Mon, 28 Oct 2024 12:23:29 +0000 Subject: [PATCH 02/11] Get rid of abseil warnings --- tests/tensorflow-notebook/units/unit_tensorflow.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/tensorflow-notebook/units/unit_tensorflow.py b/tests/tensorflow-notebook/units/unit_tensorflow.py index 96446a5d..9271a0e3 100644 --- a/tests/tensorflow-notebook/units/unit_tensorflow.py +++ b/tests/tensorflow-notebook/units/unit_tensorflow.py @@ -1,6 +1,13 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +import os + import tensorflow as tf +# Suppress logging warnings +# https://stackoverflow.com/a/78803598 +os.environ["GRPC_VERBOSITY"] = "ERROR" +os.environ["GLOG_minloglevel"] = "2" + print(tf.constant("Hello, TensorFlow")) print(tf.reduce_sum(tf.random.normal([1000, 1000]))) From f74a764584b6dfd9446a3b5b617f628ff869ecb5 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Mon, 28 Oct 2024 14:06:50 +0000 Subject: [PATCH 03/11] Do not install tensorflow 2.18.0 as it gives an error --- images/tensorflow-notebook/cuda/Dockerfile | 2 +- tests/tensorflow-notebook/units/unit_tensorflow.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/images/tensorflow-notebook/cuda/Dockerfile b/images/tensorflow-notebook/cuda/Dockerfile index 5018dee4..6a0dbd03 100644 --- a/images/tensorflow-notebook/cuda/Dockerfile +++ b/images/tensorflow-notebook/cuda/Dockerfile @@ -14,7 +14,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install TensorFlow, CUDA and cuDNN with pip RUN pip install --no-cache-dir \ "jupyter-server-proxy" \ - "tensorflow[and-cuda]" && \ + "tensorflow[and-cuda]<=2.17.1" && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/tests/tensorflow-notebook/units/unit_tensorflow.py b/tests/tensorflow-notebook/units/unit_tensorflow.py index 9271a0e3..96446a5d 100644 --- a/tests/tensorflow-notebook/units/unit_tensorflow.py +++ b/tests/tensorflow-notebook/units/unit_tensorflow.py @@ -1,13 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -import os - import tensorflow as tf -# Suppress logging warnings -# https://stackoverflow.com/a/78803598 -os.environ["GRPC_VERBOSITY"] = "ERROR" -os.environ["GLOG_minloglevel"] = "2" - print(tf.constant("Hello, TensorFlow")) print(tf.reduce_sum(tf.random.normal([1000, 1000]))) From 03e5fe572de01f0e825312b8bc49146aec8cdf6c Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 29 Oct 2024 10:11:31 +0000 Subject: [PATCH 04/11] Fix docs: we're not installing stable version of spark anymore (#2165) --- docs/using/specifics.md | 3 ++- images/pyspark-notebook/Dockerfile | 2 +- images/pyspark-notebook/setup_spark.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/using/specifics.md b/docs/using/specifics.md index 4df770f1..0ed2f55d 100644 --- a/docs/using/specifics.md +++ b/docs/using/specifics.md @@ -49,7 +49,8 @@ You can build a `pyspark-notebook` image with a different `Spark` version by ove - This version needs to match the version supported by the Spark distribution used above. - See [Spark Overview](https://spark.apache.org/docs/latest/#downloading) and [Ubuntu packages](https://packages.ubuntu.com/search?keywords=openjdk). - `spark_version` (optional): The Spark version to install, for example `3.5.0`. - If not specified (this is the default), latest stable Spark will be installed. + If not specified (this is the default), latest Spark will be installed. + Note: to support Python 3.12, we currently install Spark v4 preview versions: . - `hadoop_version`: The Hadoop version (`3` by default). Note, that _Spark < 3.3_ require to specify `major.minor` Hadoop version (i.e. `3.2`). - `scala_version` (optional): The Scala version, for example `2.13` (not specified by default). diff --git a/images/pyspark-notebook/Dockerfile b/images/pyspark-notebook/Dockerfile index 8585232c..be4bdfa4 100644 --- a/images/pyspark-notebook/Dockerfile +++ b/images/pyspark-notebook/Dockerfile @@ -24,7 +24,7 @@ RUN apt-get update --yes && \ ca-certificates-java && \ apt-get clean && rm -rf /var/lib/apt/lists/* -# If spark_version is not set, latest stable Spark will be installed +# If spark_version is not set, latest Spark will be installed ARG spark_version ARG hadoop_version="3" # If scala_version is not set, Spark without Scala will be installed diff --git a/images/pyspark-notebook/setup_spark.py b/images/pyspark-notebook/setup_spark.py index 79e571af..c5b76433 100755 --- a/images/pyspark-notebook/setup_spark.py +++ b/images/pyspark-notebook/setup_spark.py @@ -29,11 +29,11 @@ def get_all_refs(url: str) -> list[str]: def get_latest_spark_version() -> str: """ - Returns the last stable version of Spark using spark archive + Returns the last version of Spark using spark archive """ LOGGER.info("Downloading Spark versions information") all_refs = get_all_refs("https://archive.apache.org/dist/spark/") - stable_versions = [ + versions = [ ref.removeprefix("spark-").removesuffix("/") for ref in all_refs if ref.startswith("spark-") and "incubating" not in ref @@ -49,7 +49,7 @@ def get_latest_spark_version() -> str: patch, _, preview = arr[2].partition("-") return (major, minor, int(patch), preview) - latest_version = max(stable_versions, key=lambda ver: version_array(ver)) + latest_version = max(versions, key=lambda ver: version_array(ver)) LOGGER.info(f"Latest version: {latest_version}") return latest_version From 5841857378358c3a708cc4646045ee02c6da687e Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Mon, 4 Nov 2024 13:26:53 +0000 Subject: [PATCH 05/11] Fix broken link --- docs/maintaining/aarch64-runner.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/maintaining/aarch64-runner.md b/docs/maintaining/aarch64-runner.md index 1f27d084..3850760d 100644 --- a/docs/maintaining/aarch64-runner.md +++ b/docs/maintaining/aarch64-runner.md @@ -7,7 +7,7 @@ Each runner is recommended to have at least _2 cores_ and _30 GB_ of disk space. Add a new runner: - To use [Oracle OCI](https://www.oracle.com/cloud-0/), create a compute instance `VM.Standard.A1.Flex`. -- To use [Google Cloud](https://cloud.google.com), use [this instruction](https://cloud.google.com/compute/docs/instances/create-arm-vm-instance#armpublicimage). +- To use [Google Cloud](https://cloud.google.com), use [this instruction](https://cloud.google.com/compute/docs/instances/create-start-instance). Configure your runner: From c2d9d1f06a35d56ffe84d2360fb562b8b3bf5165 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Mon, 4 Nov 2024 14:55:35 +0000 Subject: [PATCH 06/11] Pin min notebook version (#2167) * [TMP] Pin min notebook version * Update Dockerfile --- images/base-notebook/Dockerfile | 6 +++++- tests/docker-stacks-foundation/test_packages.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/images/base-notebook/Dockerfile b/images/base-notebook/Dockerfile index 0cde5b5d..dd068c80 100644 --- a/images/base-notebook/Dockerfile +++ b/images/base-notebook/Dockerfile @@ -42,7 +42,11 @@ RUN mamba install --yes \ 'jupyterhub' \ 'jupyterlab' \ 'nbclassic' \ - 'notebook' && \ + # Sometimes, when the new version of `jupyterlab` is released, latest `notebook` might not support it for some time + # Old versions of `notebook` (=7.2.2' && \ jupyter server --generate-config && \ mamba clean --all -f -y && \ npm cache clean --force && \ diff --git a/tests/docker-stacks-foundation/test_packages.py b/tests/docker-stacks-foundation/test_packages.py index 9b70be38..0508ff14 100644 --- a/tests/docker-stacks-foundation/test_packages.py +++ b/tests/docker-stacks-foundation/test_packages.py @@ -76,6 +76,7 @@ EXCLUDED_PACKAGES = [ "hdf5", "jupyterlab-git", "mamba[version='<2.0.0']", + "notebook[version='>", "openssl", "pandas[version='>", "protobuf", From 2c525aff70edc2cff2dc8f664facceb6a48a6179 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Mon, 4 Nov 2024 16:57:17 +0000 Subject: [PATCH 07/11] Use pytorch with cuda12.4 (#2168) --- images/pytorch-notebook/cuda12/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/pytorch-notebook/cuda12/Dockerfile b/images/pytorch-notebook/cuda12/Dockerfile index cdfd089a..983c439a 100644 --- a/images/pytorch-notebook/cuda12/Dockerfile +++ b/images/pytorch-notebook/cuda12/Dockerfile @@ -13,7 +13,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install PyTorch with pip (https://pytorch.org/get-started/locally/) # hadolint ignore=DL3013 -RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu121' \ +RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu124' \ 'torch' \ 'torchaudio' \ 'torchvision' && \ From 62a85b43b6b372e17ee0558587eec629c195e77b Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 5 Nov 2024 19:47:43 +0000 Subject: [PATCH 08/11] Update pre-commit hooks --- .pre-commit-config.yaml | 18 +++++++++--------- README.md | 3 +-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 25658fa1..549c39b2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: # Autoupdate: Python code - repo: https://github.com/asottile/pyupgrade - rev: v3.17.0 + rev: v3.19.0 hooks: - id: pyupgrade args: [--py39-plus] @@ -28,14 +28,14 @@ repos: # Autoformat: Python code - repo: https://github.com/psf/black - rev: 24.8.0 + rev: 24.10.0 hooks: - id: black args: [--target-version=py39] # Check python code static typing - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.2 + rev: v1.13.0 hooks: - id: mypy args: [--config, ./mypy.ini] @@ -69,7 +69,7 @@ repos: # `pre-commit sample-config` default hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: end-of-file-fixer @@ -78,7 +78,7 @@ repos: # Lint: Dockerfile - repo: https://github.com/hadolint/hadolint - rev: v2.13.0-beta + rev: v2.13.1-beta hooks: - id: hadolint-docker entry: hadolint/hadolint:v2.12.1-beta hadolint @@ -86,7 +86,7 @@ repos: # Lint: Dockerfile # We're linting .dockerfile files as well - repo: https://github.com/hadolint/hadolint - rev: v2.13.0-beta + rev: v2.13.1-beta hooks: - id: hadolint-docker name: Lint *.dockerfile Dockerfiles @@ -124,14 +124,14 @@ repos: # Lint: Markdown - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.41.0 + rev: v0.42.0 hooks: - id: markdownlint args: ["--fix"] # Strip output from Jupyter notebooks - repo: https://github.com/kynan/nbstripout - rev: 0.7.1 + rev: 0.8.0 hooks: - id: nbstripout @@ -149,7 +149,7 @@ repos: # Run black on python code blocks in documentation files. - repo: https://github.com/adamchainz/blacken-docs - rev: 1.18.0 + rev: 1.19.1 hooks: - id: blacken-docs # --skip-errors is added to allow us to have python syntax highlighting even if diff --git a/README.md b/README.md index 426d7bad..1c998030 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # Jupyter Docker Stacks -[![GitHub actions badge](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml/badge.svg) -](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml?query=branch%3Amain "Docker images build status") +[![GitHub actions badge](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml/badge.svg)](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml?query=branch%3Amain "Docker images build status") [![Read the Docs badge](https://img.shields.io/readthedocs/jupyter-docker-stacks.svg)](https://jupyter-docker-stacks.readthedocs.io/en/latest/ "Documentation build status") [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/jupyter/docker-stacks/main.svg)](https://results.pre-commit.ci/latest/github/jupyter/docker-stacks/main "pre-commit.ci build status") [![Discourse badge](https://img.shields.io/discourse/users.svg?color=%23f37626&server=https%3A%2F%2Fdiscourse.jupyter.org)](https://discourse.jupyter.org/ "Jupyter Discourse Forum") From 28a0b25bcec1a6ee25421ddf8a3e2d156259a3ed Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 6 Nov 2024 16:04:07 +0100 Subject: [PATCH 09/11] Install jupyterhub-base instead of jupyterhub (#2171) * Install jupyterhub-base instead of jupyterhub * Update images/base-notebook/Dockerfile --------- Co-authored-by: Ayaz Salikhov --- docs/using/recipe_code/jupyterhub_version.dockerfile | 2 +- images/base-notebook/Dockerfile | 8 +++++++- tests/docker-stacks-foundation/test_packages.py | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/using/recipe_code/jupyterhub_version.dockerfile b/docs/using/recipe_code/jupyterhub_version.dockerfile index 7fd53018..22c9dfa1 100644 --- a/docs/using/recipe_code/jupyterhub_version.dockerfile +++ b/docs/using/recipe_code/jupyterhub_version.dockerfile @@ -1,6 +1,6 @@ FROM quay.io/jupyter/base-notebook -RUN mamba install --yes 'jupyterhub==4.0.1' && \ +RUN mamba install --yes 'jupyterhub-base==4.0.1' && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/images/base-notebook/Dockerfile b/images/base-notebook/Dockerfile index dd068c80..261cf3e6 100644 --- a/images/base-notebook/Dockerfile +++ b/images/base-notebook/Dockerfile @@ -39,9 +39,15 @@ USER ${NB_UID} # files across image layers when the permissions change WORKDIR /tmp RUN mamba install --yes \ - 'jupyterhub' \ + 'jupyterhub-base' \ 'jupyterlab' \ 'nbclassic' \ + # nodejs has historically been installed indirectly as a dependency. + # When it was no longer getting installed indirectly, + # we started installing it explicitly to avoid introducing a breaking change + # for users building on top of these images. + # See: https://github.com/jupyter/docker-stacks/pull/2171 + 'nodejs' \ # Sometimes, when the new version of `jupyterlab` is released, latest `notebook` might not support it for some time # Old versions of `notebook` (", From ea849fa18f4818364bfcee999555757a44dc0154 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Wed, 6 Nov 2024 15:12:25 +0000 Subject: [PATCH 10/11] Update description of jupyter/base-notebook regarding jupyterhub-base and update changelog --- CHANGELOG.md | 6 ++++++ docs/using/selecting.md | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcda303f..6ba70a95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ This changelog only contains breaking and/or significant changes manually introduced to this repository (using Pull Requests). All image manifests can be found in [the wiki](https://github.com/jupyter/docker-stacks/wiki). +## 2024-11-06 + +Affected: all images except `docker-stacks-foundation`. + +- **Non-breaking** Install `jupyterhub-base` and `nodejs` packages instead of `jupyterhub` package ([#2171](https://github.com/jupyter/docker-stacks/pull/2171)). + ## 2024-10-23 Affected: all images. diff --git a/docs/using/selecting.md b/docs/using/selecting.md index 3ea4e85e..c21a1036 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -54,7 +54,9 @@ It contains: - Everything in `jupyter/docker-stacks-foundation` - Minimally functional Server (e.g., no LaTeX support for saving notebooks as PDFs) -- `notebook`, `jupyterhub`, and `jupyterlab` packages +- `notebook`, `jupyterhub-base`, and `jupyterlab` packages + Note: we're also installing `nodejs` as it has historically been installed indirectly as a dependency of `jupyterhub` package, which was used before. + See more at: - A `start-notebook.py` script as the default command - A `start-singleuser.py` script useful for launching containers in JupyterHub - Options for a self-signed HTTPS certificate From 2c7e64abbb088d89d7147a4b0724da70c531491a Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Wed, 6 Nov 2024 15:20:33 +0000 Subject: [PATCH 11/11] Do not run linkcheck on https://anaconda.org/conda-forge because of frequent read timeouts --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 7957c246..5cf59db1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -67,6 +67,7 @@ linkcheck_ignore = [ r"http://127\.0\.0\.1:.*", # various examples r"https://mybinder\.org/v2/gh/.*", # lots of 500 errors r"https://packages\.ubuntu\.com/search\?keywords=openjdk", # frequent read timeouts + r"https://anaconda\.org\/conda-forge", # frequent read timeouts ] linkcheck_allowed_redirects = {