Merge branch 'main' into mathbunnyru-patch-1

2025-10-18 07:22:57 +00:00 · 2024-11-07 15:15:31 +00:00
parent d8090b9a5a 2c7e64abbb
commit e78a6ae9c5
15 changed files with 50 additions and 25 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -106,8 +106,10 @@ ipython_config.py
 #pdm.lock
 #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
 #   in version control.
-#   https://pdm.fming.dev/#use-with-ide
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
 .pdm.toml
+.pdm-python
+.pdm-build/

 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -14,7 +14,7 @@
 repos:
  # Autoupdate: Python code
  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.17.0
+    rev: v3.19.0
    hooks:
      - id: pyupgrade
        args: [--py39-plus]
@@ -28,14 +28,14 @@ repos:

  # Autoformat: Python code
  - repo: https://github.com/psf/black
-    rev: 24.8.0
+    rev: 24.10.0
    hooks:
      - id: black
        args: [--target-version=py39]

  # Check python code static typing
  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.13.0
    hooks:
      - id: mypy
        args: [--config, ./mypy.ini]
@@ -69,7 +69,7 @@ repos:

  # `pre-commit sample-config` default hooks
  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
    hooks:
      - id: check-added-large-files
      - id: end-of-file-fixer
@@ -78,7 +78,7 @@ repos:

  # Lint: Dockerfile
  - repo: https://github.com/hadolint/hadolint
-    rev: v2.13.0-beta
+    rev: v2.13.1-beta
    hooks:
      - id: hadolint-docker
        entry: hadolint/hadolint:v2.12.1-beta hadolint
@@ -86,7 +86,7 @@ repos:
  # Lint: Dockerfile
  # We're linting .dockerfile files as well
  - repo: https://github.com/hadolint/hadolint
-    rev: v2.13.0-beta
+    rev: v2.13.1-beta
    hooks:
      - id: hadolint-docker
        name: Lint *.dockerfile Dockerfiles
@@ -124,14 +124,14 @@ repos:

  # Lint: Markdown
  - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.41.0
+    rev: v0.42.0
    hooks:
      - id: markdownlint
        args: ["--fix"]

  # Strip output from Jupyter notebooks
  - repo: https://github.com/kynan/nbstripout
-    rev: 0.7.1
+    rev: 0.8.0
    hooks:
      - id: nbstripout

@@ -149,7 +149,7 @@ repos:

  # Run black on python code blocks in documentation files.
  - repo: https://github.com/adamchainz/blacken-docs
-    rev: 1.18.0
+    rev: 1.19.1
    hooks:
      - id: blacken-docs
        # --skip-errors is added to allow us to have python syntax highlighting even if
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,14 @@ All image manifests can be found in [the wiki](https://github.com/jupyter/docker
 Affected: all images.

 - **Breaking:** Switch to `mamba` v2 ([#2147](https://github.com/jupyter/docker-stacks/pull/2147)).
-  More information about changes made: <https://mamba.readthedocs.io/en/latest/developer_zone/changes-2.0.html>
+  More information about changes made: <https://mamba.readthedocs.io/en/latest/developer_zone/changes-2.0.html>.
+
+## 2024-11-06
+
+Affected: all images except `docker-stacks-foundation`.
+
+- **Non-breaking** Install `jupyterhub-base` and `nodejs` packages instead of `jupyterhub` package ([#2171](https://github.com/jupyter/docker-stacks/pull/2171)).
+

 ## 2024-10-23

--- a/README.md
+++ b/README.md
@@ -1,7 +1,6 @@
 # Jupyter Docker Stacks

-[![GitHub actions badge](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml/badge.svg)
-](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml?query=branch%3Amain "Docker images build status")
+[![GitHub actions badge](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml/badge.svg)](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml?query=branch%3Amain "Docker images build status")
 [![Read the Docs badge](https://img.shields.io/readthedocs/jupyter-docker-stacks.svg)](https://jupyter-docker-stacks.readthedocs.io/en/latest/ "Documentation build status")
 [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/jupyter/docker-stacks/main.svg)](https://results.pre-commit.ci/latest/github/jupyter/docker-stacks/main "pre-commit.ci build status")
 [![Discourse badge](https://img.shields.io/discourse/users.svg?color=%23f37626&server=https%3A%2F%2Fdiscourse.jupyter.org)](https://discourse.jupyter.org/ "Jupyter Discourse Forum")
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -67,6 +67,7 @@ linkcheck_ignore = [
    r"http://127\.0\.0\.1:.*",  # various examples
    r"https://mybinder\.org/v2/gh/.*",  # lots of 500 errors
    r"https://packages\.ubuntu\.com/search\?keywords=openjdk",  # frequent read timeouts
+    r"https://anaconda\.org\/conda-forge",  # frequent read timeouts
 ]

 linkcheck_allowed_redirects = {
--- a/docs/maintaining/aarch64-runner.md
+++ b/docs/maintaining/aarch64-runner.md
@@ -7,7 +7,7 @@ Each runner is recommended to have at least _2 cores_ and _30 GB_ of disk space.
 Add a new runner:

 - To use [Oracle OCI](https://www.oracle.com/cloud-0/), create a compute instance `VM.Standard.A1.Flex`.
- To use [Google Cloud](https://cloud.google.com), use [this instruction](https://cloud.google.com/compute/docs/instances/create-arm-vm-instance#armpublicimage).
+- To use [Google Cloud](https://cloud.google.com), use [this instruction](https://cloud.google.com/compute/docs/instances/create-start-instance).

 Configure your runner:

--- a/docs/using/recipe_code/jupyterhub_version.dockerfile
+++ b/docs/using/recipe_code/jupyterhub_version.dockerfile
@@ -1,6 +1,6 @@
 FROM quay.io/jupyter/base-notebook

-RUN mamba install --yes 'jupyterhub==4.0.1' && \
+RUN mamba install --yes 'jupyterhub-base==4.0.1' && \
    mamba clean --all -f -y && \
    fix-permissions "${CONDA_DIR}" && \
    fix-permissions "/home/${NB_USER}"
--- a/docs/using/selecting.md
+++ b/docs/using/selecting.md
@@ -54,7 +54,9 @@ It contains:

 - Everything in `jupyter/docker-stacks-foundation`
 - Minimally functional Server (e.g., no LaTeX support for saving notebooks as PDFs)
- `notebook`, `jupyterhub`, and `jupyterlab` packages
+- `notebook`, `jupyterhub-base`, and `jupyterlab` packages
+  Note: we're also installing `nodejs` as it has historically been installed indirectly as a dependency of `jupyterhub` package, which was used before.
+  See more at: <https://github.com/jupyter/docker-stacks/pull/2171>
 - A `start-notebook.py` script as the default command
 - A `start-singleuser.py` script useful for launching containers in JupyterHub
 - Options for a self-signed HTTPS certificate
--- a/docs/using/specifics.md
+++ b/docs/using/specifics.md
@@ -49,7 +49,8 @@ You can build a `pyspark-notebook` image with a different `Spark` version by ove
    - This version needs to match the version supported by the Spark distribution used above.
    - See [Spark Overview](https://spark.apache.org/docs/latest/#downloading) and [Ubuntu packages](https://packages.ubuntu.com/search?keywords=openjdk).
  - `spark_version` (optional): The Spark version to install, for example `3.5.0`.
-    If not specified (this is the default), latest stable Spark will be installed.
+    If not specified (this is the default), latest Spark will be installed.
+    Note: to support Python 3.12, we currently install Spark v4 preview versions: <https://github.com/jupyter/docker-stacks/pull/2072#issuecomment-2414123851>.
  - `hadoop_version`: The Hadoop version (`3` by default).
    Note, that _Spark < 3.3_ require to specify `major.minor` Hadoop version (i.e. `3.2`).
  - `scala_version` (optional): The Scala version, for example `2.13` (not specified by default).
--- a/images/base-notebook/Dockerfile
+++ b/images/base-notebook/Dockerfile
@@ -39,10 +39,20 @@ USER ${NB_UID}
 # files across image layers when the permissions change
 WORKDIR /tmp
 RUN mamba install --yes \
-    'jupyterhub' \
+    'jupyterhub-base' \
    'jupyterlab' \
    'nbclassic' \
-    'notebook' && \
+    # nodejs has historically been installed indirectly as a dependency.
+    # When it was no longer getting installed indirectly,
+    # we started installing it explicitly to avoid introducing a breaking change
+    # for users building on top of these images.
+    # See: https://github.com/jupyter/docker-stacks/pull/2171
+    'nodejs' \
+    # Sometimes, when the new version of `jupyterlab` is released, latest `notebook` might not support it for some time
+    # Old versions of `notebook` (<v7) didn't have a restriction on the `jupyterlab` version, and old `notebook` is getting installed
+    # That's why we have to pin the minimum notebook version
+    # More info: https://github.com/jupyter/docker-stacks/pull/2167
+    'notebook>=7.2.2' && \
    jupyter server --generate-config && \
    mamba clean --all -f -y && \
    npm cache clean --force && \
--- a/images/pyspark-notebook/Dockerfile
+++ b/images/pyspark-notebook/Dockerfile
@@ -24,7 +24,7 @@ RUN apt-get update --yes && \
    ca-certificates-java && \
    apt-get clean && rm -rf /var/lib/apt/lists/*

-# If spark_version is not set, latest stable Spark will be installed
+# If spark_version is not set, latest Spark will be installed
 ARG spark_version
 ARG hadoop_version="3"
 # If scala_version is not set, Spark without Scala will be installed
--- a/images/pyspark-notebook/setup_spark.py
+++ b/images/pyspark-notebook/setup_spark.py
@@ -29,11 +29,11 @@ def get_all_refs(url: str) -> list[str]:

 def get_latest_spark_version() -> str:
    """
-    Returns the last stable version of Spark using spark archive
+    Returns the last version of Spark using spark archive
    """
    LOGGER.info("Downloading Spark versions information")
    all_refs = get_all_refs("https://archive.apache.org/dist/spark/")
-    stable_versions = [
+    versions = [
        ref.removeprefix("spark-").removesuffix("/")
        for ref in all_refs
        if ref.startswith("spark-") and "incubating" not in ref
@@ -49,7 +49,7 @@ def get_latest_spark_version() -> str:
        patch, _, preview = arr[2].partition("-")
        return (major, minor, int(patch), preview)

-    latest_version = max(stable_versions, key=lambda ver: version_array(ver))
+    latest_version = max(versions, key=lambda ver: version_array(ver))
    LOGGER.info(f"Latest version: {latest_version}")
    return latest_version

--- a/images/pytorch-notebook/cuda12/Dockerfile
+++ b/images/pytorch-notebook/cuda12/Dockerfile
@@ -13,7 +13,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]

 # Install PyTorch with pip (https://pytorch.org/get-started/locally/)
 # hadolint ignore=DL3013
-RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu121' \
+RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu124' \
    'torch' \
    'torchaudio' \
    'torchvision' && \
--- a/images/tensorflow-notebook/cuda/Dockerfile
+++ b/images/tensorflow-notebook/cuda/Dockerfile
@@ -14,7 +14,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 # Install TensorFlow, CUDA and cuDNN with pip
 RUN pip install --no-cache-dir \
    "jupyter-server-proxy" \
-    "tensorflow[and-cuda]" && \
+    "tensorflow[and-cuda]<=2.17.1" && \
    fix-permissions "${CONDA_DIR}" && \
    fix-permissions "/home/${NB_USER}"

--- a/tests/docker-stacks-foundation/test_packages.py
+++ b/tests/docker-stacks-foundation/test_packages.py
@@ -74,8 +74,11 @@ EXCLUDED_PACKAGES = [
    "grpcio-status",
    "grpcio",
    "hdf5",
+    "jupyterhub-base",
    "jupyterlab-git",
    "mamba",
+    "nodejs",
+    "notebook[version='>",
    "openssl",
    "pandas[version='>",
    "protobuf",