diff --git a/.github/actions/create-dev-env/action.yml b/.github/actions/create-dev-env/action.yml index 7323494f..c7b008b8 100644 --- a/.github/actions/create-dev-env/action.yml +++ b/.github/actions/create-dev-env/action.yml @@ -4,14 +4,10 @@ description: Create a build environment runs: using: composite steps: - # actions/setup-python doesn't support Linux aarch64 runners - # See: https://github.com/actions/setup-python/issues/108 - # python3 is manually preinstalled in the aarch64 VM self-hosted runner - name: Set Up Python ๐Ÿ uses: actions/setup-python@v5 with: - python-version: 3.x - if: runner.arch == 'X64' + python-version: "3.12" - name: Install Dev Dependencies ๐Ÿ“ฆ run: | diff --git a/.github/workflows/aarch64-setup.yml b/.github/workflows/aarch64-setup.yml deleted file mode 100644 index bdaaf806..00000000 --- a/.github/workflows/aarch64-setup.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Test aarch64-runner setup script - -on: - schedule: - # Weekly, at 03:00 on Monday UTC - - cron: "0 3 * * 1" - pull_request: - paths: - - ".github/workflows/aarch64-setup.yml" - - "aarch64-runner/setup.sh" - push: - branches: - - main - paths: - - ".github/workflows/aarch64-setup.yml" - - "aarch64-runner/setup.sh" - workflow_dispatch: - -jobs: - test-script: - # The script itself is not aarch64-specific - # It is easier to test on x86_64 - # Using `ubuntu-22.04` as this is what our aarch64 runners currently use - runs-on: ubuntu-22.04 - - steps: - - name: Checkout Repo โšก๏ธ - uses: actions/checkout@v4 - - - name: Run setup script โœ… - run: sudo ./aarch64-runner/setup.sh diff --git a/.github/workflows/contributed-recipes.yml b/.github/workflows/contributed-recipes.yml index 6a4ba74d..2a6e57fe 100644 --- a/.github/workflows/contributed-recipes.yml +++ b/.github/workflows/contributed-recipes.yml @@ -44,7 +44,6 @@ jobs: uses: actions/checkout@v4 - name: Build recipe ๐Ÿ›  - # We're pulling here to avoid accidentally using an image that might be present on aarch64 self-hosted runner run: docker build --pull --rm --force-rm --tag my-custom-image -f ./${{ matrix.dockerfile }} ./ env: DOCKER_BUILDKIT: 1 diff --git a/.github/workflows/docker-build-test-upload.yml b/.github/workflows/docker-build-test-upload.yml index a7bb1950..9d86136f 100644 --- a/.github/workflows/docker-build-test-upload.yml +++ b/.github/workflows/docker-build-test-upload.yml @@ -41,7 +41,7 @@ jobs: steps: # Image with CUDA needs extra disk space - name: Free disk space ๐Ÿงน - if: contains(inputs.variant, 'cuda') && inputs.platform == 'x86_64' + if: contains(inputs.variant, 'cuda') && runner.arch == 'X64' uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be with: tool-cache: false @@ -57,18 +57,6 @@ jobs: - name: Create dev environment ๐Ÿ“ฆ uses: ./.github/actions/create-dev-env - # Self-hosted runners share a state (whole VM) between runs - # Also, they might have running or stopped containers, - # which are not cleaned up by `docker system prune` - - name: Reset docker state and cleanup artifacts ๐Ÿ—‘๏ธ - if: inputs.platform != 'x86_64' - run: | - docker kill $(docker ps --quiet) || true - docker rm $(docker ps --all --quiet) || true - docker system prune --all --force - rm -rf /tmp/jupyter/ - shell: bash - - name: Load parent built image to Docker ๐Ÿ“ฅ if: inputs.parent-image != '' uses: ./.github/actions/load-image diff --git a/.github/workflows/docker-merge-tags.yml b/.github/workflows/docker-merge-tags.yml index a4768794..6c76f674 100644 --- a/.github/workflows/docker-merge-tags.yml +++ b/.github/workflows/docker-merge-tags.yml @@ -41,7 +41,7 @@ jobs: with: name: ${{ inputs.image }}-aarch64-${{ inputs.variant }}-tags path: /tmp/jupyter/tags/ - if: github.repository_owner == 'jupyter' && !contains(inputs.variant, 'cuda') + if: ${{ !contains(inputs.variant, 'cuda') }} # Docker might be stuck when pulling images # https://github.com/docker/for-mac/issues/2083 diff --git a/.github/workflows/docker-tag-push.yml b/.github/workflows/docker-tag-push.yml index e4962a61..7aaf6798 100644 --- a/.github/workflows/docker-tag-push.yml +++ b/.github/workflows/docker-tag-push.yml @@ -33,7 +33,7 @@ jobs: steps: # Image with CUDA needs extra disk space - name: Free disk space ๐Ÿงน - if: contains(inputs.variant, 'cuda') && inputs.platform == 'x86_64' + if: contains(inputs.variant, 'cuda') && runner.arch == 'X64' uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be with: tool-cache: false diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e25d5e8b..50a10a45 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -66,8 +66,7 @@ jobs: parent-image: "" image: docker-stacks-foundation platform: aarch64 - runs-on: ARM64_FAST - if: github.repository_owner == 'jupyter' + runs-on: ubuntu-22.04-arm x86_64-foundation: uses: ./.github/workflows/docker-build-test-upload.yml @@ -83,9 +82,8 @@ jobs: parent-image: docker-stacks-foundation image: base-notebook platform: aarch64 - runs-on: ARM64_FAST + runs-on: ubuntu-22.04-arm needs: [aarch64-foundation] - if: github.repository_owner == 'jupyter' x86_64-base: uses: ./.github/workflows/docker-build-test-upload.yml @@ -102,9 +100,9 @@ jobs: parent-image: base-notebook image: minimal-notebook platform: aarch64 - runs-on: ARM64_FAST + runs-on: ubuntu-22.04-arm needs: [aarch64-base] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} x86_64-minimal: uses: ./.github/workflows/docker-build-test-upload.yml @@ -122,9 +120,9 @@ jobs: parent-image: minimal-notebook image: scipy-notebook platform: aarch64 - runs-on: ARM64_FAST + runs-on: ubuntu-22.04-arm needs: [aarch64-minimal] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} x86_64-scipy: uses: ./.github/workflows/docker-build-test-upload.yml @@ -142,9 +140,9 @@ jobs: parent-image: minimal-notebook image: r-notebook platform: aarch64 - runs-on: ARM64_SLOW + runs-on: ubuntu-22.04-arm needs: [aarch64-minimal] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} x86_64-r: uses: ./.github/workflows/docker-build-test-upload.yml @@ -162,9 +160,9 @@ jobs: parent-image: minimal-notebook image: julia-notebook platform: aarch64 - runs-on: ARM64_SLOW + runs-on: ubuntu-22.04-arm needs: [aarch64-minimal] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} x86_64-julia: uses: ./.github/workflows/docker-build-test-upload.yml @@ -182,9 +180,9 @@ jobs: parent-image: scipy-notebook image: tensorflow-notebook platform: aarch64 - runs-on: ARM64_SLOW + runs-on: ubuntu-22.04-arm needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} x86_64-tensorflow: uses: ./.github/workflows/docker-build-test-upload.yml @@ -213,9 +211,9 @@ jobs: parent-image: scipy-notebook image: pytorch-notebook platform: aarch64 - runs-on: ARM64_SLOW + runs-on: ubuntu-22.04-arm needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} x86_64-pytorch: uses: ./.github/workflows/docker-build-test-upload.yml @@ -255,9 +253,9 @@ jobs: parent-image: scipy-notebook image: datascience-notebook platform: aarch64 - runs-on: ARM64_SLOW + runs-on: ubuntu-22.04-arm needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} x86_64-datascience: uses: ./.github/workflows/docker-build-test-upload.yml @@ -275,9 +273,9 @@ jobs: parent-image: scipy-notebook image: pyspark-notebook platform: aarch64 - runs-on: ARM64_FAST + runs-on: ubuntu-22.04-arm needs: [aarch64-scipy] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} x86_64-pyspark: uses: ./.github/workflows/docker-build-test-upload.yml @@ -295,9 +293,9 @@ jobs: parent-image: pyspark-notebook image: all-spark-notebook platform: aarch64 - runs-on: ARM64_FAST + runs-on: ubuntu-22.04-arm needs: [aarch64-pyspark] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} x86_64-all-spark: uses: ./.github/workflows/docker-build-test-upload.yml @@ -348,7 +346,7 @@ jobs: aarch64-pyspark, aarch64-all-spark, ] - if: github.repository_owner == 'jupyter' && !contains(github.event.pull_request.title, '[FAST_BUILD]') + if: ${{ !contains(github.event.pull_request.title, '[FAST_BUILD]') }} aarch64-images-tag-push-fast: uses: ./.github/workflows/docker-tag-push.yml @@ -367,7 +365,7 @@ jobs: { image: base-notebook, variant: default }, ] needs: [aarch64-foundation, aarch64-base] - if: github.repository_owner == 'jupyter' && contains(github.event.pull_request.title, '[FAST_BUILD]') + if: contains(github.event.pull_request.title, '[FAST_BUILD]') x86_64-images-tag-push: uses: ./.github/workflows/docker-tag-push.yml @@ -505,4 +503,4 @@ jobs: contributed-recipes: uses: ./.github/workflows/contributed-recipes.yml needs: [merge-tags] - if: github.repository_owner == 'jupyter' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule') + if: github.ref == 'refs/heads/main' || github.event_name == 'schedule' diff --git a/CHANGELOG.md b/CHANGELOG.md index a8d055f7..6b54320d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ This changelog only contains breaking and/or significant changes manually introduced to this repository (using Pull Requests). All image manifests can be found in [the wiki](https://github.com/jupyter/docker-stacks/wiki). +## 2025-02-11 + +Affected: all images. + +- **Non-breaking:**: start using GitHub-hosted `aarch64` runners. + ## 2024-12-03 Affected: all images. diff --git a/README.md b/README.md index fdadc016..3e829ed8 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ more information is available in the [documentation](https://jupyter-docker-stac - Starting from `2022-07-05`, `aarch64` self-hosted runners were sponsored by [`@mathbunnyru`](https://github.com/mathbunnyru/). Please, consider [sponsoring his work](https://github.com/sponsors/mathbunnyru) on GitHub - Starting from `2023-10-31`, `aarch64` self-hosted runners are sponsored by an amazing [`2i2c non-profit organization`](https://2i2c.org) +- Starting from `2025-02-11`, we use GitHub-hosted `aarch64` runners ## CPU Architectures diff --git a/docs/conf.py b/docs/conf.py index 6ddb83d3..b3aaf551 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,7 +63,6 @@ myst_heading_anchors = 3 linkcheck_ignore = [ r".*github\.com.*#", # javascript based anchors - r"https://github\.com/jupyter/docker-stacks/settings/actions/runners/new\?arch=arm64\&os=linux", # only works for users with permissions to change runners r"http://127\.0\.0\.1:.*", # various examples r"https://mybinder\.org/v2/gh/.*", # lots of 500 errors r"https://packages\.ubuntu\.com/search\?keywords=openjdk", # frequent read timeouts diff --git a/docs/index.rst b/docs/index.rst index 5368a503..e8cd0321 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,7 +36,6 @@ Table of Contents maintaining/new-images-and-packages-policy maintaining/tasks - maintaining/aarch64-runner .. toctree:: :maxdepth: 2 diff --git a/docs/maintaining/aarch64-runner.md b/docs/maintaining/aarch64-runner.md deleted file mode 100644 index c6e129e3..00000000 --- a/docs/maintaining/aarch64-runner.md +++ /dev/null @@ -1,30 +0,0 @@ -# Self-hosted runners - -For building `aarch64` images, we use self-hosted GitHub runners. -It is recommended to have at least two runners to allow better parallelism. -Each runner is recommended to have at least _2 cores_ and _30 GB_ of disk space. - -Add a new runner: - -- To use [Oracle OCI](https://www.oracle.com/cloud/), create a compute instance `VM.Standard.A1.Flex`. -- To use [Google Cloud](https://cloud.google.com), use [this instruction](https://cloud.google.com/compute/docs/instances/create-start-instance). - -Configure your runner: - -1. Run under `root`: - - ```bash - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/jupyter/docker-stacks/HEAD/aarch64-runner/setup.sh)" - ``` - - This will perform the initial runner setup and create a user `runner-user` without `sudo` capabilities. - -2. Set up a new GitHub Runner under `runner-user` using [GitHub Instructions](https://github.com/jupyter/docker-stacks/settings/actions/runners/new?arch=arm64&os=linux). - **Do not `./run.sh` yet**. -3. Run under `root`: - - ```bash - cd /home/runner-user/actions-runner/ && ./svc.sh install runner-user - ``` - -4. Reboot the VM to apply all updates and run GitHub runner. diff --git a/docs/using/recipe_code/generate_matrix.py b/docs/using/recipe_code/generate_matrix.py index f44736d2..d3be960c 100755 --- a/docs/using/recipe_code/generate_matrix.py +++ b/docs/using/recipe_code/generate_matrix.py @@ -12,13 +12,13 @@ REPOSITORY_OWNER = os.environ["REPOSITORY_OWNER"] def generate_matrix() -> dict[str, Any]: dockerfiles = sorted(file.name for file in THIS_DIR.glob("*.dockerfile")) - runs_on = ["ubuntu-latest"] - if REPOSITORY_OWNER == "jupyter": - runs_on.append("ARM64") + runs_on = ["ubuntu-latest", "ubuntu-22.04-arm"] return { "dockerfile": dockerfiles, "runs-on": runs_on, - "exclude": [{"dockerfile": "oracledb.dockerfile", "runs-on": "ARM64"}], + "exclude": [ + {"dockerfile": "oracledb.dockerfile", "runs-on": "ubuntu-22.04-arm"} + ], } diff --git a/images/minimal-notebook/setup-scripts/setup_julia.py b/images/minimal-notebook/setup-scripts/setup_julia.py index c431d778..de122a1e 100755 --- a/images/minimal-notebook/setup-scripts/setup_julia.py +++ b/images/minimal-notebook/setup-scripts/setup_julia.py @@ -43,12 +43,6 @@ def get_latest_julia_url() -> tuple[str, str]: triplet = unify_aarch64(platform.machine()) + "-linux-gnu" file_info = [vf for vf in latest_version_files if vf["triplet"] == triplet][0] LOGGER.info(f"Latest version: {file_info['version']} url: {file_info['url']}") - BROKEN_VERSION = "1.11.3" - if file_info["version"] == BROKEN_VERSION: - LOGGER.warning( - f"Not using Julia {BROKEN_VERSION}, because it hangs in GitHub self-hosted runners" - ) - return file_info["url"].replace(BROKEN_VERSION, "1.11.1"), "1.11.1" return file_info["url"], file_info["version"]