mirror of
https://github.com/DSpace/DSpace.git
synced 2025-10-07 01:54:22 +00:00
Merge pull request #10122 from DSpace/backport-10120-to-dspace-7_x
[Port dspace-7_x] [GitHub Actions] Refactor Docker build process to use ghcr.io for build, and then copy to docker.io once build completes
This commit is contained in:
Tim Donohue
4
.github/workflows/docker.yml
vendored
4
.github/workflows/docker.yml
vendored
@@ -15,6 +15,7 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read # to fetch code (actions/checkout)
|
||||
packages: write # to write images to GitHub Container Registry (GHCR)
|
||||
|
||||
jobs:
|
||||
####################################################
|
||||
@@ -175,6 +176,9 @@ jobs:
|
||||
# Else, just use the branch name.
|
||||
# NOTE: DSPACE_VER is used because our docker compose scripts default to using the "-test" image.
|
||||
DSPACE_VER: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || (github.event_name == 'pull_request' && github.event.pull_request.base.ref) || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }}
|
||||
# Docker Registry to use for Docker compose scripts below.
|
||||
# We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub.
|
||||
DOCKER_REGISTRY: ghcr.io
|
||||
steps:
|
||||
# Checkout our codebase (to get access to Docker Compose scripts)
|
||||
- name: Checkout codebase
|
||||
|
||||
195
.github/workflows/reusable-docker-build.yml
vendored
195
.github/workflows/reusable-docker-build.yml
vendored
@@ -74,7 +74,11 @@ env:
|
||||
# Current DSpace maintenance branch (and architecture) which is deployed to demo.dspace.org / sandbox.dspace.org
|
||||
# (NOTE: No deployment branch specified for sandbox.dspace.org as it uses the default_branch)
|
||||
DEPLOY_DEMO_BRANCH: 'dspace-8_x'
|
||||
DEPLOY_SANDBOX_BRANCH: 'main'
|
||||
DEPLOY_ARCH: 'linux/amd64'
|
||||
# Registry used during building of Docker images. (All images are later copied to docker.io registry)
|
||||
# We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub.
|
||||
DOCKER_BUILD_REGISTRY: ghcr.io
|
||||
|
||||
jobs:
|
||||
docker-build:
|
||||
@@ -99,6 +103,7 @@ jobs:
|
||||
# This step converts the slashes in the "arch" matrix values above into dashes & saves to env.ARCH_NAME
|
||||
# E.g. "linux/amd64" becomes "linux-amd64"
|
||||
# This is necessary because all upload artifacts CANNOT have special chars (like slashes)
|
||||
# NOTE: The regex-like syntax below is Bash Parameter Substitution
|
||||
- name: Prepare
|
||||
run: |
|
||||
platform=${{ matrix.arch }}
|
||||
@@ -109,13 +114,14 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# https://github.com/docker/login-action
|
||||
- name: Login to DockerHub
|
||||
# Only login if not a PR, as PRs only trigger a Docker build and not a push
|
||||
if: ${{ ! matrix.isPr }}
|
||||
# NOTE: This login occurs for BOTH non-PRs or PRs. PRs *must* also login to access private images from GHCR
|
||||
# during the build process
|
||||
- name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
|
||||
registry: ${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# https://github.com/docker/setup-qemu-action
|
||||
- name: Set up QEMU emulation to build for multiple architectures
|
||||
@@ -131,19 +137,20 @@ jobs:
|
||||
id: meta_build
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.IMAGE_NAME }}
|
||||
images: ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: ${{ env.IMAGE_TAGS }}
|
||||
flavor: ${{ env.TAGS_FLAVOR }}
|
||||
|
||||
#------------------------------------------------------------
|
||||
# Build & deploy steps for new commits to a branch (non-PRs)
|
||||
#--------------------------------------------------------------------
|
||||
# First, for all branch commits (non-PRs) we build the image & upload
|
||||
# to GitHub Container Registry (GHCR). After uploading the image
|
||||
# to GHCR, we store the image digest in an artifact, so we can
|
||||
# create a merged manifest later (see 'docker-build_manifest' job).
|
||||
#
|
||||
# These steps build the images, push to DockerHub, and
|
||||
# (if necessary) redeploy demo/sandbox sites.
|
||||
#------------------------------------------------------------
|
||||
# NOTE: We use GHCR in order to avoid aggressive rate limits at DockerHub.
|
||||
#--------------------------------------------------------------------
|
||||
# https://github.com/docker/build-push-action
|
||||
- name: Build and push image to DockerHub
|
||||
# Only build & push if not a PR
|
||||
- name: Build and push image to ${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
if: ${{ ! matrix.isPr }}
|
||||
id: docker_build
|
||||
uses: docker/build-push-action@v5
|
||||
@@ -152,6 +159,9 @@ jobs:
|
||||
${{ inputs.dockerfile_additional_contexts }}
|
||||
context: ${{ inputs.dockerfile_context }}
|
||||
file: ${{ inputs.dockerfile_path }}
|
||||
# Tell DSpace's Docker files to use the build registry instead of DockerHub
|
||||
build-args:
|
||||
DOCKER_REGISTRY=${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
platforms: ${{ matrix.arch }}
|
||||
push: true
|
||||
# Use tags / labels provided by 'docker/metadata-action' above
|
||||
@@ -162,7 +172,7 @@ jobs:
|
||||
cache-from: type=gha,scope=${{ inputs.build_id }}
|
||||
cache-to: type=gha,scope=${{ inputs.build_id }},mode=max
|
||||
|
||||
# Export the digest of Docker build locally (for non PRs only)
|
||||
# Export the digest of Docker build locally
|
||||
- name: Export Docker build digest
|
||||
if: ${{ ! matrix.isPr }}
|
||||
run: |
|
||||
@@ -170,7 +180,8 @@ jobs:
|
||||
digest="${{ steps.docker_build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
# Upload digest to an artifact, so that it can be used in manifest below
|
||||
# Upload digest to an artifact, so that it can be used in combined manifest below
|
||||
# (The purpose of the combined manifest is to list both amd64 and arm64 builds under same tag)
|
||||
- name: Upload Docker build digest to artifact
|
||||
if: ${{ ! matrix.isPr }}
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -180,48 +191,31 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# If this build is NOT a PR and passed in a REDEPLOY_SANDBOX_URL secret,
|
||||
# Then redeploy https://sandbox.dspace.org if this build is for our deployment architecture and 'main' branch.
|
||||
- name: Redeploy sandbox.dspace.org (based on main branch)
|
||||
if: |
|
||||
!matrix.isPR &&
|
||||
env.REDEPLOY_SANDBOX_URL != '' &&
|
||||
matrix.arch == env.DEPLOY_ARCH &&
|
||||
github.ref_name == github.event.repository.default_branch
|
||||
run: |
|
||||
curl -X POST $REDEPLOY_SANDBOX_URL
|
||||
|
||||
# If this build is NOT a PR and passed in a REDEPLOY_DEMO_URL secret,
|
||||
# Then redeploy https://demo.dspace.org if this build is for our deployment architecture and demo branch.
|
||||
- name: Redeploy demo.dspace.org (based on maintenance branch)
|
||||
if: |
|
||||
!matrix.isPR &&
|
||||
env.REDEPLOY_DEMO_URL != '' &&
|
||||
matrix.arch == env.DEPLOY_ARCH &&
|
||||
github.ref_name == env.DEPLOY_DEMO_BRANCH
|
||||
run: |
|
||||
curl -X POST $REDEPLOY_DEMO_URL
|
||||
|
||||
#-------------------------------------------------------------
|
||||
# Shared Build steps.
|
||||
# These are used for PRs as well as new commits to a branch (non-PRs)
|
||||
#------------------------------------------------------------------------------
|
||||
# Second, we build the image again in order to store it in a local TAR file.
|
||||
# This TAR of the image is cached/saved as an artifact, so that it can be used
|
||||
# by later jobs to install the brand-new images for automated testing.
|
||||
# This TAR build is performed BOTH for PRs and for branch commits (non-PRs).
|
||||
#
|
||||
# These steps build the images and cache/store as a build artifact.
|
||||
# These artifacts can then be used by later jobs to install the
|
||||
# brand-new images for automated testing. For non-PRs, this cache is
|
||||
# also used to avoid pulling the images we just built from DockerHub.
|
||||
#--------------------------------------------------------------
|
||||
|
||||
# (This approach has the advantage of avoiding having to download the newly built
|
||||
# image from DockerHub or GHCR during automated testing.)
|
||||
#
|
||||
# See the 'docker-deploy' job in docker.yml as an example of where this TAR is used.
|
||||
#-------------------------------------------------------------------------------
|
||||
# Build local image (again) and store in a TAR file in /tmp directory
|
||||
# NOTE: This build is run for both PRs and non-PRs as it's used to "cache" our built images as artifacts.
|
||||
# NOTE #2: This cannot be combined with push to DockerHub registry above as it's a different type of output.
|
||||
# This step is only done for AMD64, as that's the only image we use in our automated testing (at this time).
|
||||
# NOTE: This step cannot be combined with the build above as it's a different type of output.
|
||||
- name: Build and push image to local TAR file
|
||||
if: ${{ matrix.arch == 'linux/amd64'}}
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
build-contexts: |
|
||||
${{ inputs.dockerfile_additional_contexts }}
|
||||
context: ${{ inputs.dockerfile_context }}
|
||||
file: ${{ inputs.dockerfile_path }}
|
||||
# Tell DSpace's Docker files to use the build registry instead of DockerHub
|
||||
build-args:
|
||||
DOCKER_REGISTRY=${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
platforms: ${{ matrix.arch }}
|
||||
tags: ${{ steps.meta_build.outputs.tags }}
|
||||
labels: ${{ steps.meta_build.outputs.labels }}
|
||||
@@ -233,7 +227,9 @@ jobs:
|
||||
outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar
|
||||
|
||||
# Upload the local docker image (in TAR file) to a build Artifact
|
||||
# This step is only done for AMD64, as that's the only image we use in our automated testing (at this time).
|
||||
- name: Upload local image TAR to artifact
|
||||
if: ${{ matrix.arch == 'linux/amd64'}}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }}
|
||||
@@ -241,10 +237,12 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Merge Docker digests (from various architectures) into a manifest.
|
||||
# This runs after all Docker builds complete above, and it tells hub.docker.com
|
||||
# that these builds should be all included in the manifest for this tag.
|
||||
# (e.g. AMD64 and ARM64 should be listed as options under the same tagged Docker image)
|
||||
##########################################################################################
|
||||
# Merge Docker digests (from various architectures) into a single manifest.
|
||||
# This runs after all Docker builds complete above. The purpose is to include all builds
|
||||
# under a single manifest for this tag.
|
||||
# (e.g. both linux/amd64 and linux/arm64 should be listed under the same tagged Docker image)
|
||||
##########################################################################################
|
||||
docker-build_manifest:
|
||||
# Only run if this is NOT a PR
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
@@ -260,11 +258,12 @@ jobs:
|
||||
pattern: digests-${{ inputs.build_id }}-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Login to Docker Hub
|
||||
- name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
|
||||
registry: ${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
@@ -272,17 +271,89 @@ jobs:
|
||||
- name: Add Docker metadata for image
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: ${{ env.IMAGE_TAGS }}
|
||||
flavor: ${{ env.TAGS_FLAVOR }}
|
||||
|
||||
- name: Create manifest list from digests and push to ${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
|
||||
|
||||
- name: Inspect manifest in ${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
|
||||
|
||||
##########################################################################################
|
||||
# Copy images / manifest to DockerHub.
|
||||
# This MUST run after *both* images (AMD64 and ARM64) are built and uploaded to GitHub
|
||||
# Container Registry (GHCR). Attempting to run this in parallel to GHCR builds can result
|
||||
# in a race condition...i.e. the copy to DockerHub may fail if GHCR image has been updated
|
||||
# at the moment when the copy occurs.
|
||||
##########################################################################################
|
||||
docker-copy_to_dockerhub:
|
||||
# Only run if this is NOT a PR
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- docker-build_manifest
|
||||
|
||||
steps:
|
||||
# 'regctl' is used to more easily copy the image to DockerHub and obtain the digest from DockerHub
|
||||
# See https://github.com/regclient/regclient/blob/main/docs/regctl.md
|
||||
- name: Install regctl for Docker registry tools
|
||||
uses: regclient/actions/regctl-installer@main
|
||||
with:
|
||||
release: 'v0.8.0'
|
||||
|
||||
# This recreates Docker tags for DockerHub
|
||||
- name: Add Docker metadata for image
|
||||
id: meta_dockerhub
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.IMAGE_NAME }}
|
||||
tags: ${{ env.IMAGE_TAGS }}
|
||||
flavor: ${{ env.TAGS_FLAVOR }}
|
||||
|
||||
- name: Create manifest list from digests and push
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ env.IMAGE_NAME }}@sha256:%s ' *)
|
||||
# Login to source registry first, as this is where we are copying *from*
|
||||
- name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.DOCKER_BUILD_REGISTRY }}
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Inspect image
|
||||
# Login to DockerHub, since this is where we are copying *to*
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
|
||||
|
||||
# Copy the image from source to DockerHub
|
||||
- name: Copy image from ${{ env.DOCKER_BUILD_REGISTRY }} to docker.io
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
|
||||
regctl image copy ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }} docker.io/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }}
|
||||
|
||||
#--------------------------------------------------------------------
|
||||
# Finally, check whether demo.dspace.org or sandbox.dspace.org need
|
||||
# to be redeployed based on these new DockerHub images.
|
||||
#--------------------------------------------------------------------
|
||||
# If this build is for the branch that Sandbox uses and passed in a REDEPLOY_SANDBOX_URL secret,
|
||||
# Then redeploy https://sandbox.dspace.org
|
||||
- name: Redeploy sandbox.dspace.org (based on main branch)
|
||||
if: |
|
||||
env.REDEPLOY_SANDBOX_URL != '' &&
|
||||
github.ref_name == env.DEPLOY_SANDBOX_BRANCH
|
||||
run: |
|
||||
curl -X POST $REDEPLOY_SANDBOX_URL
|
||||
# If this build is for the branch that Demo uses and passed in a REDEPLOY_DEMO_URL secret,
|
||||
# Then redeploy https://demo.dspace.org
|
||||
- name: Redeploy demo.dspace.org (based on maintenance branch)
|
||||
if: |
|
||||
env.REDEPLOY_DEMO_URL != '' &&
|
||||
github.ref_name == env.DEPLOY_DEMO_BRANCH
|
||||
run: |
|
||||
curl -X POST $REDEPLOY_DEMO_URL
|
||||
Reference in New Issue
Block a user