Merge pull request #10086 from tdonohue/port_github_action_docker_to_8x

[Port dspace-8_x] [GitHub Actions] Port several Docker building improvements

.github/workflows/docker.yml

@@ -149,9 +149,9 @@ jobs:
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
       DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }}
 
-  ########################################################################
+  #################################################################################
-  # Test Deployment via Docker to ensure images are working properly
+  # Test Deployment via Docker to ensure newly built images are working properly
-  ########################################################################
+  #################################################################################
   docker-deploy:
     # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace'
     if: github.repository == 'dspace/dspace'
@@ -170,27 +170,24 @@ jobs:
       signposting__P__enabled: true
       sword__D__server__P__enabled: true
       swordv2__D__server__P__enabled: true
-      # If this is a PR, force using "pr-testing" version of all Docker images. Otherwise, if on main branch, use the
+      # If this is a PR, used the base branch name. If on main branch, use the "latest" tag. Otherwise, use branch name.
-      # "latest" tag. Otherwise, use the branch name. NOTE: the "pr-testing" tag is a temporary tag that we assign to
+      # NOTE: DSPACE_VER is used because our docker compose scripts default to using the "-test" image.
-      # all PR-built docker images in reusabe-docker-build.yml
+      DSPACE_VER: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.ref) || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }}
-      DSPACE_VER: ${{ (github.event_name == 'pull_request' && 'pr-testing') || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }}
     steps:
       # Checkout our codebase (to get access to Docker Compose scripts)
       - name: Checkout codebase
         uses: actions/checkout@v4
-      # For PRs, download Docker image artifacts (built by reusable-docker-build.yml for all PRs)
+      # Download Docker image artifacts (which were just built by reusable-docker-build.yml)
-      - name: Download Docker image artifacts (for PRs)
+      - name: Download Docker image artifacts
-        if: github.event_name == 'pull_request'
         uses: actions/download-artifact@v4
         with:
-          # Download all Docker images (TAR files) into the /tmp/docker directory
+          # Download all amd64 Docker images (TAR files) into the /tmp/docker directory
-          pattern: docker-image-*
+          pattern: docker-image-*-linux-amd64
           path: /tmp/docker
           merge-multiple: true
-      # For PRs, load each of the images into Docker by calling "docker image load" for each.
+      # Load each of the images into Docker by calling "docker image load" for each.
-      # This ensures we are using the images built from this PR & not the prior versions on DockerHub
+      # This ensures we are using the images just built & not any prior versions on DockerHub
-      - name: Load all downloaded Docker images (for PRs)
+      - name: Load all downloaded Docker images
-        if: github.event_name == 'pull_request'
         run: |
           find /tmp/docker -type f -name "*.tar" -exec docker image load --input "{}" \;
           docker image ls -a

.github/workflows/reusable-docker-build.yml

@@ -54,10 +54,13 @@ env:
   # For a new commit on default branch (main), use the literal tag 'latest' on Docker image.
   # For a new commit on other branches, use the branch name as the tag for Docker image.
   # For a new tag, copy that tag name as the tag for Docker image.
+  # For a pull request, use the name of the base branch that the PR was created against or "latest" (for main).
+  #   e.g. PR against 'main' will use "latest". a PR against 'dspace-7_x' will use 'dspace-7_x'.
   IMAGE_TAGS: |
     type=raw,value=latest,enable=${{ github.ref_name == github.event.repository.default_branch }}
     type=ref,event=branch,enable=${{ github.ref_name != github.event.repository.default_branch }}
     type=ref,event=tag
+    type=raw,value=${{ (github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || github.event.pull_request.base.ref }},enable=${{ github.event_name == 'pull_request' }}
   # Define default tag "flavor" for docker/metadata-action per
   # https://github.com/docker/metadata-action#flavor-input
   # We manage the 'latest' tag ourselves to the 'main' branch (see settings above)
@@ -105,20 +108,6 @@ jobs:
       - name: Checkout codebase
         uses: actions/checkout@v4
 
-      # https://github.com/docker/setup-buildx-action
-      - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      # https://github.com/docker/setup-qemu-action
-      - name: Set up QEMU emulation to build for multiple architectures
-        uses: docker/setup-qemu-action@v3
-
-      #------------------------------------------------------------
-      # Build & deploy steps for new commits to a branch (non-PRs)
-      #
-      # These steps build the images, push to DockerHub, and
-      # (if necessary) redeploy demo/sandbox sites.
-      #------------------------------------------------------------
       # https://github.com/docker/login-action
       - name: Login to DockerHub
       # Only login if not a PR, as PRs only trigger a Docker build and not a push
@@ -128,10 +117,17 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
 
+      # https://github.com/docker/setup-qemu-action
+      - name: Set up QEMU emulation to build for multiple architectures
+        uses: docker/setup-qemu-action@v3
+
+      # https://github.com/docker/setup-buildx-action
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       # https://github.com/docker/metadata-action
-      # Get Metadata for docker_build_deps step below
+      # Extract metadata used for Docker images in all build steps below
-      - name: Sync metadata (tags, labels) from GitHub to Docker for image
+      - name: Extract metadata (tags, labels) from GitHub for Docker image
-        if: ${{ ! matrix.isPr }}
         id: meta_build
         uses: docker/metadata-action@v5
         with:
@@ -139,6 +135,12 @@ jobs:
           tags: ${{ env.IMAGE_TAGS }}
           flavor: ${{ env.TAGS_FLAVOR }}
 
+      #------------------------------------------------------------
+      # Build & deploy steps for new commits to a branch (non-PRs)
+      #
+      # These steps build the images, push to DockerHub, and
+      # (if necessary) redeploy demo/sandbox sites.
+      #------------------------------------------------------------
       # https://github.com/docker/build-push-action
       - name: Build and push image to DockerHub
         # Only build & push if not a PR
@@ -155,6 +157,10 @@ jobs:
           # Use tags / labels provided by 'docker/metadata-action' above
           tags: ${{ steps.meta_build.outputs.tags }}
           labels: ${{ steps.meta_build.outputs.labels }}
+          # Use GitHub cache to load cached Docker images and cache the results of this build
+          # This decreases the number of images we need to fetch from DockerHub
+          cache-from: type=gha,scope=${{ inputs.build_id }}
+          cache-to: type=gha,scope=${{ inputs.build_id }},mode=max
 
       # Export the digest of Docker build locally (for non PRs only)
       - name: Export Docker build digest
@@ -197,26 +203,19 @@ jobs:
           curl -X POST $REDEPLOY_DEMO_URL
 
       #-------------------------------------------------------------
-      # Build steps for PRs only
+      # Shared Build steps.
+      # These are used for PRs as well as new commits to a branch (non-PRs)
       #
-      # These steps build the images and store as a build artifact.
+      # These steps build the images and cache/store as a build artifact.
-      # These artifacts can then be used by later jobs to run the
+      # These artifacts can then be used by later jobs to install the
-      # brand-new images for automated testing.
+      # brand-new images for automated testing. For non-PRs, this cache is
+      # also used to avoid pulling the images we just built from DockerHub.
       #--------------------------------------------------------------
-      # Get Metadata for docker_build_deps step below
+
-      - name: Create metadata (tags, labels) for local Docker image
+      # Build local image (again) and store in a TAR file in /tmp directory
-        if: matrix.isPr
+      # NOTE: This build is run for both PRs and non-PRs as it's used to "cache" our built images as artifacts.
-        id: meta_build_pr
+      # NOTE #2: This cannot be combined with push to DockerHub registry above as it's a different type of output.
-        uses: docker/metadata-action@v5
+      - name: Build and push image to local TAR file
-        with:
-          images: ${{ env.IMAGE_NAME }}
-          # Hardcode to use custom "pr-testing" tag because that will allow us to spin up this PR
-          # for testing in docker.yml
-          tags: pr-testing
-          flavor: ${{ env.TAGS_FLAVOR }}
-      # Build local image and stores in a TAR file in /tmp directory
-      - name: Build and push image to local image
-        if: matrix.isPr
         uses: docker/build-push-action@v5
         with:
           build-contexts: |
@@ -224,16 +223,20 @@ jobs:
           context: ${{ inputs.dockerfile_context }}
           file: ${{ inputs.dockerfile_path }}
           platforms: ${{ matrix.arch }}
-          tags: ${{ steps.meta_build_pr.outputs.tags }}
+          tags: ${{ steps.meta_build.outputs.tags }}
-          labels: ${{ steps.meta_build_pr.outputs.labels }}
+          labels: ${{ steps.meta_build.outputs.labels }}
+          # Use GitHub cache to load cached Docker images and cache the results of this build
+          # This decreases the number of images we need to fetch from DockerHub
+          cache-from: type=gha,scope=${{ inputs.build_id }}
+          cache-to: type=gha,scope=${{ inputs.build_id }},mode=max
           # Export image to a local TAR file
           outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar
+
       # Upload the local docker image (in TAR file) to a build Artifact
-      - name: Upload local image to artifact
+      - name: Upload local image TAR to artifact
-        if: matrix.isPr
         uses: actions/upload-artifact@v4
         with:
-          name: docker-image-${{ inputs.build_id }}
+          name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }}
           path: /tmp/${{ inputs.build_id }}.tar
           if-no-files-found: error
           retention-days: 1
@@ -257,6 +260,12 @@ jobs:
           pattern: digests-${{ inputs.build_id }}-*
           merge-multiple: true
 
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -268,12 +277,6 @@ jobs:
           tags: ${{ env.IMAGE_TAGS }}
           flavor: ${{ env.TAGS_FLAVOR }}
 
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-
       - name: Create manifest list from digests and push
         working-directory: /tmp/digests
         run: |

docker-compose.yml

@@ -64,7 +64,7 @@ services:
   dspacedb:
     container_name: dspacedb
     # Uses a custom Postgres image with pgcrypto installed
-    image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-latest}"
+    image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-8_x}"
     build:
       # Must build out of subdirectory to have access to install script for pgcrypto
       context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/
@@ -84,7 +84,7 @@ services:
   # DSpace Solr container
   dspacesolr:
     container_name: dspacesolr
-    image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-latest}"
+    image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-8_x}"
     build:
       context: ./dspace/src/main/docker/dspace-solr/
       # Provide path to Solr configs necessary to build Docker image

dspace/src/main/docker-compose/db.entities.yml

@@ -8,7 +8,7 @@
 
 services:
   dspacedb:
-    image: dspace/dspace-postgres-pgcrypto:${DSPACE_VER:-latest}-loadsql
+    image: dspace/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-8_x}-loadsql
     environment:
       # This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data
       - LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-data.sql

dspace/src/main/docker-compose/db.restore.yml

@@ -12,7 +12,7 @@
 # This can be used to restore a "dspacedb" container from a pg_dump, or during upgrade to a new version of PostgreSQL.
 services:
   dspacedb:
-    image: dspace/dspace-postgres-pgcrypto:${DSPACE_VER:-latest}-loadsql
+    image: dspace/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-8_x}-loadsql
     environment:
       # Location where the dump SQL file will be available on the running container
       - LOCALSQL=/tmp/pgdump.sql

dspace/src/main/docker-compose/docker-compose-angular.yml

@@ -26,7 +26,7 @@ services:
       DSPACE_REST_HOST: localhost
       DSPACE_REST_PORT: 8080
       DSPACE_REST_NAMESPACE: /server
-    image: dspace/dspace-angular:${DSPACE_VER:-latest}
+    image: dspace/dspace-angular:${DSPACE_VER:-dspace-8_x}
     ports:
     - published: 4000
       target: 4000