diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7e7cbc1b07..64597ff568 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -149,9 +149,9 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_ACCESS_TOKEN: ${{ secrets.DOCKER_ACCESS_TOKEN }} - ######################################################################## - # Test Deployment via Docker to ensure images are working properly - ######################################################################## + ################################################################################# + # Test Deployment via Docker to ensure newly built images are working properly + ################################################################################# docker-deploy: # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace' if: github.repository == 'dspace/dspace' @@ -170,27 +170,24 @@ jobs: signposting__P__enabled: true sword__D__server__P__enabled: true swordv2__D__server__P__enabled: true - # If this is a PR, force using "pr-testing" version of all Docker images. Otherwise, if on main branch, use the - # "latest" tag. Otherwise, use the branch name. NOTE: the "pr-testing" tag is a temporary tag that we assign to - # all PR-built docker images in reusabe-docker-build.yml - DSPACE_VER: ${{ (github.event_name == 'pull_request' && 'pr-testing') || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }} + # If this is a PR, used the base branch name. If on main branch, use the "latest" tag. Otherwise, use branch name. + # NOTE: DSPACE_VER is used because our docker compose scripts default to using the "-test" image. + DSPACE_VER: ${{ (github.event_name == 'pull_request' && github.event.pull_request.base.ref) || (github.ref_name == github.event.repository.default_branch && 'latest') || github.ref_name }} steps: # Checkout our codebase (to get access to Docker Compose scripts) - name: Checkout codebase uses: actions/checkout@v4 - # For PRs, download Docker image artifacts (built by reusable-docker-build.yml for all PRs) - - name: Download Docker image artifacts (for PRs) - if: github.event_name == 'pull_request' + # Download Docker image artifacts (which were just built by reusable-docker-build.yml) + - name: Download Docker image artifacts uses: actions/download-artifact@v4 with: - # Download all Docker images (TAR files) into the /tmp/docker directory - pattern: docker-image-* + # Download all amd64 Docker images (TAR files) into the /tmp/docker directory + pattern: docker-image-*-linux-amd64 path: /tmp/docker merge-multiple: true - # For PRs, load each of the images into Docker by calling "docker image load" for each. - # This ensures we are using the images built from this PR & not the prior versions on DockerHub - - name: Load all downloaded Docker images (for PRs) - if: github.event_name == 'pull_request' + # Load each of the images into Docker by calling "docker image load" for each. + # This ensures we are using the images just built & not any prior versions on DockerHub + - name: Load all downloaded Docker images run: | find /tmp/docker -type f -name "*.tar" -exec docker image load --input "{}" \; docker image ls -a diff --git a/.github/workflows/reusable-docker-build.yml b/.github/workflows/reusable-docker-build.yml index 91aa93c54a..a6e3268460 100644 --- a/.github/workflows/reusable-docker-build.yml +++ b/.github/workflows/reusable-docker-build.yml @@ -54,10 +54,13 @@ env: # For a new commit on default branch (main), use the literal tag 'latest' on Docker image. # For a new commit on other branches, use the branch name as the tag for Docker image. # For a new tag, copy that tag name as the tag for Docker image. + # For a pull request, use the name of the base branch that the PR was created against or "latest" (for main). + # e.g. PR against 'main' will use "latest". a PR against 'dspace-7_x' will use 'dspace-7_x'. IMAGE_TAGS: | type=raw,value=latest,enable=${{ github.ref_name == github.event.repository.default_branch }} type=ref,event=branch,enable=${{ github.ref_name != github.event.repository.default_branch }} type=ref,event=tag + type=raw,value=${{ (github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || github.event.pull_request.base.ref }},enable=${{ github.event_name == 'pull_request' }} # Define default tag "flavor" for docker/metadata-action per # https://github.com/docker/metadata-action#flavor-input # We manage the 'latest' tag ourselves to the 'main' branch (see settings above) @@ -105,33 +108,26 @@ jobs: - name: Checkout codebase uses: actions/checkout@v4 - # https://github.com/docker/setup-buildx-action - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU emulation to build for multiple architectures - uses: docker/setup-qemu-action@v3 - - #------------------------------------------------------------ - # Build & deploy steps for new commits to a branch (non-PRs) - # - # These steps build the images, push to DockerHub, and - # (if necessary) redeploy demo/sandbox sites. - #------------------------------------------------------------ # https://github.com/docker/login-action - name: Login to DockerHub - # Only login if not a PR, as PRs only trigger a Docker build and not a push + # Only login if not a PR, as PRs only trigger a Docker build and not a push if: ${{ ! matrix.isPr }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU emulation to build for multiple architectures + uses: docker/setup-qemu-action@v3 + + # https://github.com/docker/setup-buildx-action + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + # https://github.com/docker/metadata-action - # Get Metadata for docker_build_deps step below - - name: Sync metadata (tags, labels) from GitHub to Docker for image - if: ${{ ! matrix.isPr }} + # Extract metadata used for Docker images in all build steps below + - name: Extract metadata (tags, labels) from GitHub for Docker image id: meta_build uses: docker/metadata-action@v5 with: @@ -139,6 +135,12 @@ jobs: tags: ${{ env.IMAGE_TAGS }} flavor: ${{ env.TAGS_FLAVOR }} + #------------------------------------------------------------ + # Build & deploy steps for new commits to a branch (non-PRs) + # + # These steps build the images, push to DockerHub, and + # (if necessary) redeploy demo/sandbox sites. + #------------------------------------------------------------ # https://github.com/docker/build-push-action - name: Build and push image to DockerHub # Only build & push if not a PR @@ -155,6 +157,10 @@ jobs: # Use tags / labels provided by 'docker/metadata-action' above tags: ${{ steps.meta_build.outputs.tags }} labels: ${{ steps.meta_build.outputs.labels }} + # Use GitHub cache to load cached Docker images and cache the results of this build + # This decreases the number of images we need to fetch from DockerHub + cache-from: type=gha,scope=${{ inputs.build_id }} + cache-to: type=gha,scope=${{ inputs.build_id }},mode=max # Export the digest of Docker build locally (for non PRs only) - name: Export Docker build digest @@ -197,26 +203,19 @@ jobs: curl -X POST $REDEPLOY_DEMO_URL #------------------------------------------------------------- - # Build steps for PRs only + # Shared Build steps. + # These are used for PRs as well as new commits to a branch (non-PRs) # - # These steps build the images and store as a build artifact. - # These artifacts can then be used by later jobs to run the - # brand-new images for automated testing. + # These steps build the images and cache/store as a build artifact. + # These artifacts can then be used by later jobs to install the + # brand-new images for automated testing. For non-PRs, this cache is + # also used to avoid pulling the images we just built from DockerHub. #-------------------------------------------------------------- - # Get Metadata for docker_build_deps step below - - name: Create metadata (tags, labels) for local Docker image - if: matrix.isPr - id: meta_build_pr - uses: docker/metadata-action@v5 - with: - images: ${{ env.IMAGE_NAME }} - # Hardcode to use custom "pr-testing" tag because that will allow us to spin up this PR - # for testing in docker.yml - tags: pr-testing - flavor: ${{ env.TAGS_FLAVOR }} - # Build local image and stores in a TAR file in /tmp directory - - name: Build and push image to local image - if: matrix.isPr + + # Build local image (again) and store in a TAR file in /tmp directory + # NOTE: This build is run for both PRs and non-PRs as it's used to "cache" our built images as artifacts. + # NOTE #2: This cannot be combined with push to DockerHub registry above as it's a different type of output. + - name: Build and push image to local TAR file uses: docker/build-push-action@v5 with: build-contexts: | @@ -224,16 +223,20 @@ jobs: context: ${{ inputs.dockerfile_context }} file: ${{ inputs.dockerfile_path }} platforms: ${{ matrix.arch }} - tags: ${{ steps.meta_build_pr.outputs.tags }} - labels: ${{ steps.meta_build_pr.outputs.labels }} + tags: ${{ steps.meta_build.outputs.tags }} + labels: ${{ steps.meta_build.outputs.labels }} + # Use GitHub cache to load cached Docker images and cache the results of this build + # This decreases the number of images we need to fetch from DockerHub + cache-from: type=gha,scope=${{ inputs.build_id }} + cache-to: type=gha,scope=${{ inputs.build_id }},mode=max # Export image to a local TAR file outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar + # Upload the local docker image (in TAR file) to a build Artifact - - name: Upload local image to artifact - if: matrix.isPr + - name: Upload local image TAR to artifact uses: actions/upload-artifact@v4 with: - name: docker-image-${{ inputs.build_id }} + name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }} path: /tmp/${{ inputs.build_id }}.tar if-no-files-found: error retention-days: 1 @@ -257,6 +260,12 @@ jobs: pattern: digests-${{ inputs.build_id }}-* merge-multiple: true + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -268,12 +277,6 @@ jobs: tags: ${{ env.IMAGE_TAGS }} flavor: ${{ env.TAGS_FLAVOR }} - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_ACCESS_TOKEN }} - - name: Create manifest list from digests and push working-directory: /tmp/digests run: | diff --git a/docker-compose.yml b/docker-compose.yml index 4322c71734..8886443824 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,7 +64,7 @@ services: dspacedb: container_name: dspacedb # Uses a custom Postgres image with pgcrypto installed - image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-latest}" + image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-8_x}" build: # Must build out of subdirectory to have access to install script for pgcrypto context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/ @@ -84,7 +84,7 @@ services: # DSpace Solr container dspacesolr: container_name: dspacesolr - image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-latest}" + image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-8_x}" build: context: ./dspace/src/main/docker/dspace-solr/ # Provide path to Solr configs necessary to build Docker image diff --git a/dspace/src/main/docker-compose/db.entities.yml b/dspace/src/main/docker-compose/db.entities.yml index 931d04942d..243511ddae 100644 --- a/dspace/src/main/docker-compose/db.entities.yml +++ b/dspace/src/main/docker-compose/db.entities.yml @@ -8,7 +8,7 @@ services: dspacedb: - image: dspace/dspace-postgres-pgcrypto:${DSPACE_VER:-latest}-loadsql + image: dspace/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-8_x}-loadsql environment: # This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data - LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-data.sql diff --git a/dspace/src/main/docker-compose/db.restore.yml b/dspace/src/main/docker-compose/db.restore.yml index ffeca1e598..dc3993a78c 100644 --- a/dspace/src/main/docker-compose/db.restore.yml +++ b/dspace/src/main/docker-compose/db.restore.yml @@ -12,7 +12,7 @@ # This can be used to restore a "dspacedb" container from a pg_dump, or during upgrade to a new version of PostgreSQL. services: dspacedb: - image: dspace/dspace-postgres-pgcrypto:${DSPACE_VER:-latest}-loadsql + image: dspace/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-8_x}-loadsql environment: # Location where the dump SQL file will be available on the running container - LOCALSQL=/tmp/pgdump.sql diff --git a/dspace/src/main/docker-compose/docker-compose-angular.yml b/dspace/src/main/docker-compose/docker-compose-angular.yml index dc837c5c99..3610d23286 100644 --- a/dspace/src/main/docker-compose/docker-compose-angular.yml +++ b/dspace/src/main/docker-compose/docker-compose-angular.yml @@ -26,7 +26,7 @@ services: DSPACE_REST_HOST: localhost DSPACE_REST_PORT: 8080 DSPACE_REST_NAMESPACE: /server - image: dspace/dspace-angular:${DSPACE_VER:-latest} + image: dspace/dspace-angular:${DSPACE_VER:-dspace-8_x} ports: - published: 4000 target: 4000