Refactor handling of artifacts (#1930)

.github/actions/load-image/action.yml

@@ -18,13 +18,12 @@ runs:
       uses: actions/download-artifact@v3
       with:
         name: ${{ inputs.image }}-${{ inputs.platform }}
-        path: /tmp/
+        path: /tmp/jupyter/images/
     - name: Load downloaded image to docker 📥
       run: |
-        docker load --input /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+        docker load --input /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar
         docker image ls --all
       shell: bash
     - name: Delete the file 🗑️
-      run: rm -f /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+      run: rm /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar
       shell: bash
-      if: always()

.github/workflows/docker-build-test-upload.yml

@@ -44,9 +44,7 @@ jobs:
           docker kill $(docker ps --quiet) || true
           docker rm $(docker ps --all --quiet) || true
           docker system prune --all --force
-          rm -rf /tmp/tags/
-          rm -rf /tmp/hist_lines/
-          rm -rf /tmp/manifests/
+          rm -rf /tmp/jupyter/
         shell: bash
 
       - name: Load parent built image to Docker 📥
@@ -70,44 +68,39 @@ jobs:
 
       - name: Write tags file 🏷
         run: |
-          python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/tags/ --owner ${{ env.OWNER }}
+          python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --owner ${{ env.OWNER }}
         shell: bash
       - name: Upload tags file 💾
         uses: actions/upload-artifact@v3
         with:
           name: ${{ inputs.image }}-${{ inputs.platform }}-tags
-          path: /tmp/tags/${{ inputs.platform }}-${{ inputs.image }}.txt
+          path: /tmp/jupyter/tags/${{ inputs.platform }}-${{ inputs.image }}.txt
           retention-days: 3
 
       - name: Write manifest and build history file 🏷
-        run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner ${{ env.OWNER }}
+        run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner ${{ env.OWNER }}
         shell: bash
       - name: Upload manifest file 💾
         uses: actions/upload-artifact@v3
         with:
           name: ${{ inputs.image }}-${{ inputs.platform }}-manifest
-          path: /tmp/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md
+          path: /tmp/jupyter/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md
           retention-days: 3
       - name: Upload build history line 💾
         uses: actions/upload-artifact@v3
         with:
           name: ${{ inputs.image }}-${{ inputs.platform }}-history_line
-          path: /tmp/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt
+          path: /tmp/jupyter/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt
           retention-days: 3
 
       - name: Save image as a tar for later use 💾
-        run: docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+        run: |
+          mkdir -p /tmp/jupyter/images/
+          docker save ${{ env.OWNER }}/${{ inputs.image }} -o /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar
         shell: bash
       - name: Upload image as artifact 💾
         uses: actions/upload-artifact@v3
         with:
           name: ${{ inputs.image }}-${{ inputs.platform }}
-          path: /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
+          path: /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar
           retention-days: 3
-
-      # Self-hosted runners share a state (whole VM) between runs
-      - name: Cleanup artifacts 🗑️
-        run: |
-          rm -f /tmp/${{ inputs.image }}-${{ inputs.platform }}.tar
-        shell: bash
-        if: always()

.github/workflows/docker-merge-tags.yml

@@ -32,12 +32,12 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: ${{ inputs.image }}-x86_64-tags
-          path: /tmp/tags/
+          path: /tmp/jupyter/tags/
       - name: Download aarch64 tags file 📥
         uses: actions/download-artifact@v3
         with:
           name: ${{ inputs.image }}-aarch64-tags
-          path: /tmp/tags/
+          path: /tmp/jupyter/tags/
 
       # Docker might be stuck when pulling images
       # https://github.com/docker/for-mac/issues/2083
@@ -57,5 +57,5 @@ jobs:
 
       - name: Merge tags for the images 🔀
         if: github.repository == 'jupyter/docker-stacks' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule')
-        run: python3 -m tagging.merge_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/tags/
+        run: python3 -m tagging.merge_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/
         shell: bash

.github/workflows/docker-tag-push.yml

@@ -48,9 +48,9 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: ${{ inputs.image }}-${{ inputs.platform }}-tags
-          path: /tmp/tags/
+          path: /tmp/jupyter/tags/
       - name: Apply tags to the loaded image 🏷
-        run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }}
+        run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --owner ${{ env.OWNER }}
 
       - name: Push Images to Docker Hub 📤
         if: github.repository == 'jupyter/docker-stacks' && (github.ref == 'refs/heads/main' || github.event_name == 'schedule')

.github/workflows/docker-wiki-update.yml

@@ -20,12 +20,12 @@ jobs:
       - name: Download all manifests and history lines 📥
         uses: ./.github/actions/download-manifests
         with:
-          histLineDir: /tmp/hist_lines/
-          manifestDir: /tmp/manifests/
+          histLineDir: /tmp/jupyter/hist_lines/
+          manifestDir: /tmp/jupyter/manifests/
       - name: Display structure of downloaded files 🔍️
         run: |
-          ls -R /tmp/hist_lines/
-          ls -R /tmp/manifests/
+          ls -R /tmp/jupyter/hist_lines/
+          ls -R /tmp/jupyter/manifests/
         shell: bash
 
       - name: Checkout Wiki Repo 📃
@@ -35,7 +35,7 @@ jobs:
           path: wiki/
 
       - name: Update wiki page 🏷
-        run: python3 -m tagging.update_wiki_page --wiki-dir wiki/ --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/
+        run: python3 -m tagging.update_wiki_page --wiki-dir wiki/ --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/
         shell: bash
 
       - name: Push Wiki to GitHub 📤

Makefile

@@ -69,7 +69,7 @@ linkcheck-docs: ## check broken links
 
 hook/%: ## run post-build hooks for an image
 	python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \
-	python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/hist_lines/ --manifest-dir /tmp/manifests/ --owner "$(OWNER)"
+	python3 -m tagging.write_manifest --short-image-name "$(notdir $@)" --hist-line-dir /tmp/jupyter/hist_lines/ --manifest-dir /tmp/jupyter/manifests/ --owner "$(OWNER)"
 hook-all: $(foreach I, $(ALL_IMAGES), hook/$(I)) ## run post-build hooks for all images