From bfb8cc7d50af7d8443770299ed57703b15c369a9 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 20 Jun 2021 22:13:15 +0300 Subject: [PATCH 1/9] Allow conda to automatically deduce package versions --- all-spark-notebook/Dockerfile | 12 ++++---- base-notebook/Dockerfile | 6 ++-- datascience-notebook/Dockerfile | 40 ++++++++++++------------ pyspark-notebook/Dockerfile | 2 +- r-notebook/Dockerfile | 40 ++++++++++++------------ scipy-notebook/Dockerfile | 54 ++++++++++++++++----------------- tensorflow-notebook/Dockerfile | 2 +- 7 files changed, 78 insertions(+), 78 deletions(-) diff --git a/all-spark-notebook/Dockerfile b/all-spark-notebook/Dockerfile index 21b54bf5..077c8798 100644 --- a/all-spark-notebook/Dockerfile +++ b/all-spark-notebook/Dockerfile @@ -23,17 +23,17 @@ USER ${NB_UID} # R packages including IRKernel which gets installed globally. RUN mamba install --quiet --yes \ - 'r-base=4.1.0' \ - 'r-ggplot2=3.3*' \ - 'r-irkernel=1.2*' \ - 'r-rcurl=1.98*' \ - 'r-sparklyr=1.7*' && \ + 'r-base' \ + 'r-ggplot2' \ + 'r-irkernel' \ + 'r-rcurl' \ + 'r-sparklyr' && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" # Spylon-kernel -RUN mamba install --quiet --yes 'spylon-kernel=0.4*' && \ +RUN mamba install --quiet --yes 'spylon-kernel' && \ mamba clean --all -f -y && \ python -m spylon_kernel install --sys-prefix && \ rm -rf "/home/${NB_USER}/.local" && \ diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 6dd06d92..55b7c887 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -130,9 +130,9 @@ RUN wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${m # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change RUN conda install --quiet --yes \ - 'notebook=6.4.0' \ - 'jupyterhub=1.4.1' \ - 'jupyterlab=3.0.16' && \ + 'notebook' \ + 'jupyterhub' \ + 'jupyterlab' && \ conda clean --all -f -y && \ npm cache clean --force && \ jupyter notebook --generate-config && \ diff --git a/datascience-notebook/Dockerfile b/datascience-notebook/Dockerfile index c930a900..c2b2eac8 100644 --- a/datascience-notebook/Dockerfile +++ b/datascience-notebook/Dockerfile @@ -54,26 +54,26 @@ USER ${NB_UID} # R packages including IRKernel which gets installed globally. RUN conda install --quiet --yes \ - 'r-base=4.1.0' \ - 'r-caret=6.0*' \ - 'r-crayon=1.4*' \ - 'r-devtools=2.4*' \ - 'r-forecast=8.15*' \ - 'r-hexbin=1.28*' \ - 'r-htmltools=0.5*' \ - 'r-htmlwidgets=1.5*' \ - 'r-irkernel=1.2*' \ - 'r-nycflights13=1.0*' \ - 'r-randomforest=4.6*' \ - 'r-rcurl=1.98*' \ - 'r-rmarkdown=2.9*' \ - 'r-rodbc=1.3*' \ - 'r-rsqlite=2.2*' \ - 'r-shiny=1.6*' \ - 'r-tidymodels=0.1*' \ - 'r-tidyverse=1.3*' \ - 'rpy2=3.4*' \ - 'unixodbc=2.3.*' && \ + 'r-base' \ + 'r-caret' \ + 'r-crayon' \ + 'r-devtools' \ + 'r-forecast' \ + 'r-hexbin' \ + 'r-htmltools' \ + 'r-htmlwidgets' \ + 'r-irkernel' \ + 'r-nycflights13' \ + 'r-randomforest' \ + 'r-rcurl' \ + 'r-rmarkdown' \ + 'r-rodbc' \ + 'r-rsqlite' \ + 'r-shiny' \ + 'r-tidymodels' \ + 'r-tidyverse' \ + 'rpy2' \ + 'unixodbc' && \ conda clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/pyspark-notebook/Dockerfile b/pyspark-notebook/Dockerfile index 8dc2dcc4..cb0e0b37 100644 --- a/pyspark-notebook/Dockerfile +++ b/pyspark-notebook/Dockerfile @@ -56,7 +56,7 @@ USER ${NB_UID} # Install pyarrow RUN mamba install --quiet --yes \ - 'pyarrow=4.0.*' && \ + 'pyarrow' && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/r-notebook/Dockerfile b/r-notebook/Dockerfile index 20053c47..131080f5 100644 --- a/r-notebook/Dockerfile +++ b/r-notebook/Dockerfile @@ -25,31 +25,31 @@ USER ${NB_UID} # R packages including IRKernel which gets installed globally. RUN conda install --quiet --yes \ - 'r-base=4.1.0' \ - 'r-caret=6.*' \ - 'r-crayon=1.4*' \ - 'r-devtools=2.4*' \ - 'r-forecast=8.15*' \ - 'r-hexbin=1.28*' \ - 'r-htmltools=0.5*' \ - 'r-htmlwidgets=1.5*' \ - 'r-irkernel=1.2*' \ - 'r-nycflights13=1.0*' \ - 'r-randomforest=4.6*' \ - 'r-rcurl=1.98*' \ - 'r-rmarkdown=2.9*' \ - 'r-rodbc=1.3*' \ - 'r-rsqlite=2.2*' \ - 'r-shiny=1.6*' \ - 'r-tidymodels=0.1*' \ - 'r-tidyverse=1.3*' \ - 'unixodbc=2.3.*' && \ + 'r-base' \ + 'r-caret' \ + 'r-crayon' \ + 'r-devtools' \ + 'r-forecast' \ + 'r-hexbin' \ + 'r-htmltools' \ + 'r-htmlwidgets' \ + 'r-irkernel' \ + 'r-nycflights13' \ + 'r-randomforest' \ + 'r-rcurl' \ + 'r-rmarkdown' \ + 'r-rodbc' \ + 'r-rsqlite' \ + 'r-shiny' \ + 'r-tidymodels' \ + 'r-tidyverse' \ + 'unixodbc' && \ conda clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" # Install e1071 R package (dependency of the caret R package) -RUN conda install --quiet --yes r-e1071 && \ +RUN conda install --quiet --yes 'r-e1071' && \ conda clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/scipy-notebook/Dockerfile b/scipy-notebook/Dockerfile index b0c41ffc..63483768 100644 --- a/scipy-notebook/Dockerfile +++ b/scipy-notebook/Dockerfile @@ -16,34 +16,34 @@ USER ${NB_UID} # Install Python 3 packages RUN conda install --quiet --yes \ - 'altair=4.1.*' \ - 'beautifulsoup4=4.9.*' \ - 'bokeh=2.3.*' \ - 'bottleneck=1.3.*' \ - 'cloudpickle=1.6.*' \ + 'altair' \ + 'beautifulsoup4' \ + 'bokeh' \ + 'bottleneck' \ + 'cloudpickle' \ 'conda-forge::blas=*=openblas' \ - 'cython=0.29.*' \ - 'dask=2021.6.*' \ - 'dill=0.3.*' \ - 'h5py=3.2.*' \ - 'ipympl=0.7.*'\ - 'ipywidgets=7.6.*' \ - 'matplotlib-base=3.4.*' \ - 'numba=0.53.*' \ - 'numexpr=2.7.*' \ - 'pandas=1.2.*' \ - 'patsy=0.5.*' \ - 'protobuf=3.17.*' \ - 'pytables=3.6.*' \ - 'scikit-image=0.18.*' \ - 'scikit-learn=0.24.*' \ - 'scipy=1.6.*' \ - 'seaborn=0.11.*' \ - 'sqlalchemy=1.4.*' \ - 'statsmodels=0.12.*' \ - 'sympy=1.8.*' \ - 'widgetsnbextension=3.5.*'\ - 'xlrd=2.0.*' && \ + 'cython' \ + 'dask' \ + 'dill' \ + 'h5py' \ + 'ipympl'\ + 'ipywidgets' \ + 'matplotlib-base' \ + 'numba' \ + 'numexpr' \ + 'pandas' \ + 'patsy' \ + 'protobuf' \ + 'pytables' \ + 'scikit-image' \ + 'scikit-learn' \ + 'scipy' \ + 'seaborn' \ + 'sqlalchemy' \ + 'statsmodels' \ + 'sympy' \ + 'widgetsnbextension'\ + 'xlrd' && \ conda clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" diff --git a/tensorflow-notebook/Dockerfile b/tensorflow-notebook/Dockerfile index 3271de75..a0545717 100644 --- a/tensorflow-notebook/Dockerfile +++ b/tensorflow-notebook/Dockerfile @@ -7,7 +7,7 @@ LABEL maintainer="Jupyter Project " # Install Tensorflow RUN mamba install --quiet --yes \ - 'tensorflow=2.4.1' && \ + 'tensorflow' && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" From 62a5ce24a4b44e7088725d7dde11be6ec49fff84 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 22 Jun 2021 13:41:06 +0300 Subject: [PATCH 2/9] Use ubuntu:focal --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 55b7c887..0378e604 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -4,7 +4,7 @@ # Ubuntu 20.04 (focal) # https://hub.docker.com/_/ubuntu/?tab=tags&name=focal # OS/ARCH: linux/amd64 -ARG ROOT_CONTAINER=ubuntu:focal-20210609@sha256:376209074d481dca0a9cf4282710cd30a9e7ff402dea8261acdaaf57a18971dd +ARG ROOT_CONTAINER=ubuntu:focal ARG BASE_CONTAINER=$ROOT_CONTAINER FROM $BASE_CONTAINER From c12b513bf21b02407d43c271fb7fa5f6a2673610 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 14:51:23 +0300 Subject: [PATCH 3/9] Try to add cron workflow --- .github/workflows/cron-docker.yaml | 26 ++++++++++++++++++++++++++ .github/workflows/docker.yml | 2 ++ 2 files changed, 28 insertions(+) create mode 100644 .github/workflows/cron-docker.yaml diff --git a/.github/workflows/cron-docker.yaml b/.github/workflows/cron-docker.yaml new file mode 100644 index 00000000..3b9f675d --- /dev/null +++ b/.github/workflows/cron-docker.yaml @@ -0,0 +1,26 @@ +name: Create a commit to update SHA tag and dispatch to launch Docker images build + +on: + schedule: + # weekly + - cron: "0 0 * * 0" + +jobs: + build-images: + name: Create a commit to update SHA tag and dispatch to launch Docker images build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Clone Main Repo + uses: actions/checkout@v2 + - name: Create an empty commit + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Empty commit to update sha tag + commit_options: "--allow-empty" + - name: Repository Dispatch + uses: peter-evans/repository-dispatch@v1 + with: + token: ${{secrets.REPO_ACCESS_TOKEN}} + event-type: cron-docker-update diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 714e705d..5efe67ee 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,8 @@ name: Build Docker Images on: + repository_dispatch: + types: [cron-docker-update] pull_request: paths-ignore: - ".github/ISSUE_TEMPLATE/**" From f1010ec6f1a856aa9cdfa33e22597d8204b029a3 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Mon, 28 Jun 2021 10:28:54 +0300 Subject: [PATCH 4/9] Pin 3rd party actions --- .github/workflows/cron-docker.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cron-docker.yaml b/.github/workflows/cron-docker.yaml index 3b9f675d..c1865852 100644 --- a/.github/workflows/cron-docker.yaml +++ b/.github/workflows/cron-docker.yaml @@ -15,12 +15,12 @@ jobs: - name: Clone Main Repo uses: actions/checkout@v2 - name: Create an empty commit - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release with: commit_message: Empty commit to update sha tag commit_options: "--allow-empty" - name: Repository Dispatch - uses: peter-evans/repository-dispatch@v1 + uses: peter-evans/repository-dispatch@ce5485de42c9b2622d2ed064be479e8ed65e76f4 # dependabot updates to latest release with: token: ${{secrets.REPO_ACCESS_TOKEN}} event-type: cron-docker-update From b2a0489e96d2e34688145edee79edb320bfdb334 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 17 Jul 2021 22:28:11 +0300 Subject: [PATCH 5/9] Improve workflow style --- .github/workflows/cron-docker.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cron-docker.yaml b/.github/workflows/cron-docker.yaml index c1865852..5ee3410f 100644 --- a/.github/workflows/cron-docker.yaml +++ b/.github/workflows/cron-docker.yaml @@ -11,14 +11,17 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + steps: - name: Clone Main Repo uses: actions/checkout@v2 + - name: Create an empty commit uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release with: commit_message: Empty commit to update sha tag commit_options: "--allow-empty" + - name: Repository Dispatch uses: peter-evans/repository-dispatch@ce5485de42c9b2622d2ed064be479e8ed65e76f4 # dependabot updates to latest release with: From 481d34a97c0a176aff3cd33539fc93ec0d994e4a Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 3 Aug 2021 23:05:30 +0300 Subject: [PATCH 6/9] Apply suggestions from code review Co-authored-by: Erik Sundell --- .github/workflows/cron-docker.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cron-docker.yaml b/.github/workflows/cron-docker.yaml index 5ee3410f..a1bced10 100644 --- a/.github/workflows/cron-docker.yaml +++ b/.github/workflows/cron-docker.yaml @@ -1,4 +1,10 @@ -name: Create a commit to update SHA tag and dispatch to launch Docker images build +# This workflows purpose is to make sure we rebuild our images regularly. +# +# This workflow makes an empty commit to ensure we get a new SHA tag, and it +# explicitly dispatches the regular workflow that builds and pushes our images +# because the commit we made in this workflow can't trigger the other workflow. +# +name: Weekly rebuild dispatch on: schedule: @@ -6,7 +12,7 @@ on: - cron: "0 0 * * 0" jobs: - build-images: + commit-and-dispatch: name: Create a commit to update SHA tag and dispatch to launch Docker images build runs-on: ubuntu-latest permissions: @@ -19,6 +25,7 @@ jobs: - name: Create an empty commit uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release with: + commit_author: GitHub Actions commit_message: Empty commit to update sha tag commit_options: "--allow-empty" From 034a7b8886c9f48766e912d7c1ef3395bdc9a7df Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 9 Aug 2021 14:28:22 +0200 Subject: [PATCH 7/9] ci: refactor cron job into existing workflow --- .github/workflows/cron-docker.yaml | 36 ------------------------------ .github/workflows/docker.yml | 11 ++++----- 2 files changed, 6 insertions(+), 41 deletions(-) delete mode 100644 .github/workflows/cron-docker.yaml diff --git a/.github/workflows/cron-docker.yaml b/.github/workflows/cron-docker.yaml deleted file mode 100644 index a1bced10..00000000 --- a/.github/workflows/cron-docker.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# This workflows purpose is to make sure we rebuild our images regularly. -# -# This workflow makes an empty commit to ensure we get a new SHA tag, and it -# explicitly dispatches the regular workflow that builds and pushes our images -# because the commit we made in this workflow can't trigger the other workflow. -# -name: Weekly rebuild dispatch - -on: - schedule: - # weekly - - cron: "0 0 * * 0" - -jobs: - commit-and-dispatch: - name: Create a commit to update SHA tag and dispatch to launch Docker images build - runs-on: ubuntu-latest - permissions: - contents: write - - steps: - - name: Clone Main Repo - uses: actions/checkout@v2 - - - name: Create an empty commit - uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release - with: - commit_author: GitHub Actions - commit_message: Empty commit to update sha tag - commit_options: "--allow-empty" - - - name: Repository Dispatch - uses: peter-evans/repository-dispatch@ce5485de42c9b2622d2ed064be479e8ed65e76f4 # dependabot updates to latest release - with: - token: ${{secrets.REPO_ACCESS_TOKEN}} - event-type: cron-docker-update diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f188862a..81c4375f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,8 +1,9 @@ name: Build, test, and publish Docker Images on: - repository_dispatch: - types: [cron-docker-update] + schedule: + # Weekly, at 03:00 on Monday UTC time + - cron: "0 3 * * 1" pull_request: paths-ignore: - ".github/ISSUE_TEMPLATE/**" @@ -80,19 +81,19 @@ jobs: run: make -C main hook-all - name: Push Wiki to GitHub - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.event_name == 'schedule' uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release with: commit_message: "[ci skip] Automated publish for ${{github.sha}}" repository: wiki/ - name: Login to Docker Hub - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.event_name == 'schedule' uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # dependabot updates to latest release with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} - name: Push Images to DockerHub - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.event_name == 'schedule' run: make -C main push-all-multi From 86f94a35197d82f47bc3179faea5caf015736739 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 9 Aug 2021 16:45:35 +0200 Subject: [PATCH 8/9] docs: adjust docs about building/versioning --- docs/using/selecting.md | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/docs/using/selecting.md b/docs/using/selecting.md index 9ea5f1c4..4b6b0a34 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -193,23 +193,21 @@ diagram](../images/inherit.svg)](http://interactive.blockdiag.com/?compression=d ### Builds -Pull requests to the `jupyter/docker-stacks` repository trigger builds of all images on GitHub Actions. -These images are for testing purposes only and are not saved for further use. -When pull requests merge to master, all images rebuild on Docker Hub and become available to `docker pull` from Docker Hub. +Every Monday and whenever a pull requests is merged, images are rebuilt and pushed to the public container registry. -### Versioning +### Versioning via image tags -The `latest` tag in each Docker Hub repository tracks the master branch `HEAD` reference on GitHub. -`latest` is a moving target, by definition, and will have backward-incompatible changes regularly. +Whenever a docker image is pushed to the container registry, it is tagged with: -Every image on Docker Hub also receives a 12-character tag which corresponds with the git commit SHA that triggered the image build. -You can inspect the state of the `jupyter/docker-stacks` repository for that commit to review the definition of the image -(e.g., images with tag `33add21fab64` were built from . +- a `latest` tag +- a 12-character git commit SHA like `b9f6ce795cfc` +- a date formatted like `2021-08-29` +- a set of software version tags like `python-3.9.6`, and `lab-3.0.16` -You must refer to git-SHA image tags when stability and reproducibility are important in your work. -(e.g. `FROM jupyter/scipy-notebook:33add21fab64`, `docker run -it --rm jupyter/scipy-notebook:33add21fab64`). -You should only use `latest` when a one-off container instance is acceptable -(e.g., you want to briefly try a new library in a notebook). +For stability and reproducibility, you should either reference a date formatted +tag from a date before the current date (in UTC time) or a git commit SHA older +than the latest git commit SHA in the default branch of the +jupyter/docker-stacks GitHub repository. ## Community Stacks From f50b2a80a50ea805dbf3e339557cbc9d2dee4a98 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 9 Aug 2021 16:53:35 +0200 Subject: [PATCH 9/9] Apply suggestions from code review --- docs/using/selecting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using/selecting.md b/docs/using/selecting.md index 4b6b0a34..8f109ec2 100644 --- a/docs/using/selecting.md +++ b/docs/using/selecting.md @@ -202,7 +202,7 @@ Whenever a docker image is pushed to the container registry, it is tagged with: - a `latest` tag - a 12-character git commit SHA like `b9f6ce795cfc` - a date formatted like `2021-08-29` -- a set of software version tags like `python-3.9.6`, and `lab-3.0.16` +- a set of software version tags like `python-3.9.6` and `lab-3.0.16` For stability and reproducibility, you should either reference a date formatted tag from a date before the current date (in UTC time) or a git commit SHA older