diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 00000000..a29f1e0e --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,3 @@ +ignored: + - DL3006 + - DL3008 diff --git a/.travis.yml b/.travis.yml index 720984b1..763376d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,11 @@ jobs: install: - pip install --upgrade pip - make dev-env + - make lint-install script: - set -e - if [ $(make n-docs-diff) -ne 0 ]; then make docs; fi; - - if [ $(make n-other-diff) -ne 0 ]; then make build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"; fi; + - if [ $(make n-other-diff) -ne 0 ]; then make lint-build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"; fi; - stage: push-tx install: - pip install --upgrade pip @@ -26,10 +27,11 @@ jobs: install: - pip install --upgrade pip - make dev-env + - make lint-install script: - set -e - make docs - - make build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1" + - make lint-build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1" stages: - name: diff-test diff --git a/Makefile b/Makefile index 802d4a2d..106ebcfd 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,9 @@ endif ALL_IMAGES:=$(ALL_STACKS) +# Linter +HADOLINT="${HOME}/hadolint" + help: # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html @echo "jupyter/docker-stacks" @@ -73,6 +76,23 @@ dev/%: ## run a foreground container for a stack dev-env: ## install libraries required to build docs and run tests pip install -r requirements-dev.txt +lint/%: ARGS?= +lint/%: ## lint the dockerfile(s) for a stack + @echo "Linting Dockerfiles in $(notdir $@)..." + @git ls-files --exclude='Dockerfile*' --ignored $(notdir $@) | grep -v ppc64 | xargs -L 1 $(HADOLINT) $(ARGS) + @echo "Linting done!" + +lint-all: $(foreach I,$(ALL_IMAGES),lint/$(I) ) ## lint all stacks + +lint-build-test-all: $(foreach I,$(ALL_IMAGES),lint/$(I) arch_patch/$(I) build/$(I) test/$(I) ) ## lint, build and test all stacks + +lint-install: ## install hadolint + @echo "Installing hadolint at $(HADOLINT) ..." + @curl -sL -o $(HADOLINT) "https://github.com/hadolint/hadolint/releases/download/v1.17.6/hadolint-$(shell uname -s)-$(shell uname -m)" + @chmod 700 $(HADOLINT) + @echo "Installation done!" + @$(HADOLINT) --version + img-clean: img-rm-dang img-rm ## clean dangling and jupyter images img-list: ## list jupyter images diff --git a/all-spark-notebook/Dockerfile b/all-spark-notebook/Dockerfile index 29c0f0ba..401a2d0a 100644 --- a/all-spark-notebook/Dockerfile +++ b/all-spark-notebook/Dockerfile @@ -30,22 +30,23 @@ RUN conda install --quiet --yes \ 'r-sparklyr=1.2*' \ && \ conda clean --all -f -y && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" # Apache Toree kernel +# hadolint ignore=DL3013 RUN pip install --no-cache-dir \ https://dist.apache.org/repos/dist/release/incubator/toree/0.3.0-incubating/toree-pip/toree-0.3.0.tar.gz \ && \ jupyter toree install --sys-prefix && \ - rm -rf /home/$NB_USER/.local && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + rm -rf "/home/${NB_USER}/.local" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" # Spylon-kernel RUN conda install --quiet --yes 'spylon-kernel=0.4*' && \ conda clean --all -f -y && \ python -m spylon_kernel install --sys-prefix && \ - rm -rf /home/$NB_USER/.local && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + rm -rf "/home/${NB_USER}/.local" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 49b30e8d..354b36c6 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -13,6 +13,9 @@ ARG NB_USER="jovyan" ARG NB_UID="1000" ARG NB_GID="100" +# Fix DL4006 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + USER root # Install all OS dependencies for notebook server that starts but lacks all @@ -76,8 +79,8 @@ ENV MINICONDA_VERSION=4.8.2 \ MINICONDA_MD5=87e77f097f6ebb5127c77662dfc3165e \ CONDA_VERSION=4.8.2 -RUN cd /tmp && \ - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \ +WORKDIR /tmp +RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \ echo "${MINICONDA_MD5} *Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \ /bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \ @@ -137,3 +140,5 @@ RUN fix-permissions /etc/jupyter/ # Switch back to jovyan to avoid accidental container runs as root USER $NB_UID + +WORKDIR $HOME diff --git a/datascience-notebook/Dockerfile b/datascience-notebook/Dockerfile index c7989245..82803f57 100644 --- a/datascience-notebook/Dockerfile +++ b/datascience-notebook/Dockerfile @@ -9,6 +9,9 @@ LABEL maintainer="Jupyter Project " # be skipped to shorten build time. ARG TEST_ONLY_BUILD +# Fix DL4006 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + USER root # R pre-requisites @@ -25,21 +28,23 @@ ENV JULIA_DEPOT_PATH=/opt/julia ENV JULIA_PKGDIR=/opt/julia ENV JULIA_VERSION=1.4.1 -RUN mkdir /opt/julia-${JULIA_VERSION} && \ - cd /tmp && \ - wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \ +WORKDIR /tmp + +# hadolint ignore=SC2046 +RUN mkdir "/opt/julia-${JULIA_VERSION}" && \ + wget -q https://julialang-s3.julialang.org/bin/linux/x64/$(echo "${JULIA_VERSION}" | cut -d. -f 1,2)"/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" && \ echo "fd6d8cadaed678174c3caefb92207a3b0e8da9f926af6703fb4d1e4e4f50610a *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \ - tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \ - rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz + tar xzf "julia-${JULIA_VERSION}-linux-x86_64.tar.gz" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 && \ + rm "/tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia # Show Julia where conda libraries are \ RUN mkdir /etc/julia && \ echo "push!(Libdl.DL_LOAD_PATH, \"$CONDA_DIR/lib\")" >> /etc/julia/juliarc.jl && \ # Create JULIA_PKGDIR \ - mkdir $JULIA_PKGDIR && \ - chown $NB_USER $JULIA_PKGDIR && \ - fix-permissions $JULIA_PKGDIR + mkdir "${JULIA_PKGDIR}" && \ + chown "${NB_USER}" "${JULIA_PKGDIR}" && \ + fix-permissions "${JULIA_PKGDIR}" USER $NB_UID @@ -66,8 +71,8 @@ RUN conda install --quiet --yes \ 'rpy2=3.1*' \ && \ conda clean --all -f -y && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" # Add Julia packages. Only add HDF5 if this is not a test-only build since # it takes roughly half the entire build time of all of the images on Travis @@ -80,7 +85,9 @@ RUN julia -e 'import Pkg; Pkg.update()' && \ (test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \ julia -e "using Pkg; pkg\"add IJulia\"; pkg\"precompile\"" && \ # move kernelspec out of home \ - mv $HOME/.local/share/jupyter/kernels/julia* $CONDA_DIR/share/jupyter/kernels/ && \ - chmod -R go+rx $CONDA_DIR/share/jupyter && \ - rm -rf $HOME/.local && \ - fix-permissions $JULIA_PKGDIR $CONDA_DIR/share/jupyter + mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" && \ + chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \ + rm -rf "${HOME}/.local" && \ + fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter" + +WORKDIR $HOME diff --git a/docs/contributing/lint.md b/docs/contributing/lint.md new file mode 100644 index 00000000..7ce07bdd --- /dev/null +++ b/docs/contributing/lint.md @@ -0,0 +1,78 @@ +# Image Lint + +To comply with [Docker best practices][dbp], we are using the [Hadolint][hadolint] tool to analyse each `Dockerfile` . + +## Installation + +There is a specific `make` target to install the linter. +By default `hadolint` will be installed in `${HOME}/hadolint`. + +```bash +$ make lint-install + +# Installing hadolint at /Users/romain/hadolint ... +# Installation done! +# Haskell Dockerfile Linter v1.17.6-0-gc918759 +``` + +## Lint + +### Per Stack + +The linter can be run per stack. + +```bash +$ make lint/scipy-notebook + +# Linting Dockerfiles in scipy-notebook... +# scipy-notebook/Dockerfile:4 DL3006 Always tag the version of an image explicitly +# scipy-notebook/Dockerfile:11 DL3008 Pin versions in apt get install. Instead of `apt-get install ` use `apt-get install =` +# scipy-notebook/Dockerfile:18 SC2086 Double quote to prevent globbing and word splitting. +# scipy-notebook/Dockerfile:68 SC2086 Double quote to prevent globbing and word splitting. +# scipy-notebook/Dockerfile:68 DL3003 Use WORKDIR to switch to a directory +# scipy-notebook/Dockerfile:79 SC2086 Double quote to prevent globbing and word splitting. +# make: *** [lint/scipy-notebook] Error 1 +``` + +Optionally you can pass arguments to the linter. + +```bash +# Use a different export format +$ make lint/scipy-notebook ARGS="--format codeclimate" +``` + +### All the Stacks + +The linter can be run against all the stacks. + +```bash +$ make lint-all +``` + +## Ignoring Rules + +Sometimes it is necessary to ignore [some rules][rules]. +The following rules are ignored by default and sor for all images in the `.hadolint.yaml` file. + +- [`DL3006`][DL3006]: We use a specific policy to manage image tags. + - `base-notebook` `FROM` clause is fixed but based on an argument (`ARG`). + - Building downstream images from (`FROM`) the latest is done on purpose. +- [`DL3008`][DL3008]: System packages are always updated (`apt-get`) to the latest version. + +For other rules, the preferred way to do it is to flag ignored rules in the `Dockerfile`. + +> It is also possible to ignore rules by using a special comment directly above the Dockerfile instruction you want to make an exception for. Ignore rule comments look like `# hadolint ignore=DL3001,SC1081`. For example: + +```dockerfile + +FROM ubuntu + +# hadolint ignore=DL3003,SC1035 +RUN cd /tmp && echo "hello!" +``` + +[hadolint]: https://github.com/hadolint/hadolint +[dbp]: https://docs.docker.com/develop/develop-images/dockerfile_best-practices +[rules]: https://github.com/hadolint/hadolint#rules +[DL3006]: https://github.com/hadolint/hadolint/wiki/DL3006 +[DL3008]: https://github.com/hadolint/hadolint/wiki/DL3008 \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 132d2aed..6cf28541 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,6 +47,7 @@ Table of Contents contributing/packages contributing/recipes contributing/translations + contributing/lint contributing/tests contributing/features contributing/stacks diff --git a/pyspark-notebook/Dockerfile b/pyspark-notebook/Dockerfile index 8e3d5a70..83f0b161 100644 --- a/pyspark-notebook/Dockerfile +++ b/pyspark-notebook/Dockerfile @@ -5,6 +5,9 @@ FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " +# Fix DL4006 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + USER root # Spark dependencies @@ -16,13 +19,16 @@ RUN apt-get -y update && \ rm -rf /var/lib/apt/lists/* # Using the preferred mirror to download Spark -RUN cd /tmp && \ - wget -q $(wget -qO- https://www.apache.org/dyn/closer.lua/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz\?as_json | \ +WORKDIR /tmp +# hadolint ignore=SC2046 +RUN wget -q $(wget -qO- https://www.apache.org/dyn/closer.lua/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz\?as_json | \ python -c "import sys, json; content=json.load(sys.stdin); print(content['preferred']+content['path_info'])") && \ echo "2426a20c548bdfc07df288cd1d18d1da6b3189d0b78dee76fa034c52a4e02895f0ad460720c526f163ba63a17efae4764c46a1cd8f9b04c60f9937a554db85d2 *spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" | sha512sum -c - && \ - tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -C /usr/local --owner root --group root --no-same-owner && \ - rm spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark + tar xzf "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" -C /usr/local --owner root --group root --no-same-owner && \ + rm "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" + +WORKDIR /usr/local +RUN ln -s "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" spark # Configure Spark ENV SPARK_HOME=/usr/local/spark @@ -35,5 +41,7 @@ USER $NB_UID # Install pyarrow RUN conda install --quiet -y 'pyarrow' && \ conda clean --all -f -y && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +WORKDIR $HOME diff --git a/r-notebook/Dockerfile b/r-notebook/Dockerfile index 641b0ed5..3be64c85 100644 --- a/r-notebook/Dockerfile +++ b/r-notebook/Dockerfile @@ -47,7 +47,7 @@ RUN conda install --quiet --yes \ 'unixodbc=2.3.*' \ && \ conda clean --all -f -y && \ - fix-permissions $CONDA_DIR + fix-permissions "${CONDA_DIR}" # Install e1071 R package (dependency of the caret R package) RUN conda install --quiet --yes r-e1071 diff --git a/scipy-notebook/Dockerfile b/scipy-notebook/Dockerfile index aa631f49..b498e534 100644 --- a/scipy-notebook/Dockerfile +++ b/scipy-notebook/Dockerfile @@ -59,24 +59,25 @@ RUN conda install --quiet --yes \ jupyter lab build -y && \ jupyter lab clean -y && \ npm cache clean --force && \ - rm -rf /home/$NB_USER/.cache/yarn && \ - rm -rf /home/$NB_USER/.node-gyp && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + rm -rf "/home/${NB_USER}/.cache/yarn" && \ + rm -rf "/home/${NB_USER}/.node-gyp" && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" # Install facets which does not have a pip or conda package at the moment -RUN cd /tmp && \ - git clone https://github.com/PAIR-code/facets.git && \ - cd facets && \ - jupyter nbextension install facets-dist/ --sys-prefix && \ - cd && \ +WORKDIR /tmp +RUN git clone https://github.com/PAIR-code/facets.git && \ + jupyter nbextension install facets/facets-dist/ --sys-prefix && \ rm -rf /tmp/facets && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" # Import matplotlib the first time to build the font cache. -ENV XDG_CACHE_HOME /home/$NB_USER/.cache/ +ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/" + RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ - fix-permissions /home/$NB_USER + fix-permissions "/home/${NB_USER}" USER $NB_UID + +WORKDIR $HOME diff --git a/tensorflow-notebook/Dockerfile b/tensorflow-notebook/Dockerfile index 880905fe..4533fbc6 100644 --- a/tensorflow-notebook/Dockerfile +++ b/tensorflow-notebook/Dockerfile @@ -8,5 +8,5 @@ LABEL maintainer="Jupyter Project " # Install Tensorflow RUN pip install --quiet --no-cache-dir \ 'tensorflow==2.2.0' && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}"