From 30a1d17085cce7a49b67d70b543e83485bfca502 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 14:40:45 +0300 Subject: [PATCH 01/47] Add a way to easily test units --- .../test/test_spark_notebooks.py | 3 +- base-notebook/test/test_package_managers.py | 11 ++++--- base-notebook/test/test_pandoc.py | 3 +- base-notebook/test/test_start_container.py | 6 +++- datascience-notebook/test/test_julia.py | 3 +- minimal-notebook/test/test_nbconvert.py | 16 ++++++--- pyspark-notebook/test/test_spark.py | 12 ------- pyspark-notebook/test/units/unit_spark.py | 4 +++ requirements-dev.txt | 1 + scipy-notebook/test/test_matplotlib.py | 18 +++++----- scipy-notebook/test/test_pandas.py | 32 ------------------ scipy-notebook/test/units/unit_pandas.py | 9 +++++ tensorflow-notebook/test/test_tensorflow.py | 30 ----------------- .../test/units/unit_tensorflow.py | 7 ++++ test/helpers.py | 3 +- test/test_units.py | 33 +++++++++++++++++++ 16 files changed, 93 insertions(+), 98 deletions(-) create mode 100644 pyspark-notebook/test/units/unit_spark.py delete mode 100644 scipy-notebook/test/test_pandas.py create mode 100644 scipy-notebook/test/units/unit_pandas.py delete mode 100644 tensorflow-notebook/test/test_tensorflow.py create mode 100644 tensorflow-notebook/test/units/unit_tensorflow.py create mode 100644 test/test_units.py diff --git a/all-spark-notebook/test/test_spark_notebooks.py b/all-spark-notebook/test/test_spark_notebooks.py index 2e232daf..1a1d8528 100644 --- a/all-spark-notebook/test/test_spark_notebooks.py +++ b/all-spark-notebook/test/test_spark_notebooks.py @@ -7,6 +7,7 @@ import pytest import os LOGGER = logging.getLogger(__name__) +THIS_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.parametrize( @@ -16,7 +17,7 @@ LOGGER = logging.getLogger(__name__) ) def test_nbconvert(container, test_file): """Check if Spark notebooks can be executed""" - host_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") + host_data_dir = os.path.join(THIS_DIR, "data") cont_data_dir = "/home/jovyan/data" output_dir = "/tmp" timeout_ms = 600 diff --git a/base-notebook/test/test_package_managers.py b/base-notebook/test/test_package_managers.py index ebd8f294..060706e3 100644 --- a/base-notebook/test/test_package_managers.py +++ b/base-notebook/test/test_package_managers.py @@ -8,22 +8,23 @@ LOGGER = logging.getLogger(__name__) @pytest.mark.parametrize( - "package_manager, cmd", + "package_manager, version_arg", [ ("apt", "--version"), ("conda", "--version"), ("mamba", "--version"), ("npm", "--version"), - ("pip", "--version"), - ], + ("pip", "--version") + ] ) -def test_package_manager(container, package_manager, cmd): +def test_package_manager(container, package_manager, version_arg): """Test the notebook start-notebook script""" LOGGER.info( f"Test that the package manager {package_manager} is working properly ..." ) c = container.run( - tty=True, command=["start.sh", "bash", "-c", f"{package_manager} {cmd}"] + tty=True, + command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"] ) rv = c.wait(timeout=5) logs = c.logs(stdout=True).decode("utf-8") diff --git a/base-notebook/test/test_pandoc.py b/base-notebook/test/test_pandoc.py index c84787c1..bc128921 100644 --- a/base-notebook/test/test_pandoc.py +++ b/base-notebook/test/test_pandoc.py @@ -9,7 +9,8 @@ LOGGER = logging.getLogger(__name__) def test_pandoc(container): """Pandoc shall be able to convert MD to HTML.""" c = container.run( - tty=True, command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc'] + tty=True, + command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc'] ) c.wait(timeout=10) logs = c.logs(stdout=True).decode("utf-8") diff --git a/base-notebook/test/test_start_container.py b/base-notebook/test/test_start_container.py index f55b55f6..3ffd4356 100644 --- a/base-notebook/test/test_start_container.py +++ b/base-notebook/test/test_start_container.py @@ -8,7 +8,11 @@ LOGGER = logging.getLogger(__name__) @pytest.mark.parametrize( - "env,expected_server", [(["JUPYTER_ENABLE_LAB=yes"], "lab"), (None, "notebook"), ], + "env,expected_server", + [ + (["JUPYTER_ENABLE_LAB=yes"], "lab"), + (None, "notebook") + ] ) def test_start_notebook(container, http_client, env, expected_server): """Test the notebook start-notebook script""" diff --git a/datascience-notebook/test/test_julia.py b/datascience-notebook/test/test_julia.py index eaabf559..5fa631a8 100644 --- a/datascience-notebook/test/test_julia.py +++ b/datascience-notebook/test/test_julia.py @@ -9,7 +9,8 @@ def test_julia(container): """Basic julia test""" LOGGER.info("Test that julia is correctly installed ...") running_container = container.run( - tty=True, command=["start.sh", "bash", "-c", "sleep infinity"] + tty=True, + command=["start.sh", "bash", "-c", "sleep infinity"] ) command = "julia --version" cmd = running_container.exec_run(command) diff --git a/minimal-notebook/test/test_nbconvert.py b/minimal-notebook/test/test_nbconvert.py index a9f3093b..e0f45d54 100644 --- a/minimal-notebook/test/test_nbconvert.py +++ b/minimal-notebook/test/test_nbconvert.py @@ -7,15 +7,21 @@ import pytest import os LOGGER = logging.getLogger(__name__) +THIS_DIR = os.path.dirname(os.path.realpath(__file__)) -@pytest.mark.parametrize("test_file, output_format,", [ - ("notebook_math", "pdf"), ("notebook_math", "html"), - ("notebook_svg", "pdf"), ("notebook_svg", "html"), -]) +@pytest.mark.parametrize( + "test_file, output_format", + [ + ("notebook_math", "pdf"), + ("notebook_math", "html"), + ("notebook_svg", "pdf"), + ("notebook_svg", "html") + ] +) def test_nbconvert(container, test_file, output_format): """Check if nbconvert is able to convert a notebook file""" - host_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") + host_data_dir = os.path.join(THIS_DIR, "data") cont_data_dir = "/home/jovyan/data" output_dir = "/tmp" LOGGER.info(f"Test that the example notebook {test_file} can be converted to {output_format.upper()} ...") diff --git a/pyspark-notebook/test/test_spark.py b/pyspark-notebook/test/test_spark.py index 09d74481..a98a2666 100644 --- a/pyspark-notebook/test/test_spark.py +++ b/pyspark-notebook/test/test_spark.py @@ -16,15 +16,3 @@ def test_spark_shell(container): logs = c.logs(stdout=True).decode('utf-8') LOGGER.debug(logs) assert 'res0: Int = 2' in logs, "spark-shell does not work" - - -def test_pyspark(container): - """PySpark should be in the Python path""" - c = container.run( - tty=True, - command=['start.sh', 'python', '-c', 'import pyspark'] - ) - rv = c.wait(timeout=30) - assert rv == 0 or rv["StatusCode"] == 0, "pyspark not in PYTHONPATH" - logs = c.logs(stdout=True).decode('utf-8') - LOGGER.debug(logs) diff --git a/pyspark-notebook/test/units/unit_spark.py b/pyspark-notebook/test/units/unit_spark.py new file mode 100644 index 00000000..80b47d43 --- /dev/null +++ b/pyspark-notebook/test/units/unit_spark.py @@ -0,0 +1,4 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +import pyspark # noqa: F401 diff --git a/requirements-dev.txt b/requirements-dev.txt index a82294f2..7eb7a7cd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ docker myst-parser +packaging plumbum pre-commit pytest diff --git a/scipy-notebook/test/test_matplotlib.py b/scipy-notebook/test/test_matplotlib.py index 34a63f79..ac6d7ceb 100644 --- a/scipy-notebook/test/test_matplotlib.py +++ b/scipy-notebook/test/test_matplotlib.py @@ -7,23 +7,23 @@ import pytest import os LOGGER = logging.getLogger(__name__) +THIS_DIR = os.path.dirname(os.path.realpath(__file__)) -@pytest.mark.parametrize("test_file,expected_file,description", - [ - ("matplotlib_1.py", "test.png", - "Test that matplotlib is able to plot a graph and write it as an image ..."), - ("matplotlib_fonts_1.py", "test_fonts.png", - "Test cm-super latex labels in matplotlib ...") - ]) +@pytest.mark.parametrize( + "test_file,expected_file,description", + [ + ("matplotlib_1.py", "test.png", "Test that matplotlib is able to plot a graph and write it as an image ..."), + ("matplotlib_fonts_1.py", "test_fonts.png", "Test cm-super latex labels in matplotlib ...") + ] +) def test_matplotlib(container, test_file, expected_file, description): """Various tests performed on matplotlib - Test that matplotlib is able to plot a graph and write it as an image - Test matplotlib latex fonts, which depend on the cm-super package """ - host_data_dir = os.path.join(os.path.dirname( - os.path.realpath(__file__)), "data") + host_data_dir = os.path.join(THIS_DIR, "data") cont_data_dir = "/home/jovyan/data" output_dir = "/tmp" LOGGER.info(description) diff --git a/scipy-notebook/test/test_pandas.py b/scipy-notebook/test/test_pandas.py deleted file mode 100644 index 410f5023..00000000 --- a/scipy-notebook/test/test_pandas.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import logging - -import pytest - -LOGGER = logging.getLogger(__name__) - - -@pytest.mark.parametrize( - "name,command_list", - [ - ( - "Sum series", - [ - "import pandas as pd", - "import numpy as np", - "np.random.seed(0)", - "print(pd.Series(np.random.randint(0, 7, size=10)).sum())" - ] - ), - ], -) -def test_pandas(container, name, command_list): - """Basic pandas tests""" - LOGGER.info(f"Testing pandas: {name} ...") - command = ';'.join(command_list) - c = container.run(tty=True, command=["start.sh", "python", "-c", command]) - rv = c.wait(timeout=30) - assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed" - logs = c.logs(stdout=True).decode("utf-8") - LOGGER.debug(logs) diff --git a/scipy-notebook/test/units/unit_pandas.py b/scipy-notebook/test/units/unit_pandas.py new file mode 100644 index 00000000..5036f823 --- /dev/null +++ b/scipy-notebook/test/units/unit_pandas.py @@ -0,0 +1,9 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +import numpy as np +import pandas as pd + + +np.random.seed(0) +print(pd.Series(np.random.randint(0, 7, size=10)).sum()) diff --git a/tensorflow-notebook/test/test_tensorflow.py b/tensorflow-notebook/test/test_tensorflow.py deleted file mode 100644 index f5a6910f..00000000 --- a/tensorflow-notebook/test/test_tensorflow.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import logging - -import pytest - -LOGGER = logging.getLogger(__name__) - - -@pytest.mark.parametrize( - "name,command", - [ - ( - "Hello world", - "import tensorflow as tf;print(tf.constant('Hello, TensorFlow'))", - ), - ( - "Sum", - "import tensorflow as tf;print(tf.reduce_sum(tf.random.normal([1000, 1000])))", - ), - ], -) -def test_tensorflow(container, name, command): - """Basic tensorflow tests""" - LOGGER.info(f"Testing tensorflow: {name} ...") - c = container.run(tty=True, command=["start.sh", "python", "-c", command]) - rv = c.wait(timeout=30) - assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed" - logs = c.logs(stdout=True).decode("utf-8") - LOGGER.debug(logs) diff --git a/tensorflow-notebook/test/units/unit_tensorflow.py b/tensorflow-notebook/test/units/unit_tensorflow.py new file mode 100644 index 00000000..3d5a60c0 --- /dev/null +++ b/tensorflow-notebook/test/units/unit_tensorflow.py @@ -0,0 +1,7 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import tensorflow as tf + + +print(tf.constant('Hello, TensorFlow')) +print(tf.reduce_sum(tf.random.normal([1000, 1000]))) diff --git a/test/helpers.py b/test/helpers.py index 70919c40..7fe41062 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -50,7 +50,8 @@ class CondaPackageHelper: """Start the TrackedContainer and return an instance of a running container""" LOGGER.info(f"Starting container {container.image_name} ...") return container.run( - tty=True, command=["start.sh", "bash", "-c", "sleep infinity"] + tty=True, + command=["start.sh", "bash", "-c", "sleep infinity"] ) @staticmethod diff --git a/test/test_units.py b/test/test_units.py new file mode 100644 index 00000000..ecd74cbd --- /dev/null +++ b/test/test_units.py @@ -0,0 +1,33 @@ +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +import logging +import os + +LOGGER = logging.getLogger(__name__) +THIS_DIR = os.path.dirname(os.path.realpath(__file__)) + + +def test_units(container): + """Various units tests + Add a py file in the {image}/test/units dir and it will be automatically tested + """ + host_data_dir = os.path.join(THIS_DIR, "../", container.image_name(), "test/units") + cont_data_dir = "/home/jovyan/data" + + if not os.path.exists(host_data_dir): + LOGGER.info(f"Not found unit tests for image: {container.image_name()}") + return + + command = "sleep infinity" + running_container = container.run( + volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}}, + tty=True, + command=["start.sh", "bash", "-c", command], + ) + for test_file in os.listdir(host_data_dir): + LOGGER.info("Running unit test: {test_file}") + command = f"python {cont_data_dir}/{test_file}" + cmd = running_container.exec_run(command) + assert cmd.exit_code == 0, f"Command {command} failed" + LOGGER.debug(cmd.output.decode("utf-8")) From bbb91c41a6c8994d13ea2c2466f2c513fdf320c2 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 14:44:01 +0300 Subject: [PATCH 02/47] Fix style --- all-spark-notebook/test/test_spark_notebooks.py | 2 +- scipy-notebook/test/test_extensions.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/all-spark-notebook/test/test_spark_notebooks.py b/all-spark-notebook/test/test_spark_notebooks.py index 1a1d8528..5c6abc3c 100644 --- a/all-spark-notebook/test/test_spark_notebooks.py +++ b/all-spark-notebook/test/test_spark_notebooks.py @@ -13,7 +13,7 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.parametrize( "test_file", # TODO: add local_sparklyr - ["local_pyspark", "local_spylon", "local_sparkR", "issue_1168"], + ["local_pyspark", "local_spylon", "local_sparkR", "issue_1168"] ) def test_nbconvert(container, test_file): """Check if Spark notebooks can be executed""" diff --git a/scipy-notebook/test/test_extensions.py b/scipy-notebook/test/test_extensions.py index 3571a73c..cf6b53eb 100644 --- a/scipy-notebook/test/test_extensions.py +++ b/scipy-notebook/test/test_extensions.py @@ -13,8 +13,8 @@ LOGGER = logging.getLogger(__name__) [ "@bokeh/jupyter_bokeh", "@jupyter-widgets/jupyterlab-manager", - "jupyter-matplotlib", - ], + "jupyter-matplotlib" + ] ) def test_check_extension(container, extension): """Basic check of each extension From 21927f8451e2694713fab2332bbd18a13ae051de Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 14:54:14 +0300 Subject: [PATCH 03/47] Remove redundant upper --- minimal-notebook/test/test_nbconvert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minimal-notebook/test/test_nbconvert.py b/minimal-notebook/test/test_nbconvert.py index e0f45d54..125c5757 100644 --- a/minimal-notebook/test/test_nbconvert.py +++ b/minimal-notebook/test/test_nbconvert.py @@ -24,7 +24,7 @@ def test_nbconvert(container, test_file, output_format): host_data_dir = os.path.join(THIS_DIR, "data") cont_data_dir = "/home/jovyan/data" output_dir = "/tmp" - LOGGER.info(f"Test that the example notebook {test_file} can be converted to {output_format.upper()} ...") + LOGGER.info(f"Test that the example notebook {test_file} can be converted to {output_format} ...") command = f"jupyter nbconvert {cont_data_dir}/{test_file}.ipynb --output-dir {output_dir} --to {output_format}" c = container.run( volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}}, From 9bf20478b6fafc5d562c8dca5f5aa97c4116f2b9 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 14:57:18 +0300 Subject: [PATCH 04/47] Fix style --- minimal-notebook/test/test_inkscape.py | 3 ++- scipy-notebook/test/test_extensions.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/minimal-notebook/test/test_inkscape.py b/minimal-notebook/test/test_inkscape.py index e8f50e2c..fdb0fbcf 100644 --- a/minimal-notebook/test/test_inkscape.py +++ b/minimal-notebook/test/test_inkscape.py @@ -10,7 +10,8 @@ def test_inkscape(container): """Inkscape shall be installed to be able to convert SVG files.""" LOGGER.info("Test that inkscape is working by printing its version ...") c = container.run( - tty=True, command=["start.sh", "bash", "-c", "inkscape --version"] + tty=True, + command=["start.sh", "bash", "-c", "inkscape --version"] ) c.wait(timeout=10) logs = c.logs(stdout=True).decode("utf-8") diff --git a/scipy-notebook/test/test_extensions.py b/scipy-notebook/test/test_extensions.py index cf6b53eb..460760a7 100644 --- a/scipy-notebook/test/test_extensions.py +++ b/scipy-notebook/test/test_extensions.py @@ -26,7 +26,8 @@ def test_check_extension(container, extension): """ LOGGER.info(f"Checking the extension: {extension} ...") c = container.run( - tty=True, command=["start.sh", "jupyter", "labextension", "check", extension] + tty=True, + command=["start.sh", "jupyter", "labextension", "check", extension] ) rv = c.wait(timeout=10) logs = c.logs(stdout=True).decode("utf-8") From 856577b35dcdd7eef93339b9fac5ba7dfddce648 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 15:07:26 +0300 Subject: [PATCH 05/47] Image name is a property --- test/test_units.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_units.py b/test/test_units.py index ecd74cbd..82da86e7 100644 --- a/test/test_units.py +++ b/test/test_units.py @@ -12,11 +12,11 @@ def test_units(container): """Various units tests Add a py file in the {image}/test/units dir and it will be automatically tested """ - host_data_dir = os.path.join(THIS_DIR, "../", container.image_name(), "test/units") + host_data_dir = os.path.join(THIS_DIR, "../", container.image_name, "test/units") cont_data_dir = "/home/jovyan/data" if not os.path.exists(host_data_dir): - LOGGER.info(f"Not found unit tests for image: {container.image_name()}") + LOGGER.info(f"Not found unit tests for image: {container.image_name}") return command = "sleep infinity" From c66273974fadc829c0252975c20367ac124863f8 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 15:07:40 +0300 Subject: [PATCH 06/47] Simplify conftest --- conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index b3c20df7..90723701 100644 --- a/conftest.py +++ b/conftest.py @@ -33,10 +33,10 @@ def docker_client(): @pytest.fixture(scope='session') def image_name(): """Image name to test""" - return os.getenv('TEST_IMAGE', 'jupyter/base-notebook') + return os.getenv('TEST_IMAGE') -class TrackedContainer(object): +class TrackedContainer: """Wrapper that collects docker container configuration and delays container creation/execution. From 847724465eaef0a8628dc2eee15a346c5d0006dc Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 15:16:46 +0300 Subject: [PATCH 07/47] Add tests README --- test/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 test/README.md diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..f0c887ea --- /dev/null +++ b/test/README.md @@ -0,0 +1,12 @@ +# Docker stacks testing + +We test our images using `pytest` module. + +There are two kinds of tests we use: + +- General tests - these are located in [this](https://github.com/jupyter/docker-stacks/blob/master/test) folder +- Image specific tests - for example, [base-notebook/test](https://github.com/jupyter/docker-stacks/blob/master/base-notebook/test) folder + +We also have a way to easily run arbitrary python files in a container. +This is useful for running unit tests of packages we use, so we put these files in `{image}/test/units` folder. +An example of such a test is [unit_pandas.py](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/test/units) From 466a5fb661efe3fb662b9bc5a361a710f978c8c2 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 15:17:54 +0300 Subject: [PATCH 08/47] Fix syntax --- test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/README.md b/test/README.md index f0c887ea..887a244a 100644 --- a/test/README.md +++ b/test/README.md @@ -9,4 +9,4 @@ There are two kinds of tests we use: We also have a way to easily run arbitrary python files in a container. This is useful for running unit tests of packages we use, so we put these files in `{image}/test/units` folder. -An example of such a test is [unit_pandas.py](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/test/units) +An example of such a test is [unit_pandas.py](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/test/units). From a93c13a8aac159786620c380f40aa7f343ca82ca Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 15:21:20 +0300 Subject: [PATCH 09/47] Add pytest info --- test/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/README.md b/test/README.md index 887a244a..26ebdf35 100644 --- a/test/README.md +++ b/test/README.md @@ -2,6 +2,9 @@ We test our images using `pytest` module. +`conftest.py` and `pytest.ini` in the root of our repository define the environment in which tests are run. +More info on pytest can be found [here](https://docs.pytest.org/en/latest/reference/index.html). + There are two kinds of tests we use: - General tests - these are located in [this](https://github.com/jupyter/docker-stacks/blob/master/test) folder From bf05f4243a2f773771e0c31e22251058845b5e9c Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 18:53:54 +0300 Subject: [PATCH 10/47] Some fixes --- test/test_units.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_units.py b/test/test_units.py index 82da86e7..a5a5f503 100644 --- a/test/test_units.py +++ b/test/test_units.py @@ -12,7 +12,9 @@ def test_units(container): """Various units tests Add a py file in the {image}/test/units dir and it will be automatically tested """ - host_data_dir = os.path.join(THIS_DIR, "../", container.image_name, "test/units") + short_image_name = container.image_name[container.image_name.find('/') + 1:] + host_data_dir = os.path.join(THIS_DIR, f"../{short_image_name}/test/units") + LOGGER.info(f"Searching for units tests in {host_data_dir}") cont_data_dir = "/home/jovyan/data" if not os.path.exists(host_data_dir): @@ -26,7 +28,7 @@ def test_units(container): command=["start.sh", "bash", "-c", command], ) for test_file in os.listdir(host_data_dir): - LOGGER.info("Running unit test: {test_file}") + LOGGER.info(f"Running unit test: {test_file}") command = f"python {cont_data_dir}/{test_file}" cmd = running_container.exec_run(command) assert cmd.exit_code == 0, f"Command {command} failed" From 1cef89a029de01b5155e58f7e88bd60309abd458 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 18:55:29 +0300 Subject: [PATCH 11/47] Fix link --- test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/README.md b/test/README.md index 26ebdf35..7cf2b805 100644 --- a/test/README.md +++ b/test/README.md @@ -3,7 +3,7 @@ We test our images using `pytest` module. `conftest.py` and `pytest.ini` in the root of our repository define the environment in which tests are run. -More info on pytest can be found [here](https://docs.pytest.org/en/latest/reference/index.html). +More info on pytest can be found [here](https://docs.pytest.org/en/latest/contents.html). There are two kinds of tests we use: From e264a5e702699fcb63c5b4017f0c6f9bc9e49acb Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sat, 26 Jun 2021 19:08:04 +0300 Subject: [PATCH 12/47] Fix link --- test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/README.md b/test/README.md index 7cf2b805..3b32fc28 100644 --- a/test/README.md +++ b/test/README.md @@ -12,4 +12,4 @@ There are two kinds of tests we use: We also have a way to easily run arbitrary python files in a container. This is useful for running unit tests of packages we use, so we put these files in `{image}/test/units` folder. -An example of such a test is [unit_pandas.py](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/test/units). +An example of such a test is [unit_pandas.py](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/test/units/unit_pandas.py). From a305eaa5564deb0bb65224bd1836348f4f2bbb8d Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 00:46:42 +0300 Subject: [PATCH 13/47] Fix container running --- test/test_units.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/test/test_units.py b/test/test_units.py index a5a5f503..0d511dcb 100644 --- a/test/test_units.py +++ b/test/test_units.py @@ -21,15 +21,14 @@ def test_units(container): LOGGER.info(f"Not found unit tests for image: {container.image_name}") return - command = "sleep infinity" - running_container = container.run( - volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}}, - tty=True, - command=["start.sh", "bash", "-c", command], - ) for test_file in os.listdir(host_data_dir): LOGGER.info(f"Running unit test: {test_file}") - command = f"python {cont_data_dir}/{test_file}" - cmd = running_container.exec_run(command) - assert cmd.exit_code == 0, f"Command {command} failed" - LOGGER.debug(cmd.output.decode("utf-8")) + + c = container.run( + tty=True, + command=['start.sh', 'python', f'{cont_data_dir}/{test_file}'] + ) + rv = c.wait(timeout=30) + assert rv == 0 or rv["StatusCode"] == 0 + logs = c.logs(stdout=True).decode('utf-8') + LOGGER.debug(logs) From a3a0a5b54bdfc03810b956a280ae54b38200fe00 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 01:14:06 +0300 Subject: [PATCH 14/47] Fix --- test/test_units.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_units.py b/test/test_units.py index 0d511dcb..1a1cacba 100644 --- a/test/test_units.py +++ b/test/test_units.py @@ -25,6 +25,7 @@ def test_units(container): LOGGER.info(f"Running unit test: {test_file}") c = container.run( + volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}}, tty=True, command=['start.sh', 'python', f'{cont_data_dir}/{test_file}'] ) From 532d3734dc1879b2920614a6acd9ea49f3f163a9 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 12:30:07 +0300 Subject: [PATCH 15/47] Apply suggestions from code review --- all-spark-notebook/test/test_spark_notebooks.py | 2 +- base-notebook/test/test_package_managers.py | 4 ++-- base-notebook/test/test_start_container.py | 4 ++-- minimal-notebook/test/test_nbconvert.py | 4 ++-- scipy-notebook/test/test_extensions.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/all-spark-notebook/test/test_spark_notebooks.py b/all-spark-notebook/test/test_spark_notebooks.py index 5c6abc3c..1a1d8528 100644 --- a/all-spark-notebook/test/test_spark_notebooks.py +++ b/all-spark-notebook/test/test_spark_notebooks.py @@ -13,7 +13,7 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.parametrize( "test_file", # TODO: add local_sparklyr - ["local_pyspark", "local_spylon", "local_sparkR", "issue_1168"] + ["local_pyspark", "local_spylon", "local_sparkR", "issue_1168"], ) def test_nbconvert(container, test_file): """Check if Spark notebooks can be executed""" diff --git a/base-notebook/test/test_package_managers.py b/base-notebook/test/test_package_managers.py index 060706e3..49b9bc8a 100644 --- a/base-notebook/test/test_package_managers.py +++ b/base-notebook/test/test_package_managers.py @@ -14,8 +14,8 @@ LOGGER = logging.getLogger(__name__) ("conda", "--version"), ("mamba", "--version"), ("npm", "--version"), - ("pip", "--version") - ] + ("pip", "--version"), + ], ) def test_package_manager(container, package_manager, version_arg): """Test the notebook start-notebook script""" diff --git a/base-notebook/test/test_start_container.py b/base-notebook/test/test_start_container.py index 3ffd4356..e1231e19 100644 --- a/base-notebook/test/test_start_container.py +++ b/base-notebook/test/test_start_container.py @@ -11,8 +11,8 @@ LOGGER = logging.getLogger(__name__) "env,expected_server", [ (["JUPYTER_ENABLE_LAB=yes"], "lab"), - (None, "notebook") - ] + (None, "notebook"), + ], ) def test_start_notebook(container, http_client, env, expected_server): """Test the notebook start-notebook script""" diff --git a/minimal-notebook/test/test_nbconvert.py b/minimal-notebook/test/test_nbconvert.py index 125c5757..6e2675eb 100644 --- a/minimal-notebook/test/test_nbconvert.py +++ b/minimal-notebook/test/test_nbconvert.py @@ -16,8 +16,8 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) ("notebook_math", "pdf"), ("notebook_math", "html"), ("notebook_svg", "pdf"), - ("notebook_svg", "html") - ] + ("notebook_svg", "html"), + ], ) def test_nbconvert(container, test_file, output_format): """Check if nbconvert is able to convert a notebook file""" diff --git a/scipy-notebook/test/test_extensions.py b/scipy-notebook/test/test_extensions.py index 460760a7..168d92ad 100644 --- a/scipy-notebook/test/test_extensions.py +++ b/scipy-notebook/test/test_extensions.py @@ -13,8 +13,8 @@ LOGGER = logging.getLogger(__name__) [ "@bokeh/jupyter_bokeh", "@jupyter-widgets/jupyterlab-manager", - "jupyter-matplotlib" - ] + "jupyter-matplotlib", + ], ) def test_check_extension(container, extension): """Basic check of each extension From 52a91a3e760e55e7028f98f803ada2a7ba76d6c1 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 14:16:48 +0300 Subject: [PATCH 16/47] assert after logs output --- test/test_units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_units.py b/test/test_units.py index 1a1cacba..ce5a9250 100644 --- a/test/test_units.py +++ b/test/test_units.py @@ -30,6 +30,6 @@ def test_units(container): command=['start.sh', 'python', f'{cont_data_dir}/{test_file}'] ) rv = c.wait(timeout=30) - assert rv == 0 or rv["StatusCode"] == 0 logs = c.logs(stdout=True).decode('utf-8') LOGGER.debug(logs) + assert rv == 0 or rv["StatusCode"] == 0 From 36b039f2ffe975d5a54553a1ecce40ee5ff8f517 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 17:10:37 +0300 Subject: [PATCH 17/47] Use stefanzweifel/git-auto-commit-action to push to github --- .github/workflows/docker.yml | 9 ++++----- .github/workflows/sphinx.yml | 8 +++----- Makefile | 16 ---------------- 3 files changed, 7 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 714e705d..1a6382b7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -57,11 +57,10 @@ jobs: run: make -C main hook-all - name: Push Wiki to GitHub if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' - # Pass GITHUB_REPOSITORY directly to avoid conflict with GitHub Actions built-in env var - run: make -C main git-commit GITHUB_REPOSITORY='${{github.repository}}.wiki' - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - LOCAL_PATH: ../wiki + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: "[ci skip] Automated publish for ${{ GITHUB_SHA }}" + repository: ../wiki - name: Login to Docker Hub if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' uses: docker/login-action@v1 diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 796c3acc..bdd9747c 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -42,8 +42,6 @@ jobs: sphinx-intl update -p ./_build/gettext -l en - name: Push Strings to Master if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' - run: make git-commit - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - GITHUB_REPOSITORY: ${{github.repository}} - LOCAL_PATH: ./docs/locale/en + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: "[ci skip] Automated publish for ${{ GITHUB_SHA }}" diff --git a/Makefile b/Makefile index 907aec71..35722294 100644 --- a/Makefile +++ b/Makefile @@ -62,22 +62,6 @@ dev-env: ## install libraries required to build docs and run tests docs: ## build HTML documentation sphinx-build docs/ docs/_build/ -git-commit: LOCAL_PATH?=. -git-commit: GITHUB_SHA?=$(shell git rev-parse HEAD) -git-commit: GITHUB_REPOSITORY?=jupyter/docker-stacks -git-commit: GITHUB_TOKEN?= -git-commit: ## commit outstading git changes and push to remote - @git config --global user.name "GitHub Actions" - @git config --global user.email "actions@users.noreply.github.com" - - @echo "Publishing outstanding changes in $(LOCAL_PATH) to $(GITHUB_REPOSITORY)" - @cd $(LOCAL_PATH) && \ - git remote add publisher https://$(GITHUB_TOKEN)@github.com/$(GITHUB_REPOSITORY).git && \ - git checkout master && \ - git add -A -- . && \ - git commit -m "[ci skip] Automated publish for $(GITHUB_SHA)" || exit 0 - @cd $(LOCAL_PATH) && git push -u publisher master - hook/%: WIKI_PATH?=../wiki hook/%: ## run post-build hooks for an image python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \ From 8ff5fa630be52199d3bacdfcf6257dfc13f8de78 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 17:14:25 +0300 Subject: [PATCH 18/47] Try to fix var name --- .github/workflows/docker.yml | 2 +- .github/workflows/sphinx.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1a6382b7..0a5da289 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -59,7 +59,7 @@ jobs: if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: "[ci skip] Automated publish for ${{ GITHUB_SHA }}" + commit_message: "[ci skip] Automated publish for $GITHUB_SHA" repository: ../wiki - name: Login to Docker Hub if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index bdd9747c..4ad22b5e 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -44,4 +44,4 @@ jobs: if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: "[ci skip] Automated publish for ${{ GITHUB_SHA }}" + commit_message: "[ci skip] Automated publish for $GITHUB_SHA" From 71a49a2a818df7134e757bad8b3730fd2ba1e097 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 17:16:06 +0300 Subject: [PATCH 19/47] Test sha --- .github/workflows/sphinx.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 4ad22b5e..8f0eaae3 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -33,6 +33,9 @@ jobs: run: | python -m pip install --upgrade pip make dev-env + - name: Echo sha + run: | + echo "SHA_TAG $GITHUB_SHA" - name: Build Documentation run: make docs - name: Extract Source Strings From 9632da4ce1ea3821cfcb4ac2bf300c0228d19728 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 17:34:26 +0300 Subject: [PATCH 20/47] Fix sha tag --- .github/workflows/sphinx.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 8f0eaae3..f906b939 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -33,9 +33,6 @@ jobs: run: | python -m pip install --upgrade pip make dev-env - - name: Echo sha - run: | - echo "SHA_TAG $GITHUB_SHA" - name: Build Documentation run: make docs - name: Extract Source Strings @@ -43,8 +40,16 @@ jobs: run: | sphinx-build -M gettext ./ ./_build/ sphinx-intl update -p ./_build/gettext -l en + - name: Get last commit hash + id: last-commit-hash + run: | + echo "::set-output name=hash::$(git rev-parse HEAD)" + - name: Echo sha + run: | + echo "SHA_TAG $GITHUB_SHA" + echo "Calculates SHA_TAG ${{steps.last-commit-hash.outputs.hash}}" - name: Push Strings to Master if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: "[ci skip] Automated publish for $GITHUB_SHA" + commit_message: "[ci skip] Automated publish for ${{steps.last-commit-hash.outputs.hash}}" From 35e67277bde874205be2587ba7b1f308f0fbd2d8 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 17:38:08 +0300 Subject: [PATCH 21/47] Actually, GITHUB_SHA seems to work the same way --- .github/workflows/sphinx.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index f906b939..4ad22b5e 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -40,16 +40,8 @@ jobs: run: | sphinx-build -M gettext ./ ./_build/ sphinx-intl update -p ./_build/gettext -l en - - name: Get last commit hash - id: last-commit-hash - run: | - echo "::set-output name=hash::$(git rev-parse HEAD)" - - name: Echo sha - run: | - echo "SHA_TAG $GITHUB_SHA" - echo "Calculates SHA_TAG ${{steps.last-commit-hash.outputs.hash}}" - name: Push Strings to Master if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: "[ci skip] Automated publish for ${{steps.last-commit-hash.outputs.hash}}" + commit_message: "[ci skip] Automated publish for $GITHUB_SHA" From f2f44d1736158a63dc4f9299b10fbd9fcb9b2e30 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 17:58:36 +0300 Subject: [PATCH 22/47] Fix --- .github/workflows/docker.yml | 2 +- .github/workflows/sphinx.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0a5da289..cb45257e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -59,7 +59,7 @@ jobs: if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: "[ci skip] Automated publish for $GITHUB_SHA" + commit_message: "[ci skip] Automated publish for ${{github.sha}}" repository: ../wiki - name: Login to Docker Hub if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 4ad22b5e..8b682041 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -44,4 +44,4 @@ jobs: if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: "[ci skip] Automated publish for $GITHUB_SHA" + commit_message: "[ci skip] Automated publish for ${{github.sha}}" From 17967ce3366677f7da1476051e45fb7ae527398d Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 18:33:59 +0300 Subject: [PATCH 23/47] Fix repository path --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cb45257e..b198f191 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -60,7 +60,7 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: "[ci skip] Automated publish for ${{github.sha}}" - repository: ../wiki + repository: wiki/ - name: Login to Docker Hub if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' uses: docker/login-action@v1 From bd8ffe6bdedff3c8642f5d51f3a227156ab48b39 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 18:54:32 +0300 Subject: [PATCH 24/47] Create dependabot.yml for GitHub actions --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..dd9ad17a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" From 3336adc9b5de063a9179a9becde8bd3eea079b8f Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 18:59:26 +0300 Subject: [PATCH 25/47] Pin GitHub action sha tag --- .github/workflows/docker.yml | 2 +- .github/workflows/sphinx.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b198f191..5f4dbb3e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -57,7 +57,7 @@ jobs: run: make -C main hook-all - name: Push Wiki to GitHub if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@ 5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release with: commit_message: "[ci skip] Automated publish for ${{github.sha}}" repository: wiki/ diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 8b682041..cb26a06a 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -42,6 +42,6 @@ jobs: sphinx-intl update -p ./_build/gettext -l en - name: Push Strings to Master if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@ 5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release with: commit_message: "[ci skip] Automated publish for ${{github.sha}}" From 93726eff884e1e44342135259551200d3b21088a Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Sun, 27 Jun 2021 19:01:39 +0300 Subject: [PATCH 26/47] Fix typo --- .github/workflows/docker.yml | 2 +- .github/workflows/sphinx.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5f4dbb3e..7f230139 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -57,7 +57,7 @@ jobs: run: make -C main hook-all - name: Push Wiki to GitHub if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' - uses: stefanzweifel/git-auto-commit-action@ 5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release + uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release with: commit_message: "[ci skip] Automated publish for ${{github.sha}}" repository: wiki/ diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index cb26a06a..7a008728 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -42,6 +42,6 @@ jobs: sphinx-intl update -p ./_build/gettext -l en - name: Push Strings to Master if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' - uses: stefanzweifel/git-auto-commit-action@ 5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release + uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release with: commit_message: "[ci skip] Automated publish for ${{github.sha}}" From 7cdc14ecd146cb7bac427c3801c8ada7ad6a5215 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Mon, 28 Jun 2021 10:53:32 +0300 Subject: [PATCH 27/47] Pin docker/login-action sha tag --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7f230139..1a6d9d19 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -63,7 +63,7 @@ jobs: repository: wiki/ - name: Login to Docker Hub if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' - uses: docker/login-action@v1 + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # dependabot updates to latest release with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} From e87d910e37fd28324ccf361daf1b99c5e20b5731 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Jun 2021 17:49:12 +0000 Subject: [PATCH 28/47] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-prettier: v2.3.1 → v2.3.2](https://github.com/pre-commit/mirrors-prettier/compare/v2.3.1...v2.3.2) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 20ec3a92..80c69208 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,6 +44,6 @@ repos: - id: markdownlint args: ["--fix"] - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.3.1 + rev: v2.3.2 hooks: - id: prettier From c73a1d1b5d96e01cb0a5f5d9cda4b16f2c13727e Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Mon, 28 Jun 2021 22:29:59 +0300 Subject: [PATCH 29/47] Update test/test_units.py Co-authored-by: Erik Sundell --- test/test_units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_units.py b/test/test_units.py index ce5a9250..8edef5c9 100644 --- a/test/test_units.py +++ b/test/test_units.py @@ -12,7 +12,7 @@ def test_units(container): """Various units tests Add a py file in the {image}/test/units dir and it will be automatically tested """ - short_image_name = container.image_name[container.image_name.find('/') + 1:] + short_image_name = container.image_name[container.image_name.rfind('/') + 1:] host_data_dir = os.path.join(THIS_DIR, f"../{short_image_name}/test/units") LOGGER.info(f"Searching for units tests in {host_data_dir}") cont_data_dir = "/home/jovyan/data" From b7087c505348180464403bdc2232af783af383ed Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 26 Jun 2021 19:16:43 +0200 Subject: [PATCH 30/47] pre-commit: add black, a Python autoformatter --- .pre-commit-config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80c69208..e48bc9fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,6 +3,12 @@ ci: skip: [hadolint-docker] repos: + # Autoformat: Python code + - repo: https://github.com/ambv/black + rev: 21.6b0 + hooks: + - id: black + args: [--target-version=py36] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: From bb756586b85f68d66a09819c264c79337568442f Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 26 Jun 2021 19:25:39 +0200 Subject: [PATCH 31/47] pre-commit: refactor config and add inline comment --- .pre-commit-config.yaml | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e48bc9fc..2e6c55d8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,47 +9,65 @@ repos: hooks: - id: black args: [--target-version=py36] + + # Autoformat: Python code + - repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v1.5.7 + hooks: + - id: autopep8 + + # Autoformat: YAML, JSON, Markdown, etc. + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.3.2 + hooks: + - id: prettier + + # Check: YAML - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: - id: check-yaml files: .*\.(yaml|yml)$ + + # Lint: Dockerfile - repo: https://github.com/hadolint/hadolint.git rev: v2.5.0 hooks: - id: hadolint-docker # FIXME: remove after https://github.com/hadolint/hadolint/issues/628 is resolved entry: hadolint/hadolint:v2.5.0 hadolint + + # Lint: YAML - repo: https://github.com/adrienverge/yamllint.git rev: v1.26.1 hooks: - id: yamllint args: ["-d {extends: relaxed, rules: {line-length: disable}}", "-s"] files: \.(yaml|yml)$ + + # Lint: Bash scripts - repo: https://github.com/openstack-dev/bashate.git rev: 2.0.0 hooks: - id: bashate args: ["--ignore=E006"] + + # Lint: Shell scripts - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.7.2.1 hooks: - id: shellcheck args: ["-x"] + + # Lint: Python - repo: https://github.com/PyCQA/flake8 rev: 3.9.2 hooks: - id: flake8 - - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.5.7 - hooks: - - id: autopep8 + + # Lint: Markdown - repo: https://github.com/igorshubovych/markdownlint-cli rev: v0.27.1 hooks: - id: markdownlint args: ["--fix"] - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.3.2 - hooks: - - id: prettier From 744cb368d04fed6a291af7d210da8131811a65f1 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 26 Jun 2021 19:27:13 +0200 Subject: [PATCH 32/47] pre-commit: remove check-yaml, prettier will cover that --- .pre-commit-config.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2e6c55d8..9ab1331a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,13 +22,6 @@ repos: hooks: - id: prettier - # Check: YAML - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 - hooks: - - id: check-yaml - files: .*\.(yaml|yml)$ - # Lint: Dockerfile - repo: https://github.com/hadolint/hadolint.git rev: v2.5.0 From 58e7a5a01a6d0d270ee0c755873ff77a9d057e54 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 26 Jun 2021 19:39:01 +0200 Subject: [PATCH 33/47] pre-commit: remove autopep8, black will cover that --- .pre-commit-config.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ab1331a..2b0d850a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,12 +10,6 @@ repos: - id: black args: [--target-version=py36] - # Autoformat: Python code - - repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v1.5.7 - hooks: - - id: autopep8 - # Autoformat: YAML, JSON, Markdown, etc. - repo: https://github.com/pre-commit/mirrors-prettier rev: v2.3.2 From a99a1829401baa661e3fdf3e2448230416c4cc09 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 26 Jun 2021 19:40:07 +0200 Subject: [PATCH 34/47] pre-commit: use black compatible flake8 config This configuration is taken from the recommendation in https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length. --- .flake8 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.flake8 b/.flake8 index d5c9935a..6f094a9d 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,4 @@ [flake8] -ignore = W605,W503,W504,H306,H238,H301,H202 -max-line-length = 120 -per-file-ignores = - test/test_packages.py:E501 +max-line-length = 88 +select = C,E,F,W,B,B950 +extend-ignore = E203, E501 From fe3968efe0c89148360733b565525833ca95783b Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 28 Jun 2021 12:32:21 +0200 Subject: [PATCH 35/47] pre-commit: run black autoformatter on all files --- .../test/test_spark_notebooks.py | 10 +- base-notebook/jupyter_notebook_config.py | 46 ++++--- base-notebook/test/test_container_options.py | 118 ++++++++--------- base-notebook/test/test_package_managers.py | 3 +- base-notebook/test/test_pandoc.py | 3 +- conftest.py | 25 ++-- datascience-notebook/test/test_julia.py | 3 +- docs/conf.py | 92 ++++++------- minimal-notebook/test/test_inkscape.py | 3 +- minimal-notebook/test/test_nbconvert.py | 4 +- pyspark-notebook/test/test_spark.py | 7 +- scipy-notebook/test/data/matplotlib_1.py | 5 +- .../test/data/matplotlib_fonts_1.py | 18 ++- scipy-notebook/test/test_extensions.py | 3 +- scipy-notebook/test/test_matplotlib.py | 14 +- tagging/create_manifests.py | 31 +++-- tagging/docker_runner.py | 11 +- tagging/get_taggers_and_manifests.py | 4 +- tagging/images_hierarchy.py | 64 +++++---- tagging/manifests.py | 124 ++++++++++-------- tagging/tag_image.py | 8 +- tagging/taggers.py | 17 ++- .../test/units/unit_tensorflow.py | 2 +- test/helpers.py | 22 ++-- test/test_packages.py | 7 +- test/test_units.py | 6 +- 26 files changed, 359 insertions(+), 291 deletions(-) diff --git a/all-spark-notebook/test/test_spark_notebooks.py b/all-spark-notebook/test/test_spark_notebooks.py index cd949b8e..a87f43f8 100644 --- a/all-spark-notebook/test/test_spark_notebooks.py +++ b/all-spark-notebook/test/test_spark_notebooks.py @@ -22,10 +22,12 @@ def test_nbconvert(container, test_file): output_dir = "/tmp" timeout_ms = 600 LOGGER.info(f"Test that {test_file} notebook can be executed ...") - command = "jupyter nbconvert --to markdown " + \ - f"--ExecutePreprocessor.timeout={timeout_ms} " + \ - f"--output-dir {output_dir} " + \ - f"--execute {cont_data_dir}/{test_file}.ipynb" + command = ( + "jupyter nbconvert --to markdown " + + f"--ExecutePreprocessor.timeout={timeout_ms} " + + f"--output-dir {output_dir} " + + f"--execute {cont_data_dir}/{test_file}.ipynb" + ) c = container.run( volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}}, tty=True, diff --git a/base-notebook/jupyter_notebook_config.py b/base-notebook/jupyter_notebook_config.py index 19b5e8b4..e0c7d12a 100644 --- a/base-notebook/jupyter_notebook_config.py +++ b/base-notebook/jupyter_notebook_config.py @@ -8,7 +8,7 @@ import errno import stat c = get_config() # noqa: F821 -c.NotebookApp.ip = '0.0.0.0' +c.NotebookApp.ip = "0.0.0.0" c.NotebookApp.port = 8888 c.NotebookApp.open_browser = False @@ -16,9 +16,9 @@ c.NotebookApp.open_browser = False c.FileContentsManager.delete_to_trash = False # Generate a self-signed certificate -if 'GEN_CERT' in os.environ: +if "GEN_CERT" in os.environ: dir_name = jupyter_data_dir() - pem_file = os.path.join(dir_name, 'notebook.pem') + pem_file = os.path.join(dir_name, "notebook.pem") try: os.makedirs(dir_name) except OSError as exc: # Python >2.5 @@ -28,28 +28,42 @@ if 'GEN_CERT' in os.environ: raise # Generate an openssl.cnf file to set the distinguished name - cnf_file = os.path.join(os.getenv('CONDA_DIR', '/usr/lib'), 'ssl', 'openssl.cnf') + cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf") if not os.path.isfile(cnf_file): - with open(cnf_file, 'w') as fh: - fh.write('''\ + with open(cnf_file, "w") as fh: + fh.write( + """\ [req] distinguished_name = req_distinguished_name [req_distinguished_name] -''') +""" + ) # Generate a certificate if one doesn't exist on disk - subprocess.check_call(['openssl', 'req', '-new', - '-newkey', 'rsa:2048', - '-days', '365', - '-nodes', '-x509', - '-subj', '/C=XX/ST=XX/L=XX/O=generated/CN=generated', - '-keyout', pem_file, - '-out', pem_file]) + subprocess.check_call( + [ + "openssl", + "req", + "-new", + "-newkey", + "rsa:2048", + "-days", + "365", + "-nodes", + "-x509", + "-subj", + "/C=XX/ST=XX/L=XX/O=generated/CN=generated", + "-keyout", + pem_file, + "-out", + pem_file, + ] + ) # Restrict access to the file os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) c.NotebookApp.certfile = pem_file # Change default umask for all subprocesses of the notebook server if set in # the environment -if 'NB_UMASK' in os.environ: - os.umask(int(os.environ['NB_UMASK'], 8)) +if "NB_UMASK" in os.environ: + os.umask(int(os.environ["NB_UMASK"], 8)) diff --git a/base-notebook/test/test_container_options.py b/base-notebook/test/test_container_options.py index 595a5119..2187d8f4 100644 --- a/base-notebook/test/test_container_options.py +++ b/base-notebook/test/test_container_options.py @@ -11,59 +11,52 @@ LOGGER = logging.getLogger(__name__) def test_cli_args(container, http_client): """Container should respect notebook server command line args (e.g., disabling token security)""" - c = container.run( - command=["start-notebook.sh", "--NotebookApp.token=''"] - ) - resp = http_client.get('http://localhost:8888') + c = container.run(command=["start-notebook.sh", "--NotebookApp.token=''"]) + resp = http_client.get("http://localhost:8888") resp.raise_for_status() - logs = c.logs(stdout=True).decode('utf-8') + logs = c.logs(stdout=True).decode("utf-8") LOGGER.debug(logs) - assert 'login_submit' not in resp.text + assert "login_submit" not in resp.text -@pytest.mark.filterwarnings('ignore:Unverified HTTPS request') +@pytest.mark.filterwarnings("ignore:Unverified HTTPS request") def test_unsigned_ssl(container, http_client): """Container should generate a self-signed SSL certificate and notebook server should use it to enable HTTPS. """ - container.run( - environment=['GEN_CERT=yes'] - ) + container.run(environment=["GEN_CERT=yes"]) # NOTE: The requests.Session backing the http_client fixture does not retry # properly while the server is booting up. An SSL handshake error seems to # abort the retry logic. Forcing a long sleep for the moment until I have # time to dig more. time.sleep(5) - resp = http_client.get('https://localhost:8888', verify=False) + resp = http_client.get("https://localhost:8888", verify=False) resp.raise_for_status() - assert 'login_submit' in resp.text + assert "login_submit" in resp.text def test_uid_change(container): """Container should change the UID of the default user.""" c = container.run( tty=True, - user='root', - environment=['NB_UID=1010'], - command=['start.sh', 'bash', '-c', 'id && touch /opt/conda/test-file'] + user="root", + environment=["NB_UID=1010"], + command=["start.sh", "bash", "-c", "id && touch /opt/conda/test-file"], ) # usermod is slow so give it some time c.wait(timeout=120) - assert 'uid=1010(jovyan)' in c.logs(stdout=True).decode('utf-8') + assert "uid=1010(jovyan)" in c.logs(stdout=True).decode("utf-8") def test_gid_change(container): """Container should change the GID of the default user.""" c = container.run( - tty=True, - user='root', - environment=['NB_GID=110'], - command=['start.sh', 'id'] + tty=True, user="root", environment=["NB_GID=110"], command=["start.sh", "id"] ) c.wait(timeout=10) - logs = c.logs(stdout=True).decode('utf-8') - assert 'gid=110(jovyan)' in logs - assert 'groups=110(jovyan),100(users)' in logs + logs = c.logs(stdout=True).decode("utf-8") + assert "gid=110(jovyan)" in logs + assert "groups=110(jovyan),100(users)" in logs def test_nb_user_change(container): @@ -72,11 +65,8 @@ def test_nb_user_change(container): running_container = container.run( tty=True, user="root", - environment=[ - f"NB_USER={nb_user}", - "CHOWN_HOME=yes" - ], - command=['start.sh', 'bash', '-c', 'sleep infinity'] + environment=[f"NB_USER={nb_user}", "CHOWN_HOME=yes"], + command=["start.sh", "bash", "-c", "sleep infinity"], ) # Give the chown time to complete. Use sleep, not wait, because the @@ -98,25 +88,27 @@ def test_nb_user_change(container): expected_output = f"{nb_user} users" cmd = running_container.exec_run(command, workdir=f"/home/{nb_user}") output = cmd.output.decode("utf-8").strip("\n") - assert output == expected_output, f"Bad owner for the {nb_user} home folder {output}, expected {expected_output}" + assert ( + output == expected_output + ), f"Bad owner for the {nb_user} home folder {output}, expected {expected_output}" def test_chown_extra(container): """Container should change the UID/GID of CHOWN_EXTRA.""" c = container.run( tty=True, - user='root', + user="root", environment=[ - 'NB_UID=1010', - 'NB_GID=101', - 'CHOWN_EXTRA=/opt/conda', - 'CHOWN_EXTRA_OPTS=-R' + "NB_UID=1010", + "NB_GID=101", + "CHOWN_EXTRA=/opt/conda", + "CHOWN_EXTRA_OPTS=-R", ], - command=['start.sh', 'bash', '-c', 'stat -c \'%n:%u:%g\' /opt/conda/LICENSE.txt'] + command=["start.sh", "bash", "-c", "stat -c '%n:%u:%g' /opt/conda/LICENSE.txt"], ) # chown is slow so give it some time c.wait(timeout=120) - assert '/opt/conda/LICENSE.txt:1010:101' in c.logs(stdout=True).decode('utf-8') + assert "/opt/conda/LICENSE.txt:1010:101" in c.logs(stdout=True).decode("utf-8") def test_chown_home(container): @@ -124,64 +116,66 @@ def test_chown_home(container): group to the current value of NB_UID and NB_GID.""" c = container.run( tty=True, - user='root', - environment=[ - 'CHOWN_HOME=yes', - 'CHOWN_HOME_OPTS=-R' + user="root", + environment=["CHOWN_HOME=yes", "CHOWN_HOME_OPTS=-R"], + command=[ + "start.sh", + "bash", + "-c", + "chown root:root /home/jovyan && ls -alsh /home", ], - command=['start.sh', 'bash', '-c', 'chown root:root /home/jovyan && ls -alsh /home'] ) c.wait(timeout=120) - assert "Changing ownership of /home/jovyan to 1000:100 with options '-R'" in c.logs(stdout=True).decode('utf-8') + assert "Changing ownership of /home/jovyan to 1000:100 with options '-R'" in c.logs( + stdout=True + ).decode("utf-8") def test_sudo(container): """Container should grant passwordless sudo to the default user.""" c = container.run( tty=True, - user='root', - environment=['GRANT_SUDO=yes'], - command=['start.sh', 'sudo', 'id'] + user="root", + environment=["GRANT_SUDO=yes"], + command=["start.sh", "sudo", "id"], ) rv = c.wait(timeout=10) assert rv == 0 or rv["StatusCode"] == 0 - assert 'uid=0(root)' in c.logs(stdout=True).decode('utf-8') + assert "uid=0(root)" in c.logs(stdout=True).decode("utf-8") def test_sudo_path(container): """Container should include /opt/conda/bin in the sudo secure_path.""" c = container.run( tty=True, - user='root', - environment=['GRANT_SUDO=yes'], - command=['start.sh', 'sudo', 'which', 'jupyter'] + user="root", + environment=["GRANT_SUDO=yes"], + command=["start.sh", "sudo", "which", "jupyter"], ) rv = c.wait(timeout=10) assert rv == 0 or rv["StatusCode"] == 0 - assert c.logs(stdout=True).decode('utf-8').rstrip().endswith('/opt/conda/bin/jupyter') + assert ( + c.logs(stdout=True).decode("utf-8").rstrip().endswith("/opt/conda/bin/jupyter") + ) def test_sudo_path_without_grant(container): """Container should include /opt/conda/bin in the sudo secure_path.""" - c = container.run( - tty=True, - user='root', - command=['start.sh', 'which', 'jupyter'] - ) + c = container.run(tty=True, user="root", command=["start.sh", "which", "jupyter"]) rv = c.wait(timeout=10) assert rv == 0 or rv["StatusCode"] == 0 - assert c.logs(stdout=True).decode('utf-8').rstrip().endswith('/opt/conda/bin/jupyter') + assert ( + c.logs(stdout=True).decode("utf-8").rstrip().endswith("/opt/conda/bin/jupyter") + ) def test_group_add(container, tmpdir): """Container should run with the specified uid, gid, and secondary group. """ - c = container.run( - user='1010:1010', - group_add=['users'], - command=['start.sh', 'id'] - ) + c = container.run(user="1010:1010", group_add=["users"], command=["start.sh", "id"]) rv = c.wait(timeout=5) assert rv == 0 or rv["StatusCode"] == 0 - assert 'uid=1010 gid=1010 groups=1010,100(users)' in c.logs(stdout=True).decode('utf-8') + assert "uid=1010 gid=1010 groups=1010,100(users)" in c.logs(stdout=True).decode( + "utf-8" + ) diff --git a/base-notebook/test/test_package_managers.py b/base-notebook/test/test_package_managers.py index 1355a9c1..e673078c 100644 --- a/base-notebook/test/test_package_managers.py +++ b/base-notebook/test/test_package_managers.py @@ -23,8 +23,7 @@ def test_package_manager(container, package_manager, version_arg): f"Test that the package manager {package_manager} is working properly ..." ) c = container.run( - tty=True, - command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"] + tty=True, command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"] ) rv = c.wait(timeout=5) logs = c.logs(stdout=True).decode("utf-8") diff --git a/base-notebook/test/test_pandoc.py b/base-notebook/test/test_pandoc.py index bc128921..c84787c1 100644 --- a/base-notebook/test/test_pandoc.py +++ b/base-notebook/test/test_pandoc.py @@ -9,8 +9,7 @@ LOGGER = logging.getLogger(__name__) def test_pandoc(container): """Pandoc shall be able to convert MD to HTML.""" c = container.run( - tty=True, - command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc'] + tty=True, command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc'] ) c.wait(timeout=10) logs = c.logs(stdout=True).decode("utf-8") diff --git a/conftest.py b/conftest.py index 90723701..d13654bf 100644 --- a/conftest.py +++ b/conftest.py @@ -14,26 +14,26 @@ from requests.adapters import HTTPAdapter LOGGER = logging.getLogger(__name__) -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def http_client(): """Requests session with retries and backoff.""" s = requests.Session() retries = Retry(total=5, backoff_factor=1) - s.mount('http://', HTTPAdapter(max_retries=retries)) - s.mount('https://', HTTPAdapter(max_retries=retries)) + s.mount("http://", HTTPAdapter(max_retries=retries)) + s.mount("https://", HTTPAdapter(max_retries=retries)) return s -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def docker_client(): """Docker client configured based on the host environment""" return docker.from_env() -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def image_name(): """Image name to test""" - return os.getenv('TEST_IMAGE') + return os.getenv("TEST_IMAGE") class TrackedContainer: @@ -78,7 +78,9 @@ class TrackedContainer: all_kwargs.update(self.kwargs) all_kwargs.update(kwargs) LOGGER.info(f"Running {self.image_name} with args {all_kwargs} ...") - self.container = self.docker_client.containers.run(self.image_name, **all_kwargs) + self.container = self.docker_client.containers.run( + self.image_name, **all_kwargs + ) return self.container def remove(self): @@ -87,7 +89,7 @@ class TrackedContainer: self.container.remove(force=True) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def container(docker_client, image_name): """Notebook container with initial configuration appropriate for testing (e.g., HTTP port exposed to the host for HTTP calls). @@ -95,12 +97,7 @@ def container(docker_client, image_name): Yields the container instance and kills it when the caller is done with it. """ container = TrackedContainer( - docker_client, - image_name, - detach=True, - ports={ - '8888/tcp': 8888 - } + docker_client, image_name, detach=True, ports={"8888/tcp": 8888} ) yield container container.remove() diff --git a/datascience-notebook/test/test_julia.py b/datascience-notebook/test/test_julia.py index 887764ed..dd2cf15e 100644 --- a/datascience-notebook/test/test_julia.py +++ b/datascience-notebook/test/test_julia.py @@ -9,8 +9,7 @@ def test_julia(container): """Basic julia test""" LOGGER.info("Test that julia is correctly installed ...") running_container = container.run( - tty=True, - command=["start.sh", "bash", "-c", "sleep infinity"] + tty=True, command=["start.sh", "bash", "-c", "sleep infinity"] ) command = "julia --version" cmd = running_container.exec_run(command) diff --git a/docs/conf.py b/docs/conf.py index 0c06232f..17b16941 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,40 +25,37 @@ # If your documentation needs a minimal Sphinx version, state it here. # -needs_sphinx = '2.1' +needs_sphinx = "2.1" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [ - 'myst_parser', - 'sphinx_copybutton' -] +extensions = ["myst_parser", "sphinx_copybutton"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] source_suffix = { - '.rst': 'restructuredtext', - '.md': 'markdown', + ".rst": "restructuredtext", + ".md": "markdown", } # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'docker-stacks' -copyright = '2018- Project Jupyter' -author = 'Project Jupyter' +project = "docker-stacks" +copyright = "2018- Project Jupyter" +author = "Project Jupyter" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = 'latest' +version = "latest" # The full version, including alpha/beta/rc tags. -release = 'latest' +release = "latest" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -70,10 +67,10 @@ language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -84,27 +81,27 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { - 'description': "Jupyter Docker Stacks", - 'fixed_sidebar': False, - 'show_relbars': True, - 'github_user': 'jupyter', - 'github_repo': 'docker-stacks', - 'github_type': 'star', - 'logo': 'jupyter-logo.svg', - 'logo_text_align': 'left' + "description": "Jupyter Docker Stacks", + "fixed_sidebar": False, + "show_relbars": True, + "github_user": "jupyter", + "github_repo": "docker-stacks", + "github_type": "star", + "logo": "jupyter-logo.svg", + "logo_text_align": "left", } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -115,18 +112,18 @@ html_static_path = ['_static'] # 'searchbox.html']``. # html_sidebars = { - '**': [ - 'about.html', - 'navigation.html', - 'relations.html', - 'searchbox.html', + "**": [ + "about.html", + "navigation.html", + "relations.html", + "searchbox.html", ] } # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'docker-stacksdoc' +htmlhelp_basename = "docker-stacksdoc" # -- Options for LaTeX output --------------------------------------------- @@ -135,15 +132,12 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -153,8 +147,13 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'docker-stacks.tex', 'docker-stacks Documentation', - 'Project Jupyter', 'manual'), + ( + master_doc, + "docker-stacks.tex", + "docker-stacks Documentation", + "Project Jupyter", + "manual", + ), ] @@ -162,10 +161,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'docker-stacks', 'docker-stacks Documentation', - [author], 1) -] +man_pages = [(master_doc, "docker-stacks", "docker-stacks Documentation", [author], 1)] # -- Options for Texinfo output ------------------------------------------- @@ -174,9 +170,15 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'docker-stacks', 'docker-stacks Documentation', - author, 'docker-stacks', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "docker-stacks", + "docker-stacks Documentation", + author, + "docker-stacks", + "One line description of project.", + "Miscellaneous", + ), ] # -- Extension configuration ---------------------------------------------- @@ -187,4 +189,4 @@ linkcheck_anchors = False # -- Translation ---------------------------------------------------------- gettext_uuid = True -locale_dirs = ['locale/'] +locale_dirs = ["locale/"] diff --git a/minimal-notebook/test/test_inkscape.py b/minimal-notebook/test/test_inkscape.py index fdb0fbcf..e8f50e2c 100644 --- a/minimal-notebook/test/test_inkscape.py +++ b/minimal-notebook/test/test_inkscape.py @@ -10,8 +10,7 @@ def test_inkscape(container): """Inkscape shall be installed to be able to convert SVG files.""" LOGGER.info("Test that inkscape is working by printing its version ...") c = container.run( - tty=True, - command=["start.sh", "bash", "-c", "inkscape --version"] + tty=True, command=["start.sh", "bash", "-c", "inkscape --version"] ) c.wait(timeout=10) logs = c.logs(stdout=True).decode("utf-8") diff --git a/minimal-notebook/test/test_nbconvert.py b/minimal-notebook/test/test_nbconvert.py index eff7fd0e..e5d28086 100644 --- a/minimal-notebook/test/test_nbconvert.py +++ b/minimal-notebook/test/test_nbconvert.py @@ -24,7 +24,9 @@ def test_nbconvert(container, test_file, output_format): host_data_dir = os.path.join(THIS_DIR, "data") cont_data_dir = "/home/jovyan/data" output_dir = "/tmp" - LOGGER.info(f"Test that the example notebook {test_file} can be converted to {output_format} ...") + LOGGER.info( + f"Test that the example notebook {test_file} can be converted to {output_format} ..." + ) command = f"jupyter nbconvert {cont_data_dir}/{test_file}.ipynb --output-dir {output_dir} --to {output_format}" c = container.run( volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}}, diff --git a/pyspark-notebook/test/test_spark.py b/pyspark-notebook/test/test_spark.py index a98a2666..a25a542d 100644 --- a/pyspark-notebook/test/test_spark.py +++ b/pyspark-notebook/test/test_spark.py @@ -9,10 +9,9 @@ LOGGER = logging.getLogger(__name__) def test_spark_shell(container): """Checking if Spark (spark-shell) is running properly""" c = container.run( - tty=True, - command=['start.sh', 'bash', '-c', 'spark-shell <<< "1+1"'] + tty=True, command=["start.sh", "bash", "-c", 'spark-shell <<< "1+1"'] ) c.wait(timeout=60) - logs = c.logs(stdout=True).decode('utf-8') + logs = c.logs(stdout=True).decode("utf-8") LOGGER.debug(logs) - assert 'res0: Int = 2' in logs, "spark-shell does not work" + assert "res0: Int = 2" in logs, "spark-shell does not work" diff --git a/scipy-notebook/test/data/matplotlib_1.py b/scipy-notebook/test/data/matplotlib_1.py index 1d32aae2..aab63ea7 100644 --- a/scipy-notebook/test/data/matplotlib_1.py +++ b/scipy-notebook/test/data/matplotlib_1.py @@ -14,8 +14,9 @@ s = 1 + np.sin(2 * np.pi * t) fig, ax = plt.subplots() ax.plot(t, s) -ax.set(xlabel='time (s)', ylabel='voltage (mV)', - title='About as simple as it gets, folks') +ax.set( + xlabel="time (s)", ylabel="voltage (mV)", title="About as simple as it gets, folks" +) ax.grid() # Note that the test can be run headless by checking if an image is produced file_path = os.path.join("/tmp", "test.png") diff --git a/scipy-notebook/test/data/matplotlib_fonts_1.py b/scipy-notebook/test/data/matplotlib_fonts_1.py index 7a37a0ba..7e77e1f9 100644 --- a/scipy-notebook/test/data/matplotlib_fonts_1.py +++ b/scipy-notebook/test/data/matplotlib_fonts_1.py @@ -3,16 +3,22 @@ import matplotlib import matplotlib.pyplot as plt import os -matplotlib.rcParams['pgf.texsystem'] = 'pdflatex' -matplotlib.rcParams.update({'font.family': 'serif', 'font.size': 18, - 'axes.labelsize': 20, 'axes.titlesize': 24, - 'figure.titlesize': 28}) -matplotlib.rcParams['text.usetex'] = True +matplotlib.rcParams["pgf.texsystem"] = "pdflatex" +matplotlib.rcParams.update( + { + "font.family": "serif", + "font.size": 18, + "axes.labelsize": 20, + "axes.titlesize": 24, + "figure.titlesize": 28, + } +) +matplotlib.rcParams["text.usetex"] = True fig, ax = plt.subplots(1, 1) x = [1, 2] y = [1, 2] -ax.plot(x, y, label='a label') +ax.plot(x, y, label="a label") ax.legend(fontsize=15) file_path = os.path.join("/tmp", "test_fonts.png") diff --git a/scipy-notebook/test/test_extensions.py b/scipy-notebook/test/test_extensions.py index 168d92ad..3571a73c 100644 --- a/scipy-notebook/test/test_extensions.py +++ b/scipy-notebook/test/test_extensions.py @@ -26,8 +26,7 @@ def test_check_extension(container, extension): """ LOGGER.info(f"Checking the extension: {extension} ...") c = container.run( - tty=True, - command=["start.sh", "jupyter", "labextension", "check", extension] + tty=True, command=["start.sh", "jupyter", "labextension", "check", extension] ) rv = c.wait(timeout=10) logs = c.logs(stdout=True).decode("utf-8") diff --git a/scipy-notebook/test/test_matplotlib.py b/scipy-notebook/test/test_matplotlib.py index a0e73af3..e6233182 100644 --- a/scipy-notebook/test/test_matplotlib.py +++ b/scipy-notebook/test/test_matplotlib.py @@ -13,9 +13,17 @@ THIS_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.parametrize( "test_file,expected_file,description", [ - ("matplotlib_1.py", "test.png", "Test that matplotlib is able to plot a graph and write it as an image ..."), - ("matplotlib_fonts_1.py", "test_fonts.png", "Test cm-super latex labels in matplotlib ...") - ] + ( + "matplotlib_1.py", + "test.png", + "Test that matplotlib is able to plot a graph and write it as an image ...", + ), + ( + "matplotlib_fonts_1.py", + "test_fonts.png", + "Test cm-super latex labels in matplotlib ...", + ), + ], ) def test_matplotlib(container, test_file, expected_file, description): """Various tests performed on matplotlib diff --git a/tagging/create_manifests.py b/tagging/create_manifests.py index 5c7228e4..03f984aa 100755 --- a/tagging/create_manifests.py +++ b/tagging/create_manifests.py @@ -19,7 +19,9 @@ BUILD_TIMESTAMP = datetime.datetime.utcnow().isoformat()[:-7] + "Z" MARKDOWN_LINE_BREAK = "
" -def append_build_history_line(short_image_name: str, owner: str, wiki_path: str, all_tags: List[str]) -> None: +def append_build_history_line( + short_image_name: str, owner: str, wiki_path: str, all_tags: List[str] +) -> None: logger.info("Appending build history line") date_column = f"`{BUILD_TIMESTAMP}`" @@ -28,11 +30,13 @@ def append_build_history_line(short_image_name: str, owner: str, wiki_path: str, ) commit_hash = GitHelper.commit_hash() commit_hash_tag = GitHelper.commit_hash_tag() - links_column = MARKDOWN_LINE_BREAK.join([ - f"[Git diff](https://github.com/jupyter/docker-stacks/commit/{commit_hash})", - f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/{short_image_name}/Dockerfile)", - f"[Build manifest](./{short_image_name}-{commit_hash_tag})" - ]) + links_column = MARKDOWN_LINE_BREAK.join( + [ + f"[Git diff](https://github.com/jupyter/docker-stacks/commit/{commit_hash})", + f"[Dockerfile](https://github.com/jupyter/docker-stacks/blob/{commit_hash}/{short_image_name}/Dockerfile)", + f"[Build manifest](./{short_image_name}-{commit_hash_tag})", + ] + ) build_history_line = "|".join([date_column, image_column, links_column]) + "|" home_wiki_file = os.path.join(wiki_path, "Home.md") @@ -49,16 +53,19 @@ def create_manifest_file( owner: str, wiki_path: str, manifests: List[ManifestInterface], - container + container, ) -> None: manifest_names = [manifest.__name__ for manifest in manifests] logger.info(f"Using manifests: {manifest_names}") commit_hash_tag = GitHelper.commit_hash_tag() - manifest_file = os.path.join(wiki_path, f"manifests/{short_image_name}-{commit_hash_tag}.md") + manifest_file = os.path.join( + wiki_path, f"manifests/{short_image_name}-{commit_hash_tag}.md" + ) - markdown_pieces = [ManifestHeader.create_header(short_image_name, owner, BUILD_TIMESTAMP)] + \ - [manifest.markdown_piece(container) for manifest in manifests] + markdown_pieces = [ + ManifestHeader.create_header(short_image_name, owner, BUILD_TIMESTAMP) + ] + [manifest.markdown_piece(container) for manifest in manifests] markdown_content = "\n\n".join(markdown_pieces) + "\n" with open(manifest_file, "w") as f: @@ -81,7 +88,9 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO) arg_parser = argparse.ArgumentParser() - arg_parser.add_argument("--short-image-name", required=True, help="Short image name to apply tags for") + arg_parser.add_argument( + "--short-image-name", required=True, help="Short image name to apply tags for" + ) arg_parser.add_argument("--owner", required=True, help="Owner of the image") arg_parser.add_argument("--wiki-path", required=True, help="Path to the wiki pages") args = arg_parser.parse_args() diff --git a/tagging/docker_runner.py b/tagging/docker_runner.py index c391e963..15780b7e 100644 --- a/tagging/docker_runner.py +++ b/tagging/docker_runner.py @@ -8,7 +8,12 @@ logger = logging.getLogger(__name__) class DockerRunner: - def __init__(self, image_name: str, docker_client=docker.from_env(), command: str = "sleep infinity"): + def __init__( + self, + image_name: str, + docker_client=docker.from_env(), + command: str = "sleep infinity", + ): self.container = None self.image_name = image_name self.command = command @@ -17,7 +22,9 @@ class DockerRunner: def __enter__(self): logger.info(f"Creating container for image {self.image_name} ...") self.container = self.docker_client.containers.run( - image=self.image_name, command=self.command, detach=True, + image=self.image_name, + command=self.command, + detach=True, ) logger.info(f"Container {self.container.name} created") return self.container diff --git a/tagging/get_taggers_and_manifests.py b/tagging/get_taggers_and_manifests.py index 49c90160..da8c201c 100644 --- a/tagging/get_taggers_and_manifests.py +++ b/tagging/get_taggers_and_manifests.py @@ -6,7 +6,9 @@ from .manifests import ManifestInterface from .taggers import TaggerInterface -def get_taggers_and_manifests(short_image_name: str) -> Tuple[List[TaggerInterface], List[ManifestInterface]]: +def get_taggers_and_manifests( + short_image_name: str, +) -> Tuple[List[TaggerInterface], List[ManifestInterface]]: taggers: List[TaggerInterface] = [] manifests: List[ManifestInterface] = [] while short_image_name is not None: diff --git a/tagging/images_hierarchy.py b/tagging/images_hierarchy.py index 21c39b8e..4154c9d8 100644 --- a/tagging/images_hierarchy.py +++ b/tagging/images_hierarchy.py @@ -2,15 +2,29 @@ # Distributed under the terms of the Modified BSD License. from dataclasses import dataclass, field from typing import Optional, List -from .taggers import TaggerInterface, \ - SHATagger, \ - UbuntuVersionTagger, PythonVersionTagger, \ - JupyterNotebookVersionTagger, JupyterLabVersionTagger, JupyterHubVersionTagger, \ - RVersionTagger, TensorflowVersionTagger, JuliaVersionTagger, \ - SparkVersionTagger, HadoopVersionTagger, JavaVersionTagger -from .manifests import ManifestInterface, \ - CondaEnvironmentManifest, AptPackagesManifest, \ - RPackagesManifest, JuliaPackagesManifest, SparkInfoManifest +from .taggers import ( + TaggerInterface, + SHATagger, + UbuntuVersionTagger, + PythonVersionTagger, + JupyterNotebookVersionTagger, + JupyterLabVersionTagger, + JupyterHubVersionTagger, + RVersionTagger, + TensorflowVersionTagger, + JuliaVersionTagger, + SparkVersionTagger, + HadoopVersionTagger, + JavaVersionTagger, +) +from .manifests import ( + ManifestInterface, + CondaEnvironmentManifest, + AptPackagesManifest, + RPackagesManifest, + JuliaPackagesManifest, + SparkInfoManifest, +) @dataclass @@ -25,41 +39,37 @@ ALL_IMAGES = { parent_image=None, taggers=[ SHATagger, - UbuntuVersionTagger, PythonVersionTagger, - JupyterNotebookVersionTagger, JupyterLabVersionTagger, JupyterHubVersionTagger + UbuntuVersionTagger, + PythonVersionTagger, + JupyterNotebookVersionTagger, + JupyterLabVersionTagger, + JupyterHubVersionTagger, ], - manifests=[ - CondaEnvironmentManifest, AptPackagesManifest - ] - ), - "minimal-notebook": ImageDescription( - parent_image="base-notebook" - ), - "scipy-notebook": ImageDescription( - parent_image="minimal-notebook" + manifests=[CondaEnvironmentManifest, AptPackagesManifest], ), + "minimal-notebook": ImageDescription(parent_image="base-notebook"), + "scipy-notebook": ImageDescription(parent_image="minimal-notebook"), "r-notebook": ImageDescription( parent_image="minimal-notebook", taggers=[RVersionTagger], - manifests=[RPackagesManifest] + manifests=[RPackagesManifest], ), "tensorflow-notebook": ImageDescription( - parent_image="scipy-notebook", - taggers=[TensorflowVersionTagger] + parent_image="scipy-notebook", taggers=[TensorflowVersionTagger] ), "datascience-notebook": ImageDescription( parent_image="scipy-notebook", taggers=[RVersionTagger, JuliaVersionTagger], - manifests=[RPackagesManifest, JuliaPackagesManifest] + manifests=[RPackagesManifest, JuliaPackagesManifest], ), "pyspark-notebook": ImageDescription( parent_image="scipy-notebook", taggers=[SparkVersionTagger, HadoopVersionTagger, JavaVersionTagger], - manifests=[SparkInfoManifest] + manifests=[SparkInfoManifest], ), "all-spark-notebook": ImageDescription( parent_image="pyspark-notebook", taggers=[RVersionTagger], - manifests=[RPackagesManifest] - ) + manifests=[RPackagesManifest], + ), } diff --git a/tagging/manifests.py b/tagging/manifests.py index c704dd01..70664d35 100644 --- a/tagging/manifests.py +++ b/tagging/manifests.py @@ -10,41 +10,49 @@ logger = logging.getLogger(__name__) def quoted_output(container, cmd: str) -> str: - return "\n".join([ - "```", - DockerRunner.run_simple_command(container, cmd, print_result=False), - "```" - ]) + return "\n".join( + [ + "```", + DockerRunner.run_simple_command(container, cmd, print_result=False), + "```", + ] + ) class ManifestHeader: """ManifestHeader doesn't fall under common interface and we run it separately""" + @staticmethod def create_header(short_image_name: str, owner: str, build_timestamp: str) -> str: commit_hash = GitHelper.commit_hash() commit_hash_tag = GitHelper.commit_hash_tag() commit_message = GitHelper.commit_message() - image_size = docker["images", f"{owner}/{short_image_name}:latest", "--format", "{{.Size}}"]().rstrip() + image_size = docker[ + "images", f"{owner}/{short_image_name}:latest", "--format", "{{.Size}}" + ]().rstrip() - return "\n".join([ - f"# Build manifest for image: {short_image_name}:{commit_hash_tag}", - "", - "## Build Info", - "", - f"* Build datetime: {build_timestamp}", - f"* Docker image: {owner}/{short_image_name}:{commit_hash_tag}", - f"* Docker image size: {image_size}", - f"* Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash})", - "* Git commit message:", - "```", - f"{commit_message}", - "```" - ]) + return "\n".join( + [ + f"# Build manifest for image: {short_image_name}:{commit_hash_tag}", + "", + "## Build Info", + "", + f"* Build datetime: {build_timestamp}", + f"* Docker image: {owner}/{short_image_name}:{commit_hash_tag}", + f"* Docker image size: {image_size}", + f"* Git commit SHA: [{commit_hash}](https://github.com/jupyter/docker-stacks/commit/{commit_hash})", + "* Git commit message:", + "```", + f"{commit_message}", + "```", + ] + ) class ManifestInterface: """Common interface for all manifests""" + @staticmethod def markdown_piece(container) -> str: raise NotImplementedError @@ -53,56 +61,66 @@ class ManifestInterface: class CondaEnvironmentManifest(ManifestInterface): @staticmethod def markdown_piece(container) -> str: - return "\n".join([ - "## Python Packages", - "", - quoted_output(container, "python --version"), - "", - quoted_output(container, "conda info"), - "", - quoted_output(container, "conda list") - ]) + return "\n".join( + [ + "## Python Packages", + "", + quoted_output(container, "python --version"), + "", + quoted_output(container, "conda info"), + "", + quoted_output(container, "conda list"), + ] + ) class AptPackagesManifest(ManifestInterface): @staticmethod def markdown_piece(container) -> str: - return "\n".join([ - "## Apt Packages", - "", - quoted_output(container, "apt list --installed") - ]) + return "\n".join( + ["## Apt Packages", "", quoted_output(container, "apt list --installed")] + ) class RPackagesManifest(ManifestInterface): @staticmethod def markdown_piece(container) -> str: - return "\n".join([ - "## R Packages", - "", - quoted_output(container, "R --version"), - "", - quoted_output(container, "R --silent -e 'installed.packages(.Library)[, c(1,3)]'") - ]) + return "\n".join( + [ + "## R Packages", + "", + quoted_output(container, "R --version"), + "", + quoted_output( + container, "R --silent -e 'installed.packages(.Library)[, c(1,3)]'" + ), + ] + ) class JuliaPackagesManifest(ManifestInterface): @staticmethod def markdown_piece(container) -> str: - return "\n".join([ - "## Julia Packages", - "", - quoted_output(container, "julia -E 'using InteractiveUtils; versioninfo()'"), - "", - quoted_output(container, "julia -E 'import Pkg; Pkg.status()'") - ]) + return "\n".join( + [ + "## Julia Packages", + "", + quoted_output( + container, "julia -E 'using InteractiveUtils; versioninfo()'" + ), + "", + quoted_output(container, "julia -E 'import Pkg; Pkg.status()'"), + ] + ) class SparkInfoManifest(ManifestInterface): @staticmethod def markdown_piece(container) -> str: - return "\n".join([ - "## Apache Spark", - "", - quoted_output(container, "/usr/local/spark/bin/spark-submit --version"), - ]) + return "\n".join( + [ + "## Apache Spark", + "", + quoted_output(container, "/usr/local/spark/bin/spark-submit --version"), + ] + ) diff --git a/tagging/tag_image.py b/tagging/tag_image.py index beb92d12..0c38f0e7 100755 --- a/tagging/tag_image.py +++ b/tagging/tag_image.py @@ -21,7 +21,9 @@ def tag_image(short_image_name: str, owner: str) -> None: for tagger in taggers: tagger_name = tagger.__name__ tag_value = tagger.tag_value(container) - logger.info(f"Applying tag tagger_name: {tagger_name} tag_value: {tag_value}") + logger.info( + f"Applying tag tagger_name: {tagger_name} tag_value: {tag_value}" + ) docker["tag", image, f"{owner}/{short_image_name}:{tag_value}"]() @@ -29,7 +31,9 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO) arg_parser = argparse.ArgumentParser() - arg_parser.add_argument("--short-image-name", required=True, help="Short image name to apply tags for") + arg_parser.add_argument( + "--short-image-name", required=True, help="Short image name to apply tags for" + ) arg_parser.add_argument("--owner", required=True, help="Owner of the image") args = arg_parser.parse_args() diff --git a/tagging/taggers.py b/tagging/taggers.py index d2c99148..392b2179 100644 --- a/tagging/taggers.py +++ b/tagging/taggers.py @@ -14,30 +14,27 @@ def _get_program_version(container, program: str) -> str: def _get_env_variable(container, variable: str) -> str: env = DockerRunner.run_simple_command( - container, - cmd="env", - print_result=False + container, cmd="env", print_result=False ).split() for env_entry in env: if env_entry.startswith(variable): - return env_entry[len(variable) + 1:] + return env_entry[len(variable) + 1 :] raise KeyError(variable) def _get_pip_package_version(container, package: str) -> str: VERSION_PREFIX = "Version: " package_info = DockerRunner.run_simple_command( - container, - cmd=f"pip show {package}", - print_result=False + container, cmd=f"pip show {package}", print_result=False ) version_line = package_info.split("\n")[1] assert version_line.startswith(VERSION_PREFIX) - return version_line[len(VERSION_PREFIX):] + return version_line[len(VERSION_PREFIX) :] class TaggerInterface: """Common interface for all taggers""" + @staticmethod def tag_value(container) -> str: raise NotImplementedError @@ -52,7 +49,9 @@ class SHATagger(TaggerInterface): class UbuntuVersionTagger(TaggerInterface): @staticmethod def tag_value(container) -> str: - os_release = DockerRunner.run_simple_command(container, "cat /etc/os-release").split("\n") + os_release = DockerRunner.run_simple_command( + container, "cat /etc/os-release" + ).split("\n") for line in os_release: if line.startswith("VERSION_ID"): return "ubuntu-" + line.split("=")[1].strip('"') diff --git a/tensorflow-notebook/test/units/unit_tensorflow.py b/tensorflow-notebook/test/units/unit_tensorflow.py index 3d5a60c0..0f5911ca 100644 --- a/tensorflow-notebook/test/units/unit_tensorflow.py +++ b/tensorflow-notebook/test/units/unit_tensorflow.py @@ -3,5 +3,5 @@ import tensorflow as tf -print(tf.constant('Hello, TensorFlow')) +print(tf.constant("Hello, TensorFlow")) print(tf.reduce_sum(tf.random.normal([1000, 1000]))) diff --git a/test/helpers.py b/test/helpers.py index 7fe41062..b48ef70a 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -34,8 +34,7 @@ LOGGER = logging.getLogger(__name__) class CondaPackageHelper: - """Conda package helper permitting to get information about packages - """ + """Conda package helper permitting to get information about packages""" def __init__(self, container): # if isinstance(container, TrackedContainer): @@ -50,8 +49,7 @@ class CondaPackageHelper: """Start the TrackedContainer and return an instance of a running container""" LOGGER.info(f"Starting container {container.image_name} ...") return container.run( - tty=True, - command=["start.sh", "bash", "-c", "sleep infinity"] + tty=True, command=["start.sh", "bash", "-c", "sleep infinity"] ) @staticmethod @@ -76,7 +74,9 @@ class CondaPackageHelper: if self.specs is None: LOGGER.info("Grabing the list of specifications ...") self.specs = CondaPackageHelper._packages_from_json( - self._execute_command(CondaPackageHelper._conda_export_command(from_history=True)) + self._execute_command( + CondaPackageHelper._conda_export_command(from_history=True) + ) ) return self.specs @@ -112,9 +112,7 @@ class CondaPackageHelper: def available_packages(self): """Return the available packages""" if self.available is None: - LOGGER.info( - "Grabing the list of available packages (can take a while) ..." - ) + LOGGER.info("Grabing the list of available packages (can take a while) ...") # Keeping command line output since `conda search --outdated --json` is way too long ... self.available = CondaPackageHelper._extract_available( self._execute_command(["conda", "search", "--outdated"]) @@ -146,10 +144,9 @@ class CondaPackageHelper: current = min(inst_vs, key=CondaPackageHelper.semantic_cmp) newest = avail_vs[-1] if avail_vs and current != newest: - if ( - CondaPackageHelper.semantic_cmp(current) < - CondaPackageHelper.semantic_cmp(newest) - ): + if CondaPackageHelper.semantic_cmp( + current + ) < CondaPackageHelper.semantic_cmp(newest): self.comparison.append( {"Package": pkg, "Current": current, "Newest": newest} ) @@ -162,6 +159,7 @@ class CondaPackageHelper: def mysplit(string): def version_substrs(x): return re.findall(r"([A-z]+|\d+)", x) + return list(chain(map(version_substrs, string.split(".")))) def str_ord(string): diff --git a/test/test_packages.py b/test/test_packages.py index ca6da5e7..0aa12edc 100644 --- a/test/test_packages.py +++ b/test/test_packages.py @@ -68,7 +68,7 @@ EXCLUDED_PACKAGES = [ "protobuf", "r-irkernel", "unixodbc", - "bzip2" + "bzip2", ] @@ -133,8 +133,9 @@ def _import_packages(package_helper, filtered_packages, check_function, max_fail for package in filtered_packages: LOGGER.info(f"Trying to import {package}") try: - assert check_function(package_helper, package) == 0, \ - f"Package [{package}] import failed" + assert ( + check_function(package_helper, package) == 0 + ), f"Package [{package}] import failed" except AssertionError as err: failures[package] = err if len(failures) > max_failures: diff --git a/test/test_units.py b/test/test_units.py index 8edef5c9..bfa120f9 100644 --- a/test/test_units.py +++ b/test/test_units.py @@ -12,7 +12,7 @@ def test_units(container): """Various units tests Add a py file in the {image}/test/units dir and it will be automatically tested """ - short_image_name = container.image_name[container.image_name.rfind('/') + 1:] + short_image_name = container.image_name[container.image_name.rfind("/") + 1 :] host_data_dir = os.path.join(THIS_DIR, f"../{short_image_name}/test/units") LOGGER.info(f"Searching for units tests in {host_data_dir}") cont_data_dir = "/home/jovyan/data" @@ -27,9 +27,9 @@ def test_units(container): c = container.run( volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}}, tty=True, - command=['start.sh', 'python', f'{cont_data_dir}/{test_file}'] + command=["start.sh", "python", f"{cont_data_dir}/{test_file}"], ) rv = c.wait(timeout=30) - logs = c.logs(stdout=True).decode('utf-8') + logs = c.logs(stdout=True).decode("utf-8") LOGGER.debug(logs) assert rv == 0 or rv["StatusCode"] == 0 From a582c24e1992cbd276221295b5f2eca6482ede95 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 28 Jun 2021 13:03:40 +0200 Subject: [PATCH 36/47] black style tweaks: manually added commas etc to tweak formatting --- base-notebook/jupyter_notebook_config.py | 15 +++++---------- conftest.py | 5 ++++- scipy-notebook/test/data/matplotlib_1.py | 4 +++- tagging/create_manifests.py | 5 ++++- tagging/taggers.py | 4 +++- test/helpers.py | 16 +++++++++------- 6 files changed, 28 insertions(+), 21 deletions(-) diff --git a/base-notebook/jupyter_notebook_config.py b/base-notebook/jupyter_notebook_config.py index e0c7d12a..ad41dafa 100644 --- a/base-notebook/jupyter_notebook_config.py +++ b/base-notebook/jupyter_notebook_config.py @@ -45,18 +45,13 @@ distinguished_name = req_distinguished_name "openssl", "req", "-new", - "-newkey", - "rsa:2048", - "-days", - "365", + "-newkey=rsa:2048", + "-days=365", "-nodes", "-x509", - "-subj", - "/C=XX/ST=XX/L=XX/O=generated/CN=generated", - "-keyout", - pem_file, - "-out", - pem_file, + "-subj=/C=XX/ST=XX/L=XX/O=generated/CN=generated", + f"-keyout={pem_file}", + f"-out={pem_file}", ] ) # Restrict access to the file diff --git a/conftest.py b/conftest.py index d13654bf..4c3056ad 100644 --- a/conftest.py +++ b/conftest.py @@ -97,7 +97,10 @@ def container(docker_client, image_name): Yields the container instance and kills it when the caller is done with it. """ container = TrackedContainer( - docker_client, image_name, detach=True, ports={"8888/tcp": 8888} + docker_client, + image_name, + detach=True, + ports={"8888/tcp": 8888}, ) yield container container.remove() diff --git a/scipy-notebook/test/data/matplotlib_1.py b/scipy-notebook/test/data/matplotlib_1.py index aab63ea7..57a93e70 100644 --- a/scipy-notebook/test/data/matplotlib_1.py +++ b/scipy-notebook/test/data/matplotlib_1.py @@ -15,7 +15,9 @@ fig, ax = plt.subplots() ax.plot(t, s) ax.set( - xlabel="time (s)", ylabel="voltage (mV)", title="About as simple as it gets, folks" + xlabel="time (s)", + ylabel="voltage (mV)", + title="About as simple as it gets, folks", ) ax.grid() # Note that the test can be run headless by checking if an image is produced diff --git a/tagging/create_manifests.py b/tagging/create_manifests.py index 03f984aa..b4064d82 100755 --- a/tagging/create_manifests.py +++ b/tagging/create_manifests.py @@ -20,7 +20,10 @@ MARKDOWN_LINE_BREAK = "
" def append_build_history_line( - short_image_name: str, owner: str, wiki_path: str, all_tags: List[str] + short_image_name: str, + owner: str, + wiki_path: str, + all_tags: List[str], ) -> None: logger.info("Appending build history line") diff --git a/tagging/taggers.py b/tagging/taggers.py index 392b2179..2f95d09b 100644 --- a/tagging/taggers.py +++ b/tagging/taggers.py @@ -25,7 +25,9 @@ def _get_env_variable(container, variable: str) -> str: def _get_pip_package_version(container, package: str) -> str: VERSION_PREFIX = "Version: " package_info = DockerRunner.run_simple_command( - container, cmd=f"pip show {package}", print_result=False + container, + cmd=f"pip show {package}", + print_result=False, ) version_line = package_info.split("\n")[1] assert version_line.startswith(VERSION_PREFIX) diff --git a/test/helpers.py b/test/helpers.py index b48ef70a..a36cab06 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -143,13 +143,15 @@ class CondaPackageHelper: continue current = min(inst_vs, key=CondaPackageHelper.semantic_cmp) newest = avail_vs[-1] - if avail_vs and current != newest: - if CondaPackageHelper.semantic_cmp( - current - ) < CondaPackageHelper.semantic_cmp(newest): - self.comparison.append( - {"Package": pkg, "Current": current, "Newest": newest} - ) + if ( + avail_vs + and current != newest + and CondaPackageHelper.semantic_cmp(current) + < CondaPackageHelper.semantic_cmp(newest) + ): + self.comparison.append( + {"Package": pkg, "Current": current, "Newest": newest} + ) return self.comparison @staticmethod From 962a630f531e8ec9c1f1dec22f1e25f435805580 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 28 Jun 2021 15:08:32 +0200 Subject: [PATCH 37/47] Apply suggestions from code review Co-authored-by: Ayaz Salikhov --- .pre-commit-config.yaml | 2 +- base-notebook/test/test_container_options.py | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b0d850a..7e0c098d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ repos: rev: 21.6b0 hooks: - id: black - args: [--target-version=py36] + args: [--target-version=py39] # Autoformat: YAML, JSON, Markdown, etc. - repo: https://github.com/pre-commit/mirrors-prettier diff --git a/base-notebook/test/test_container_options.py b/base-notebook/test/test_container_options.py index 2187d8f4..6ca616a3 100644 --- a/base-notebook/test/test_container_options.py +++ b/base-notebook/test/test_container_options.py @@ -154,9 +154,8 @@ def test_sudo_path(container): ) rv = c.wait(timeout=10) assert rv == 0 or rv["StatusCode"] == 0 - assert ( - c.logs(stdout=True).decode("utf-8").rstrip().endswith("/opt/conda/bin/jupyter") - ) + logs = c.logs(stdout=True).decode("utf-8") + assert logs.rstrip().endswith("/opt/conda/bin/jupyter") def test_sudo_path_without_grant(container): @@ -164,9 +163,8 @@ def test_sudo_path_without_grant(container): c = container.run(tty=True, user="root", command=["start.sh", "which", "jupyter"]) rv = c.wait(timeout=10) assert rv == 0 or rv["StatusCode"] == 0 - assert ( - c.logs(stdout=True).decode("utf-8").rstrip().endswith("/opt/conda/bin/jupyter") - ) + logs = c.logs(stdout=True).decode("utf-8") + assert logs.rstrip().endswith("/opt/conda/bin/jupyter") def test_group_add(container, tmpdir): @@ -176,6 +174,5 @@ def test_group_add(container, tmpdir): c = container.run(user="1010:1010", group_add=["users"], command=["start.sh", "id"]) rv = c.wait(timeout=5) assert rv == 0 or rv["StatusCode"] == 0 - assert "uid=1010 gid=1010 groups=1010,100(users)" in c.logs(stdout=True).decode( - "utf-8" - ) + logs = c.logs(stdout=True).decode("utf-8") + assert "uid=1010 gid=1010 groups=1010,100(users)" in logs From 411663b5d4ee4632e980cc9b2efdbb410cca919d Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 28 Jun 2021 15:09:13 +0200 Subject: [PATCH 38/47] black style tweaks: add trailing commas --- base-notebook/test/test_container_options.py | 17 ++++++++++++++--- base-notebook/test/test_package_managers.py | 3 ++- base-notebook/test/test_pandoc.py | 3 ++- base-notebook/test/test_python.py | 5 ++++- base-notebook/test/test_start_container.py | 11 +++++++++-- datascience-notebook/test/test_julia.py | 3 ++- minimal-notebook/test/test_inkscape.py | 3 ++- pyspark-notebook/test/test_spark.py | 3 ++- scipy-notebook/test/test_extensions.py | 3 ++- test/helpers.py | 3 ++- 10 files changed, 41 insertions(+), 13 deletions(-) diff --git a/base-notebook/test/test_container_options.py b/base-notebook/test/test_container_options.py index 6ca616a3..91cce1a8 100644 --- a/base-notebook/test/test_container_options.py +++ b/base-notebook/test/test_container_options.py @@ -51,7 +51,10 @@ def test_uid_change(container): def test_gid_change(container): """Container should change the GID of the default user.""" c = container.run( - tty=True, user="root", environment=["NB_GID=110"], command=["start.sh", "id"] + tty=True, + user="root", + environment=["NB_GID=110"], + command=["start.sh", "id"], ) c.wait(timeout=10) logs = c.logs(stdout=True).decode("utf-8") @@ -160,7 +163,11 @@ def test_sudo_path(container): def test_sudo_path_without_grant(container): """Container should include /opt/conda/bin in the sudo secure_path.""" - c = container.run(tty=True, user="root", command=["start.sh", "which", "jupyter"]) + c = container.run( + tty=True, + user="root", + command=["start.sh", "which", "jupyter"], + ) rv = c.wait(timeout=10) assert rv == 0 or rv["StatusCode"] == 0 logs = c.logs(stdout=True).decode("utf-8") @@ -171,7 +178,11 @@ def test_group_add(container, tmpdir): """Container should run with the specified uid, gid, and secondary group. """ - c = container.run(user="1010:1010", group_add=["users"], command=["start.sh", "id"]) + c = container.run( + user="1010:1010", + group_add=["users"], + command=["start.sh", "id"], + ) rv = c.wait(timeout=5) assert rv == 0 or rv["StatusCode"] == 0 logs = c.logs(stdout=True).decode("utf-8") diff --git a/base-notebook/test/test_package_managers.py b/base-notebook/test/test_package_managers.py index e673078c..7c96a73d 100644 --- a/base-notebook/test/test_package_managers.py +++ b/base-notebook/test/test_package_managers.py @@ -23,7 +23,8 @@ def test_package_manager(container, package_manager, version_arg): f"Test that the package manager {package_manager} is working properly ..." ) c = container.run( - tty=True, command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"] + tty=True, + command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"], ) rv = c.wait(timeout=5) logs = c.logs(stdout=True).decode("utf-8") diff --git a/base-notebook/test/test_pandoc.py b/base-notebook/test/test_pandoc.py index c84787c1..f0e33ab8 100644 --- a/base-notebook/test/test_pandoc.py +++ b/base-notebook/test/test_pandoc.py @@ -9,7 +9,8 @@ LOGGER = logging.getLogger(__name__) def test_pandoc(container): """Pandoc shall be able to convert MD to HTML.""" c = container.run( - tty=True, command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc'] + tty=True, + command=["start.sh", "bash", "-c", 'echo "**BOLD**" | pandoc'], ) c.wait(timeout=10) logs = c.logs(stdout=True).decode("utf-8") diff --git a/base-notebook/test/test_python.py b/base-notebook/test/test_python.py index 3e35d2b8..7dc8ef1f 100644 --- a/base-notebook/test/test_python.py +++ b/base-notebook/test/test_python.py @@ -10,7 +10,10 @@ LOGGER = logging.getLogger(__name__) def test_python_version(container, python_next_version="3.10"): """Check that python version is lower than the next version""" LOGGER.info(f"Checking that python version is lower than {python_next_version}") - c = container.run(tty=True, command=["start.sh"]) + c = container.run( + tty=True, + command=["start.sh"], + ) cmd = c.exec_run("python --version") output = cmd.output.decode("utf-8") actual_python_version = version.parse(output.split()[1]) diff --git a/base-notebook/test/test_start_container.py b/base-notebook/test/test_start_container.py index 32760185..9d3a8580 100644 --- a/base-notebook/test/test_start_container.py +++ b/base-notebook/test/test_start_container.py @@ -19,7 +19,11 @@ def test_start_notebook(container, http_client, env, expected_server): LOGGER.info( f"Test that the start-notebook launches the {expected_server} server from the env {env} ..." ) - c = container.run(tty=True, environment=env, command=["start-notebook.sh"]) + c = container.run( + tty=True, + environment=env, + command=["start-notebook.sh"], + ) resp = http_client.get("http://localhost:8888") logs = c.logs(stdout=True).decode("utf-8") LOGGER.debug(logs) @@ -40,7 +44,10 @@ def test_tini_entrypoint(container, pid=1, command="tini"): https://superuser.com/questions/632979/if-i-know-the-pid-number-of-a-process-how-can-i-get-its-name """ LOGGER.info(f"Test that {command} is launched as PID {pid} ...") - c = container.run(tty=True, command=["start.sh"]) + c = container.run( + tty=True, + command=["start.sh"], + ) # Select the PID 1 and get the corresponding command cmd = c.exec_run(f"ps -p {pid} -o comm=") output = cmd.output.decode("utf-8").strip("\n") diff --git a/datascience-notebook/test/test_julia.py b/datascience-notebook/test/test_julia.py index dd2cf15e..610fab3d 100644 --- a/datascience-notebook/test/test_julia.py +++ b/datascience-notebook/test/test_julia.py @@ -9,7 +9,8 @@ def test_julia(container): """Basic julia test""" LOGGER.info("Test that julia is correctly installed ...") running_container = container.run( - tty=True, command=["start.sh", "bash", "-c", "sleep infinity"] + tty=True, + command=["start.sh", "bash", "-c", "sleep infinity"], ) command = "julia --version" cmd = running_container.exec_run(command) diff --git a/minimal-notebook/test/test_inkscape.py b/minimal-notebook/test/test_inkscape.py index e8f50e2c..a39bcb15 100644 --- a/minimal-notebook/test/test_inkscape.py +++ b/minimal-notebook/test/test_inkscape.py @@ -10,7 +10,8 @@ def test_inkscape(container): """Inkscape shall be installed to be able to convert SVG files.""" LOGGER.info("Test that inkscape is working by printing its version ...") c = container.run( - tty=True, command=["start.sh", "bash", "-c", "inkscape --version"] + tty=True, + command=["start.sh", "bash", "-c", "inkscape --version"], ) c.wait(timeout=10) logs = c.logs(stdout=True).decode("utf-8") diff --git a/pyspark-notebook/test/test_spark.py b/pyspark-notebook/test/test_spark.py index a25a542d..1cfba7e0 100644 --- a/pyspark-notebook/test/test_spark.py +++ b/pyspark-notebook/test/test_spark.py @@ -9,7 +9,8 @@ LOGGER = logging.getLogger(__name__) def test_spark_shell(container): """Checking if Spark (spark-shell) is running properly""" c = container.run( - tty=True, command=["start.sh", "bash", "-c", 'spark-shell <<< "1+1"'] + tty=True, + command=["start.sh", "bash", "-c", 'spark-shell <<< "1+1"'], ) c.wait(timeout=60) logs = c.logs(stdout=True).decode("utf-8") diff --git a/scipy-notebook/test/test_extensions.py b/scipy-notebook/test/test_extensions.py index 3571a73c..03f80d5f 100644 --- a/scipy-notebook/test/test_extensions.py +++ b/scipy-notebook/test/test_extensions.py @@ -26,7 +26,8 @@ def test_check_extension(container, extension): """ LOGGER.info(f"Checking the extension: {extension} ...") c = container.run( - tty=True, command=["start.sh", "jupyter", "labextension", "check", extension] + tty=True, + command=["start.sh", "jupyter", "labextension", "check", extension], ) rv = c.wait(timeout=10) logs = c.logs(stdout=True).decode("utf-8") diff --git a/test/helpers.py b/test/helpers.py index a36cab06..98b8d75c 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -49,7 +49,8 @@ class CondaPackageHelper: """Start the TrackedContainer and return an instance of a running container""" LOGGER.info(f"Starting container {container.image_name} ...") return container.run( - tty=True, command=["start.sh", "bash", "-c", "sleep infinity"] + tty=True, + command=["start.sh", "bash", "-c", "sleep infinity"], ) @staticmethod From 967f0c2f7a3e3db3fd9f6cb026c9e7baf53b27ce Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 29 Jun 2021 01:50:30 +0300 Subject: [PATCH 39/47] Update conftest.py --- conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index 4c3056ad..3c957142 100644 --- a/conftest.py +++ b/conftest.py @@ -79,7 +79,8 @@ class TrackedContainer: all_kwargs.update(kwargs) LOGGER.info(f"Running {self.image_name} with args {all_kwargs} ...") self.container = self.docker_client.containers.run( - self.image_name, **all_kwargs + self.image_name, + **all_kwargs, ) return self.container From 9159c549df5f4d99534b0b9e0806c6cee42cbc54 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 29 Jun 2021 01:50:38 +0300 Subject: [PATCH 40/47] Update tagging/create_manifests.py --- tagging/create_manifests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tagging/create_manifests.py b/tagging/create_manifests.py index b4064d82..caba1f97 100755 --- a/tagging/create_manifests.py +++ b/tagging/create_manifests.py @@ -63,7 +63,8 @@ def create_manifest_file( commit_hash_tag = GitHelper.commit_hash_tag() manifest_file = os.path.join( - wiki_path, f"manifests/{short_image_name}-{commit_hash_tag}.md" + wiki_path, + f"manifests/{short_image_name}-{commit_hash_tag}.md", ) markdown_pieces = [ From 45c8443cc839390db46ca733232f8b9940a08b60 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 29 Jun 2021 01:50:46 +0300 Subject: [PATCH 41/47] Update tagging/create_manifests.py --- tagging/create_manifests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tagging/create_manifests.py b/tagging/create_manifests.py index caba1f97..e7c380e3 100755 --- a/tagging/create_manifests.py +++ b/tagging/create_manifests.py @@ -93,7 +93,9 @@ if __name__ == "__main__": arg_parser = argparse.ArgumentParser() arg_parser.add_argument( - "--short-image-name", required=True, help="Short image name to apply tags for" + "--short-image-name", + required=True, + help="Short image name to apply tags for", ) arg_parser.add_argument("--owner", required=True, help="Owner of the image") arg_parser.add_argument("--wiki-path", required=True, help="Path to the wiki pages") From a050b9454b85515a974f4f8196c8246f8b20175a Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 29 Jun 2021 01:50:53 +0300 Subject: [PATCH 42/47] Update tagging/manifests.py --- tagging/manifests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tagging/manifests.py b/tagging/manifests.py index 70664d35..6ebd2ea4 100644 --- a/tagging/manifests.py +++ b/tagging/manifests.py @@ -92,7 +92,8 @@ class RPackagesManifest(ManifestInterface): quoted_output(container, "R --version"), "", quoted_output( - container, "R --silent -e 'installed.packages(.Library)[, c(1,3)]'" + container, + "R --silent -e 'installed.packages(.Library)[, c(1,3)]'", ), ] ) From 9dbf3cdbeaec54f47f7811492aa45b64c7c86c35 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 29 Jun 2021 01:51:00 +0300 Subject: [PATCH 43/47] Update tagging/manifests.py --- tagging/manifests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tagging/manifests.py b/tagging/manifests.py index 6ebd2ea4..5be19832 100644 --- a/tagging/manifests.py +++ b/tagging/manifests.py @@ -107,7 +107,8 @@ class JuliaPackagesManifest(ManifestInterface): "## Julia Packages", "", quoted_output( - container, "julia -E 'using InteractiveUtils; versioninfo()'" + container, + "julia -E 'using InteractiveUtils; versioninfo()'", ), "", quoted_output(container, "julia -E 'import Pkg; Pkg.status()'"), From e52fcda7c1d41e63d715b9534d5f20abb59abb51 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 29 Jun 2021 01:51:08 +0300 Subject: [PATCH 44/47] Update tagging/tag_image.py --- tagging/tag_image.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tagging/tag_image.py b/tagging/tag_image.py index 0c38f0e7..3aef779a 100755 --- a/tagging/tag_image.py +++ b/tagging/tag_image.py @@ -32,7 +32,9 @@ if __name__ == "__main__": arg_parser = argparse.ArgumentParser() arg_parser.add_argument( - "--short-image-name", required=True, help="Short image name to apply tags for" + "--short-image-name", + required=True, + help="Short image name to apply tags for", ) arg_parser.add_argument("--owner", required=True, help="Owner of the image") args = arg_parser.parse_args() From 54d4c06c8069524aacb7cfd9521f58cf19bd5a87 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 29 Jun 2021 01:51:16 +0300 Subject: [PATCH 45/47] Update tagging/taggers.py --- tagging/taggers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tagging/taggers.py b/tagging/taggers.py index 2f95d09b..0dcb4508 100644 --- a/tagging/taggers.py +++ b/tagging/taggers.py @@ -14,7 +14,9 @@ def _get_program_version(container, program: str) -> str: def _get_env_variable(container, variable: str) -> str: env = DockerRunner.run_simple_command( - container, cmd="env", print_result=False + container, + cmd="env", + print_result=False, ).split() for env_entry in env: if env_entry.startswith(variable): From 054aff20ee6c5e5e455ecf53b923ebb028044061 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 29 Jun 2021 01:51:33 +0300 Subject: [PATCH 46/47] Update tagging/taggers.py --- tagging/taggers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tagging/taggers.py b/tagging/taggers.py index 0dcb4508..ceea21e1 100644 --- a/tagging/taggers.py +++ b/tagging/taggers.py @@ -54,7 +54,8 @@ class UbuntuVersionTagger(TaggerInterface): @staticmethod def tag_value(container) -> str: os_release = DockerRunner.run_simple_command( - container, "cat /etc/os-release" + container, + "cat /etc/os-release", ).split("\n") for line in os_release: if line.startswith("VERSION_ID"): From f28efe481e415e3a2cd32aae4e0b5ff046dc8813 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Tue, 29 Jun 2021 01:51:43 +0300 Subject: [PATCH 47/47] Update tagging/manifests.py --- tagging/manifests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tagging/manifests.py b/tagging/manifests.py index 5be19832..a7d1edc3 100644 --- a/tagging/manifests.py +++ b/tagging/manifests.py @@ -78,7 +78,11 @@ class AptPackagesManifest(ManifestInterface): @staticmethod def markdown_piece(container) -> str: return "\n".join( - ["## Apt Packages", "", quoted_output(container, "apt list --installed")] + [ + "## Apt Packages", + "", + quoted_output(container, "apt list --installed"), + ] )