Merge pull request #1269 from mathbunnyru/asalikhov/small_updates

Bunch of small refactorings and updates

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.1.0
+    rev: v3.4.0
     hooks:
       - id: check-yaml
         files: .*\.(yaml|yml)$
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.23.0
+    rev: v1.26.1
     hooks:
       - id: yamllint
         args: ['-d {extends: relaxed, rules: {line-length: disable}}', '-s']
@@ -17,10 +17,10 @@ repos:
       - id: bashate
         args: ['--ignore=E006']
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+    rev: 3.9.1
     hooks:
       - id: flake8
   - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.5.4
+    rev: v1.5.6
     hooks:
       - id: autopep8

.readthedocs.yml

@@ -12,6 +12,6 @@ sphinx:
 formats: all
 
 python:
-  version: 3.7
+  version: 3.8
   install:
     - requirements: requirements-dev.txt

CODE_OF_CONDUCT.md

@@ -1 +1 @@
-Please see the [Project Jupyter Code of Conduct](https://github.com/jupyter/governance/blob/master/conduct/code_of_conduct.md).
+Please see the [Project Jupyter Code of Conduct](https://github.com/jupyter/governance/blob/master/conduct/code_of_conduct.md).

Makefile

@@ -90,7 +90,7 @@ git-commit: ## commit outstading git changes and push to remote
 	@git config --global user.name "GitHub Actions"
 	@git config --global user.email "actions@users.noreply.github.com"
 
-	@echo "Publishing outstanding changes in $(LOCAL_PATH) to $(GITHUB_REPOSITORY)" 
+	@echo "Publishing outstanding changes in $(LOCAL_PATH) to $(GITHUB_REPOSITORY)"
 	@cd $(LOCAL_PATH) && \
 		git remote add publisher https://$(GITHUB_TOKEN)@github.com/$(GITHUB_REPOSITORY).git && \
 		git checkout master && \
@@ -152,6 +152,8 @@ pull/%: DARGS?=
 pull/%: ## pull a jupyter image
 	docker pull $(DARGS) $(OWNER)/$(notdir $@)
 
+pull-all: $(foreach I,$(ALL_IMAGES),pull/$(I) ) ## pull all images
+
 push/%: DARGS?=
 push/%: ## push all tags for a jupyter image
 	docker push --all-tags $(DARGS) $(OWNER)/$(notdir $@)

README.md

@@ -31,7 +31,7 @@ in working on the project.
 ## Jupyter Notebook Deprecation Notice
 
 Following [Jupyter Notebook notice](https://github.com/jupyter/notebook#notice), we encourage users to transition to JupyterLab.
-This can be done by passing the environment variable `JUPYTER_ENABLE_LAB=yes` at container startup, 
+This can be done by passing the environment variable `JUPYTER_ENABLE_LAB=yes` at container startup,
 more information is available in the [documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html#docker-options).
 
 At some point, JupyterLab will become the default for all of the Jupyter Docker stack images, however a new environment variable will be introduced to switch back to Jupyter Notebook if needed.

all-spark-notebook/hooks/run_hook

@@ -58,4 +58,4 @@ $(docker run --rm ${IMAGE_NAME} R --silent -e 'installed.packages(.Library)[, c(
 \`\`\`
 $(docker run --rm ${IMAGE_NAME} apt list --installed)
 \`\`\`
-EOF
+EOF

all-spark-notebook/test/data/local_pyspark.ipynb

@@ -57,4 +57,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

all-spark-notebook/test/data/local_sparkR.ipynb

@@ -38,4 +38,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

all-spark-notebook/test/data/local_sparklyr.ipynb

@@ -40,4 +40,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

all-spark-notebook/test/data/local_spylon.ipynb

@@ -60,4 +60,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

base-notebook/Dockerfile

@@ -79,7 +79,7 @@ RUN chmod a+rx /usr/local/bin/fix-permissions
 # hadolint ignore=SC2016
 RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \
    # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
-   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc 
+   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
 
 # Create NB_USER with name jovyan user with UID=1000 and in the 'users' group
 # and make sure these dirs are writable by the `users` group.

datascience-notebook/Dockerfile

@@ -62,10 +62,10 @@ RUN conda install --quiet --yes \
     'r-caret=6.0*' \
     'r-crayon=1.4*' \
     'r-devtools=2.4*' \
-    'r-forecast=8.14*' \ 
+    'r-forecast=8.14*' \
     'r-hexbin=1.28*' \
     'r-htmltools=0.5*' \
-    'r-htmlwidgets=1.5*' \ 
+    'r-htmlwidgets=1.5*' \
     'r-irkernel=1.1*' \
     'r-nycflights13=1.0*' \
     'r-randomforest=4.6*' \

docs/Makefile

@@ -17,4 +17,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/custom.css

@@ -1,3 +1,3 @@
 body div.sphinxsidebarwrapper p.logo {
     text-align: left;
-}
+}

docs/contributing/issues.md

@@ -22,4 +22,4 @@ your problem.
 * If you have a general question about how to use the Jupyter Docker Stacks in
   your environment, in conjunction with other tools, with customizations, and so
   on, please post your question on the [Jupyter Discourse
-  site](https://discourse.jupyter.org).
+  site](https://discourse.jupyter.org).

docs/contributing/lint.md

@@ -111,4 +111,4 @@ RUN cd /tmp && echo "hello!"
 [rules]: https://github.com/hadolint/hadolint#rules
 [DL3006]: https://github.com/hadolint/hadolint/wiki/DL3006
 [DL3008]: https://github.com/hadolint/hadolint/wiki/DL3008
-[pre-commit]: https://pre-commit.com/
+[pre-commit]: https://pre-commit.com/

docs/contributing/translations.md

@@ -4,4 +4,4 @@ We are delighted when members of the Jupyter community want to help translate th
 
 1. Follow the steps documented on the [Getting Started as a Translator](https://docs.transifex.com/getting-started-1/translators) page.
 2. Look for *jupyter-docker-stacks* when prompted to choose a translation team. Alternatively, visit https://www.transifex.com/project-jupyter/jupyter-docker-stacks-1 after creating your account and request to join the project.
-3. See [Translating with the Web Editor](https://docs.transifex.com/translation/translating-with-the-web-editor) in the Transifex documentation.
+3. See [Translating with the Web Editor](https://docs.transifex.com/translation/translating-with-the-web-editor) in the Transifex documentation.

docs/using/common.md

@@ -116,7 +116,7 @@ mamba install some-package
 
 ### Using alternative channels
 
-Conda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel. 
+Conda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel.
 However, alternative channels can be used either one shot by overwriting the default channel in the installation command or by configuring `conda` to use different channels.
 The examples below show how to use the [anaconda default channels](https://repo.anaconda.com/pkgs/main) instead of `conda-forge` to install packages.
 
@@ -127,4 +127,4 @@ conda install --channel defaults humanize
 conda config --system --prepend channels defaults
 # install a package
 conda install humanize
-```
+```

docs/using/recipes.md

@@ -130,7 +130,7 @@ RUN $CONDA_DIR/envs/${conda_env}/bin/python -m ipykernel install --user --name=$
     fix-permissions /home/$NB_USER
 
 # any additional pip installs can be added by uncommenting the following line
-# RUN $CONDA_DIR/envs/${conda_env}/bin/pip install 
+# RUN $CONDA_DIR/envs/${conda_env}/bin/pip install
 
 # prepend conda environment to path
 ENV PATH $CONDA_DIR/envs/${conda_env}/bin:$PATH

docs/using/running.md

@@ -76,7 +76,7 @@ Executing the command: jupyter notebook
 
 Pressing `Ctrl-C` shuts down the notebook server and immediately destroys the Docker container. Files written to `~/work` in the container remain touched. Any other changes made in the container are lost.
 
-**Example 3** This command pulls the `jupyter/all-spark-notebook` image currently tagged `latest` from Docker Hub if an image tagged `latest` is not already present on the local host. It then starts a container named `notebook` running a JupyterLab server and exposes the server on a randomly selected port. 
+**Example 3** This command pulls the `jupyter/all-spark-notebook` image currently tagged `latest` from Docker Hub if an image tagged `latest` is not already present on the local host. It then starts a container named `notebook` running a JupyterLab server and exposes the server on a randomly selected port.
 
 ```
 docker run -d -P --name notebook jupyter/all-spark-notebook

docs/using/specifics.md

@@ -82,7 +82,7 @@ sc <- sparkR.session("local")
 # Sum of the first 100 whole numbers
 sdf <- createDataFrame(list(1:100))
 dapplyCollect(sdf,
-              function(x) 
+              function(x)
               { x <- sum(x)}
              )
 # 5050
@@ -102,7 +102,7 @@ conf$spark.sql.catalogImplementation <- "in-memory"
 sc <- spark_connect(master = "local", config = conf)
 
 # Sum of the first 100 whole numbers
-sdf_len(sc, 100, repartition = 1) %>% 
+sdf_len(sc, 100, repartition = 1) %>%
     spark_apply(function(e) sum(e))
 # 5050
 ```
@@ -171,7 +171,7 @@ sc <- sparkR.session("spark://master:7077")
 # Sum of the first 100 whole numbers
 sdf <- createDataFrame(list(1:100))
 dapplyCollect(sdf,
-              function(x) 
+              function(x)
               { x <- sum(x)}
              )
 # 5050
@@ -190,7 +190,7 @@ conf$spark.sql.catalogImplementation <- "in-memory"
 sc <- spark_connect(master = "spark://master:7077", config = conf)
 
 # Sum of the first 100 whole numbers
-sdf_len(sc, 100, repartition = 1) %>% 
+sdf_len(sc, 100, repartition = 1) %>%
     spark_apply(function(e) sum(e))
 # 5050
 ```
@@ -249,4 +249,4 @@ sess.run(hello)
 
 [sparkr]: https://spark.apache.org/docs/latest/sparkr.html
 [sparklyr]: https://spark.rstudio.com/
-[spark-conf]: https://spark.apache.org/docs/latest/configuration.html
+[spark-conf]: https://spark.apache.org/docs/latest/configuration.html

examples/docker-compose/README.md

@@ -85,7 +85,7 @@ NAME=your-notebook PORT=9001 WORK_VOLUME=our-work notebook/up.sh
 
 ### How do I run over HTTPS?
 
-To run the notebook server with a self-signed certificate, pass the `--secure` option to the `up.sh` script.  You must also provide a password, which will be used to secure the notebook server.  You can specify the password by setting the `PASSWORD` environment variable, or by passing it to the `up.sh` script.  
+To run the notebook server with a self-signed certificate, pass the `--secure` option to the `up.sh` script.  You must also provide a password, which will be used to secure the notebook server.  You can specify the password by setting the `PASSWORD` environment variable, or by passing it to the `up.sh` script.
 
 ```bash
 PASSWORD=a_secret notebook/up.sh --secure

examples/make-deploy/Makefile

@@ -41,4 +41,4 @@ include softlayer.makefile
 
 # Preset notebook configurations
 include self-signed.makefile
-include letsencrypt.makefile
+include letsencrypt.makefile

minimal-notebook/test/data/notebook_math.ipynb

@@ -146,4 +146,4 @@
    "metadata": {}
   }
  ]
-}
+}

pyspark-notebook/hooks/run_hook

@@ -48,4 +48,4 @@ $(docker run --rm ${IMAGE_NAME} conda list)
 \`\`\`
 $(docker run --rm ${IMAGE_NAME} apt list --installed)
 \`\`\`
-EOF
+EOF

pytest.ini

@@ -5,4 +5,4 @@ log_cli_level = INFO
 log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)
 log_cli_date_format=%Y-%m-%d %H:%M:%S
 markers =
-    info: marks tests as info (deselect with '-m "not info"')
+    info: marks tests as info (deselect with '-m "not info"')

scipy-notebook/hooks/run_hook

@@ -42,4 +42,4 @@ $(docker run --rm ${IMAGE_NAME} conda list)
 \`\`\`
 $(docker run --rm ${IMAGE_NAME} apt list --installed)
 \`\`\`
-EOF
+EOF

test/helpers.py

@@ -75,7 +75,7 @@ class CondaPackageHelper:
         if self.specs is None:
             LOGGER.info("Grabing the list of specifications ...")
             self.specs = CondaPackageHelper._packages_from_json(
-                self._execute_command(CondaPackageHelper._conda_export_command(True))
+                self._execute_command(CondaPackageHelper._conda_export_command(from_history=True))
             )
         return self.specs
 
@@ -130,7 +130,7 @@ class CondaPackageHelper:
         return ddict
 
     def check_updatable_packages(self, specifications_only=True):
-        """Check the updatables packages including or not dependencies"""
+        """Check the updatable packages including or not dependencies"""
         specs = self.specified_packages()
         installed = self.installed_packages()
         available = self.available_packages()
@@ -145,9 +145,10 @@ class CondaPackageHelper:
                 current = min(inst_vs, key=CondaPackageHelper.semantic_cmp)
                 newest = avail_vs[-1]
                 if avail_vs and current != newest:
-                    if CondaPackageHelper.semantic_cmp(
+                    if (
-                        current
+                        CondaPackageHelper.semantic_cmp(current) <
-                    ) < CondaPackageHelper.semantic_cmp(newest):
+                        CondaPackageHelper.semantic_cmp(newest)
+                    ):
                         self.comparison.append(
                             {"Package": pkg, "Current": current, "Newest": newest}
                         )
@@ -180,10 +181,7 @@ class CondaPackageHelper:
 
     def get_outdated_summary(self, specifications_only=True):
         """Return a summary of outdated packages"""
-        if specifications_only:
+        nb_packages = len(self.specs if specifications_only else self.installed)
-            nb_packages = len(self.specs)
-        else:
-            nb_packages = len(self.installed)
         nb_updatable = len(self.comparison)
         updatable_ratio = nb_updatable / nb_packages
         return f"{nb_updatable}/{nb_packages} ({updatable_ratio:.0%}) packages could be updated"

test/test_packages.py

@@ -7,12 +7,12 @@ test_packages
 This test module tests if R and Python packages installed can be imported.
 It's a basic test aiming to prove that the package is working properly.
 
-The goal is to detect import errors that can be caused by incompatibilities between packages for example:
+The goal is to detect import errors that can be caused by incompatibilities between packages, for example:
 
 - #1012: issue importing `sympy`
 - #966: isssue importing `pyarrow`
 
-This module checks dynmamically, through the `CondaPackageHelper`, only the specified packages i.e. packages requested by `conda install` in the `Dockerfiles`.
+This module checks dynamically, through the `CondaPackageHelper`, only the specified packages i.e. packages requested by `conda install` in the `Dockerfile`s.
 This means that it does not check dependencies. This choice is a tradeoff to cover the main requirements while achieving reasonable test duration.
 However it could be easily changed (or completed) to cover also dependencies `package_helper.installed_packages()` instead of `package_helper.specified_packages()`.
 
@@ -86,10 +86,7 @@ def packages(package_helper):
 
 def package_map(package):
     """Perform a mapping between the python package name and the name used for the import"""
-    _package = package
+    return PACKAGE_MAPPING.get(package, package)
-    if _package in PACKAGE_MAPPING:
-        _package = PACKAGE_MAPPING.get(_package)
-    return _package
 
 
 def excluded_package_predicate(package):
@@ -136,9 +133,8 @@ def _import_packages(package_helper, filtered_packages, check_function, max_fail
     for package in filtered_packages:
         LOGGER.info(f"Trying to import {package}")
         try:
-            assert (
+            assert check_function(package_helper, package) == 0, \
-                check_function(package_helper, package) == 0
+                f"Package [{package}] import failed"
-            ), f"Package [{package}] import failed"
         except AssertionError as err:
             failures[package] = err
     if len(failures) > max_failures: