Bunch of small refactorings and updates

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.1.0
+    rev: v3.4.0
     hooks:
       - id: check-yaml
         files: .*\.(yaml|yml)$
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.23.0
+    rev: v1.26.1
     hooks:
       - id: yamllint
         args: ['-d {extends: relaxed, rules: {line-length: disable}}', '-s']
@@ -17,10 +17,10 @@ repos:
       - id: bashate
         args: ['--ignore=E006']
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.3
+    rev: 3.9.1
     hooks:
       - id: flake8
   - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.5.4
+    rev: v1.5.6
     hooks:
       - id: autopep8

.readthedocs.yml

@@ -12,6 +12,6 @@ sphinx:
 formats: all
 
 python:
-  version: 3.7
+  version: 3.8
   install:
     - requirements: requirements-dev.txt

Makefile

@@ -90,7 +90,7 @@ git-commit: ## commit outstading git changes and push to remote
 	@git config --global user.name "GitHub Actions"
 	@git config --global user.email "actions@users.noreply.github.com"
 
-	@echo "Publishing outstanding changes in $(LOCAL_PATH) to $(GITHUB_REPOSITORY)" 
+	@echo "Publishing outstanding changes in $(LOCAL_PATH) to $(GITHUB_REPOSITORY)"
 	@cd $(LOCAL_PATH) && \
 		git remote add publisher https://$(GITHUB_TOKEN)@github.com/$(GITHUB_REPOSITORY).git && \
 		git checkout master && \
@@ -152,6 +152,8 @@ pull/%: DARGS?=
 pull/%: ## pull a jupyter image
 	docker pull $(DARGS) $(OWNER)/$(notdir $@)
 
+pull-all: $(foreach I,$(ALL_IMAGES),pull/$(I) ) ## pull all images
+
 push/%: DARGS?=
 push/%: ## push all tags for a jupyter image
 	docker push --all-tags $(DARGS) $(OWNER)/$(notdir $@)

README.md

@@ -31,7 +31,7 @@ in working on the project.
 ## Jupyter Notebook Deprecation Notice
 
 Following [Jupyter Notebook notice](https://github.com/jupyter/notebook#notice), we encourage users to transition to JupyterLab.
-This can be done by passing the environment variable `JUPYTER_ENABLE_LAB=yes` at container startup, 
+This can be done by passing the environment variable `JUPYTER_ENABLE_LAB=yes` at container startup,
 more information is available in the [documentation](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/common.html#docker-options).
 
 At some point, JupyterLab will become the default for all of the Jupyter Docker stack images, however a new environment variable will be introduced to switch back to Jupyter Notebook if needed.

base-notebook/Dockerfile

@@ -79,7 +79,7 @@ RUN chmod a+rx /usr/local/bin/fix-permissions
 # hadolint ignore=SC2016
 RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \
    # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
-   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc 
+   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
 
 # Create NB_USER with name jovyan user with UID=1000 and in the 'users' group
 # and make sure these dirs are writable by the `users` group.

datascience-notebook/Dockerfile

@@ -62,10 +62,10 @@ RUN conda install --quiet --yes \
     'r-caret=6.0*' \
     'r-crayon=1.4*' \
     'r-devtools=2.4*' \
-    'r-forecast=8.14*' \ 
+    'r-forecast=8.14*' \
     'r-hexbin=1.28*' \
     'r-htmltools=0.5*' \
-    'r-htmlwidgets=1.5*' \ 
+    'r-htmlwidgets=1.5*' \
     'r-irkernel=1.1*' \
     'r-nycflights13=1.0*' \
     'r-randomforest=4.6*' \

docs/using/common.md

@@ -116,7 +116,7 @@ mamba install some-package
 
 ### Using alternative channels
 
-Conda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel. 
+Conda is configured by default to use only the [`conda-forge`](https://anaconda.org/conda-forge) channel.
 However, alternative channels can be used either one shot by overwriting the default channel in the installation command or by configuring `conda` to use different channels.
 The examples below show how to use the [anaconda default channels](https://repo.anaconda.com/pkgs/main) instead of `conda-forge` to install packages.
 

docs/using/recipes.md

@@ -130,7 +130,7 @@ RUN $CONDA_DIR/envs/${conda_env}/bin/python -m ipykernel install --user --name=$
     fix-permissions /home/$NB_USER
 
 # any additional pip installs can be added by uncommenting the following line
-# RUN $CONDA_DIR/envs/${conda_env}/bin/pip install 
+# RUN $CONDA_DIR/envs/${conda_env}/bin/pip install
 
 # prepend conda environment to path
 ENV PATH $CONDA_DIR/envs/${conda_env}/bin:$PATH

docs/using/running.md

@@ -76,7 +76,7 @@ Executing the command: jupyter notebook
 
 Pressing `Ctrl-C` shuts down the notebook server and immediately destroys the Docker container. Files written to `~/work` in the container remain touched. Any other changes made in the container are lost.
 
-**Example 3** This command pulls the `jupyter/all-spark-notebook` image currently tagged `latest` from Docker Hub if an image tagged `latest` is not already present on the local host. It then starts a container named `notebook` running a JupyterLab server and exposes the server on a randomly selected port. 
+**Example 3** This command pulls the `jupyter/all-spark-notebook` image currently tagged `latest` from Docker Hub if an image tagged `latest` is not already present on the local host. It then starts a container named `notebook` running a JupyterLab server and exposes the server on a randomly selected port.
 
 ```
 docker run -d -P --name notebook jupyter/all-spark-notebook

docs/using/specifics.md

@@ -82,7 +82,7 @@ sc <- sparkR.session("local")
 # Sum of the first 100 whole numbers
 sdf <- createDataFrame(list(1:100))
 dapplyCollect(sdf,
-              function(x) 
+              function(x)
               { x <- sum(x)}
              )
 # 5050
@@ -102,7 +102,7 @@ conf$spark.sql.catalogImplementation <- "in-memory"
 sc <- spark_connect(master = "local", config = conf)
 
 # Sum of the first 100 whole numbers
-sdf_len(sc, 100, repartition = 1) %>% 
+sdf_len(sc, 100, repartition = 1) %>%
     spark_apply(function(e) sum(e))
 # 5050
 ```
@@ -171,7 +171,7 @@ sc <- sparkR.session("spark://master:7077")
 # Sum of the first 100 whole numbers
 sdf <- createDataFrame(list(1:100))
 dapplyCollect(sdf,
-              function(x) 
+              function(x)
               { x <- sum(x)}
              )
 # 5050
@@ -190,7 +190,7 @@ conf$spark.sql.catalogImplementation <- "in-memory"
 sc <- spark_connect(master = "spark://master:7077", config = conf)
 
 # Sum of the first 100 whole numbers
-sdf_len(sc, 100, repartition = 1) %>% 
+sdf_len(sc, 100, repartition = 1) %>%
     spark_apply(function(e) sum(e))
 # 5050
 ```

examples/docker-compose/README.md

@@ -85,7 +85,7 @@ NAME=your-notebook PORT=9001 WORK_VOLUME=our-work notebook/up.sh
 
 ### How do I run over HTTPS?
 
-To run the notebook server with a self-signed certificate, pass the `--secure` option to the `up.sh` script.  You must also provide a password, which will be used to secure the notebook server.  You can specify the password by setting the `PASSWORD` environment variable, or by passing it to the `up.sh` script.  
+To run the notebook server with a self-signed certificate, pass the `--secure` option to the `up.sh` script.  You must also provide a password, which will be used to secure the notebook server.  You can specify the password by setting the `PASSWORD` environment variable, or by passing it to the `up.sh` script.
 
 ```bash
 PASSWORD=a_secret notebook/up.sh --secure

test/helpers.py

@@ -75,7 +75,7 @@ class CondaPackageHelper:
         if self.specs is None:
             LOGGER.info("Grabing the list of specifications ...")
             self.specs = CondaPackageHelper._packages_from_json(
-                self._execute_command(CondaPackageHelper._conda_export_command(True))
+                self._execute_command(CondaPackageHelper._conda_export_command(from_history=True))
             )
         return self.specs
 
@@ -130,7 +130,7 @@ class CondaPackageHelper:
         return ddict
 
     def check_updatable_packages(self, specifications_only=True):
-        """Check the updatables packages including or not dependencies"""
+        """Check the updatable packages including or not dependencies"""
         specs = self.specified_packages()
         installed = self.installed_packages()
         available = self.available_packages()
@@ -145,9 +145,10 @@ class CondaPackageHelper:
                 current = min(inst_vs, key=CondaPackageHelper.semantic_cmp)
                 newest = avail_vs[-1]
                 if avail_vs and current != newest:
-                    if CondaPackageHelper.semantic_cmp(
+                    if (
-                        current
+                        CondaPackageHelper.semantic_cmp(current) <
-                    ) < CondaPackageHelper.semantic_cmp(newest):
+                        CondaPackageHelper.semantic_cmp(newest)
+                    ):
                         self.comparison.append(
                             {"Package": pkg, "Current": current, "Newest": newest}
                         )
@@ -180,10 +181,7 @@ class CondaPackageHelper:
 
     def get_outdated_summary(self, specifications_only=True):
         """Return a summary of outdated packages"""
-        if specifications_only:
+        nb_packages = len(self.specs if specifications_only else self.installed)
-            nb_packages = len(self.specs)
-        else:
-            nb_packages = len(self.installed)
         nb_updatable = len(self.comparison)
         updatable_ratio = nb_updatable / nb_packages
         return f"{nb_updatable}/{nb_packages} ({updatable_ratio:.0%}) packages could be updated"

test/test_packages.py

@@ -7,12 +7,12 @@ test_packages
 This test module tests if R and Python packages installed can be imported.
 It's a basic test aiming to prove that the package is working properly.
 
-The goal is to detect import errors that can be caused by incompatibilities between packages for example:
+The goal is to detect import errors that can be caused by incompatibilities between packages, for example:
 
 - #1012: issue importing `sympy`
 - #966: isssue importing `pyarrow`
 
-This module checks dynmamically, through the `CondaPackageHelper`, only the specified packages i.e. packages requested by `conda install` in the `Dockerfiles`.
+This module checks dynamically, through the `CondaPackageHelper`, only the specified packages i.e. packages requested by `conda install` in the `Dockerfile`s.
 This means that it does not check dependencies. This choice is a tradeoff to cover the main requirements while achieving reasonable test duration.
 However it could be easily changed (or completed) to cover also dependencies `package_helper.installed_packages()` instead of `package_helper.specified_packages()`.
 
@@ -86,10 +86,7 @@ def packages(package_helper):
 
 def package_map(package):
     """Perform a mapping between the python package name and the name used for the import"""
-    _package = package
+    return PACKAGE_MAPPING.get(package, package)
-    if _package in PACKAGE_MAPPING:
-        _package = PACKAGE_MAPPING.get(_package)
-    return _package
 
 
 def excluded_package_predicate(package):
@@ -136,9 +133,8 @@ def _import_packages(package_helper, filtered_packages, check_function, max_fail
     for package in filtered_packages:
         LOGGER.info(f"Trying to import {package}")
         try:
-            assert (
+            assert check_function(package_helper, package) == 0, \
-                check_function(package_helper, package) == 0
+                f"Package [{package}] import failed"
-            ), f"Package [{package}] import failed"
         except AssertionError as err:
             failures[package] = err
     if len(failures) > max_failures: