diff --git a/.github/workflows/docker-amd64.yml b/.github/workflows/docker-amd64.yml index c95ad8b2..b853e4a4 100644 --- a/.github/workflows/docker-amd64.yml +++ b/.github/workflows/docker-amd64.yml @@ -66,7 +66,7 @@ jobs: - name: Install Dev Dependencies run: | python -m pip install --upgrade pip - make -C main dev-env + make -C main install-dev-env - name: Build Docker Images run: make -C main build-all diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 232625e4..be1d1eb0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -83,7 +83,7 @@ jobs: - name: Install Dev Dependencies run: | python -m pip install --upgrade pip - make -C main dev-env + make -C main install-dev-env - name: Build Docker Images run: make -C main build-all-multi diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 2ceca229..617bafe4 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -7,7 +7,7 @@ on: - "docs/**" - ".readthedocs.yaml" - - "requirements-dev.txt" + - "requirements-docs.txt" push: branches: - main @@ -17,7 +17,7 @@ on: - "docs/**" - ".readthedocs.yaml" - - "requirements-dev.txt" + - "requirements-docs.txt" workflow_dispatch: jobs: @@ -39,7 +39,7 @@ jobs: - name: Install Dev Dependencies run: | python -m pip install --upgrade pip - make dev-env + make install-docs-env - name: Build Documentation run: make docs diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 678760a9..406a33fa 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -27,4 +27,4 @@ sphinx: # Optionally declare the Python requirements required to build your docs python: install: - - requirements: requirements-dev.txt + - requirements: requirements-docs.txt diff --git a/Makefile b/Makefile index d7ad2de7..4406e785 100644 --- a/Makefile +++ b/Makefile @@ -132,7 +132,7 @@ dev/%: PORT?=8888 dev/%: ## run a foreground container for a stack docker run -it --rm -p $(PORT):8888 $(DARGS) $(OWNER)/$(notdir $@) -dev-env: ## install libraries required to build docs and run tests +install-dev-env: ## install libraries required to build images and run tests @pip install -r requirements-dev.txt @@ -140,6 +140,9 @@ dev-env: ## install libraries required to build docs and run tests docs: ## build HTML documentation sphinx-build docs/ docs/_build/ +install-docs-env: ## install libraries required to build docs + @pip install -r requirements-docs.txt + hook/%: WIKI_PATH?=../wiki diff --git a/docs/_static/using/troubleshooting/vscode-jupyter-settings.png b/docs/_static/using/troubleshooting/vscode-jupyter-settings.png new file mode 100644 index 00000000..177c3855 Binary files /dev/null and b/docs/_static/using/troubleshooting/vscode-jupyter-settings.png differ diff --git a/docs/contributing/lint.md b/docs/contributing/lint.md index 1c22417e..c5862bc4 100644 --- a/docs/contributing/lint.md +++ b/docs/contributing/lint.md @@ -13,7 +13,7 @@ This can be achieved by using the generic task used to install all Python develo ```sh # Install all development dependencies for the project -make dev-env +make install-dev-env # It can also be installed directly pip install pre-commit ``` diff --git a/docs/index.rst b/docs/index.rst index f9a912f2..91271b16 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,6 +13,7 @@ Table of Contents using/common using/specifics using/recipes + using/troubleshooting .. toctree:: :maxdepth: 2 diff --git a/docs/maintaining/tasks.md b/docs/maintaining/tasks.md index f490bc2a..eb569b4c 100644 --- a/docs/maintaining/tasks.md +++ b/docs/maintaining/tasks.md @@ -24,7 +24,7 @@ Pushing `Run Workflow` button will trigger this process. ## Adding a New Core Image to Docker Hub ```{note} -In general, we do not add new core images and ask contributors to either create a [recipe](../using/recipes.md) or [community stack](../using/stacks.md). +In general, we do not add new core images and ask contributors to either create a [recipe](../using/recipes.md) or [community stack](../contributing/stacks.md). ``` When there's a new stack definition, do the following before merging the PR with the new stack: diff --git a/docs/using/common.md b/docs/using/common.md index 6bd6fcbd..ed393504 100644 --- a/docs/using/common.md +++ b/docs/using/common.md @@ -36,7 +36,7 @@ You do so by passing arguments to the `docker run` command. The default value is `jovyan`. Setting `NB_USER` refits the `jovyan` default user and ensures that the desired user has the correct file permissions for the new home directory created at `/home/`. - For this option to take effect, you must run the container with `--user root`, set the working directory `-w "/home/${NB_USER}"` + For this option to take effect, you **must** run the container with `--user root`, set the working directory `-w "/home/${NB_USER}"` and set the environment variable `-e CHOWN_HOME=yes`. Example usage: @@ -54,7 +54,7 @@ You do so by passing arguments to the `docker run` command. - `-e NB_UID=` - Instructs the startup script to switch the numeric user ID of `${NB_USER}` to the given value. The default value is `1000`. This feature is useful when mounting host volumes with specific owner permissions. - For this option to take effect, you must run the container with `--user root`. + For this option to take effect, you **must** run the container with `--user root`. (The startup script will `su ${NB_USER}` after adjusting the user ID.) Instead, you might consider using the modern Docker-native options [`--user`](https://docs.docker.com/engine/reference/run/#user) and [`--group-add`](https://docs.docker.com/engine/reference/run/#additional-groups) - see the last bullet in this section for more details. @@ -63,7 +63,7 @@ You do so by passing arguments to the `docker run` command. - `-e NB_GID=` - Instructs the startup script to change the primary group of `${NB_USER}` to `${NB_GID}` (the new group is added with a name of `${NB_GROUP}` if it is defined. Otherwise, the group is named `${NB_USER}`). This feature is useful when mounting host volumes with specific group permissions. - For this option to take effect, you must run the container with `--user root`. + For this option to take effect, you **must** run the container with `--user root`. (The startup script will `su ${NB_USER}` after adjusting the group ID.) You might consider using modern Docker options `--user` and `--group-add` instead. See bullet points regarding `--user` and `--group-add`. @@ -87,7 +87,7 @@ You do so by passing arguments to the `docker run` command. `NB_UMASK` when set only applies to the Jupyter process itself - you cannot use it to set a `umask` for additional files created during run-hooks. For example, via `pip` or `conda`. - If you need to set a `umask` for these, you must set the `umask` value for each command._ + If you need to set a `umask` for these, you **must** set the `umask` value for each command._ ``` - `-e CHOWN_HOME=yes` - Instructs the startup script to change the `${NB_USER}` home directory owner and group to the current value of `${NB_UID}` and `${NB_GID}`. diff --git a/docs/using/troubleshooting.md b/docs/using/troubleshooting.md new file mode 100644 index 00000000..c36a16b2 --- /dev/null +++ b/docs/using/troubleshooting.md @@ -0,0 +1,308 @@ +# Troubleshooting Common Problems + +When troubleshooting, you may see unexpected behaviors or receive an error message. +This section provides advice on +how to identify and fix some of the most commonly encountered issues. + +Most of the `docker run` flags used in this document are explained in detail in the [Common Features, Docker Options section](../using/common.html#Docker-Options) of the documentation. + +## Permission denied when mounting volumes + +If you are running a Docker container while mounting a local volume or host directory using the `-v` flag like so: + +```bash +docker run -it --rm \ + -p 8888:8888 \ + -v : \ + jupyter/minimal-notebook:latest +``` + +you might face permissions issues when trying to access the mounted volume: + +```bash +# assuming we mounted the volume in /home/jovyan/stagingarea +# root is the owner of the mounted volume +ls -ld ~/stagingarea/ +# drwxr-xr-x 2 root root 4096 Feb 1 12:55 stagingarea/ + +touch stagingarea/kale.txt +# touch: cannot touch 'stagingarea/kale.txt': Permission denied +``` + +In this case, the user of the container (`jovyan`) and the owner of the mounted volume (`root`) have different permission levels and ownership over the container's directories and mounts. +The following sections cover a few of these scenarios and how to fix them. + +**Some things to try:** + +1. **Change ownership of the volume mount** + + You can change the ownership of the volume mount using the `chown` command. In the case of the docker-stacks images, you can set the `CHOWN_EXTRA` and `CHOWN_EXTRA_OPTS` environment variables. + + For example, to change the ownership of the volume mount to the jovyan user (non-privileged default user in the Docker images): + + ```bash + # running in detached mode - can also be run in interactive mode + docker run -d \ + -v : \ + -p 8888:8888 \ + --user root \ + -e CHOWN_EXTRA="" \ + -e CHOWN_EXTRA_OPTS="-R" \ + jupyter/minimal-notebook + ``` + + where: + + - `CHOWN_EXTRA=,`: will change the ownership and group of the specified container directory (non-recursive by default). You need to provide full paths starting with `/`. + - `CHOWN_EXTRA_OPTS="-R"`: will recursively change the ownership and group of the directory specified in `CHOWN_EXTRA`. + - `--user root`: you **must** run the container with the root user to change ownership at runtime. + + now accessing the mount should work as expected: + + ```bash + # assuming we mounted the volume in /home/jovyan/stagingarea + ls -ld ~/stagingarea + # drwxr-xr-x 2 jovyan users 4096 Feb 1 12:55 stagingarea/ + + touch stagingarea/kale.txt + # jovyan is now the owner of /home/jovyan/stagingarea + # ls -la ~/stagingarea/ + # -rw-r--r-- 1 jovyan users 0 Feb 1 14:41 kale.txt + ``` + + ```{admonition} Additional notes + - If you are mounting your volume inside the `/home/` directory, you can use the `-e CHOWN_HOME=yes` and `CHOWN_HOME_OPTS="-R"` flags + instead of the `-e CHOWN_EXTRA` and `-e CHOWN_EXTRA_OPTS` in the example above. + - This solution should work in most cases where you have created a docker volume + (i.e. using the [`docker volume create --name `command](https://docs.docker.com/storage/volumes/#create-and-manage-volumes)) and mounted it using the`-v` flag in `docker run`. + ``` + +2. **Matching the container's UID/GID with the host's** + + Docker handles mounting host directories differently from mounting volumes, even though the syntax is essentially the same (i.e. `-v`). + + When you initialize a Docker container using the flag `-v`, the host directories are bind-mounted directly into the container. + Therefore, the permissions and ownership are copied over and will be **the same** as the ones in your local host + (including user ids) which may result in permissions errors when trying to access directories or create/modify files inside. + + Suppose your local user has a `UID` and `GID` of `1234` and `5678`, respectively. To fix the UID discrepancies between your local directories and the container's + directories, you can run the container with an explicit `NB_UID` and `NB_GID` to match the that of the local user: + + ```bash + docker run -it --rm \ + --user root \ + -p 8888:8888 \ + -e NB_UID=1234 \ + -e NB_GID=5678 \ + -v "${PWD}"/test:/home/jovyan/work \ + jupyter/minimal-notebook:latest + + # you should see an output similar to this + # Update jovyan's UID:GID to 1234:5678 + # Running as jovyan: bash + ``` + + where: + + - `NB_IUD` and `NB_GID` should match the local user's UID and GID. + - You **must** use `--user root` to ensure that the `UID` and `GID` are updated at runtime. + +````{admonition} Additional notes +- The caveat with this approach is that since these changes are applied at runtime, you will need to re-run the same command + with the appropriate flags and environment variables if you need to recreate the container (i.e. after removing/destroying it). + - If you pass a numeric UID, it **must** be in the range of 0-2147483647 + - This approach only updates the UID and GID of the **existing `jovyan` user** instead of creating a new user. From the above example: + ```bash + id + # uid=1234(jovyan) gid=5678(jovyan) groups=5678(jovyan),100(users) + ``` +```` + +## Permission issues after changing the UID/GID and USER in the container + +If you have also **created a new user**, you might be experiencing any of the following issues: + +- `root` is the owner of `/home` or a mounted volume +- when starting the container, you get an error such as `Failed to change ownership of the home directory.` +- getting permission denied when trying to `conda install` packages + +**Some things to try:** + +1. **Ensure the new user has ownership of `/home` and volume mounts** + + For example, say you want to create a user `callisto` with a `GID` and `UID` of `1234`, you will have to add the following flags to the docker run command: + + ```bash + docker run -it --rm \ + -p 8888:8888 \ + --user root \ + -e NB_USER=callisto \ + -e NB_UID=1234 \ + -e NB_GID=1234 \ + -e CHOWN_HOME=yes \ + -e CHOWN_HOME_OPTS="-R" \ + -w "/home/${NB_USER}" \ + -v "${PWD}"/test:/home/callisto/work \ + jupyter/minimal-notebook + + # Updated the jovyan user: + # - username: jovyan -> callisto + # - home dir: /home/jovyan -> /home/callisto + # Update callisto UID:GID to 1234:1234 + # Attempting to copy /home/jovyan to /home/callisto... + # Success! + # Ensuring /home/callisto is owned by 1234:1234 + # Running as callisto: bash + ``` + + where: + + - `-e NB_USER=callisto`: will create a new user `callisto` and automatically add it to the `users` group (does not delete jovyan) + - `-e NB_UID=1234` and `-e NB_GID=1234`: will set the `UID` and `GID` of the new user (`callisto`) to `1234` + - `-e CHOWN_HOME_OPTS="-R"` and `-e CHOWN_HOME=yes`: ensure that the new user is the owner of the `/home` directory and subdirectories + (setting `CHOWN_HOME_OPTS="-R` will ensure this change is applied recursively) + - `-w "/home/${NB_USER}"` sets the working directory to be the new user's home + + ```{admonition} Additional notes + In the example above, the `-v` flag is used to mount the local volume onto the new user's `/home` directory. + + However, if you are mounting a volume elsewhere, you also need to use the `-e CHOWN_EXTRA=` flag to avoid any permission + issues (see the section [Permission denied when mounting volumes](../using/troubleshooting.html#permission-denied-when-mounting-volumes) in this page). + ``` + +2. **Dynamically assign the user ID and GID** + + The above case ensures that the `/home` directory is owned by the a newly created user with an specific `UID` and `GID`, but if you want to assign the `UID` and `GID` + of the new user dynamically you can make the following adjustments: + + ```bash + docker run -it --rm \ + -p 8888:8888 \ + --user root \ + -e NB_USER=callisto \ + -e NB_UID="$(id -u)" \ + -e NB_GID="$(id -g)" \ + -e CHOWN_HOME=yes \ + -e CHOWN_HOME_OPTS="-R" \ + -w "/home/${NB_USER}" \ + -v "${PWD}"/test:/home/callisto/work \ + jupyter/minimal-notebook + ``` + + where: + + - `"$(id -u)" and "$(id -g)"` will dynamically assign the `UID` and `GID` of the user executing the `docker run` command to the new user (`callisto`) + +## Additional tips and troubleshooting commands for permission-related errors + +- Pass absolute paths to the `-v` flag: + + ```bash + -v "${PWD}"/:/home/jovyan/work + ``` + + This example uses the syntax `$(PWD)`, which is replaced with the full path to the current directory at runtime. The destination + path should also be an absolute path starting with a `/` such as `home/jovyan/work`. + +- You might want to consider using the Docker native `--user ` and `--group-add users` flags instead of `-e NB_UID` and `-e NB_GID`: + + ```bash + # note this will use the same UID from + # the user calling the command, thus matching the local host + + docker run -it --rm \ + -p 8888:8888 \ + --user "$(id -u)" --group-add users \ + -v :/home/jovyan/work jupyter/datascience-notebook + ``` + + This command will launch the container with a specific user UID and add that user to the `users` group to modify the files in the default `/home` and `/opt/conda` directories. + Further avoiding issues when trying to `conda install` additional packages. + +- Use `docker inspect ` and look for the [`Mounts` section](https://docs.docker.com/storage/volumes/#start-a-container-with-a-volume) to verify that the volume was created and mounted accordingly: + + ```json + "Mounts": [ + { + "Type": "volume", + "Name": "my-vol", + "Source": "/var/lib/docker/volumes/stagingarea/_data", + "Destination": "/home/jovyan/stagingarea", + "Driver": "local", + "Mode": "z", + "RW": true, + "Propagation": "" + } + ], + ``` + +## Problems installing conda packages from specific channels + +By default, the docker-stacks images have the conda channels priority set to `strict`. This may cause problems when trying to install packages from a channel with lower priority. + +```bash +conda config --show | grep priority +# channel_priority: strict + +# to see its meaning +conda config --describe channel_priority + +# checking the current channels +conda config --show default_channels +# default_channels: +# - https://repo.anaconda.com/pkgs/main +# - https://repo.anaconda.com/pkgs/r +``` + +**Installing packages from alternative channels:** + +You can install packages from other conda channels (e.g. bioconda) by disabling the `channel_priority` setting: + +```bash +# install by disabling channel priority at command level +conda install --no-channel-priority -c bioconda bioconductor-geoquery +``` + +## Tokens are being rejected + +If you are a regular user of VSCode and the Jupyter extension, you might experience either of these issues when using any of the docker-stacks images: + +- when clicking on the URL displayed on your command line logs, you face a "This site cannot be reached" page on your web browser +- using the produced token and/or URL results in an "Invalid credentials" error on the Jupyter "Token authentication is enabled" page + + ```bash + # example log output from the docker run command + + # [...] + # Or copy and paste one of these URLs: + # http://3d4cf3809e3f:8888/?token=996426e890f8dc22fa6835a44442b6026cba02ee61fee6a2 + # or http://127.0.0.1:8888/?token=996426e890f8dc22fa6835a44442b6026cba02ee61fee6a2 + ``` + +**Some things to try:** + +1. **Find lingering Jupyter processes in the background** + + The first thing you want to try is to check that no other Jupyter processes are running in the background: + + ```bash + ps aux | grep jupyter + ``` + + If there are existing processes running, you can kill them with: + + ```bash + # example output from the above command + # my-user 3412 ... --daemon-module=vscode_datascience_helpers.jupyter_daemon + + # using the pid from the above log + kill 3412 + ``` + +2. **Turn off Jupyter auto start in VSCode** + + Alternatively - you might want to ensure that the `Jupyter: Auto Start` setting is turned off to avoid this issue in the future. + + You can achieve this from the `Preferences > Jupyter` menu in VScode: + + ![VSCode Preferences UI - Jupyter: Disable Jupyter Auto Start checkbox unchecked](../_static/using/troubleshooting/vscode-jupyter-settings.png) diff --git a/minimal-notebook/Dockerfile b/minimal-notebook/Dockerfile index 4b1a4939..11ec31e4 100644 --- a/minimal-notebook/Dockerfile +++ b/minimal-notebook/Dockerfile @@ -17,15 +17,13 @@ RUN apt-get update --yes && \ tzdata \ unzip \ vim-tiny \ + # Inkscape is installed to be able to convert SVG files + inkscape \ # git-over-ssh openssh-client \ - # TODO: check if these are needed and describe - inkscape \ - libsm6 \ - libxext-dev \ - libxrender1 \ - lmodern \ - netcat \ + # less is needed to run help in R + # see: https://github.com/jupyter/docker-stacks/issues/1588 + less \ # nbconvert dependencies # https://nbconvert.readthedocs.io/en/latest/install.html#installing-tex texlive-xetex \ diff --git a/requirements-dev.txt b/requirements-dev.txt index 73ca5eb8..23cb6bcc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,10 +1,7 @@ docker -myst-parser packaging plumbum pre-commit pytest requests -sphinx -sphinx-copybutton tabulate diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 00000000..ba6908ee --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,3 @@ +myst-parser +sphinx +sphinx-copybutton