start.sh: tests, docs, and refinements

base-notebook/start.sh

@@ -12,9 +12,8 @@ else
     cmd=( "$@" )
 fi
 
-# The run-hooks function looks for .sh scripts and executable files
+# The run-hooks function looks for .sh scripts to source and executable files to
-# - .sh scripts will be sourced
+# run within a passed directory.
-# - executable files (+x) will be executed
 run-hooks () {
     if [[ ! -d "${1}" ]] ; then
         return
@@ -41,99 +40,101 @@ run-hooks () {
 }
 
 
-
 # NOTE: This hook will run as the user the container was started with!
 run-hooks /usr/local/bin/start-notebook.d
 
-
+# If the container started as the root user, then we have permission to refit
-
+# the jovyan user, and ensure file permissions, grant sudo rights, and such
-# If we have started the container as the root user, then we have permission to
+# things before we run the command passed to start.sh as the desired user
-# manipulate user identities and storage file permissions before we start the
+# (NB_USER).
-# server as a non-root user.
 #
-# Environment variables of relevance:
-# - NB_UID: the user we want to run as, uniquely identified by an id
-# - NB_USER: the username and associated home folder we want for our user
-# - NB_GID: a group we want our user to belong to, uniquely identified by an id
-# - NB_GROUP: the groupname we want for the group
 if [ "$(id -u)" == 0 ] ; then
-
-    # Optionally ensure the user we want to run as (NB_UID) get filesystem
-    # ownership of it's home folder and additional folders. This can be relevant
-    # for attached NFS storage.
-    #
     # Environment variables:
+    # - NB_USER: the desired username and associated home folder
+    # - NB_UID: the desired user id
+    # - NB_GID: a group id we want our user to belong to
+    # - NB_GROUP: the groupname we want for the group
+    # - GRANT_SUDO: a boolean ("1" or "yes") to grant the user sudo rights
     # - CHOWN_HOME: a boolean ("1" or "yes") to chown the user's home folder
     # - CHOWN_EXTRA: a comma separated list of paths to chown
-    # - CHOWN_HOME_OPTS / CHOWN_EXTRA_OPTS: arguments to the chown command
+    # - CHOWN_HOME_OPTS / CHOWN_EXTRA_OPTS: arguments to the chown commands
-    if [[ "${CHOWN_HOME}" == "1" || "${CHOWN_HOME}" == "yes" ]]; then
-        echo "Updating ownership of /home/${NB_USER} to ${NB_UID}:${NB_GID} with options '${CHOWN_HOME_OPTS}'"
-        # shellcheck disable=SC2086
-        chown $CHOWN_HOME_OPTS "${NB_UID}:${NB_GID}" "/home/${NB_USER}"
-    fi
-    if [ -n "${CHOWN_EXTRA}" ]; then
-        for extra_dir in $(echo "${CHOWN_EXTRA}" | tr ',' ' '); do
-            echo "Updating ownership of ${extra_dir} to ${NB_UID}:${NB_GID} with options '${CHOWN_EXTRA_OPTS}'"
-            # shellcheck disable=SC2086
-            chown ${CHOWN_EXTRA_OPTS} "${NB_UID}:${NB_GID}" "${extra_dir}"
-        done
-    fi
 
-    # Update the jovyan identity to get desired username and its associated home folder.
+    # Refit the jovyan user to the desired the user (NB_USER)
     if id jovyan &> /dev/null ; then
         if ! usermod --home "/home/${NB_USER}" --login "${NB_USER}" jovyan 2>&1 | grep "no changes" > /dev/null; then
             echo "Updated the jovyan user:"
-            echo "username: jovyan -> ${NB_USER}"
+            echo "- username: jovyan       -> ${NB_USER}"
-            echo "home dir location: /home/jovyan -> /home/${NB_USER}"
+            echo "- home dir: /home/jovyan -> /home/${NB_USER}"
         fi
+    elif ! id -u "${NB_USER}" &> /dev/null; then
+        echo "ERROR: Neither the jovyan user or '$NB_USER' exists."
+        echo "       This could be the result of stopping and starting, the"
+        echo "       container with a different NB_USER environment variable."
+        exit 1
+    fi
+    # Ensure the desired user (NB_USER) gets its desired user id (NB_UID) and is
+    # a member of the desired group (NB_GROUP, NB_GID)
+    if [ "${NB_UID}" != "$(id -u "${NB_USER}")" ] || [ "${NB_GID}" != "$(id -g "${NB_USER}")" ]; then
+        echo "Update ${NB_USER}'s UID:GID to ${NB_UID}:${NB_GID}"
+        # Ensure the desired group's existence
+        if [ "${NB_GID}" != "$(id -g "${NB_USER}")" ]; then
+            groupadd --force --gid "${NB_GID}" --non-unique "${NB_GROUP:-${NB_USER}}"
+        fi
+        # Recreate the desired user as we want it
+        userdel "${NB_USER}"
+        useradd --home "/home/${NB_USER}" --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 --no-log-init "${NB_USER}"
     fi
-    # Update env vars set during image build with the current NB_USER value
-    export XDG_CACHE_HOME=/home/$NB_USER/.cache
 
-    # For non-jovyan username's, populate their home directory with the jovyan's
+    # Move or symlink the jovyan home directory to the desired users home
-    # home directory as a fallback if they don't have one mounted already.
+    # directory if it doesn't already exist, and update the current working
+    # directory to the new location if needed.
     if [[ "${NB_USER}" != "jovyan" ]]; then
         if [[ ! -e "/home/${NB_USER}" ]]; then
             echo "Attempting to copy /home/jovyan to /home/${NB_USER}..."
             mkdir "/home/${NB_USER}"
             if cp -a /home/jovyan/. "/home/${NB_USER}/"; then
-                echo "Done"
+                echo "Success!"
             else
-                echo "Failed. Attempting to symlink /home/jovyan to /home/${NB_USER}..."
+                echo "Failed!"
-                ln -s /home/jovyan "/home/${NB_USER}" && echo "Done"
+                echo "Attempting to symlink /home/jovyan to /home/${NB_USER}..."
+                if ln -s /home/jovyan "/home/${NB_USER}"; then
+                    echo "Success!"
+                else
+                    echo "Failed!"
+                fi
             fi
         fi
-        # Ensure the current working directory is updated
+        # Ensure the current working directory is updated to the new path
         if [[ "${PWD}/" == "/home/jovyan/"* ]]; then
-            newcwd="/home/${NB_USER}/${PWD:13}"
+            new_wd="/home/${NB_USER}/${PWD:13}"
-            echo "Changing working directory to ${newcwd}"
+            echo "Changing working directory to ${new_wd}"
-            cd "${newcwd}"
+            cd "${new_wd}"
         fi
     fi
 
-    # Ensure NB_USER gets the NB_UID user id and is a member of the NB_GID group
+    # Optionally ensure the desired user get filesystem ownership of it's home
-    if [ "${NB_UID}" != "$(id -u "${NB_USER}")" ] || [ "${NB_GID}" != "$(id -g "${NB_USER}")" ]; then
+    # folder and/or additional folders
-        echo "Update ${NB_USER}'s UID:GID to ${NB_UID}:${NB_GID}"
+    if [[ "${CHOWN_HOME}" == "1" || "${CHOWN_HOME}" == "yes" ]]; then
-        # Ensure the group's existence
+        echo "Ensuring /home/${NB_USER} is owned by ${NB_UID}:${NB_GID} ${CHOWN_HOME_OPTS:+chown options: ${CHOWN_HOME_OPTS}}"
-        if [ "${NB_GID}" != "$(id -g "${NB_USER}")" ]; then
+        # shellcheck disable=SC2086
-            groupadd --force --gid "$NB_GID" --non-unique "${NB_GROUP:-${NB_USER}}"
+        chown ${CHOWN_HOME_OPTS} "${NB_UID}:${NB_GID}" "/home/${NB_USER}"
-        fi
+    fi
-        # Recreate the user as we want it
+    if [ -n "${CHOWN_EXTRA}" ]; then
-        userdel "${NB_USER}"
+        for extra_dir in $(echo "${CHOWN_EXTRA}" | tr ',' ' '); do
-        useradd --home "/home/${NB_USER}" --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 --no-log-init "${NB_USER}"
+            echo "Ensuring ${extra_dir} is owned by ${NB_UID}:${NB_GID} ${CHOWN_HOME_OPTS:+(chown options: ${CHOWN_HOME_OPTS})}"
+            # shellcheck disable=SC2086
+            chown ${CHOWN_EXTRA_OPTS} "${NB_UID}:${NB_GID}" "${extra_dir}"
+        done
     fi
 
-    # Conditionally enable passwordless sudo usage for the jovyan user
+    # Update potentially outdated environment variables since image build
-    if [[ "${GRANT_SUDO}" == "1" || "${GRANT_SUDO}" == 'yes' ]]; then
+    export XDG_CACHE_HOME=/home/$NB_USER/.cache
-        echo "Granting ${NB_USER} passwordless sudo rights!"
-        echo "${NB_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/added-by-start-script
-    fi
 
-    # Ensure that the initial environment that this container is started with
+    # Notes on how we ensure that the environment that this container is started
-    # is preserved when we run transition from running as root to running as
+    # with is preserved (except vars listen in JUPYTER_ENV_VARS_TO_UNSET) when
-    # NB_USER.
+    # we transition from running as root to running as NB_USER.
     #
-    # - We use the sudo command to execute the command as NB_USER. But, what
+    # - We use `sudo` to execute the command as NB_USER. What then
     #   happens to the environment will be determined by configuration in
     #   /etc/sudoers and /etc/sudoers.d/* as well as flags we pass to the sudo
     #   command. The behavior can be inspected with `sudo -V` run as root.
@@ -141,19 +142,26 @@ if [ "$(id -u)" == 0 ] ; then
     #   ref: `man sudo`    https://linux.die.net/man/8/sudo
     #   ref: `man sudoers` https://www.sudo.ws/man/1.8.15/sudoers.man.html
     #
-    # - We use the `--preserve-env` flag to pass through most environment, but
+    # - We use the `--preserve-env` flag to pass through most environment
-    #   understand that exceptions are caused by the sudoers configuration:
+    #   variables, but understand that exceptions are caused by the sudoers
-    #   `env_delete`, `env_check`, and `secure_path`.
+    #   configuration: `env_delete`, `env_check`, and `secure_path`.
     #
-    # - We reduce the `env_delete` list of default variables to be deleted by
+    # - We use the `--set-home` flag to set the HOME variable appropriatly.
-    #   default which would ignore the `--preserve-env` flag and `env_keep`
+    #
+    # - We reduce the `env_delete` list of default variables to be deleted. It
+    #   has higher priority than the `--preserve-env` flag and `env_keep`
     #   configuration.
     #
-    # - We manage the PATH variable specifically as `secure_path` is set by
+    # - We disable the `secure_path` which is set by default in /etc/sudoers as
-    #   default in /etc/sudoers and would override the PATH variable. So we
+    #   it would override the PATH variable.
-    #   disable that default.
+    echo 'Defaults !secure_path' > /etc/sudoers.d/added-by-start-script
     echo 'Defaults env_delete -= "PATH LD_* PYTHON*"' >> /etc/sudoers.d/added-by-start-script
-    echo 'Defaults !secure_path' >> /etc/sudoers.d/added-by-start-script
+
+    # Optionally grant passwordless sudo rights for the desired user
+    if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then
+        echo "Granting $NB_USER passwordless sudo rights!"
+        echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/added-by-start-script
+    fi
 
     # NOTE: This hook is run as the root user!
     run-hooks /usr/local/bin/before-notebook.d
@@ -161,48 +169,49 @@ if [ "$(id -u)" == 0 ] ; then
     echo "Running as ${NB_USER}:" "${cmd[@]}"
     exec sudo --preserve-env --set-home --user "${NB_USER}" "${cmd[@]}"
 
-
+# The container didn't start as the root user, so we will have to act as the
-
+# user we started as.
-# The container didn't start as the root user.
 else
-    if [[ "${NB_UID}" == "$(id -u jovyan 2>/dev/null)" && "${NB_GID}" == "$(id -g jovyan 2>/dev/null)" ]]; then
+    # Warn about misconfiguration of: desired username, user id, or group id
-        # User is not attempting to override user/group via environment
+    if [[ -n "${NB_USER}" && "${NB_USER}" != "$(id -un)" ]]; then
-        # variables, but they could still have overridden the uid/gid that
+        echo "WARNING: container must be started as root to change the desired user's name with NB_USER!"
-        # container runs as. Check that the user has an entry in the passwd
+    fi
-        # file and if not add an entry.
+    if [[ -n "${NB_UID}" && "${NB_UID}" != "$(id -u)" ]]; then
-        STATUS=0 && whoami &> /dev/null || STATUS=$? && true
+        echo "WARNING: container must be started as root to change the desired user's id with NB_UID!"
-        if [[ "${STATUS}" != "0" ]]; then
+    fi
-            if [[ -w /etc/passwd ]]; then
+    if [[ -n "${NB_GID}" && "${NB_GID}" != "$(id -g)" ]]; then
-                echo "Adding passwd file entry for $(id -u)"
+        echo "WARNING: container must be started as root to change the desired user's group id with NB_GID!"
-                sed -e "s/^jovyan:/nayvoj:/" /etc/passwd > /tmp/passwd
-                echo "jovyan:x:$(id -u):$(id -g):,,,:/home/jovyan:/bin/bash" >> /tmp/passwd
-                cat /tmp/passwd > /etc/passwd
-                rm /tmp/passwd
-            else
-                echo "WARNING: Container must be run with group 'root' to update passwd file"
-            fi
-        fi
-
-        # Warn if the user isn't going to be able to write files to ${HOME}.
-        if [[ ! -w /home/jovyan ]]; then
-            echo "WARNING: Container must be run with group 'users' to update files"
-        fi
-    else
-        # Warn if looks like user want to override uid/gid but hasn't
-        # run the container as root.
-        if [[ -n "${NB_UID}" && "${NB_UID}" != "$(id -u)" ]]; then
-            echo "Container must be run as root to set NB_UID to ${NB_UID}"
-        fi
-        if [[ -n "${NB_GID}" && "${NB_GID}" != "$(id -g)" ]]; then
-            echo "Container must be run as root to set NB_GID to ${NB_GID}"
-        fi
     fi
 
-    # Warn about a probable misconfiguration of sudo
+    # Warn about misconfiguration of: granting sudo rights
     if [[ "${GRANT_SUDO}" == "1" || "${GRANT_SUDO}" == 'yes' ]]; then
-        echo "WARNING: container must be started up as root to grant sudo permissions."
+        echo "WARNING: container must be started as root to grant sudo permissions!"
     fi
 
+    # Attempt to ensure the user uid we currently run as has a named entry in
+    # the /etc/passwd file, as it avoids software crashing on hard assumptions
+    # on such entry. Writing to the /etc/passwd was allowed for the root group
+    # from the Dockerfile during build.
+    #
+    # ref: https://github.com/jupyter/docker-stacks/issues/552
+    if ! whoami &> /dev/null; then
+        if [[ -w /etc/passwd ]]; then
+            sed --in-place "s/^jovyan:/nayvoj:/" /etc/passwd
+            echo "Renamed old jovyan user to nayvoj (1000:100)"
+
+            echo "jovyan:x:$(id -u):$(id -g):,,,:/home/jovyan:/bin/bash" >> /etc/passwd
+            echo "Added new jovyan user ($(id -u):$(id -g))"
+        else
+            echo "WARNING: container must be started with group 'root' (0) to add a user entry in /etc/passwd!"
+        fi
+    fi
+
+    # Warn if the user isn't able to write files to ${HOME}
+    if [[ ! -w /home/jovyan ]]; then
+        echo "WARNING: container must be started with group 'users' (100) to be allowed to write to /home/jovyan!"
+    fi
+
+    # NOTE: This hook is run as the user we started the container as!
     run-hooks /usr/local/bin/before-notebook.d
     echo "Executing the command:" "${cmd[@]}"
     exec "${cmd[@]}"

base-notebook/test/test_container_options.py

@@ -44,7 +44,8 @@ def test_uid_change(container):
         command=["start.sh", "bash", "-c", "id && touch /opt/conda/test-file"],
     )
     # usermod is slow so give it some time
-    c.wait(timeout=120)
+    rv = c.wait(timeout=120)
+    assert rv == 0 or rv["StatusCode"] == 0
     assert "uid=1010(jovyan)" in c.logs(stdout=True).decode("utf-8")
 
 
@@ -56,7 +57,8 @@ def test_gid_change(container):
         environment=["NB_GID=110"],
         command=["start.sh", "id"],
     )
-    c.wait(timeout=10)
+    rv = c.wait(timeout=10)
+    assert rv == 0 or rv["StatusCode"] == 0
     logs = c.logs(stdout=True).decode("utf-8")
     assert "gid=110(jovyan)" in logs
     assert "groups=110(jovyan),100(users)" in logs
@@ -77,7 +79,9 @@ def test_nb_user_change(container):
     time.sleep(10)
     LOGGER.info(f"Checking if the user is changed to {nb_user} by the start script ...")
     output = running_container.logs(stdout=True).decode("utf-8")
-    assert f"Set username to: {nb_user}" in output, f"User is not changed to {nb_user}"
+    assert (
+        f"username: jovyan       -> {nb_user}" in output
+    ), f"User is not changed to {nb_user}"
 
     LOGGER.info(f"Checking {nb_user} id ...")
     command = "id"
@@ -108,21 +112,30 @@ def test_nb_user_change(container):
 
 
 def test_chown_extra(container):
-    """Container should change the UID/GID of CHOWN_EXTRA."""
+    """Container should change the UID/GID of a comma separated
+    CHOWN_EXTRA list of folders."""
     c = container.run(
         tty=True,
         user="root",
         environment=[
             "NB_UID=1010",
             "NB_GID=101",
-            "CHOWN_EXTRA=/opt/conda",
+            "CHOWN_EXTRA=/home/jovyan,/opt/conda/bin",
             "CHOWN_EXTRA_OPTS=-R",
         ],
-        command=["start.sh", "bash", "-c", "stat -c '%n:%u:%g' /opt/conda/LICENSE.txt"],
+        command=[
+            "start.sh",
+            "bash",
+            "-c",
+            "stat -c '%n:%u:%g' /home/jovyan/.bashrc /opt/conda/bin/jupyter",
+        ],
     )
     # chown is slow so give it some time
-    c.wait(timeout=120)
+    rv = c.wait(timeout=120)
-    assert "/opt/conda/LICENSE.txt:1010:101" in c.logs(stdout=True).decode("utf-8")
+    assert rv == 0 or rv["StatusCode"] == 0
+    logs = c.logs(stdout=True).decode("utf-8")
+    assert "/home/jovyan/.bashrc:1010:101" in logs
+    assert "/opt/conda/bin/jupyter:1010:101" in logs
 
 
 def test_chown_home(container):
@@ -131,18 +144,19 @@ def test_chown_home(container):
     c = container.run(
         tty=True,
         user="root",
-        environment=["CHOWN_HOME=yes", "CHOWN_HOME_OPTS=-R"],
+        environment=[
-        command=[
+            "CHOWN_HOME=yes",
-            "start.sh",
+            "CHOWN_HOME_OPTS=-R",
-            "bash",
+            "NB_USER=kitten",
-            "-c",
+            "NB_UID=1010",
-            "chown root:root /home/jovyan && ls -alsh /home",
+            "NB_GID=101",
         ],
+        command=["start.sh", "bash", "-c", "stat -c '%n:%u:%g' /home/kitten/.bashrc"],
     )
-    c.wait(timeout=120)
+    rv = c.wait(timeout=120)
-    assert "Changing ownership of /home/jovyan to 1000:100 with options '-R'" in c.logs(
+    assert rv == 0 or rv["StatusCode"] == 0
-        stdout=True
+    logs = c.logs(stdout=True).decode("utf-8")
-    ).decode("utf-8")
+    assert "/home/kitten/.bashrc:1010:101" in logs
 
 
 def test_sudo(container):
@@ -224,3 +238,29 @@ def test_container_not_delete_bind_mount(container, tmp_path):
     assert rv == 0 or rv["StatusCode"] == 0
     assert p.read_text() == "some-content"
     assert len(list(tmp_path.iterdir())) == 1
+
+
+@pytest.mark.skip(reason="not yet implemented; TODO: cherry-pick b44b7ab")
+def test_jupyter_env_vars_to_unset_as_root(container):
+    """Environment variables names listed in JUPYTER_ENV_VARS_TO_UNSET
+    should be unset in the final environment."""
+    c = container.run(
+        tty=True,
+        user="root",
+        environment=[
+            "JUPYTER_ENV_VARS_TO_UNSET=SECRET_ANIMAL,UNUSED_ENV,SECRET_FRUIT",
+            "FRUIT=bananas",
+            "SECRET_FRUIT=mango",
+            "SECRET_ANIMAL=cats",
+        ],
+        command=[
+            "start.sh",
+            "bash",
+            "-c",
+            "echo I like $FRUIT and ${SECRET_FRUIT:-stuff}, and love ${SECRET_LOVE:-to keep secrets}!",
+        ],
+    )
+    rv = c.wait(timeout=10)
+    assert rv == 0 or rv["StatusCode"] == 0
+    logs = c.logs(stdout=True).decode("utf-8")
+    assert "I like bananas and stuff, and love to keep secrets!" in logs