Merge pull request #3270 from meeseeksmachine/auto-backport-of-pr-3246-on-1.2.x

Backport PR #3246 on branch 1.2.x (Migrate from travis to GitHub actions)

.github/workflows/test.yml

@@ -0,0 +1,208 @@
+# This is a GitHub workflow defining a set of jobs with a set of steps.
+# ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions
+#
+name: Run tests
+
+# Trigger the workflow's on all PRs but only on pushed tags or commits to
+# main/master branch to avoid PRs developed in a GitHub fork's dedicated branch
+# to trigger.
+on:
+  pull_request:
+  push:
+    branches: [main, master, pr/migrate-to-gh-actions-from-travis]
+    tags:
+
+defaults:
+  run:
+    # Declare bash be used by default in this workflow's "run" steps.
+    #
+    # NOTE: bash will by default run with:
+    #   --noprofile: Ignore ~/.profile etc.
+    #   --norc:      Ignore ~/.bashrc etc.
+    #   -e:          Exit directly on errors
+    #   -o pipefail: Don't mask errors from a command piped into another command
+    shell: bash
+
+env:
+  # UTF-8 content may be interpreted as ascii and causes errors without this.
+  LANG: C.UTF-8
+
+jobs:
+  # Run "pre-commit run --all-files"
+  pre-commit:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 2
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      # ref: https://github.com/pre-commit/action
+      - uses: pre-commit/action@v2.0.0
+      - name: Help message if pre-commit fail
+        if: ${{ failure() }}
+        run: |
+          echo "You can install pre-commit hooks to automatically run formatting"
+          echo "on each commit with:"
+          echo "    pre-commit install"
+          echo "or you can run by hand on staged files with"
+          echo "    pre-commit run"
+          echo "or after-the-fact on already committed files with"
+          echo "    pre-commit run --all-files"
+
+
+  # Run "pytest jupyterhub/tests" in various configurations
+  pytest:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 10
+
+    strategy:
+      # Keep running even if one variation of the job fail
+      fail-fast: false
+      matrix:
+        # We run this job multiple times with different parameterization
+        # specified below, these parameters have no meaning on their own and
+        # gain meaning on how job steps use them.
+        #
+        # subdomain:
+        #   Tests everything when JupyterHub is configured to add routes for
+        #   users with dedicated subdomains like user1.jupyter.example.com
+        #   rather than jupyter.example.com/user/user1.
+        #
+        # db: [mysql/postgres]
+        #   Tests everything when JupyterHub works against a dedicated mysql or
+        #   postgresql server.
+        #
+        # jupyter_server:
+        #   Tests everything when the user instances are started with
+        #   jupyter_server instead of notebook.
+        #
+        # main_dependencies:
+        #   Tests everything when the we use the latest available dependencies
+        #   from: ipytraitlets.
+        #
+        # NOTE: Since only the value of these parameters are presented in the
+        #       GitHub UI when the workflow run, we avoid using true/false as
+        #       values by instead duplicating the name to signal true.
+        include:
+          - python: "3.6"
+            subdomain: subdomain
+          - python: "3.7"
+            db: mysql
+          - python: "3.8"
+            db: postgres
+          - python: "3.8"
+            jupyter_server: jupyter_server
+          - python: "3.9"
+            main_dependencies: main_dependencies
+
+    steps:
+      # NOTE: In GitHub workflows, environment variables are set by writing
+      #       assignment statements to a file. They will be set in the following
+      #       steps as if would used `export MY_ENV=my-value`.
+      - name: Configure environment variables
+        run: |
+          if [ "${{ matrix.subdomain }}" != "" ]; then
+              echo "JUPYTERHUB_TEST_SUBDOMAIN_HOST=http://localhost.jovyan.org:8000" >> $GITHUB_ENV
+          fi
+          if [ "${{ matrix.db }}" == "mysql" ]; then
+              echo "MYSQL_HOST=127.0.0.1" >> $GITHUB_ENV
+              echo "JUPYTERHUB_TEST_DB_URL=mysql+mysqlconnector://root@127.0.0.1:3306/jupyterhub" >> $GITHUB_ENV
+          fi
+          if [ "${{ matrix.db }}" == "postgres" ]; then
+              echo "PGHOST=127.0.0.1" >> $GITHUB_ENV
+              echo "PGUSER=test_user" >> $GITHUB_ENV
+              echo "PGPASSWORD=hub[test/:?" >> $GITHUB_ENV
+              echo "JUPYTERHUB_TEST_DB_URL=postgresql://test_user:hub%5Btest%2F%3A%3F@127.0.0.1:5432/jupyterhub" >> $GITHUB_ENV
+          fi
+          if [ "${{ matrix.jupyter_server }}" != "" ]; then
+              echo "JUPYTERHUB_SINGLEUSER_APP=jupyterhub.tests.mockserverapp.MockServerApp" >> $GITHUB_ENV
+          fi
+      - uses: actions/checkout@v2
+
+      # NOTE: actions/setup-node@v1 make use of a cache within the GitHub base
+      #       environment and setup in a fraction of a second.
+      - name: Install Node v14
+        uses: actions/setup-node@v1
+        with:
+          node-version: "14"
+      - name: Install Node dependencies
+        run: |
+          npm install
+          npm install -g configurable-http-proxy
+          npm list
+
+      # NOTE: actions/setup-python@v2 make use of a cache within the GitHub base
+      #       environment and setup in a fraction of a second.
+      - name: Install Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install --upgrade . -r dev-requirements.txt
+
+          if [ "${{ matrix.main_dependencies }}" != "" ]; then
+              pip install git+https://github.com/ipython/traitlets#egg=traitlets --force
+          fi
+          if [ "${{ matrix.jupyter_server }}" != "" ]; then
+              pip uninstall notebook --yes
+              pip install jupyter_server
+          fi
+          if [ "${{ matrix.db }}" == "mysql" ]; then
+              pip install mysql-connector-python
+          fi
+          if [ "${{ matrix.db }}" == "postgres" ]; then
+              pip install psycopg2-binary
+          fi
+
+          pip freeze
+
+      # NOTE: If you need to debug this DB setup step, consider the following.
+      #
+      # 1. mysql/postgressql are database servers we start as docker containers,
+      #    and we use clients named mysql/psql.
+      #
+      # 2. When we start a database server we need to pass environment variables
+      #    explicitly as part of the `docker run` command. These environment
+      #    variables are named differently from the similarly named environment
+      #    variables used by the clients.
+      #
+      #    - mysql    server ref: https://hub.docker.com/_/mysql/
+      #    - mysql    client ref: https://dev.mysql.com/doc/refman/5.7/en/environment-variables.html
+      #    - postgres server ref: https://hub.docker.com/_/postgres/
+      #    - psql     client ref: https://www.postgresql.org/docs/9.5/libpq-envars.html
+      #
+      # 3. When we connect, they should use 127.0.0.1 rather than the
+      #    default way of connecting which leads to errors like below both for
+      #    mysql and postgresql unless we set MYSQL_HOST/PGHOST to 127.0.0.1.
+      #
+      #    - ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2)
+      #
+      - name: Start a database server (${{ matrix.db }})
+        if: ${{ matrix.db }}
+        run: |
+          if [ "${{ matrix.db }}" == "mysql" ]; then
+              sudo apt-get update
+              sudo apt-get install -y mysql-client
+              DB=mysql bash ci/docker-db.sh
+              DB=mysql bash ci/init-db.sh
+          fi
+          if [ "${{ matrix.db }}" == "postgres" ]; then
+              sudo apt-get update
+              sudo apt-get install -y postgresql-client
+              DB=postgres bash ci/docker-db.sh
+              DB=postgres bash ci/init-db.sh
+          fi
+
+      - name: Run pytest
+        # FIXME: --color=yes explicitly set because:
+        #        https://github.com/actions/runner/issues/241
+        run: |
+          pytest -v --maxfail=2 --color=yes --cov=jupyterhub jupyterhub/tests
+      - name: Submit codecov report
+        run: |
+          codecov

.travis.yml

@@ -1,120 +0,0 @@
-dist: bionic
-language: python
-cache:
-  - pip
-env:
-  global:
-    - MYSQL_HOST=127.0.0.1
-    - MYSQL_TCP_PORT=13306
-
-# request additional services for the jobs to access
-services:
-  - postgresql
-  - docker
-
-# install dependencies for running pytest (but not linting)
-before_install:
-  - set -e
-  - nvm install 6; nvm use 6
-  - npm install
-  - npm install -g configurable-http-proxy
-  - |
-    # setup database
-    if [[ $JUPYTERHUB_TEST_DB_URL == mysql* ]]; then
-      unset MYSQL_UNIX_PORT
-      DB=mysql bash ci/docker-db.sh
-      DB=mysql bash ci/init-db.sh
-      # FIXME: mysql-connector-python 8.0.16 incorrectly decodes bytes to str
-      # ref: https://bugs.mysql.com/bug.php?id=94944
-      pip install 'mysql-connector-python==8.0.11'
-    elif [[ $JUPYTERHUB_TEST_DB_URL == postgresql* ]]; then
-      psql -c "CREATE USER $PGUSER WITH PASSWORD '$PGPASSWORD';" -U postgres
-      DB=postgres bash ci/init-db.sh
-      pip install psycopg2-binary
-    fi
-
-# install general dependencies
-install:
-  - pip install --upgrade pip
-  - pip install --upgrade --pre -r dev-requirements.txt .
-  - |
-    if [[ "$MASTER_DEPENDENCIES" == "True" ]]; then
-       pip install git+https://github.com/ipython/traitlets#egg=traitlets --force
-    fi
-  - |
-    if [[ "$TEST" == "jupyter_server" ]]; then
-       pip uninstall notebook --yes
-       pip install jupyter_server
-    fi
-  - pip freeze
-
-# run tests
-script:
-  - pytest -v --maxfail=2 --cov=jupyterhub jupyterhub/tests
-
-# collect test coverage information
-after_success:
-  - codecov
-
-# list the jobs
-jobs:
-  include:
-    - name: autoformatting check
-      python: 3.6
-      # NOTE: It does not suffice to override to: null, [], or [""]. Travis will
-      #       fall back to the default if we do.
-      before_install: echo "Do nothing before install."
-      script:
-        - pre-commit run --all-files
-      after_success: echo "Do nothing after success."
-      after_failure:
-        - |
-          echo "You can install pre-commit hooks to automatically run formatting"
-          echo "on each commit with:"
-          echo "    pre-commit install"
-          echo "or you can run by hand on staged files with"
-          echo "    pre-commit run"
-          echo "or after-the-fact on already committed files with"
-          echo "    pre-commit run --all-files"
-    # When we run pytest, we want to run it with python>=3.5 as well as with
-    # various configurations. We increment the python version at the same time
-    # as we test new configurations in order to reduce the number of test jobs.
-    - name: python:3.5 + dist:xenial
-      python: 3.5
-      dist: xenial
-    - name: python:3.6 + subdomain
-      python: 3.6
-      env: JUPYTERHUB_TEST_SUBDOMAIN_HOST=http://localhost.jovyan.org:8000
-    - name: python:3.7 + mysql
-      python: 3.7
-      env:
-        - JUPYTERHUB_TEST_DB_URL=mysql+mysqlconnector://root@127.0.0.1:$MYSQL_TCP_PORT/jupyterhub
-    - name: python:3.8 + postgresql
-      python: 3.8
-      env:
-        - PGUSER=jupyterhub
-        - PGPASSWORD=hub[test/:?
-        # The password in url below is url-encoded with: urllib.parse.quote($PGPASSWORD, safe='')
-        - JUPYTERHUB_TEST_DB_URL=postgresql://jupyterhub:hub%5Btest%2F%3A%3F@127.0.0.1/jupyterhub
-    - name: python:3.8 + master dependencies
-      python: 3.8
-      env:
-        - PGUSER=jupyterhub
-        - PGPASSWORD=hub[test/:?
-        # The password in url below is url-encoded with: urllib.parse.quote($PGPASSWORD, safe='')
-        - JUPYTERHUB_TEST_DB_URL=postgresql://jupyterhub:hub%5Btest%2F%3A%3F@127.0.0.1/jupyterhub
-        - MASTER_DEPENDENCIES=True
-    - name: python:3.8 + jupyter_server
-      python: 3.8
-      env:
-        - TEST=jupyter_server
-        - JUPYTERHUB_SINGLEUSER_APP=jupyterhub.tests.mockserverapp.MockServerApp
-
-    - name: python:nightly
-      python: nightly
-  allow_failures:
-    - name: python:nightly
-    # https://github.com/jupyterhub/jupyterhub/issues/3141
-    # The latest traitlets is close to release so it should not fail
-    # - name: python:3.8 + master dependencies
-  fast_finish: true

ci/docker-db.sh

@@ -1,36 +1,55 @@
 #!/usr/bin/env bash
-# source this file to setup postgres and mysql
+# The goal of this script is to start a database server as a docker container.
-# for local testing (as similar as possible to docker)
+#
+# Required environment variables:
+# - DB: The database server to start, either "postgres" or "mysql".
+#
+# - PGUSER/PGPASSWORD: For the creation of a postgresql user with associated
+#   password.
 
 set -eu
 
-export MYSQL_HOST=127.0.0.1
+# Stop and remove any existing database container
-export MYSQL_TCP_PORT=${MYSQL_TCP_PORT:-13306}
+DOCKER_CONTAINER="hub-test-$DB"
-export PGHOST=127.0.0.1
+docker rm -f "$DOCKER_CONTAINER" 2>/dev/null || true
-NAME="hub-test-$DB"
-DOCKER_RUN="docker run -d --name $NAME"
 
-docker rm -f "$NAME" 2>/dev/null || true
+# Prepare environment variables to startup and await readiness of either a mysql
-
+# or postgresql server.
-case "$DB" in
+if [[ "$DB" == "mysql" ]]; then
-"mysql")
+    # Environment variables can influence both the mysql server in the docker
-  RUN_ARGS="-e MYSQL_ALLOW_EMPTY_PASSWORD=1 -p $MYSQL_TCP_PORT:3306 mysql:5.7"
+    # container and the mysql client.
-  CHECK="mysql --host $MYSQL_HOST --port $MYSQL_TCP_PORT --user root -e \q"
+    #
-  ;;
+    # ref server: https://hub.docker.com/_/mysql/
-"postgres")
+    # ref client: https://dev.mysql.com/doc/refman/5.7/en/setting-environment-variables.html
-  RUN_ARGS="-p 5432:5432 postgres:9.5"
+    #
-  CHECK="psql --user postgres -c \q"
+    DOCKER_RUN_ARGS="-p 3306:3306 --env MYSQL_ALLOW_EMPTY_PASSWORD=1 mysql:5.7"
-  ;;
+    READINESS_CHECK="mysql --user root --execute \q"
-*)
+elif [[ "$DB" == "postgres" ]]; then
+    # Environment variables can influence both the postgresql server in the
+    # docker container and the postgresql client (psql).
+    #
+    # ref server: https://hub.docker.com/_/postgres/
+    # ref client: https://www.postgresql.org/docs/9.5/libpq-envars.html
+    #
+    # POSTGRES_USER / POSTGRES_PASSWORD will create a user on startup of the
+    # postgres server, but PGUSER and PGPASSWORD are the environment variables
+    # used by the postgresql client psql, so we configure the user based on how
+    # we want to connect.
+    #
+    DOCKER_RUN_ARGS="-p 5432:5432 --env "POSTGRES_USER=${PGUSER}" --env "POSTGRES_PASSWORD=${PGPASSWORD}" postgres:9.5"
+    READINESS_CHECK="psql --command \q"
+else
     echo '$DB must be mysql or postgres'
     exit 1
-esac
+fi
 
-$DOCKER_RUN $RUN_ARGS
+# Start the database server
+docker run --detach --name "$DOCKER_CONTAINER" $DOCKER_RUN_ARGS
 
+# Wait for the database server to start
 echo -n "waiting for $DB "
 for i in {1..60}; do
-  if $CHECK; then
+    if $READINESS_CHECK; then
         echo 'done'
         break
     else
@@ -38,22 +57,4 @@ for i in {1..60}; do
         sleep 1
     fi
 done
-$CHECK
+$READINESS_CHECK
-
-case "$DB" in
-"mysql")
-  ;;
-"postgres")
-  # create the user
-  psql --user postgres -c "CREATE USER $PGUSER WITH PASSWORD '$PGPASSWORD';"
-  ;;
-*)
-esac
-
-echo -e "
-Set these environment variables:
-
-    export MYSQL_HOST=127.0.0.1
-    export MYSQL_TCP_PORT=$MYSQL_TCP_PORT
-    export PGHOST=127.0.0.1
-"

ci/init-db.sh

@@ -1,27 +1,26 @@
 #!/usr/bin/env bash
-# initialize jupyterhub databases for testing
+# The goal of this script is to initialize a running database server with clean
+# databases for use during tests.
+#
+# Required environment variables:
+# - DB: The database server to start, either "postgres" or "mysql".
 
 set -eu
 
-MYSQL="mysql --user root --host $MYSQL_HOST --port $MYSQL_TCP_PORT -e "
+# Prepare env vars SQL_CLIENT and EXTRA_CREATE_DATABASE_ARGS
-PSQL="psql --user postgres -c "
+if [[ "$DB" == "mysql" ]]; then
-
+    SQL_CLIENT="mysql --user root --execute "
-case "$DB" in
+    EXTRA_CREATE_DATABASE_ARGS='CHARACTER SET utf8 COLLATE utf8_general_ci'
-"mysql")
+elif [[ "$DB" == "postgres" ]]; then
-  EXTRA_CREATE='CHARACTER SET utf8 COLLATE utf8_general_ci'
+    SQL_CLIENT="psql --command "
-  SQL="$MYSQL"
+else
-  ;;
-"postgres")
-  SQL="$PSQL"
-  ;;
-*)
     echo '$DB must be mysql or postgres'
     exit 1
-esac
+fi
 
+# Configure a set of databases in the database server for upgrade tests
 set -x
-
 for SUFFIX in '' _upgrade_072 _upgrade_081 _upgrade_094; do
-    $SQL "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true
+    $SQL_CLIENT "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true
-    $SQL "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE:-};"
+    $SQL_CLIENT "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE_DATABASE_ARGS:-};"
 done