From 39eea2f0536acd3d3d79c980b3c4d898dfc0c413 Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 11 Nov 2020 09:06:58 +0100 Subject: [PATCH] Backport PR #3246: Migrate from travis to GitHub actions --- .github/workflows/test.yml | 208 +++++++++++++++++++++++++++++++++++++ .travis.yml | 120 --------------------- ci/docker-db.sh | 99 +++++++++--------- ci/init-db.sh | 37 ++++--- 4 files changed, 276 insertions(+), 188 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..2a4d5760 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,208 @@ +# This is a GitHub workflow defining a set of jobs with a set of steps. +# ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions +# +name: Run tests + +# Trigger the workflow's on all PRs but only on pushed tags or commits to +# main/master branch to avoid PRs developed in a GitHub fork's dedicated branch +# to trigger. +on: + pull_request: + push: + branches: [main, master, pr/migrate-to-gh-actions-from-travis] + tags: + +defaults: + run: + # Declare bash be used by default in this workflow's "run" steps. + # + # NOTE: bash will by default run with: + # --noprofile: Ignore ~/.profile etc. + # --norc: Ignore ~/.bashrc etc. + # -e: Exit directly on errors + # -o pipefail: Don't mask errors from a command piped into another command + shell: bash + +env: + # UTF-8 content may be interpreted as ascii and causes errors without this. + LANG: C.UTF-8 + +jobs: + # Run "pre-commit run --all-files" + pre-commit: + runs-on: ubuntu-20.04 + timeout-minutes: 2 + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.8 + + # ref: https://github.com/pre-commit/action + - uses: pre-commit/action@v2.0.0 + - name: Help message if pre-commit fail + if: ${{ failure() }} + run: | + echo "You can install pre-commit hooks to automatically run formatting" + echo "on each commit with:" + echo " pre-commit install" + echo "or you can run by hand on staged files with" + echo " pre-commit run" + echo "or after-the-fact on already committed files with" + echo " pre-commit run --all-files" + + + # Run "pytest jupyterhub/tests" in various configurations + pytest: + runs-on: ubuntu-20.04 + timeout-minutes: 10 + + strategy: + # Keep running even if one variation of the job fail + fail-fast: false + matrix: + # We run this job multiple times with different parameterization + # specified below, these parameters have no meaning on their own and + # gain meaning on how job steps use them. + # + # subdomain: + # Tests everything when JupyterHub is configured to add routes for + # users with dedicated subdomains like user1.jupyter.example.com + # rather than jupyter.example.com/user/user1. + # + # db: [mysql/postgres] + # Tests everything when JupyterHub works against a dedicated mysql or + # postgresql server. + # + # jupyter_server: + # Tests everything when the user instances are started with + # jupyter_server instead of notebook. + # + # main_dependencies: + # Tests everything when the we use the latest available dependencies + # from: ipytraitlets. + # + # NOTE: Since only the value of these parameters are presented in the + # GitHub UI when the workflow run, we avoid using true/false as + # values by instead duplicating the name to signal true. + include: + - python: "3.6" + subdomain: subdomain + - python: "3.7" + db: mysql + - python: "3.8" + db: postgres + - python: "3.8" + jupyter_server: jupyter_server + - python: "3.9" + main_dependencies: main_dependencies + + steps: + # NOTE: In GitHub workflows, environment variables are set by writing + # assignment statements to a file. They will be set in the following + # steps as if would used `export MY_ENV=my-value`. + - name: Configure environment variables + run: | + if [ "${{ matrix.subdomain }}" != "" ]; then + echo "JUPYTERHUB_TEST_SUBDOMAIN_HOST=http://localhost.jovyan.org:8000" >> $GITHUB_ENV + fi + if [ "${{ matrix.db }}" == "mysql" ]; then + echo "MYSQL_HOST=127.0.0.1" >> $GITHUB_ENV + echo "JUPYTERHUB_TEST_DB_URL=mysql+mysqlconnector://root@127.0.0.1:3306/jupyterhub" >> $GITHUB_ENV + fi + if [ "${{ matrix.db }}" == "postgres" ]; then + echo "PGHOST=127.0.0.1" >> $GITHUB_ENV + echo "PGUSER=test_user" >> $GITHUB_ENV + echo "PGPASSWORD=hub[test/:?" >> $GITHUB_ENV + echo "JUPYTERHUB_TEST_DB_URL=postgresql://test_user:hub%5Btest%2F%3A%3F@127.0.0.1:5432/jupyterhub" >> $GITHUB_ENV + fi + if [ "${{ matrix.jupyter_server }}" != "" ]; then + echo "JUPYTERHUB_SINGLEUSER_APP=jupyterhub.tests.mockserverapp.MockServerApp" >> $GITHUB_ENV + fi + - uses: actions/checkout@v2 + + # NOTE: actions/setup-node@v1 make use of a cache within the GitHub base + # environment and setup in a fraction of a second. + - name: Install Node v14 + uses: actions/setup-node@v1 + with: + node-version: "14" + - name: Install Node dependencies + run: | + npm install + npm install -g configurable-http-proxy + npm list + + # NOTE: actions/setup-python@v2 make use of a cache within the GitHub base + # environment and setup in a fraction of a second. + - name: Install Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install Python dependencies + run: | + pip install --upgrade pip + pip install --upgrade . -r dev-requirements.txt + + if [ "${{ matrix.main_dependencies }}" != "" ]; then + pip install git+https://github.com/ipython/traitlets#egg=traitlets --force + fi + if [ "${{ matrix.jupyter_server }}" != "" ]; then + pip uninstall notebook --yes + pip install jupyter_server + fi + if [ "${{ matrix.db }}" == "mysql" ]; then + pip install mysql-connector-python + fi + if [ "${{ matrix.db }}" == "postgres" ]; then + pip install psycopg2-binary + fi + + pip freeze + + # NOTE: If you need to debug this DB setup step, consider the following. + # + # 1. mysql/postgressql are database servers we start as docker containers, + # and we use clients named mysql/psql. + # + # 2. When we start a database server we need to pass environment variables + # explicitly as part of the `docker run` command. These environment + # variables are named differently from the similarly named environment + # variables used by the clients. + # + # - mysql server ref: https://hub.docker.com/_/mysql/ + # - mysql client ref: https://dev.mysql.com/doc/refman/5.7/en/environment-variables.html + # - postgres server ref: https://hub.docker.com/_/postgres/ + # - psql client ref: https://www.postgresql.org/docs/9.5/libpq-envars.html + # + # 3. When we connect, they should use 127.0.0.1 rather than the + # default way of connecting which leads to errors like below both for + # mysql and postgresql unless we set MYSQL_HOST/PGHOST to 127.0.0.1. + # + # - ERROR 2002 (HY000): Can't connect to local MySQL server through socket '/var/run/mysqld/mysqld.sock' (2) + # + - name: Start a database server (${{ matrix.db }}) + if: ${{ matrix.db }} + run: | + if [ "${{ matrix.db }}" == "mysql" ]; then + sudo apt-get update + sudo apt-get install -y mysql-client + DB=mysql bash ci/docker-db.sh + DB=mysql bash ci/init-db.sh + fi + if [ "${{ matrix.db }}" == "postgres" ]; then + sudo apt-get update + sudo apt-get install -y postgresql-client + DB=postgres bash ci/docker-db.sh + DB=postgres bash ci/init-db.sh + fi + + - name: Run pytest + # FIXME: --color=yes explicitly set because: + # https://github.com/actions/runner/issues/241 + run: | + pytest -v --maxfail=2 --color=yes --cov=jupyterhub jupyterhub/tests + - name: Submit codecov report + run: | + codecov diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 389f2088..00000000 --- a/.travis.yml +++ /dev/null @@ -1,120 +0,0 @@ -dist: bionic -language: python -cache: - - pip -env: - global: - - MYSQL_HOST=127.0.0.1 - - MYSQL_TCP_PORT=13306 - -# request additional services for the jobs to access -services: - - postgresql - - docker - -# install dependencies for running pytest (but not linting) -before_install: - - set -e - - nvm install 6; nvm use 6 - - npm install - - npm install -g configurable-http-proxy - - | - # setup database - if [[ $JUPYTERHUB_TEST_DB_URL == mysql* ]]; then - unset MYSQL_UNIX_PORT - DB=mysql bash ci/docker-db.sh - DB=mysql bash ci/init-db.sh - # FIXME: mysql-connector-python 8.0.16 incorrectly decodes bytes to str - # ref: https://bugs.mysql.com/bug.php?id=94944 - pip install 'mysql-connector-python==8.0.11' - elif [[ $JUPYTERHUB_TEST_DB_URL == postgresql* ]]; then - psql -c "CREATE USER $PGUSER WITH PASSWORD '$PGPASSWORD';" -U postgres - DB=postgres bash ci/init-db.sh - pip install psycopg2-binary - fi - -# install general dependencies -install: - - pip install --upgrade pip - - pip install --upgrade --pre -r dev-requirements.txt . - - | - if [[ "$MASTER_DEPENDENCIES" == "True" ]]; then - pip install git+https://github.com/ipython/traitlets#egg=traitlets --force - fi - - | - if [[ "$TEST" == "jupyter_server" ]]; then - pip uninstall notebook --yes - pip install jupyter_server - fi - - pip freeze - -# run tests -script: - - pytest -v --maxfail=2 --cov=jupyterhub jupyterhub/tests - -# collect test coverage information -after_success: - - codecov - -# list the jobs -jobs: - include: - - name: autoformatting check - python: 3.6 - # NOTE: It does not suffice to override to: null, [], or [""]. Travis will - # fall back to the default if we do. - before_install: echo "Do nothing before install." - script: - - pre-commit run --all-files - after_success: echo "Do nothing after success." - after_failure: - - | - echo "You can install pre-commit hooks to automatically run formatting" - echo "on each commit with:" - echo " pre-commit install" - echo "or you can run by hand on staged files with" - echo " pre-commit run" - echo "or after-the-fact on already committed files with" - echo " pre-commit run --all-files" - # When we run pytest, we want to run it with python>=3.5 as well as with - # various configurations. We increment the python version at the same time - # as we test new configurations in order to reduce the number of test jobs. - - name: python:3.5 + dist:xenial - python: 3.5 - dist: xenial - - name: python:3.6 + subdomain - python: 3.6 - env: JUPYTERHUB_TEST_SUBDOMAIN_HOST=http://localhost.jovyan.org:8000 - - name: python:3.7 + mysql - python: 3.7 - env: - - JUPYTERHUB_TEST_DB_URL=mysql+mysqlconnector://root@127.0.0.1:$MYSQL_TCP_PORT/jupyterhub - - name: python:3.8 + postgresql - python: 3.8 - env: - - PGUSER=jupyterhub - - PGPASSWORD=hub[test/:? - # The password in url below is url-encoded with: urllib.parse.quote($PGPASSWORD, safe='') - - JUPYTERHUB_TEST_DB_URL=postgresql://jupyterhub:hub%5Btest%2F%3A%3F@127.0.0.1/jupyterhub - - name: python:3.8 + master dependencies - python: 3.8 - env: - - PGUSER=jupyterhub - - PGPASSWORD=hub[test/:? - # The password in url below is url-encoded with: urllib.parse.quote($PGPASSWORD, safe='') - - JUPYTERHUB_TEST_DB_URL=postgresql://jupyterhub:hub%5Btest%2F%3A%3F@127.0.0.1/jupyterhub - - MASTER_DEPENDENCIES=True - - name: python:3.8 + jupyter_server - python: 3.8 - env: - - TEST=jupyter_server - - JUPYTERHUB_SINGLEUSER_APP=jupyterhub.tests.mockserverapp.MockServerApp - - - name: python:nightly - python: nightly - allow_failures: - - name: python:nightly - # https://github.com/jupyterhub/jupyterhub/issues/3141 - # The latest traitlets is close to release so it should not fail - # - name: python:3.8 + master dependencies - fast_finish: true diff --git a/ci/docker-db.sh b/ci/docker-db.sh index 14e06388..9bb69b73 100755 --- a/ci/docker-db.sh +++ b/ci/docker-db.sh @@ -1,59 +1,60 @@ #!/usr/bin/env bash -# source this file to setup postgres and mysql -# for local testing (as similar as possible to docker) +# The goal of this script is to start a database server as a docker container. +# +# Required environment variables: +# - DB: The database server to start, either "postgres" or "mysql". +# +# - PGUSER/PGPASSWORD: For the creation of a postgresql user with associated +# password. set -eu -export MYSQL_HOST=127.0.0.1 -export MYSQL_TCP_PORT=${MYSQL_TCP_PORT:-13306} -export PGHOST=127.0.0.1 -NAME="hub-test-$DB" -DOCKER_RUN="docker run -d --name $NAME" +# Stop and remove any existing database container +DOCKER_CONTAINER="hub-test-$DB" +docker rm -f "$DOCKER_CONTAINER" 2>/dev/null || true -docker rm -f "$NAME" 2>/dev/null || true +# Prepare environment variables to startup and await readiness of either a mysql +# or postgresql server. +if [[ "$DB" == "mysql" ]]; then + # Environment variables can influence both the mysql server in the docker + # container and the mysql client. + # + # ref server: https://hub.docker.com/_/mysql/ + # ref client: https://dev.mysql.com/doc/refman/5.7/en/setting-environment-variables.html + # + DOCKER_RUN_ARGS="-p 3306:3306 --env MYSQL_ALLOW_EMPTY_PASSWORD=1 mysql:5.7" + READINESS_CHECK="mysql --user root --execute \q" +elif [[ "$DB" == "postgres" ]]; then + # Environment variables can influence both the postgresql server in the + # docker container and the postgresql client (psql). + # + # ref server: https://hub.docker.com/_/postgres/ + # ref client: https://www.postgresql.org/docs/9.5/libpq-envars.html + # + # POSTGRES_USER / POSTGRES_PASSWORD will create a user on startup of the + # postgres server, but PGUSER and PGPASSWORD are the environment variables + # used by the postgresql client psql, so we configure the user based on how + # we want to connect. + # + DOCKER_RUN_ARGS="-p 5432:5432 --env "POSTGRES_USER=${PGUSER}" --env "POSTGRES_PASSWORD=${PGPASSWORD}" postgres:9.5" + READINESS_CHECK="psql --command \q" +else + echo '$DB must be mysql or postgres' + exit 1 +fi -case "$DB" in -"mysql") - RUN_ARGS="-e MYSQL_ALLOW_EMPTY_PASSWORD=1 -p $MYSQL_TCP_PORT:3306 mysql:5.7" - CHECK="mysql --host $MYSQL_HOST --port $MYSQL_TCP_PORT --user root -e \q" - ;; -"postgres") - RUN_ARGS="-p 5432:5432 postgres:9.5" - CHECK="psql --user postgres -c \q" - ;; -*) - echo '$DB must be mysql or postgres' - exit 1 -esac - -$DOCKER_RUN $RUN_ARGS +# Start the database server +docker run --detach --name "$DOCKER_CONTAINER" $DOCKER_RUN_ARGS +# Wait for the database server to start echo -n "waiting for $DB " for i in {1..60}; do - if $CHECK; then - echo 'done' - break - else - echo -n '.' - sleep 1 - fi + if $READINESS_CHECK; then + echo 'done' + break + else + echo -n '.' + sleep 1 + fi done -$CHECK - -case "$DB" in -"mysql") - ;; -"postgres") - # create the user - psql --user postgres -c "CREATE USER $PGUSER WITH PASSWORD '$PGPASSWORD';" - ;; -*) -esac - -echo -e " -Set these environment variables: - - export MYSQL_HOST=127.0.0.1 - export MYSQL_TCP_PORT=$MYSQL_TCP_PORT - export PGHOST=127.0.0.1 -" +$READINESS_CHECK diff --git a/ci/init-db.sh b/ci/init-db.sh index dfeb12b4..b510f549 100755 --- a/ci/init-db.sh +++ b/ci/init-db.sh @@ -1,27 +1,26 @@ #!/usr/bin/env bash -# initialize jupyterhub databases for testing +# The goal of this script is to initialize a running database server with clean +# databases for use during tests. +# +# Required environment variables: +# - DB: The database server to start, either "postgres" or "mysql". set -eu -MYSQL="mysql --user root --host $MYSQL_HOST --port $MYSQL_TCP_PORT -e " -PSQL="psql --user postgres -c " - -case "$DB" in -"mysql") - EXTRA_CREATE='CHARACTER SET utf8 COLLATE utf8_general_ci' - SQL="$MYSQL" - ;; -"postgres") - SQL="$PSQL" - ;; -*) - echo '$DB must be mysql or postgres' - exit 1 -esac +# Prepare env vars SQL_CLIENT and EXTRA_CREATE_DATABASE_ARGS +if [[ "$DB" == "mysql" ]]; then + SQL_CLIENT="mysql --user root --execute " + EXTRA_CREATE_DATABASE_ARGS='CHARACTER SET utf8 COLLATE utf8_general_ci' +elif [[ "$DB" == "postgres" ]]; then + SQL_CLIENT="psql --command " +else + echo '$DB must be mysql or postgres' + exit 1 +fi +# Configure a set of databases in the database server for upgrade tests set -x - for SUFFIX in '' _upgrade_072 _upgrade_081 _upgrade_094; do - $SQL "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true - $SQL "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE:-};" + $SQL_CLIENT "DROP DATABASE jupyterhub${SUFFIX};" 2>/dev/null || true + $SQL_CLIENT "CREATE DATABASE jupyterhub${SUFFIX} ${EXTRA_CREATE_DATABASE_ARGS:-};" done