Run tests in parallel

Makefile

@@ -207,7 +207,7 @@ run-sudo-shell/%: ## run a bash in interactive mode as root in a stack
 
 test/%: ## run tests against a stack (only common tests or common tests + specific tests)
 	@echo "::group::test/$(OWNER)/$(notdir $@)"
-	@if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test; \
+	@if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -n auto -m "not info" test; \
-	else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test $(notdir $@)/test; fi
+	else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -n auto -m "not info" test $(notdir $@)/test; fi
 	@echo "::endgroup::"
 test-all: $(foreach I, $(ALL_IMAGES), test/$(I)) ## test all stacks

base-notebook/test/test_container_options.py

@@ -7,7 +7,7 @@ import logging
 import pytest  # type: ignore
 import requests
 
-from conftest import TrackedContainer
+from conftest import TrackedContainer, find_free_port
 
 LOGGER = logging.getLogger(__name__)
 
@@ -15,11 +15,11 @@ LOGGER = logging.getLogger(__name__)
 def test_cli_args(container: TrackedContainer, http_client: requests.Session) -> None:
     """Container should respect notebook server command line args
     (e.g., disabling token security)"""
+    host_port = find_free_port()
     running_container = container.run_detached(
         command=["start-notebook.sh", "--NotebookApp.token=''"],
-        ports={"8888/tcp": None},
+        ports={"8888/tcp": host_port},
     )
-    host_port = container.get_host_port("8888/tcp")
     resp = http_client.get(f"http://localhost:{host_port}")
     resp.raise_for_status()
     logs = running_container.logs().decode("utf-8")
@@ -37,11 +37,11 @@ def test_unsigned_ssl(
     """Container should generate a self-signed SSL certificate
     and notebook server should use it to enable HTTPS.
     """
+    host_port = find_free_port()
     running_container = container.run_detached(
         environment=["GEN_CERT=yes"],
-        ports={"8888/tcp": None},
+        ports={"8888/tcp": host_port},
     )
-    host_port = container.get_host_port("8888/tcp")
     # NOTE: The requests.Session backing the http_client fixture does not retry
     # properly while the server is booting up. An SSL handshake error seems to
     # abort the retry logic. Forcing a long sleep for the moment until I have

base-notebook/test/test_start_container.py

@@ -7,7 +7,7 @@ import pytest  # type: ignore
 import requests
 import time
 
-from conftest import TrackedContainer
+from conftest import TrackedContainer, find_free_port
 
 LOGGER = logging.getLogger(__name__)
 
@@ -47,11 +47,12 @@ def test_start_notebook(
     LOGGER.info(
         f"Test that the start-notebook launches the {expected_command} server from the env {env} ..."
     )
+    host_port = find_free_port()
     running_container = container.run_detached(
         tty=True,
         environment=env,
         command=["start-notebook.sh"],
-        ports={"8888/tcp": None},
+        ports={"8888/tcp": host_port},
     )
     # sleeping some time to let the server start
     time.sleep(3)
@@ -69,7 +70,6 @@ def test_start_notebook(
     assert len(expected_warnings) == len(warnings)
     # checking if the server is listening
     if expected_start:
-        host_port = container.get_host_port("8888/tcp")
         resp = http_client.get(f"http://localhost:{host_port}")
         assert resp.status_code == 200, "Server is not listening"
 

conftest.py

@@ -1,7 +1,9 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
+from contextlib import closing
 import os
 import logging
+import socket
 from typing import Any, Optional
 
 import docker
@@ -16,6 +18,14 @@ from requests.adapters import HTTPAdapter
 LOGGER = logging.getLogger(__name__)
 
 
+def find_free_port() -> str:
+    """Returns the available host port. Can be called in multiple threads/processes."""
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        s.bind(("", 0))
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        return s.getsockname()[1]
+
+
 @pytest.fixture(scope="session")
 def http_client() -> requests.Session:
     """Requests session with retries and backoff."""
@@ -108,14 +118,6 @@ class TrackedContainer:
         assert rv == 0 or rv["StatusCode"] == 0
         return logs
 
-    def get_host_port(self, container_port: str) -> str:
-        """Returns the host port associated with the tracked container's port."""
-        assert isinstance(self.container, Container)
-        self.container.reload()
-        return self.container.attrs["NetworkSettings"]["Ports"][container_port][0][
-            "HostPort"
-        ]
-
     @staticmethod
     def get_errors(logs: str) -> list[str]:
         return TrackedContainer._lines_starting_with(logs, "ERROR")

requirements-dev.txt

@@ -3,5 +3,6 @@ packaging
 plumbum
 pre-commit
 pytest
+pytest-xdist
 requests
 tabulate

test/test_notebook.py

@@ -3,15 +3,15 @@
 
 
 import requests
-from conftest import TrackedContainer
+from conftest import TrackedContainer, find_free_port
 
 
 def test_secured_server(
     container: TrackedContainer, http_client: requests.Session
 ) -> None:
     """Notebook server should eventually request user login."""
-    container.run_detached(ports={"8888/tcp": None})
+    host_port = find_free_port()
-    host_port = container.get_host_port("8888/tcp")
+    container.run_detached(ports={"8888/tcp": host_port})
     resp = http_client.get(f"http://localhost:{host_port}")
     resp.raise_for_status()
     assert "login_submit" in resp.text, "User login not requested"