test token page with html parsing

dev-requirements.txt

@@ -1,5 +1,6 @@
 -r requirements.txt
 mock
+beautifulsoup4
 codecov
 cryptography
 pytest-cov

jupyterhub/tests/test_pages.py

@@ -1,7 +1,9 @@
 """Tests for HTML pages"""
 
+import sys
 from urllib.parse import urlencode, urlparse
 
+from bs4 import BeautifulSoup
 from tornado import gen
 from tornado.httputil import url_concat
 
@@ -605,20 +607,42 @@ def test_token_page(app):
     r = yield get_page("token", app, cookies=cookies)
     r.raise_for_status()
     assert urlparse(r.url).path.endswith('/hub/token')
-    assert "Request new API token" in r.text
+    def extract_body(r):
-    assert "API Tokens" in r.text
+        soup = BeautifulSoup(r.text, "html5lib")
-    assert "Server at %s" % app.users[name].url in r.text
+        import re
-    # no oauth tokens yet, shouldn't have that section
+        # trim empty lines
-    assert "Authorized Applications" not in r.text
+        return re.sub(r"(\n\s*)+", "\n", soup.body.find(class_="container").text)
+    body = extract_body(r)
+    assert "Request new API token" in body, body
+    # no tokens yet, no lists
+    assert "API Tokens" not in body, body
+    assert "Authorized Applications" not in body, body
 
-    # spawn the user to trigger oauth, etc.
+    # request an API token
-    r = yield get_page("spawn", app, cookies=cookies)
+    user = app.users[name]
-    r.raise_fo_status()
+    token = user.new_api_token(expires_in=60, note="my-test-token")
+    app.db.commit()
 
     r = yield get_page("token", app, cookies=cookies)
     r.raise_for_status()
-    assert "API Tokens" in r.text
+    body = extract_body(r)
-    assert "Authorized Applications" not in r.text
+    assert "API Tokens" in body, body
+    assert "my-test-token" in body, body
+    # no oauth tokens yet, shouldn't have that section
+    assert "Authorized Applications" not in body, body
+
+    # spawn the user to trigger oauth, etc.
+    # request an oauth token
+    user.spawner.cmd = [sys.executable, '-m', 'jupyterhub.singleuser']
+    r = yield get_page("spawn", app, cookies=cookies)
+    r.raise_for_status()
+
+    r = yield get_page("token", app, cookies=cookies)
+    r.raise_for_status()
+    body = extract_body(r)
+    assert "API Tokens" in body, body
+    assert "Server at %s" % user.base_url in body, body
+    assert "Authorized Applications" in body, body
 
 
 @pytest.mark.gen_test