Add prometheus metric to measure proxy route deletion times

jupyterhub/handlers/base.py

@@ -33,7 +33,9 @@ from tornado.web import RequestHandler
 from .. import __version__
 from .. import orm
 from ..metrics import PROXY_ADD_DURATION_SECONDS
+from ..metrics import PROXY_DELETE_DURATION_SECONDS
 from ..metrics import ProxyAddStatus
+from ..metrics import ProxyDeleteStatus
 from ..metrics import RUNNING_SERVERS
 from ..metrics import SERVER_POLL_DURATION_SECONDS
 from ..metrics import SERVER_SPAWN_DURATION_SECONDS
@@ -1005,7 +1007,17 @@ class BaseHandler(RequestHandler):
         self.log.warning(
             "User %s server stopped, with exit code: %s", user.name, status
         )
+        proxy_deletion_start_time = time.perf_counter()
+        try:
             await self.proxy.delete_user(user, server_name)
+            PROXY_DELETE_DURATION_SECONDS.labels(
+                status=ProxyDeleteStatus.success
+            ).observe(time.perf_counter() - proxy_deletion_start_time)
+        except:
+            PROXY_DELETE_DURATION_SECONDS.labels(
+                status=ProxyDeleteStatus.failure
+            ).observe(time.perf_counter() - proxy_deletion_start_time)
+
         await user.stop(server_name)
 
     async def stop_single_user(self, user, server_name=''):
@@ -1028,6 +1040,10 @@ class BaseHandler(RequestHandler):
             tic = time.perf_counter()
             try:
                 await self.proxy.delete_user(user, server_name)
+                PROXY_DELETE_DURATION_SECONDS.labels(
+                    status=ProxyDeleteStatus.success
+                ).observe(time.perf_counter() - tic)
+
                 await user.stop(server_name)
                 toc = time.perf_counter()
                 self.log.info(
@@ -1047,6 +1063,9 @@ class BaseHandler(RequestHandler):
                     },
                 )
             except:
+                PROXY_DELETE_DURATION_SECONDS.labels(
+                    status=ProxyDeleteStatus.failure
+                ).observe(time.perf_counter() - tic)
                 SERVER_STOP_DURATION_SECONDS.labels(
                     status=ServerStopStatus.failure
                 ).observe(time.perf_counter() - tic)

jupyterhub/metrics.py

@@ -135,6 +135,29 @@ for s in ServerStopStatus:
     SERVER_STOP_DURATION_SECONDS.labels(status=s)
 
 
+PROXY_DELETE_DURATION_SECONDS = Histogram(
+    'proxy_delete_duration_seconds',
+    'duration for deleting user routes from proxy',
+    ['status'],
+)
+
+
+class ProxyDeleteStatus(Enum):
+    """
+    Possible values for 'status' label of PROXY_DELETE_DURATION_SECONDS
+    """
+
+    success = 'success'
+    failure = 'failure'
+
+    def __str__(self):
+        return self.value
+
+
+for s in ProxyDeleteStatus:
+    PROXY_DELETE_DURATION_SECONDS.labels(status=s)
+
+
 def prometheus_log_method(handler):
     """
     Tornado log handler for recording RED metrics.