From 4ccf4fa4cf0f52f074fcc902956bcdf783b40775 Mon Sep 17 00:00:00 2001 From: kireetb Date: Sat, 21 Dec 2024 21:17:56 -0500 Subject: [PATCH 1/6] Allow configuration of bucket sizes in metrics - #4833 --- jupyterhub/metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jupyterhub/metrics.py b/jupyterhub/metrics.py index 3237fa7b..1b001ee0 100644 --- a/jupyterhub/metrics.py +++ b/jupyterhub/metrics.py @@ -37,6 +37,7 @@ from . import orm from .utils import utcnow metrics_prefix = os.getenv('JUPYTERHUB_METRICS_PREFIX', 'jupyterhub') +bucket_sizes = os.getenv('JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKET_SIZES', [0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600, float("inf")]) REQUEST_DURATION_SECONDS = Histogram( 'request_duration_seconds', @@ -51,7 +52,7 @@ SERVER_SPAWN_DURATION_SECONDS = Histogram( ['status'], # Use custom bucket sizes, since the default bucket ranges # are meant for quick running processes. Spawns can take a while! - buckets=[0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600, float("inf")], + buckets=bucket_sizes, namespace=metrics_prefix, ) From 413321beee20f0be145fd926b399c6d2a1b12bc5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 22 Dec 2024 02:21:47 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyterhub/metrics.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyterhub/metrics.py b/jupyterhub/metrics.py index 1b001ee0..ddb7e0e1 100644 --- a/jupyterhub/metrics.py +++ b/jupyterhub/metrics.py @@ -37,7 +37,10 @@ from . import orm from .utils import utcnow metrics_prefix = os.getenv('JUPYTERHUB_METRICS_PREFIX', 'jupyterhub') -bucket_sizes = os.getenv('JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKET_SIZES', [0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600, float("inf")]) +bucket_sizes = os.getenv( + 'JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKET_SIZES', + [0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600, float("inf")], +) REQUEST_DURATION_SECONDS = Histogram( 'request_duration_seconds', From 4eb2d6d8a4b3da562b99e05e2e9505e6d5aab7c6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 01:51:56 +0000 Subject: [PATCH 3/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyterhub/metrics.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/jupyterhub/metrics.py b/jupyterhub/metrics.py index 6e87c3fc..adf6efd1 100644 --- a/jupyterhub/metrics.py +++ b/jupyterhub/metrics.py @@ -30,20 +30,23 @@ from enum import Enum from prometheus_client import Gauge, Histogram from tornado.ioloop import PeriodicCallback -from traitlets import Any, Bool, Dict, Float, Integer, List, TraitError, HasTraits +from traitlets import Any, Bool, Dict, Float, HasTraits, Integer, List, TraitError from traitlets.config import LoggingConfigurable from . import orm from .utils import utcnow + class CustomEnvVars(HasTraits): bucket_sizes = List().tag(config=True) metrics_prefix = os.getenv('JUPYTERHUB_METRICS_PREFIX', 'jupyterhub') - + def load_from_env(self): - env_var_bucket_sizes = os.getenv('JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKET_SIZES', - '[0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600]') - + env_var_bucket_sizes = os.getenv( + 'JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKET_SIZES', + '[0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600]', + ) + try: # Manually convert the string to a list # Here we assume the string is in the format '[1, 2, 3]' @@ -53,6 +56,7 @@ class CustomEnvVars(HasTraits): # Handle if conversion fails raise TraitError("Failed to convert environment variable to list.") + custom_env = CustomEnvVars() custom_env.load_from_env() metrics_prefix = custom_env.metrics_prefix From ee4a8e593dbc95b0a83cb42897640a405460ea7a Mon Sep 17 00:00:00 2001 From: kireetb Date: Mon, 3 Feb 2025 22:50:58 -0500 Subject: [PATCH 4/6] simpler custom spawn bucket sizes for prometheus --- docs/source/reference/monitoring.md | 10 +++++++++ jupyterhub/metrics.py | 32 +++++++---------------------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/docs/source/reference/monitoring.md b/docs/source/reference/monitoring.md index 622f52aa..8208d799 100644 --- a/docs/source/reference/monitoring.md +++ b/docs/source/reference/monitoring.md @@ -33,6 +33,16 @@ export JUPYTERHUB_METRICS_PREFIX=jupyterhub_prod would result in the metric `jupyterhub_prod_active_users`, etc. +## Customizing spawn bucket sizes + +As of JupyterHub >= 5.2.1, override `JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKETS` env variable in Hub's environment to allow custom bucket sizes. Otherwise default to, [0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600, float("inf")] + +For example, + +```bash +export JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKETS="1,2,4,6,12,30,60,120" +``` + ## Configuring metrics ```{eval-rst} diff --git a/jupyterhub/metrics.py b/jupyterhub/metrics.py index adf6efd1..90bb41d4 100644 --- a/jupyterhub/metrics.py +++ b/jupyterhub/metrics.py @@ -36,31 +36,13 @@ from traitlets.config import LoggingConfigurable from . import orm from .utils import utcnow +metrics_prefix = os.getenv('JUPYTERHUB_METRICS_PREFIX', 'jupyterhub') +_env_buckets = os.environ.get('JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKETS', "").strip() -class CustomEnvVars(HasTraits): - bucket_sizes = List().tag(config=True) - metrics_prefix = os.getenv('JUPYTERHUB_METRICS_PREFIX', 'jupyterhub') - - def load_from_env(self): - env_var_bucket_sizes = os.getenv( - 'JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKET_SIZES', - '[0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600]', - ) - - try: - # Manually convert the string to a list - # Here we assume the string is in the format '[1, 2, 3]' - cleaned = env_var_bucket_sizes.strip('[]').replace(' ', '') - self.bucket_sizes = [str(i) for i in cleaned.split(',')] - except (ValueError, AttributeError): - # Handle if conversion fails - raise TraitError("Failed to convert environment variable to list.") - - -custom_env = CustomEnvVars() -custom_env.load_from_env() -metrics_prefix = custom_env.metrics_prefix -bucket_sizes = custom_env.bucket_sizes +if _env_buckets: + spawn_duration_buckets = [float(_s) for _s in _env_buckets.split(",")] +else: + spawn_duration_buckets = [0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600, float("inf")] REQUEST_DURATION_SECONDS = Histogram( 'request_duration_seconds', @@ -75,7 +57,7 @@ SERVER_SPAWN_DURATION_SECONDS = Histogram( ['status'], # Use custom bucket sizes, since the default bucket ranges # are meant for quick running processes. Spawns can take a while! - buckets=bucket_sizes, + buckets=spawn_duration_buckets, namespace=metrics_prefix, ) From 18d0270af11dde32ef787fc791653cceb1d787a9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 03:51:20 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- jupyterhub/metrics.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/jupyterhub/metrics.py b/jupyterhub/metrics.py index 90bb41d4..fbff828e 100644 --- a/jupyterhub/metrics.py +++ b/jupyterhub/metrics.py @@ -30,19 +30,35 @@ from enum import Enum from prometheus_client import Gauge, Histogram from tornado.ioloop import PeriodicCallback -from traitlets import Any, Bool, Dict, Float, HasTraits, Integer, List, TraitError +from traitlets import Any, Bool, Dict, Float, Integer from traitlets.config import LoggingConfigurable from . import orm from .utils import utcnow metrics_prefix = os.getenv('JUPYTERHUB_METRICS_PREFIX', 'jupyterhub') -_env_buckets = os.environ.get('JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKETS', "").strip() +_env_buckets = os.environ.get( + 'JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKETS', "" +).strip() if _env_buckets: spawn_duration_buckets = [float(_s) for _s in _env_buckets.split(",")] else: - spawn_duration_buckets = [0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600, float("inf")] + spawn_duration_buckets = [ + 0.5, + 1, + 2.5, + 5, + 10, + 15, + 30, + 60, + 120, + 180, + 300, + 600, + float("inf"), + ] REQUEST_DURATION_SECONDS = Histogram( 'request_duration_seconds', From 3ce05d42b67bbf1153f5ee110600cd8a971c403e Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 19 Mar 2025 13:18:56 +0100 Subject: [PATCH 6/6] note version for bucket env change --- docs/source/reference/monitoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference/monitoring.md b/docs/source/reference/monitoring.md index 8208d799..36078877 100644 --- a/docs/source/reference/monitoring.md +++ b/docs/source/reference/monitoring.md @@ -35,7 +35,7 @@ would result in the metric `jupyterhub_prod_active_users`, etc. ## Customizing spawn bucket sizes -As of JupyterHub >= 5.2.1, override `JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKETS` env variable in Hub's environment to allow custom bucket sizes. Otherwise default to, [0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600, float("inf")] +As of JupyterHub 5.3, override `JUPYTERHUB_SERVER_SPAWN_DURATION_SECONDS_BUCKETS` env variable in Hub's environment to allow custom bucket sizes. Otherwise default to, [0.5, 1, 2.5, 5, 10, 15, 30, 60, 120, 180, 300, 600, float("inf")] For example,