From d5a9d0f59bbbbb55ac347e6126d01b1456dbd94f Mon Sep 17 00:00:00 2001 From: Guy Stone Date: Fri, 11 Sep 2026 15:12:36 +0100 Subject: [PATCH] [Metrics] Consolidate Prometheus histogram bucket defaults into a single module (#48866) Signed-off-by: Guy Stone Co-authored-by: Claude Fable 5 --- tests/v1/metrics/test_histogram_buckets.py | 273 +++++++++++++++++++++ vllm/v1/metrics/buckets.py | 228 +++++++++++++++++ vllm/v1/metrics/loggers.py | 170 ++----------- 3 files changed, 524 insertions(+), 147 deletions(-) create mode 100644 tests/v1/metrics/test_histogram_buckets.py create mode 100644 vllm/v1/metrics/buckets.py diff --git a/tests/v1/metrics/test_histogram_buckets.py b/tests/v1/metrics/test_histogram_buckets.py new file mode 100644 index 0000000000..54cc31da94 --- /dev/null +++ b/tests/v1/metrics/test_histogram_buckets.py @@ -0,0 +1,273 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Snapshot tests locking the default Prometheus histogram buckets. + +The expected lists are deliberately hard-coded literals rather than imports: +any change to a default bucket boundary must show up as a failing test here. +""" + +import prometheus_client +import pytest + +from vllm.config import ModelConfig, VllmConfig +from vllm.config.observability import ObservabilityConfig +from vllm.v1.metrics.buckets import ( + BUCKET_FAMILY_KEYS, + build_1_2_5_buckets, + histogram_buckets, +) +from vllm.v1.metrics.loggers import PrometheusStatLogger +from vllm.v1.metrics.prometheus import unregister_vllm_metrics + +pytestmark = pytest.mark.cpu_test + +DEFAULT_BUCKET_SNAPSHOTS: dict[str, list[float]] = { + "request_latency": [ + 0.3, + 0.5, + 0.8, + 1.0, + 1.5, + 2.0, + 2.5, + 5.0, + 10.0, + 15.0, + 20.0, + 30.0, + 40.0, + 50.0, + 60.0, + 120.0, + 240.0, + 480.0, + 960.0, + 1920.0, + 7680.0, + ], + "time_to_first_token": [ + 0.001, + 0.005, + 0.01, + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.25, + 0.5, + 0.75, + 1.0, + 2.5, + 5.0, + 7.5, + 10.0, + 20.0, + 40.0, + 80.0, + 160.0, + 640.0, + 2560.0, + ], + "inter_token_latency": [ + 0.01, + 0.025, + 0.05, + 0.075, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4, + 0.5, + 0.75, + 1.0, + 2.5, + 5.0, + 7.5, + 10.0, + 20.0, + 40.0, + 80.0, + ], + "iteration_tokens": [ + 1, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1024, + 2048, + 4096, + 8192, + 16384, + ], + "request_params_n": [1, 2, 5, 10, 20], + "request_num_preemptions": [1, 2, 3, 4, 5, 10, 20], + "kv_cache_residency": [ + 0.001, + 0.002, + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.2, + 0.5, + 1, + 2, + 5, + 10, + 20, + 30, + 60, + 120, + 300, + 600, + 1200, + 1800, + ], +} + +METRIC_FAMILIES: dict[str, str] = { + "vllm:e2e_request_latency_seconds": "request_latency", + "vllm:request_queue_time_seconds": "request_latency", + "vllm:request_inference_time_seconds": "request_latency", + "vllm:request_prefill_time_seconds": "request_latency", + "vllm:request_decode_time_seconds": "request_latency", + "vllm:time_to_first_token_seconds": "time_to_first_token", + "vllm:inter_token_latency_seconds": "inter_token_latency", + "vllm:request_time_per_output_token_seconds": "inter_token_latency", + "vllm:iteration_tokens_total": "iteration_tokens", + "vllm:request_params_n": "request_params_n", + "vllm:request_num_preemptions": "request_num_preemptions", + "vllm:request_prompt_tokens": "request_tokens", + "vllm:request_generation_tokens": "request_tokens", + "vllm:request_max_num_generation_tokens": "request_tokens", + "vllm:request_params_max_tokens": "request_tokens", + "vllm:request_prefill_kv_computed_tokens": "request_tokens", + "vllm:kv_block_lifetime_seconds": "kv_cache_residency", + "vllm:kv_block_idle_before_evict_seconds": "kv_cache_residency", + "vllm:kv_block_reuse_gap_seconds": "kv_cache_residency", +} + +TEST_MODEL = "distilbert/distilgpt2" +TEST_MODEL_MAX_LEN = 1024 +REQUEST_TOKENS_SNAPSHOT_FOR_TEST_MODEL: list[float] = [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, +] + + +@pytest.mark.parametrize("family", sorted(DEFAULT_BUCKET_SNAPSHOTS)) +def test_default_bucket_snapshots(family): + """Static families must expose exactly today's default boundaries.""" + assert histogram_buckets(family) == DEFAULT_BUCKET_SNAPSHOTS[family] + + +def test_request_tokens_bucket_snapshot(): + """The 1-2-5 token-count series must be stable for a given model len.""" + assert histogram_buckets("request_tokens", max_model_len=32768) == [ + 1, + 2, + 5, + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 5000, + 10000, + 20000, + ] + assert build_1_2_5_buckets(100) == [1, 2, 5, 10, 20, 50, 100] + + +def test_request_tokens_requires_max_model_len(): + with pytest.raises(ValueError, match="max_model_len is required"): + histogram_buckets("request_tokens") + + +def test_buckets_return_fresh_copies(): + """Mutating a returned list must not corrupt the shared defaults.""" + first = histogram_buckets("request_latency") + first.append(1e9) + second = histogram_buckets("request_latency") + assert second == DEFAULT_BUCKET_SNAPSHOTS["request_latency"] + assert second is not first + + +def test_bucket_family_keys(): + """The declared families are exactly these, and each one feeds a histogram.""" + assert { + "request_latency", + "time_to_first_token", + "inter_token_latency", + "iteration_tokens", + "request_params_n", + "request_num_preemptions", + "request_tokens", + "kv_cache_residency", + } == BUCKET_FAMILY_KEYS + assert set(METRIC_FAMILIES.values()) == BUCKET_FAMILY_KEYS + + +def collect_histogram_buckets() -> dict[str, list[float]]: + """Read back each vllm:* histogram's finite bucket bounds by metric name.""" + found: dict[str, list[float]] = {} + for metric in prometheus_client.REGISTRY.collect(): + if metric.name.startswith("vllm:") and metric.type == "histogram": + found[metric.name] = sorted( + { + float(sample.labels["le"]) + for sample in metric.samples + if sample.name.endswith("_bucket") and sample.labels["le"] != "+Inf" + } + ) + return found + + +def build_logger_config( + observability_config: ObservabilityConfig, +) -> VllmConfig: + return VllmConfig( + model_config=ModelConfig(model=TEST_MODEL), + observability_config=observability_config, + ) + + +def test_prometheus_logger_default_buckets(): + """Every core engine histogram must use its family's defaults. + + The config deliberately sets no kv_transfer_config: a KV connector would + register its own histograms, which own their boundaries and are not + covered by METRIC_FAMILIES. + """ + config = build_logger_config(ObservabilityConfig(kv_cache_metrics=True)) + assert config.model_config.max_model_len == TEST_MODEL_MAX_LEN + try: + PrometheusStatLogger(config) + found = collect_histogram_buckets() + assert set(found) == set(METRIC_FAMILIES) + for metric_name, family in METRIC_FAMILIES.items(): + if family == "request_tokens": + expected = REQUEST_TOKENS_SNAPSHOT_FOR_TEST_MODEL + else: + expected = DEFAULT_BUCKET_SNAPSHOTS[family] + assert found[metric_name] == [float(b) for b in expected], metric_name + finally: + unregister_vllm_metrics() diff --git a/vllm/v1/metrics/buckets.py b/vllm/v1/metrics/buckets.py new file mode 100644 index 0000000000..375d730729 --- /dev/null +++ b/vllm/v1/metrics/buckets.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Default bucket boundaries for the Prometheus histograms emitted by vLLM. + +Every core engine histogram created by the Prometheus stat logger draws its +bucket boundaries from exactly one of the families defined here, so each +default list has a single source of truth. Histograms owned by other +subsystems, such as the KV connector and offloading metrics, keep their own +boundaries. +""" + +from typing import Literal, get_args + +BucketFamilyKey = Literal[ + "request_latency", + "time_to_first_token", + "inter_token_latency", + "iteration_tokens", + "request_params_n", + "request_num_preemptions", + "request_tokens", + "kv_cache_residency", +] +"""Canonical keys for the histogram bucket families.""" + +BUCKET_FAMILY_KEYS: frozenset[str] = frozenset(get_args(BucketFamilyKey)) +"""All bucket family keys, for membership checks.""" + +REQUEST_LATENCY_BUCKETS: tuple[float, ...] = ( + 0.3, + 0.5, + 0.8, + 1.0, + 1.5, + 2.0, + 2.5, + 5.0, + 10.0, + 15.0, + 20.0, + 30.0, + 40.0, + 50.0, + 60.0, + 120.0, + 240.0, + 480.0, + 960.0, + 1920.0, + 7680.0, +) +"""Sub-second scheduling delays through multi-hour batch requests; shared by +the accumulated request phase-timing histograms (e2e, queue, inference, +prefill, decode).""" + +TIME_TO_FIRST_TOKEN_BUCKETS: tuple[float, ...] = ( + 0.001, + 0.005, + 0.01, + 0.02, + 0.04, + 0.06, + 0.08, + 0.1, + 0.25, + 0.5, + 0.75, + 1.0, + 2.5, + 5.0, + 7.5, + 10.0, + 20.0, + 40.0, + 80.0, + 160.0, + 640.0, + 2560.0, +) +"""Millisecond-scale prefill for tiny prompts up to ~40-minute worst cases, +with the densest resolution around interactive (sub-second) latencies.""" + +INTER_TOKEN_LATENCY_BUCKETS: tuple[float, ...] = ( + 0.01, + 0.025, + 0.05, + 0.075, + 0.1, + 0.15, + 0.2, + 0.3, + 0.4, + 0.5, + 0.75, + 1.0, + 2.5, + 5.0, + 7.5, + 10.0, + 20.0, + 40.0, + 80.0, +) +"""Per-decode-step latencies: 10 ms fast decode up to multi-second stalls.""" + +ITERATION_TOKENS_BUCKETS: tuple[float, ...] = ( + 1, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1024, + 2048, + 4096, + 8192, + 16384, +) +"""Tokens processed per engine step: powers of two up to a typical +max-num-batched-tokens budget.""" + +REQUEST_PARAMS_N_BUCKETS: tuple[float, ...] = (1, 2, 5, 10, 20) +"""Small integer counts for the ``n`` sampling parameter.""" + +REQUEST_NUM_PREEMPTIONS_BUCKETS: tuple[float, ...] = (1, 2, 3, 4, 5, 10, 20) +"""Per-request preemption counts, recorded for every finished request: the +first bucket therefore holds never-preempted requests together with +singly-preempted ones, 2 through 5 are separated individually, and the tail +coarsens to 20. Distinct from `REQUEST_PARAMS_N_BUCKETS` because these are the +boundaries the metric shipped with, not a harmonized choice.""" + +KV_CACHE_RESIDENCY_BUCKETS: tuple[float, ...] = ( + 0.001, + 0.002, + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.2, + 0.5, + 1, + 2, + 5, + 10, + 20, + 30, + 60, + 120, + 300, + 600, + 1200, + 1800, +) +"""KV cache block residency times: millisecond-scale reuse gaps up to +30-minute block lifetimes.""" + +_STATIC_FAMILY_DEFAULTS: dict[str, tuple[float, ...]] = { + "request_latency": REQUEST_LATENCY_BUCKETS, + "time_to_first_token": TIME_TO_FIRST_TOKEN_BUCKETS, + "inter_token_latency": INTER_TOKEN_LATENCY_BUCKETS, + "iteration_tokens": ITERATION_TOKENS_BUCKETS, + "request_params_n": REQUEST_PARAMS_N_BUCKETS, + "request_num_preemptions": REQUEST_NUM_PREEMPTIONS_BUCKETS, + "kv_cache_residency": KV_CACHE_RESIDENCY_BUCKETS, +} + + +def build_buckets(mantissa_lst: list[int], max_value: int) -> list[float]: + """Build buckets from mantissas scaled by increasing powers of 10. + + Args: + mantissa_lst: Mantissa values multiplied by each power of 10. + max_value: Largest bucket value to include. + + Returns: + Bucket values in increasing order, capped at `max_value`. + """ + exponent = 0 + buckets: list[float] = [] + while True: + for m in mantissa_lst: + value = m * 10**exponent + if value <= max_value: + buckets.append(value) + else: + return buckets + exponent += 1 + + +def build_1_2_5_buckets(max_value: int) -> list[float]: + """Build a 1-2-5 series of buckets capped at `max_value`. + + Example: + >>> build_1_2_5_buckets(100) + [1, 2, 5, 10, 20, 50, 100] + """ + return build_buckets([1, 2, 5], max_value) + + +def histogram_buckets( + family: BucketFamilyKey, + max_model_len: int | None = None, +) -> list[float]: + """Return the default bucket boundaries for a histogram family. + + Args: + family: Canonical bucket family key. + max_model_len: Cap for the token-count series of the + `request_tokens` family; required for that family and ignored + otherwise. + + Returns: + A fresh list of bucket upper bounds; callers may mutate it freely. + + Raises: + ValueError: If `family` is `request_tokens` and `max_model_len` + is None. + """ + if family == "request_tokens": + if max_model_len is None: + raise ValueError( + "max_model_len is required for the 'request_tokens' bucket family" + ) + return build_1_2_5_buckets(max_model_len) + return list(_STATIC_FAMILY_DEFAULTS[family]) diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index 4d0b77add7..9e3f9c63a6 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( from vllm.logger import init_logger from vllm.plugins import STAT_LOGGER_PLUGINS_GROUP, load_plugins_by_group from vllm.v1.engine import FinishReason +from vllm.v1.metrics.buckets import histogram_buckets from vllm.v1.metrics.perf import PerfMetricsLogging, PerfMetricsProm from vllm.v1.metrics.prometheus import unregister_vllm_metrics from vllm.v1.metrics.stats import ( @@ -724,13 +725,23 @@ class PrometheusStatLogger(AggregateStatLoggerBase): for idx in engine_indexes } + request_tokens_buckets = histogram_buckets( + "request_tokens", max_model_len=max_model_len + ) + iteration_tokens_buckets = histogram_buckets("iteration_tokens") + request_params_n_buckets = histogram_buckets("request_params_n") + time_to_first_token_buckets = histogram_buckets("time_to_first_token") + inter_token_latency_buckets = histogram_buckets("inter_token_latency") + request_latency_buckets = histogram_buckets("request_latency") + request_num_preemptions_buckets = histogram_buckets("request_num_preemptions") + # # Histograms of counts # histogram_num_prompt_tokens_request = self._histogram_cls( name="vllm:request_prompt_tokens", documentation="Number of prefill tokens processed.", - buckets=build_1_2_5_buckets(max_model_len), + buckets=request_tokens_buckets, labelnames=labelnames, ) self.histogram_num_prompt_tokens_request = create_metric_per_engine( @@ -740,7 +751,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): histogram_num_generation_tokens_request = self._histogram_cls( name="vllm:request_generation_tokens", documentation="Number of generation tokens processed.", - buckets=build_1_2_5_buckets(max_model_len), + buckets=request_tokens_buckets, labelnames=labelnames, ) self.histogram_num_generation_tokens_request = create_metric_per_engine( @@ -753,7 +764,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): histogram_iteration_tokens = self._histogram_cls( name="vllm:iteration_tokens_total", documentation="Histogram of number of tokens per engine_step.", - buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384], + buckets=iteration_tokens_buckets, labelnames=labelnames, ) self.histogram_iteration_tokens = create_metric_per_engine( @@ -763,7 +774,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): histogram_max_num_generation_tokens_request = self._histogram_cls( name="vllm:request_max_num_generation_tokens", documentation="Histogram of maximum number of requested generation tokens.", - buckets=build_1_2_5_buckets(max_model_len), + buckets=request_tokens_buckets, labelnames=labelnames, ) self.histogram_max_num_generation_tokens_request = create_metric_per_engine( @@ -773,7 +784,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): histogram_n_request = self._histogram_cls( name="vllm:request_params_n", documentation="Histogram of the n request parameter.", - buckets=[1, 2, 5, 10, 20], + buckets=request_params_n_buckets, labelnames=labelnames, ) self.histogram_n_request = create_metric_per_engine( @@ -783,7 +794,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): histogram_max_tokens_request = self._histogram_cls( name="vllm:request_params_max_tokens", documentation="Histogram of the max_tokens request parameter.", - buckets=build_1_2_5_buckets(max_model_len), + buckets=request_tokens_buckets, labelnames=labelnames, ) self.histogram_max_tokens_request = create_metric_per_engine( @@ -796,30 +807,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): histogram_time_to_first_token = self._histogram_cls( name="vllm:time_to_first_token_seconds", documentation="Histogram of time to first token in seconds.", - buckets=[ - 0.001, - 0.005, - 0.01, - 0.02, - 0.04, - 0.06, - 0.08, - 0.1, - 0.25, - 0.5, - 0.75, - 1.0, - 2.5, - 5.0, - 7.5, - 10.0, - 20.0, - 40.0, - 80.0, - 160.0, - 640.0, - 2560.0, - ], + buckets=time_to_first_token_buckets, labelnames=labelnames, ) self.histogram_time_to_first_token = create_metric_per_engine( @@ -829,27 +817,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): histogram_inter_token_latency = self._histogram_cls( name="vllm:inter_token_latency_seconds", documentation="Histogram of inter-token latency in seconds.", - buckets=[ - 0.01, - 0.025, - 0.05, - 0.075, - 0.1, - 0.15, - 0.2, - 0.3, - 0.4, - 0.5, - 0.75, - 1.0, - 2.5, - 5.0, - 7.5, - 10.0, - 20.0, - 40.0, - 80.0, - ], + buckets=inter_token_latency_buckets, labelnames=labelnames, ) self.histogram_inter_token_latency = create_metric_per_engine( @@ -859,56 +827,13 @@ class PrometheusStatLogger(AggregateStatLoggerBase): histogram_request_time_per_output_token = self._histogram_cls( name="vllm:request_time_per_output_token_seconds", documentation="Histogram of time_per_output_token_seconds per request.", - buckets=[ - 0.01, - 0.025, - 0.05, - 0.075, - 0.1, - 0.15, - 0.2, - 0.3, - 0.4, - 0.5, - 0.75, - 1.0, - 2.5, - 5.0, - 7.5, - 10.0, - 20.0, - 40.0, - 80.0, - ], + buckets=inter_token_latency_buckets, labelnames=labelnames, ) self.histogram_request_time_per_output_token = create_metric_per_engine( histogram_request_time_per_output_token, per_engine_labelvalues ) - request_latency_buckets = [ - 0.3, - 0.5, - 0.8, - 1.0, - 1.5, - 2.0, - 2.5, - 5.0, - 10.0, - 15.0, - 20.0, - 30.0, - 40.0, - 50.0, - 60.0, - 120.0, - 240.0, - 480.0, - 960.0, - 1920.0, - 7680.0, - ] histogram_e2e_time_request = self._histogram_cls( name="vllm:e2e_request_latency_seconds", documentation="Histogram of e2e request latency in seconds.", @@ -962,7 +887,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): histogram_request_num_preemptions = self._histogram_cls( name="vllm:request_num_preemptions", documentation="Histogram of the number of times a request was preempted.", - buckets=[1, 2, 3, 4, 5, 10, 20], + buckets=request_num_preemptions_buckets, labelnames=labelnames, ) self.histogram_request_num_preemptions = create_metric_per_engine( @@ -975,7 +900,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): "Histogram of new KV tokens computed during prefill " "(excluding cached tokens)." ), - buckets=build_1_2_5_buckets(max_model_len), + buckets=request_tokens_buckets, labelnames=labelnames, ) self.histogram_prefill_kv_computed_request = create_metric_per_engine( @@ -986,29 +911,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase): # KV Cache residency metrics # if self.kv_cache_metrics_enabled: - kv_cache_residency_buckets = [ - 0.001, - 0.002, - 0.005, - 0.01, - 0.02, - 0.05, - 0.1, - 0.2, - 0.5, - 1, - 2, - 5, - 10, - 20, - 30, - 60, - 120, - 300, - 600, - 1200, - 1800, - ] + kv_cache_residency_buckets = histogram_buckets("kv_cache_residency") histogram_kv_block_lifetime = self._histogram_cls( name="vllm:kv_block_lifetime_seconds", @@ -1294,33 +1197,6 @@ class PrometheusStatLogger(AggregateStatLoggerBase): self.log_metrics_info("cache_config", self.vllm_config.cache_config) -def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]: - """ - Builds a list of buckets with increasing powers of 10 multiplied by - mantissa values until the value exceeds the specified maximum. - - """ - exponent = 0 - buckets: list[int] = [] - while True: - for m in mantissa_lst: - value = m * 10**exponent - if value <= max_value: - buckets.append(value) - else: - return buckets - exponent += 1 - - -def build_1_2_5_buckets(max_value: int) -> list[int]: - """ - Example: - >>> build_1_2_5_buckets(100) - [1, 2, 5, 10, 20, 50, 100] - """ - return build_buckets([1, 2, 5], max_value) - - class StatLoggerManager: """ StatLoggerManager: