From 37d6174096fd99a8d362c9f5b379f7ec59deedb8 Mon Sep 17 00:00:00 2001 From: Guy Stone Date: Wed, 30 Sep 2026 21:03:39 -0400 Subject: [PATCH] [Metrics] Add --custom-histogram-buckets to override histogram bucket families (#48867) The core engine histograms ship with default bucket boundaries tuned for typical serving workloads. The `--custom-histogram-buckets` option replaces the boundaries of one or more bucket families. Use it, for example, to track sub-300ms latency SLOs with the request-phase histograms, whose smallest default boundary is 0.3s. Signed-off-by: Guy Stone Co-authored-by: Claude Fable 5 --- docs/usage/metrics.md | 42 +++++++++++++++ tests/config/test_observability.py | 53 ++++++++++++++++++ tests/v1/engine/test_engine_args.py | 31 +++++++++++ tests/v1/metrics/test_histogram_buckets.py | 46 ++++++++++++++++ vllm/config/observability.py | 63 ++++++++++++++++++++++ vllm/engine/arg_utils.py | 8 +++ vllm/v1/metrics/buckets.py | 15 ++++-- vllm/v1/metrics/loggers.py | 31 ++++++++--- 8 files changed, 278 insertions(+), 11 deletions(-) create mode 100644 tests/config/test_observability.py diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md index 8254a7f420..5cb5eebc55 100644 --- a/docs/usage/metrics.md +++ b/docs/usage/metrics.md @@ -75,6 +75,48 @@ These metrics are available via `--enable-mfu-metrics`: --8<-- "gen:metrics-mfu" +## Custom Histogram Buckets + +The core engine histograms ship with default bucket boundaries tuned for +typical serving workloads. The `--custom-histogram-buckets` option replaces +the boundaries of one or more *bucket families* — exactly the histograms +listed in the table below — with your own list; histograms owned by other +subsystems (for example, the NIXL connector metrics) are not affected. Use it, +for example, to track sub-300ms latency SLOs with the request-phase +histograms, whose smallest default boundary is 0.3s: + +```bash +vllm serve Qwen/Qwen3-0.6B \ + --custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 30.0]}' +``` + +Each family key overrides a group of related histograms: + +| Family key | Histograms | +| --- | --- | +| `request_latency` | `vllm:e2e_request_latency_seconds`, `vllm:request_queue_time_seconds`, `vllm:request_inference_time_seconds`, `vllm:request_prefill_time_seconds`, `vllm:request_decode_time_seconds` | +| `time_to_first_token` | `vllm:time_to_first_token_seconds` | +| `inter_token_latency` | `vllm:inter_token_latency_seconds`, `vllm:request_time_per_output_token_seconds` | +| `iteration_tokens` | `vllm:iteration_tokens_total` | +| `request_params_n` | `vllm:request_params_n` | +| `request_num_preemptions` | `vllm:request_num_preemptions` | +| `request_tokens` | `vllm:request_prompt_tokens`, `vllm:request_generation_tokens`, `vllm:request_max_num_generation_tokens`, `vllm:request_params_max_tokens`, `vllm:request_prefill_kv_computed_tokens` | +| `kv_cache_residency` | `vllm:kv_block_lifetime_seconds`, `vllm:kv_block_idle_before_evict_seconds`, `vllm:kv_block_reuse_gap_seconds` | + +Bucket values must be positive, finite, and strictly increasing; unknown +family keys are rejected at startup. Families you do not list keep their +default boundaries. The `request_tokens` defaults normally scale with +`--max-model-len`; an override replaces that computed list. The +`kv_cache_residency` family only takes effect when `--kv-cache-metrics` is +enabled. + +!!! warning "Bucket cardinality" + Every bucket boundary creates one extra time series per metric and per + label combination (model and engine index, multiplied under data-parallel + deployments). Long bucket lists inflate Prometheus storage, scrape sizes, + and query costs. Keep custom lists short, and only override the families + you actively monitor. + ## Deprecation Policy Note: when metrics are deprecated in version `X.Y`, they are hidden in version `X.Y+1` diff --git a/tests/config/test_observability.py b/tests/config/test_observability.py new file mode 100644 index 0000000000..c62f01cc05 --- /dev/null +++ b/tests/config/test_observability.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for ObservabilityConfig validation.""" + +import pytest +from pydantic import ValidationError + +from vllm.config.observability import ObservabilityConfig + + +def test_custom_histogram_buckets_default_none(): + assert ObservabilityConfig().custom_histogram_buckets is None + + +def test_custom_histogram_buckets_valid(): + config = ObservabilityConfig( + custom_histogram_buckets={ + "request_latency": [0.01, 0.05, 0.1, 0.5], + "time_to_first_token": [1, 2, 5], + } + ) + assert config.custom_histogram_buckets == { + "request_latency": [0.01, 0.05, 0.1, 0.5], + "time_to_first_token": [1.0, 2.0, 5.0], + } + + +@pytest.mark.parametrize( + ("buckets", "match"), + [ + ({"bogus": [1.0, 2.0]}, "unknown bucket family 'bogus'"), + ({"request_latency": []}, "must not be empty"), + ({"request_latency": [0.0, 1.0]}, "must be finite and greater than 0"), + ({"request_latency": [-1.0, 1.0]}, "must be finite and greater than 0"), + ( + {"request_latency": [1.0, float("inf")]}, + "must be finite and greater than 0", + ), + ({"request_latency": [float("nan")]}, "must be finite and greater than 0"), + ({"request_latency": [1.0, 1.0]}, "must be strictly increasing"), + ({"request_latency": [2.0, 1.0]}, "must be strictly increasing"), + ({"request_latency": [True, 2.0]}, "must be a number, not a boolean"), + ({"request_latency": [False]}, "must be a number, not a boolean"), + ( + {"request_latency": [1.0, 2.0], "request_tokens": [16, 8]}, + r"request_tokens.*must be strictly increasing", + ), + ], +) +def test_custom_histogram_buckets_rejects_invalid(buckets, match): + """Each validation rule must name the field, the value, and the rule.""" + with pytest.raises(ValidationError, match=match): + ObservabilityConfig(custom_histogram_buckets=buckets) diff --git a/tests/v1/engine/test_engine_args.py b/tests/v1/engine/test_engine_args.py index 3f9f617bc7..5516ed840e 100644 --- a/tests/v1/engine/test_engine_args.py +++ b/tests/v1/engine/test_engine_args.py @@ -108,6 +108,37 @@ def test_mm_prefix_lm_raises_batched_tokens_floor(): assert vllm_config.scheduler_config.max_num_batched_tokens >= 2496 +def test_custom_histogram_buckets_from_cli(): + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + + # Unset: byte-identical default behavior. + args = parser.parse_args([]) + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() + assert vllm_config.observability_config.custom_histogram_buckets is None + + # A JSON mapping round-trips into the config verbatim. + args = parser.parse_args( + [ + "--custom-histogram-buckets", + '{"request_latency": [0.01, 0.05, 0.1, 0.5]}', + ] + ) + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() + assert vllm_config.observability_config.custom_histogram_buckets == { + "request_latency": [0.01, 0.05, 0.1, 0.5] + } + + # Malformed JSON is rejected at argument-parsing time. + parser.exit_on_error = False + with pytest.raises(ArgumentError): + parser.parse_args(["--custom-histogram-buckets", "{not json"]) + + # An unknown family key parses as JSON but fails config validation. + args = parser.parse_args(["--custom-histogram-buckets", '{"bogus": [1.0, 2.0]}']) + with pytest.raises(ValueError, match="unknown bucket family"): + EngineArgs.from_cli_args(args=args).create_engine_config() + + def test_data_parallel_start_rank_zero_infers_hybrid_lb(): """An explicit --data-parallel-start-rank 0 must be treated the same as any other explicit start rank when inferring hybrid LB mode, not as diff --git a/tests/v1/metrics/test_histogram_buckets.py b/tests/v1/metrics/test_histogram_buckets.py index 54cc31da94..cf905b5d8b 100644 --- a/tests/v1/metrics/test_histogram_buckets.py +++ b/tests/v1/metrics/test_histogram_buckets.py @@ -271,3 +271,49 @@ def test_prometheus_logger_default_buckets(): assert found[metric_name] == [float(b) for b in expected], metric_name finally: unregister_vllm_metrics() + + +def test_histogram_buckets_override_precedence(): + """A present family key wins verbatim; absent keys keep defaults.""" + overrides = { + "request_latency": [0.5, 1.0], + "request_tokens": [16.0, 4096.0], + } + got = histogram_buckets("request_latency", overrides=overrides) + assert got == [0.5, 1.0] + assert got is not overrides["request_latency"] + # An override replaces the max_model_len-derived series entirely. + assert histogram_buckets("request_tokens", overrides=overrides) == [ + 16.0, + 4096.0, + ] + assert ( + histogram_buckets("time_to_first_token", overrides=overrides) + == DEFAULT_BUCKET_SNAPSHOTS["time_to_first_token"] + ) + + +def test_prometheus_logger_applies_overrides(): + """Every family must honour an override, with a distinct list each. + + Overriding all of them at once is what catches a histogram wired to the + defaults instead of the override, and a family wired to the wrong list. + """ + overridden: dict[str, list[float]] = { + family: [index + 1.0, index + 2.0] + for index, family in enumerate(sorted(BUCKET_FAMILY_KEYS)) + } + config = build_logger_config( + ObservabilityConfig( + kv_cache_metrics=True, + custom_histogram_buckets=overridden, + ) + ) + try: + PrometheusStatLogger(config) + found = collect_histogram_buckets() + assert set(found) == set(METRIC_FAMILIES) + for metric_name, family in METRIC_FAMILIES.items(): + assert found[metric_name] == overridden[family], metric_name + finally: + unregister_vllm_metrics() diff --git a/vllm/config/observability.py b/vllm/config/observability.py index a39962a075..28a9313603 100644 --- a/vllm/config/observability.py +++ b/vllm/config/observability.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import math from functools import cached_property +from itertools import pairwise from typing import Any, Literal from packaging.version import parse @@ -65,6 +67,18 @@ class ObservabilityConfig: kv_cache_metrics_sample: float = Field(default=0.01, gt=0, le=1) """Sampling rate for KV cache metrics (0.0, 1.0]. Default 0.01 = 1% of blocks.""" + custom_histogram_buckets: dict[str, list[float]] | None = None + """Custom Prometheus histogram bucket boundaries, as a JSON mapping from + bucket-family key to a list of strictly increasing, positive, finite + upper bounds. When a family key is present, its list replaces the default + buckets for every histogram in that family; families not listed keep + their defaults. Known families: `request_latency`, `time_to_first_token`, + `inter_token_latency`, `iteration_tokens`, `request_params_n`, + `request_num_preemptions`, `request_tokens`, `kv_cache_residency`. Example: + `--custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.5]}'`. + Note that every extra bucket adds one time series per metric and label + set.""" + cudagraph_metrics: bool = False """Enable CUDA graph metrics (number of padded/unpadded tokens, runtime cudagraph dispatch modes, and their observed frequencies at every logging interval).""" @@ -150,6 +164,55 @@ class ObservabilityConfig: ) return value + @field_validator("custom_histogram_buckets", mode="before") + @classmethod + def _reject_bool_histogram_bounds(cls, value: object) -> object: + """Reject booleans before pydantic silently coerces them to floats.""" + if isinstance(value, dict): + for family, buckets in value.items(): + if not isinstance(buckets, list): + continue + for bound in buckets: + if isinstance(bound, bool): + raise ValueError( + f"custom_histogram_buckets[{family!r}]: bound " + f"{bound!r} must be a number, not a boolean" + ) + return value + + @field_validator("custom_histogram_buckets") + @classmethod + def _validate_custom_histogram_buckets( + cls, value: dict[str, list[float]] | None + ) -> dict[str, list[float]] | None: + if value is None: + return value + from vllm.v1.metrics.buckets import BUCKET_FAMILY_KEYS + + for family, buckets in value.items(): + if family not in BUCKET_FAMILY_KEYS: + raise ValueError( + f"custom_histogram_buckets: unknown bucket family " + f"{family!r}; known families: {sorted(BUCKET_FAMILY_KEYS)}" + ) + if not buckets: + raise ValueError( + f"custom_histogram_buckets[{family!r}]: bucket list " + "must not be empty" + ) + for bound in buckets: + if not math.isfinite(bound) or bound <= 0: + raise ValueError( + f"custom_histogram_buckets[{family!r}]: bound " + f"{bound!r} must be finite and greater than 0" + ) + if any(a >= b for a, b in pairwise(buckets)): + raise ValueError( + f"custom_histogram_buckets[{family!r}]: bounds {buckets} " + "must be strictly increasing" + ) + return value + @model_validator(mode="after") def _validate_tracing_config(self): if self.collect_detailed_traces and not self.otlp_traces_endpoint: diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index d641b1fbab..ce24ac28f2 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -723,6 +723,9 @@ class EngineArgs: kv_cache_metrics_sample: float = get_field( ObservabilityConfig, "kv_cache_metrics_sample" ) + custom_histogram_buckets: dict[str, list[float]] | None = ( + ObservabilityConfig.custom_histogram_buckets + ) cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics enable_layerwise_nvtx_tracing: bool = ( ObservabilityConfig.enable_layerwise_nvtx_tracing @@ -1633,6 +1636,10 @@ class EngineArgs: "--kv-cache-metrics-sample", **observability_kwargs["kv_cache_metrics_sample"], ) + observability_group.add_argument( + "--custom-histogram-buckets", + **observability_kwargs["custom_histogram_buckets"], + ) observability_group.add_argument( "--cudagraph-metrics", **observability_kwargs["cudagraph_metrics"], @@ -2126,6 +2133,7 @@ class EngineArgs: per_request_spec_decode_metrics=self.per_request_spec_decode_metrics, kv_cache_metrics=self.kv_cache_metrics, kv_cache_metrics_sample=self.kv_cache_metrics_sample, + custom_histogram_buckets=self.custom_histogram_buckets, cudagraph_metrics=self.cudagraph_metrics, enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing, enable_mfu_metrics=self.enable_mfu_metrics, diff --git a/vllm/v1/metrics/buckets.py b/vllm/v1/metrics/buckets.py index 7b9efb8f6d..213caabdfb 100644 --- a/vllm/v1/metrics/buckets.py +++ b/vllm/v1/metrics/buckets.py @@ -9,6 +9,7 @@ subsystems, such as the KV connector and offloading metrics, keep their own boundaries. """ +from collections.abc import Mapping from typing import Literal, get_args BucketFamilyKey = Literal[ @@ -205,23 +206,31 @@ def build_1_2_5_buckets(max_value: int) -> list[float]: def histogram_buckets( family: BucketFamilyKey, max_model_len: int | None = None, + overrides: Mapping[str, list[float]] | None = None, ) -> list[float]: - """Return the default bucket boundaries for a histogram family. + """Return the bucket boundaries for a histogram family. Args: family: Canonical bucket family key. max_model_len: Cap for the token-count series of the `request_tokens` family; required for that family and ignored otherwise. + overrides: Mapping from family key to replacement bucket bounds. + A present key replaces the family's defaults verbatim. Values + must already be validated; the single validation point is + `ObservabilityConfig.custom_histogram_buckets`. Returns: A fresh list of bucket upper bounds; callers may mutate it freely. Raises: - ValueError: If `family` is `request_tokens` and `max_model_len` - is None. + ValueError: If `family` is `request_tokens`, no override is given, + and `max_model_len` is None. + """ + if overrides is not None and (custom := overrides.get(family)) is not None: + return list(custom) if family == "request_tokens": if max_model_len is None: raise ValueError( diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index 26a4142df9..ac93b26a8f 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -739,15 +739,28 @@ class PrometheusStatLogger(AggregateStatLoggerBase): for idx in engine_indexes } + custom_buckets = vllm_config.observability_config.custom_histogram_buckets request_tokens_buckets = histogram_buckets( - "request_tokens", max_model_len=max_model_len + "request_tokens", max_model_len=max_model_len, overrides=custom_buckets + ) + iteration_tokens_buckets = histogram_buckets( + "iteration_tokens", overrides=custom_buckets + ) + request_params_n_buckets = histogram_buckets( + "request_params_n", overrides=custom_buckets + ) + time_to_first_token_buckets = histogram_buckets( + "time_to_first_token", overrides=custom_buckets + ) + inter_token_latency_buckets = histogram_buckets( + "inter_token_latency", overrides=custom_buckets + ) + request_latency_buckets = histogram_buckets( + "request_latency", overrides=custom_buckets + ) + request_num_preemptions_buckets = histogram_buckets( + "request_num_preemptions", overrides=custom_buckets ) - iteration_tokens_buckets = histogram_buckets("iteration_tokens") - request_params_n_buckets = histogram_buckets("request_params_n") - time_to_first_token_buckets = histogram_buckets("time_to_first_token") - inter_token_latency_buckets = histogram_buckets("inter_token_latency") - request_latency_buckets = histogram_buckets("request_latency") - request_num_preemptions_buckets = histogram_buckets("request_num_preemptions") # # Histograms of counts @@ -925,7 +938,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase): # KV Cache residency metrics # if self.kv_cache_metrics_enabled: - kv_cache_residency_buckets = histogram_buckets("kv_cache_residency") + kv_cache_residency_buckets = histogram_buckets( + "kv_cache_residency", overrides=custom_buckets + ) histogram_kv_block_lifetime = self._histogram_cls( name="vllm:kv_block_lifetime_seconds",