[Metrics] Add --custom-histogram-buckets to override histogram bucket families (#48867)

The core engine histograms ship with default bucket boundaries tuned for
typical serving workloads. The `--custom-histogram-buckets` option replaces
the boundaries of one or more bucket families. Use it, for example, to track
sub-300ms latency SLOs with the request-phase histograms, whose smallest
default boundary is 0.3s.

Signed-off-by: Guy Stone <guys@spotify.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Guy Stone
2026-10-01 01:03:39 +00:00
committed by GitHub
co-authored by Claude Fable 5
parent a47e4a5568
commit 37d6174096
8 changed files with 278 additions and 11 deletions
+42
View File
@@ -75,6 +75,48 @@ These metrics are available via `--enable-mfu-metrics`:
--8<-- "gen:metrics-mfu"
## Custom Histogram Buckets
The core engine histograms ship with default bucket boundaries tuned for
typical serving workloads. The `--custom-histogram-buckets` option replaces
the boundaries of one or more *bucket families* — exactly the histograms
listed in the table below — with your own list; histograms owned by other
subsystems (for example, the NIXL connector metrics) are not affected. Use it,
for example, to track sub-300ms latency SLOs with the request-phase
histograms, whose smallest default boundary is 0.3s:
```bash
vllm serve Qwen/Qwen3-0.6B \
--custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 30.0]}'
```
Each family key overrides a group of related histograms:
| Family key | Histograms |
| --- | --- |
| `request_latency` | `vllm:e2e_request_latency_seconds`, `vllm:request_queue_time_seconds`, `vllm:request_inference_time_seconds`, `vllm:request_prefill_time_seconds`, `vllm:request_decode_time_seconds` |
| `time_to_first_token` | `vllm:time_to_first_token_seconds` |
| `inter_token_latency` | `vllm:inter_token_latency_seconds`, `vllm:request_time_per_output_token_seconds` |
| `iteration_tokens` | `vllm:iteration_tokens_total` |
| `request_params_n` | `vllm:request_params_n` |
| `request_num_preemptions` | `vllm:request_num_preemptions` |
| `request_tokens` | `vllm:request_prompt_tokens`, `vllm:request_generation_tokens`, `vllm:request_max_num_generation_tokens`, `vllm:request_params_max_tokens`, `vllm:request_prefill_kv_computed_tokens` |
| `kv_cache_residency` | `vllm:kv_block_lifetime_seconds`, `vllm:kv_block_idle_before_evict_seconds`, `vllm:kv_block_reuse_gap_seconds` |
Bucket values must be positive, finite, and strictly increasing; unknown
family keys are rejected at startup. Families you do not list keep their
default boundaries. The `request_tokens` defaults normally scale with
`--max-model-len`; an override replaces that computed list. The
`kv_cache_residency` family only takes effect when `--kv-cache-metrics` is
enabled.
!!! warning "Bucket cardinality"
Every bucket boundary creates one extra time series per metric and per
label combination (model and engine index, multiplied under data-parallel
deployments). Long bucket lists inflate Prometheus storage, scrape sizes,
and query costs. Keep custom lists short, and only override the families
you actively monitor.
## Deprecation Policy
Note: when metrics are deprecated in version `X.Y`, they are hidden in version `X.Y+1`
+53
View File
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ObservabilityConfig validation."""
import pytest
from pydantic import ValidationError
from vllm.config.observability import ObservabilityConfig
def test_custom_histogram_buckets_default_none():
assert ObservabilityConfig().custom_histogram_buckets is None
def test_custom_histogram_buckets_valid():
config = ObservabilityConfig(
custom_histogram_buckets={
"request_latency": [0.01, 0.05, 0.1, 0.5],
"time_to_first_token": [1, 2, 5],
}
)
assert config.custom_histogram_buckets == {
"request_latency": [0.01, 0.05, 0.1, 0.5],
"time_to_first_token": [1.0, 2.0, 5.0],
}
@pytest.mark.parametrize(
("buckets", "match"),
[
({"bogus": [1.0, 2.0]}, "unknown bucket family 'bogus'"),
({"request_latency": []}, "must not be empty"),
({"request_latency": [0.0, 1.0]}, "must be finite and greater than 0"),
({"request_latency": [-1.0, 1.0]}, "must be finite and greater than 0"),
(
{"request_latency": [1.0, float("inf")]},
"must be finite and greater than 0",
),
({"request_latency": [float("nan")]}, "must be finite and greater than 0"),
({"request_latency": [1.0, 1.0]}, "must be strictly increasing"),
({"request_latency": [2.0, 1.0]}, "must be strictly increasing"),
({"request_latency": [True, 2.0]}, "must be a number, not a boolean"),
({"request_latency": [False]}, "must be a number, not a boolean"),
(
{"request_latency": [1.0, 2.0], "request_tokens": [16, 8]},
r"request_tokens.*must be strictly increasing",
),
],
)
def test_custom_histogram_buckets_rejects_invalid(buckets, match):
"""Each validation rule must name the field, the value, and the rule."""
with pytest.raises(ValidationError, match=match):
ObservabilityConfig(custom_histogram_buckets=buckets)
+31
View File
@@ -108,6 +108,37 @@ def test_mm_prefix_lm_raises_batched_tokens_floor():
assert vllm_config.scheduler_config.max_num_batched_tokens >= 2496
def test_custom_histogram_buckets_from_cli():
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
# Unset: byte-identical default behavior.
args = parser.parse_args([])
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
assert vllm_config.observability_config.custom_histogram_buckets is None
# A JSON mapping round-trips into the config verbatim.
args = parser.parse_args(
[
"--custom-histogram-buckets",
'{"request_latency": [0.01, 0.05, 0.1, 0.5]}',
]
)
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
assert vllm_config.observability_config.custom_histogram_buckets == {
"request_latency": [0.01, 0.05, 0.1, 0.5]
}
# Malformed JSON is rejected at argument-parsing time.
parser.exit_on_error = False
with pytest.raises(ArgumentError):
parser.parse_args(["--custom-histogram-buckets", "{not json"])
# An unknown family key parses as JSON but fails config validation.
args = parser.parse_args(["--custom-histogram-buckets", '{"bogus": [1.0, 2.0]}'])
with pytest.raises(ValueError, match="unknown bucket family"):
EngineArgs.from_cli_args(args=args).create_engine_config()
def test_data_parallel_start_rank_zero_infers_hybrid_lb():
"""An explicit --data-parallel-start-rank 0 must be treated the same as
any other explicit start rank when inferring hybrid LB mode, not as
@@ -271,3 +271,49 @@ def test_prometheus_logger_default_buckets():
assert found[metric_name] == [float(b) for b in expected], metric_name
finally:
unregister_vllm_metrics()
def test_histogram_buckets_override_precedence():
"""A present family key wins verbatim; absent keys keep defaults."""
overrides = {
"request_latency": [0.5, 1.0],
"request_tokens": [16.0, 4096.0],
}
got = histogram_buckets("request_latency", overrides=overrides)
assert got == [0.5, 1.0]
assert got is not overrides["request_latency"]
# An override replaces the max_model_len-derived series entirely.
assert histogram_buckets("request_tokens", overrides=overrides) == [
16.0,
4096.0,
]
assert (
histogram_buckets("time_to_first_token", overrides=overrides)
== DEFAULT_BUCKET_SNAPSHOTS["time_to_first_token"]
)
def test_prometheus_logger_applies_overrides():
"""Every family must honour an override, with a distinct list each.
Overriding all of them at once is what catches a histogram wired to the
defaults instead of the override, and a family wired to the wrong list.
"""
overridden: dict[str, list[float]] = {
family: [index + 1.0, index + 2.0]
for index, family in enumerate(sorted(BUCKET_FAMILY_KEYS))
}
config = build_logger_config(
ObservabilityConfig(
kv_cache_metrics=True,
custom_histogram_buckets=overridden,
)
)
try:
PrometheusStatLogger(config)
found = collect_histogram_buckets()
assert set(found) == set(METRIC_FAMILIES)
for metric_name, family in METRIC_FAMILIES.items():
assert found[metric_name] == overridden[family], metric_name
finally:
unregister_vllm_metrics()
+63
View File
@@ -1,7 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from functools import cached_property
from itertools import pairwise
from typing import Any, Literal
from packaging.version import parse
@@ -65,6 +67,18 @@ class ObservabilityConfig:
kv_cache_metrics_sample: float = Field(default=0.01, gt=0, le=1)
"""Sampling rate for KV cache metrics (0.0, 1.0]. Default 0.01 = 1% of blocks."""
custom_histogram_buckets: dict[str, list[float]] | None = None
"""Custom Prometheus histogram bucket boundaries, as a JSON mapping from
bucket-family key to a list of strictly increasing, positive, finite
upper bounds. When a family key is present, its list replaces the default
buckets for every histogram in that family; families not listed keep
their defaults. Known families: `request_latency`, `time_to_first_token`,
`inter_token_latency`, `iteration_tokens`, `request_params_n`,
`request_num_preemptions`, `request_tokens`, `kv_cache_residency`. Example:
`--custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.5]}'`.
Note that every extra bucket adds one time series per metric and label
set."""
cudagraph_metrics: bool = False
"""Enable CUDA graph metrics (number of padded/unpadded tokens, runtime cudagraph
dispatch modes, and their observed frequencies at every logging interval)."""
@@ -150,6 +164,55 @@ class ObservabilityConfig:
)
return value
@field_validator("custom_histogram_buckets", mode="before")
@classmethod
def _reject_bool_histogram_bounds(cls, value: object) -> object:
"""Reject booleans before pydantic silently coerces them to floats."""
if isinstance(value, dict):
for family, buckets in value.items():
if not isinstance(buckets, list):
continue
for bound in buckets:
if isinstance(bound, bool):
raise ValueError(
f"custom_histogram_buckets[{family!r}]: bound "
f"{bound!r} must be a number, not a boolean"
)
return value
@field_validator("custom_histogram_buckets")
@classmethod
def _validate_custom_histogram_buckets(
cls, value: dict[str, list[float]] | None
) -> dict[str, list[float]] | None:
if value is None:
return value
from vllm.v1.metrics.buckets import BUCKET_FAMILY_KEYS
for family, buckets in value.items():
if family not in BUCKET_FAMILY_KEYS:
raise ValueError(
f"custom_histogram_buckets: unknown bucket family "
f"{family!r}; known families: {sorted(BUCKET_FAMILY_KEYS)}"
)
if not buckets:
raise ValueError(
f"custom_histogram_buckets[{family!r}]: bucket list "
"must not be empty"
)
for bound in buckets:
if not math.isfinite(bound) or bound <= 0:
raise ValueError(
f"custom_histogram_buckets[{family!r}]: bound "
f"{bound!r} must be finite and greater than 0"
)
if any(a >= b for a, b in pairwise(buckets)):
raise ValueError(
f"custom_histogram_buckets[{family!r}]: bounds {buckets} "
"must be strictly increasing"
)
return value
@model_validator(mode="after")
def _validate_tracing_config(self):
if self.collect_detailed_traces and not self.otlp_traces_endpoint:
+8
View File
@@ -723,6 +723,9 @@ class EngineArgs:
kv_cache_metrics_sample: float = get_field(
ObservabilityConfig, "kv_cache_metrics_sample"
)
custom_histogram_buckets: dict[str, list[float]] | None = (
ObservabilityConfig.custom_histogram_buckets
)
cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics
enable_layerwise_nvtx_tracing: bool = (
ObservabilityConfig.enable_layerwise_nvtx_tracing
@@ -1633,6 +1636,10 @@ class EngineArgs:
"--kv-cache-metrics-sample",
**observability_kwargs["kv_cache_metrics_sample"],
)
observability_group.add_argument(
"--custom-histogram-buckets",
**observability_kwargs["custom_histogram_buckets"],
)
observability_group.add_argument(
"--cudagraph-metrics",
**observability_kwargs["cudagraph_metrics"],
@@ -2126,6 +2133,7 @@ class EngineArgs:
per_request_spec_decode_metrics=self.per_request_spec_decode_metrics,
kv_cache_metrics=self.kv_cache_metrics,
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
custom_histogram_buckets=self.custom_histogram_buckets,
cudagraph_metrics=self.cudagraph_metrics,
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
enable_mfu_metrics=self.enable_mfu_metrics,
+12 -3
View File
@@ -9,6 +9,7 @@ subsystems, such as the KV connector and offloading metrics, keep their own
boundaries.
"""
from collections.abc import Mapping
from typing import Literal, get_args
BucketFamilyKey = Literal[
@@ -205,23 +206,31 @@ def build_1_2_5_buckets(max_value: int) -> list[float]:
def histogram_buckets(
family: BucketFamilyKey,
max_model_len: int | None = None,
overrides: Mapping[str, list[float]] | None = None,
) -> list[float]:
"""Return the default bucket boundaries for a histogram family.
"""Return the bucket boundaries for a histogram family.
Args:
family: Canonical bucket family key.
max_model_len: Cap for the token-count series of the
`request_tokens` family; required for that family and ignored
otherwise.
overrides: Mapping from family key to replacement bucket bounds.
A present key replaces the family's defaults verbatim. Values
must already be validated; the single validation point is
`ObservabilityConfig.custom_histogram_buckets`.
Returns:
A fresh list of bucket upper bounds; callers may mutate it freely.
Raises:
ValueError: If `family` is `request_tokens` and `max_model_len`
is None.
ValueError: If `family` is `request_tokens`, no override is given,
and `max_model_len` is None.
"""
if overrides is not None and (custom := overrides.get(family)) is not None:
return list(custom)
if family == "request_tokens":
if max_model_len is None:
raise ValueError(
+23 -8
View File
@@ -739,15 +739,28 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
for idx in engine_indexes
}
custom_buckets = vllm_config.observability_config.custom_histogram_buckets
request_tokens_buckets = histogram_buckets(
"request_tokens", max_model_len=max_model_len
"request_tokens", max_model_len=max_model_len, overrides=custom_buckets
)
iteration_tokens_buckets = histogram_buckets(
"iteration_tokens", overrides=custom_buckets
)
request_params_n_buckets = histogram_buckets(
"request_params_n", overrides=custom_buckets
)
time_to_first_token_buckets = histogram_buckets(
"time_to_first_token", overrides=custom_buckets
)
inter_token_latency_buckets = histogram_buckets(
"inter_token_latency", overrides=custom_buckets
)
request_latency_buckets = histogram_buckets(
"request_latency", overrides=custom_buckets
)
request_num_preemptions_buckets = histogram_buckets(
"request_num_preemptions", overrides=custom_buckets
)
iteration_tokens_buckets = histogram_buckets("iteration_tokens")
request_params_n_buckets = histogram_buckets("request_params_n")
time_to_first_token_buckets = histogram_buckets("time_to_first_token")
inter_token_latency_buckets = histogram_buckets("inter_token_latency")
request_latency_buckets = histogram_buckets("request_latency")
request_num_preemptions_buckets = histogram_buckets("request_num_preemptions")
#
# Histograms of counts
@@ -925,7 +938,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
# KV Cache residency metrics
#
if self.kv_cache_metrics_enabled:
kv_cache_residency_buckets = histogram_buckets("kv_cache_residency")
kv_cache_residency_buckets = histogram_buckets(
"kv_cache_residency", overrides=custom_buckets
)
histogram_kv_block_lifetime = self._histogram_cls(
name="vllm:kv_block_lifetime_seconds",