[Metrics] Add --custom-histogram-buckets to override histogram bucket families (#48867)

The core engine histograms ship with default bucket boundaries tuned for
typical serving workloads. The `--custom-histogram-buckets` option replaces
the boundaries of one or more bucket families. Use it, for example, to track
sub-300ms latency SLOs with the request-phase histograms, whose smallest
default boundary is 0.3s.

Signed-off-by: Guy Stone <guys@spotify.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Guy Stone
2026-10-01 01:03:39 +00:00
committed by GitHub
co-authored by Claude Fable 5
parent a47e4a5568
commit 37d6174096
8 changed files with 278 additions and 11 deletions
+63
View File
@@ -1,7 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from functools import cached_property
from itertools import pairwise
from typing import Any, Literal
from packaging.version import parse
@@ -65,6 +67,18 @@ class ObservabilityConfig:
kv_cache_metrics_sample: float = Field(default=0.01, gt=0, le=1)
"""Sampling rate for KV cache metrics (0.0, 1.0]. Default 0.01 = 1% of blocks."""
custom_histogram_buckets: dict[str, list[float]] | None = None
"""Custom Prometheus histogram bucket boundaries, as a JSON mapping from
bucket-family key to a list of strictly increasing, positive, finite
upper bounds. When a family key is present, its list replaces the default
buckets for every histogram in that family; families not listed keep
their defaults. Known families: `request_latency`, `time_to_first_token`,
`inter_token_latency`, `iteration_tokens`, `request_params_n`,
`request_num_preemptions`, `request_tokens`, `kv_cache_residency`. Example:
`--custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.5]}'`.
Note that every extra bucket adds one time series per metric and label
set."""
cudagraph_metrics: bool = False
"""Enable CUDA graph metrics (number of padded/unpadded tokens, runtime cudagraph
dispatch modes, and their observed frequencies at every logging interval)."""
@@ -150,6 +164,55 @@ class ObservabilityConfig:
)
return value
@field_validator("custom_histogram_buckets", mode="before")
@classmethod
def _reject_bool_histogram_bounds(cls, value: object) -> object:
"""Reject booleans before pydantic silently coerces them to floats."""
if isinstance(value, dict):
for family, buckets in value.items():
if not isinstance(buckets, list):
continue
for bound in buckets:
if isinstance(bound, bool):
raise ValueError(
f"custom_histogram_buckets[{family!r}]: bound "
f"{bound!r} must be a number, not a boolean"
)
return value
@field_validator("custom_histogram_buckets")
@classmethod
def _validate_custom_histogram_buckets(
cls, value: dict[str, list[float]] | None
) -> dict[str, list[float]] | None:
if value is None:
return value
from vllm.v1.metrics.buckets import BUCKET_FAMILY_KEYS
for family, buckets in value.items():
if family not in BUCKET_FAMILY_KEYS:
raise ValueError(
f"custom_histogram_buckets: unknown bucket family "
f"{family!r}; known families: {sorted(BUCKET_FAMILY_KEYS)}"
)
if not buckets:
raise ValueError(
f"custom_histogram_buckets[{family!r}]: bucket list "
"must not be empty"
)
for bound in buckets:
if not math.isfinite(bound) or bound <= 0:
raise ValueError(
f"custom_histogram_buckets[{family!r}]: bound "
f"{bound!r} must be finite and greater than 0"
)
if any(a >= b for a, b in pairwise(buckets)):
raise ValueError(
f"custom_histogram_buckets[{family!r}]: bounds {buckets} "
"must be strictly increasing"
)
return value
@model_validator(mode="after")
def _validate_tracing_config(self):
if self.collect_detailed_traces and not self.otlp_traces_endpoint:
+8
View File
@@ -723,6 +723,9 @@ class EngineArgs:
kv_cache_metrics_sample: float = get_field(
ObservabilityConfig, "kv_cache_metrics_sample"
)
custom_histogram_buckets: dict[str, list[float]] | None = (
ObservabilityConfig.custom_histogram_buckets
)
cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics
enable_layerwise_nvtx_tracing: bool = (
ObservabilityConfig.enable_layerwise_nvtx_tracing
@@ -1633,6 +1636,10 @@ class EngineArgs:
"--kv-cache-metrics-sample",
**observability_kwargs["kv_cache_metrics_sample"],
)
observability_group.add_argument(
"--custom-histogram-buckets",
**observability_kwargs["custom_histogram_buckets"],
)
observability_group.add_argument(
"--cudagraph-metrics",
**observability_kwargs["cudagraph_metrics"],
@@ -2126,6 +2133,7 @@ class EngineArgs:
per_request_spec_decode_metrics=self.per_request_spec_decode_metrics,
kv_cache_metrics=self.kv_cache_metrics,
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
custom_histogram_buckets=self.custom_histogram_buckets,
cudagraph_metrics=self.cudagraph_metrics,
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
enable_mfu_metrics=self.enable_mfu_metrics,
+12 -3
View File
@@ -9,6 +9,7 @@ subsystems, such as the KV connector and offloading metrics, keep their own
boundaries.
"""
from collections.abc import Mapping
from typing import Literal, get_args
BucketFamilyKey = Literal[
@@ -205,23 +206,31 @@ def build_1_2_5_buckets(max_value: int) -> list[float]:
def histogram_buckets(
family: BucketFamilyKey,
max_model_len: int | None = None,
overrides: Mapping[str, list[float]] | None = None,
) -> list[float]:
"""Return the default bucket boundaries for a histogram family.
"""Return the bucket boundaries for a histogram family.
Args:
family: Canonical bucket family key.
max_model_len: Cap for the token-count series of the
`request_tokens` family; required for that family and ignored
otherwise.
overrides: Mapping from family key to replacement bucket bounds.
A present key replaces the family's defaults verbatim. Values
must already be validated; the single validation point is
`ObservabilityConfig.custom_histogram_buckets`.
Returns:
A fresh list of bucket upper bounds; callers may mutate it freely.
Raises:
ValueError: If `family` is `request_tokens` and `max_model_len`
is None.
ValueError: If `family` is `request_tokens`, no override is given,
and `max_model_len` is None.
"""
if overrides is not None and (custom := overrides.get(family)) is not None:
return list(custom)
if family == "request_tokens":
if max_model_len is None:
raise ValueError(
+23 -8
View File
@@ -739,15 +739,28 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
for idx in engine_indexes
}
custom_buckets = vllm_config.observability_config.custom_histogram_buckets
request_tokens_buckets = histogram_buckets(
"request_tokens", max_model_len=max_model_len
"request_tokens", max_model_len=max_model_len, overrides=custom_buckets
)
iteration_tokens_buckets = histogram_buckets(
"iteration_tokens", overrides=custom_buckets
)
request_params_n_buckets = histogram_buckets(
"request_params_n", overrides=custom_buckets
)
time_to_first_token_buckets = histogram_buckets(
"time_to_first_token", overrides=custom_buckets
)
inter_token_latency_buckets = histogram_buckets(
"inter_token_latency", overrides=custom_buckets
)
request_latency_buckets = histogram_buckets(
"request_latency", overrides=custom_buckets
)
request_num_preemptions_buckets = histogram_buckets(
"request_num_preemptions", overrides=custom_buckets
)
iteration_tokens_buckets = histogram_buckets("iteration_tokens")
request_params_n_buckets = histogram_buckets("request_params_n")
time_to_first_token_buckets = histogram_buckets("time_to_first_token")
inter_token_latency_buckets = histogram_buckets("inter_token_latency")
request_latency_buckets = histogram_buckets("request_latency")
request_num_preemptions_buckets = histogram_buckets("request_num_preemptions")
#
# Histograms of counts
@@ -925,7 +938,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
# KV Cache residency metrics
#
if self.kv_cache_metrics_enabled:
kv_cache_residency_buckets = histogram_buckets("kv_cache_residency")
kv_cache_residency_buckets = histogram_buckets(
"kv_cache_residency", overrides=custom_buckets
)
histogram_kv_block_lifetime = self._histogram_cls(
name="vllm:kv_block_lifetime_seconds",