[Metrics] Add --custom-histogram-buckets to override histogram bucket families (#48867)
The core engine histograms ship with default bucket boundaries tuned for typical serving workloads. The `--custom-histogram-buckets` option replaces the boundaries of one or more bucket families. Use it, for example, to track sub-300ms latency SLOs with the request-phase histograms, whose smallest default boundary is 0.3s. Signed-off-by: Guy Stone <guys@spotify.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
a47e4a5568
commit
37d6174096
@@ -1,7 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from functools import cached_property
|
||||
from itertools import pairwise
|
||||
from typing import Any, Literal
|
||||
|
||||
from packaging.version import parse
|
||||
@@ -65,6 +67,18 @@ class ObservabilityConfig:
|
||||
kv_cache_metrics_sample: float = Field(default=0.01, gt=0, le=1)
|
||||
"""Sampling rate for KV cache metrics (0.0, 1.0]. Default 0.01 = 1% of blocks."""
|
||||
|
||||
custom_histogram_buckets: dict[str, list[float]] | None = None
|
||||
"""Custom Prometheus histogram bucket boundaries, as a JSON mapping from
|
||||
bucket-family key to a list of strictly increasing, positive, finite
|
||||
upper bounds. When a family key is present, its list replaces the default
|
||||
buckets for every histogram in that family; families not listed keep
|
||||
their defaults. Known families: `request_latency`, `time_to_first_token`,
|
||||
`inter_token_latency`, `iteration_tokens`, `request_params_n`,
|
||||
`request_num_preemptions`, `request_tokens`, `kv_cache_residency`. Example:
|
||||
`--custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.5]}'`.
|
||||
Note that every extra bucket adds one time series per metric and label
|
||||
set."""
|
||||
|
||||
cudagraph_metrics: bool = False
|
||||
"""Enable CUDA graph metrics (number of padded/unpadded tokens, runtime cudagraph
|
||||
dispatch modes, and their observed frequencies at every logging interval)."""
|
||||
@@ -150,6 +164,55 @@ class ObservabilityConfig:
|
||||
)
|
||||
return value
|
||||
|
||||
@field_validator("custom_histogram_buckets", mode="before")
|
||||
@classmethod
|
||||
def _reject_bool_histogram_bounds(cls, value: object) -> object:
|
||||
"""Reject booleans before pydantic silently coerces them to floats."""
|
||||
if isinstance(value, dict):
|
||||
for family, buckets in value.items():
|
||||
if not isinstance(buckets, list):
|
||||
continue
|
||||
for bound in buckets:
|
||||
if isinstance(bound, bool):
|
||||
raise ValueError(
|
||||
f"custom_histogram_buckets[{family!r}]: bound "
|
||||
f"{bound!r} must be a number, not a boolean"
|
||||
)
|
||||
return value
|
||||
|
||||
@field_validator("custom_histogram_buckets")
|
||||
@classmethod
|
||||
def _validate_custom_histogram_buckets(
|
||||
cls, value: dict[str, list[float]] | None
|
||||
) -> dict[str, list[float]] | None:
|
||||
if value is None:
|
||||
return value
|
||||
from vllm.v1.metrics.buckets import BUCKET_FAMILY_KEYS
|
||||
|
||||
for family, buckets in value.items():
|
||||
if family not in BUCKET_FAMILY_KEYS:
|
||||
raise ValueError(
|
||||
f"custom_histogram_buckets: unknown bucket family "
|
||||
f"{family!r}; known families: {sorted(BUCKET_FAMILY_KEYS)}"
|
||||
)
|
||||
if not buckets:
|
||||
raise ValueError(
|
||||
f"custom_histogram_buckets[{family!r}]: bucket list "
|
||||
"must not be empty"
|
||||
)
|
||||
for bound in buckets:
|
||||
if not math.isfinite(bound) or bound <= 0:
|
||||
raise ValueError(
|
||||
f"custom_histogram_buckets[{family!r}]: bound "
|
||||
f"{bound!r} must be finite and greater than 0"
|
||||
)
|
||||
if any(a >= b for a, b in pairwise(buckets)):
|
||||
raise ValueError(
|
||||
f"custom_histogram_buckets[{family!r}]: bounds {buckets} "
|
||||
"must be strictly increasing"
|
||||
)
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_tracing_config(self):
|
||||
if self.collect_detailed_traces and not self.otlp_traces_endpoint:
|
||||
|
||||
@@ -723,6 +723,9 @@ class EngineArgs:
|
||||
kv_cache_metrics_sample: float = get_field(
|
||||
ObservabilityConfig, "kv_cache_metrics_sample"
|
||||
)
|
||||
custom_histogram_buckets: dict[str, list[float]] | None = (
|
||||
ObservabilityConfig.custom_histogram_buckets
|
||||
)
|
||||
cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics
|
||||
enable_layerwise_nvtx_tracing: bool = (
|
||||
ObservabilityConfig.enable_layerwise_nvtx_tracing
|
||||
@@ -1633,6 +1636,10 @@ class EngineArgs:
|
||||
"--kv-cache-metrics-sample",
|
||||
**observability_kwargs["kv_cache_metrics_sample"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--custom-histogram-buckets",
|
||||
**observability_kwargs["custom_histogram_buckets"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--cudagraph-metrics",
|
||||
**observability_kwargs["cudagraph_metrics"],
|
||||
@@ -2126,6 +2133,7 @@ class EngineArgs:
|
||||
per_request_spec_decode_metrics=self.per_request_spec_decode_metrics,
|
||||
kv_cache_metrics=self.kv_cache_metrics,
|
||||
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
|
||||
custom_histogram_buckets=self.custom_histogram_buckets,
|
||||
cudagraph_metrics=self.cudagraph_metrics,
|
||||
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
|
||||
enable_mfu_metrics=self.enable_mfu_metrics,
|
||||
|
||||
@@ -9,6 +9,7 @@ subsystems, such as the KV connector and offloading metrics, keep their own
|
||||
boundaries.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Literal, get_args
|
||||
|
||||
BucketFamilyKey = Literal[
|
||||
@@ -205,23 +206,31 @@ def build_1_2_5_buckets(max_value: int) -> list[float]:
|
||||
def histogram_buckets(
|
||||
family: BucketFamilyKey,
|
||||
max_model_len: int | None = None,
|
||||
overrides: Mapping[str, list[float]] | None = None,
|
||||
) -> list[float]:
|
||||
"""Return the default bucket boundaries for a histogram family.
|
||||
"""Return the bucket boundaries for a histogram family.
|
||||
|
||||
Args:
|
||||
family: Canonical bucket family key.
|
||||
max_model_len: Cap for the token-count series of the
|
||||
`request_tokens` family; required for that family and ignored
|
||||
otherwise.
|
||||
overrides: Mapping from family key to replacement bucket bounds.
|
||||
A present key replaces the family's defaults verbatim. Values
|
||||
must already be validated; the single validation point is
|
||||
`ObservabilityConfig.custom_histogram_buckets`.
|
||||
|
||||
Returns:
|
||||
A fresh list of bucket upper bounds; callers may mutate it freely.
|
||||
|
||||
Raises:
|
||||
ValueError: If `family` is `request_tokens` and `max_model_len`
|
||||
is None.
|
||||
ValueError: If `family` is `request_tokens`, no override is given,
|
||||
and `max_model_len` is None.
|
||||
|
||||
|
||||
"""
|
||||
if overrides is not None and (custom := overrides.get(family)) is not None:
|
||||
return list(custom)
|
||||
if family == "request_tokens":
|
||||
if max_model_len is None:
|
||||
raise ValueError(
|
||||
|
||||
@@ -739,15 +739,28 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
for idx in engine_indexes
|
||||
}
|
||||
|
||||
custom_buckets = vllm_config.observability_config.custom_histogram_buckets
|
||||
request_tokens_buckets = histogram_buckets(
|
||||
"request_tokens", max_model_len=max_model_len
|
||||
"request_tokens", max_model_len=max_model_len, overrides=custom_buckets
|
||||
)
|
||||
iteration_tokens_buckets = histogram_buckets(
|
||||
"iteration_tokens", overrides=custom_buckets
|
||||
)
|
||||
request_params_n_buckets = histogram_buckets(
|
||||
"request_params_n", overrides=custom_buckets
|
||||
)
|
||||
time_to_first_token_buckets = histogram_buckets(
|
||||
"time_to_first_token", overrides=custom_buckets
|
||||
)
|
||||
inter_token_latency_buckets = histogram_buckets(
|
||||
"inter_token_latency", overrides=custom_buckets
|
||||
)
|
||||
request_latency_buckets = histogram_buckets(
|
||||
"request_latency", overrides=custom_buckets
|
||||
)
|
||||
request_num_preemptions_buckets = histogram_buckets(
|
||||
"request_num_preemptions", overrides=custom_buckets
|
||||
)
|
||||
iteration_tokens_buckets = histogram_buckets("iteration_tokens")
|
||||
request_params_n_buckets = histogram_buckets("request_params_n")
|
||||
time_to_first_token_buckets = histogram_buckets("time_to_first_token")
|
||||
inter_token_latency_buckets = histogram_buckets("inter_token_latency")
|
||||
request_latency_buckets = histogram_buckets("request_latency")
|
||||
request_num_preemptions_buckets = histogram_buckets("request_num_preemptions")
|
||||
|
||||
#
|
||||
# Histograms of counts
|
||||
@@ -925,7 +938,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
# KV Cache residency metrics
|
||||
#
|
||||
if self.kv_cache_metrics_enabled:
|
||||
kv_cache_residency_buckets = histogram_buckets("kv_cache_residency")
|
||||
kv_cache_residency_buckets = histogram_buckets(
|
||||
"kv_cache_residency", overrides=custom_buckets
|
||||
)
|
||||
|
||||
histogram_kv_block_lifetime = self._histogram_cls(
|
||||
name="vllm:kv_block_lifetime_seconds",
|
||||
|
||||
Reference in New Issue
Block a user