[Metrics] Add --custom-histogram-buckets to override histogram bucket families (#48867)
The core engine histograms ship with default bucket boundaries tuned for typical serving workloads. The `--custom-histogram-buckets` option replaces the boundaries of one or more bucket families. Use it, for example, to track sub-300ms latency SLOs with the request-phase histograms, whose smallest default boundary is 0.3s. Signed-off-by: Guy Stone <guys@spotify.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
a47e4a5568
commit
37d6174096
@@ -75,6 +75,48 @@ These metrics are available via `--enable-mfu-metrics`:
|
||||
|
||||
--8<-- "gen:metrics-mfu"
|
||||
|
||||
## Custom Histogram Buckets
|
||||
|
||||
The core engine histograms ship with default bucket boundaries tuned for
|
||||
typical serving workloads. The `--custom-histogram-buckets` option replaces
|
||||
the boundaries of one or more *bucket families* — exactly the histograms
|
||||
listed in the table below — with your own list; histograms owned by other
|
||||
subsystems (for example, the NIXL connector metrics) are not affected. Use it,
|
||||
for example, to track sub-300ms latency SLOs with the request-phase
|
||||
histograms, whose smallest default boundary is 0.3s:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-0.6B \
|
||||
--custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 30.0]}'
|
||||
```
|
||||
|
||||
Each family key overrides a group of related histograms:
|
||||
|
||||
| Family key | Histograms |
|
||||
| --- | --- |
|
||||
| `request_latency` | `vllm:e2e_request_latency_seconds`, `vllm:request_queue_time_seconds`, `vllm:request_inference_time_seconds`, `vllm:request_prefill_time_seconds`, `vllm:request_decode_time_seconds` |
|
||||
| `time_to_first_token` | `vllm:time_to_first_token_seconds` |
|
||||
| `inter_token_latency` | `vllm:inter_token_latency_seconds`, `vllm:request_time_per_output_token_seconds` |
|
||||
| `iteration_tokens` | `vllm:iteration_tokens_total` |
|
||||
| `request_params_n` | `vllm:request_params_n` |
|
||||
| `request_num_preemptions` | `vllm:request_num_preemptions` |
|
||||
| `request_tokens` | `vllm:request_prompt_tokens`, `vllm:request_generation_tokens`, `vllm:request_max_num_generation_tokens`, `vllm:request_params_max_tokens`, `vllm:request_prefill_kv_computed_tokens` |
|
||||
| `kv_cache_residency` | `vllm:kv_block_lifetime_seconds`, `vllm:kv_block_idle_before_evict_seconds`, `vllm:kv_block_reuse_gap_seconds` |
|
||||
|
||||
Bucket values must be positive, finite, and strictly increasing; unknown
|
||||
family keys are rejected at startup. Families you do not list keep their
|
||||
default boundaries. The `request_tokens` defaults normally scale with
|
||||
`--max-model-len`; an override replaces that computed list. The
|
||||
`kv_cache_residency` family only takes effect when `--kv-cache-metrics` is
|
||||
enabled.
|
||||
|
||||
!!! warning "Bucket cardinality"
|
||||
Every bucket boundary creates one extra time series per metric and per
|
||||
label combination (model and engine index, multiplied under data-parallel
|
||||
deployments). Long bucket lists inflate Prometheus storage, scrape sizes,
|
||||
and query costs. Keep custom lists short, and only override the families
|
||||
you actively monitor.
|
||||
|
||||
## Deprecation Policy
|
||||
|
||||
Note: when metrics are deprecated in version `X.Y`, they are hidden in version `X.Y+1`
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ObservabilityConfig validation."""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.config.observability import ObservabilityConfig
|
||||
|
||||
|
||||
def test_custom_histogram_buckets_default_none():
|
||||
assert ObservabilityConfig().custom_histogram_buckets is None
|
||||
|
||||
|
||||
def test_custom_histogram_buckets_valid():
|
||||
config = ObservabilityConfig(
|
||||
custom_histogram_buckets={
|
||||
"request_latency": [0.01, 0.05, 0.1, 0.5],
|
||||
"time_to_first_token": [1, 2, 5],
|
||||
}
|
||||
)
|
||||
assert config.custom_histogram_buckets == {
|
||||
"request_latency": [0.01, 0.05, 0.1, 0.5],
|
||||
"time_to_first_token": [1.0, 2.0, 5.0],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("buckets", "match"),
|
||||
[
|
||||
({"bogus": [1.0, 2.0]}, "unknown bucket family 'bogus'"),
|
||||
({"request_latency": []}, "must not be empty"),
|
||||
({"request_latency": [0.0, 1.0]}, "must be finite and greater than 0"),
|
||||
({"request_latency": [-1.0, 1.0]}, "must be finite and greater than 0"),
|
||||
(
|
||||
{"request_latency": [1.0, float("inf")]},
|
||||
"must be finite and greater than 0",
|
||||
),
|
||||
({"request_latency": [float("nan")]}, "must be finite and greater than 0"),
|
||||
({"request_latency": [1.0, 1.0]}, "must be strictly increasing"),
|
||||
({"request_latency": [2.0, 1.0]}, "must be strictly increasing"),
|
||||
({"request_latency": [True, 2.0]}, "must be a number, not a boolean"),
|
||||
({"request_latency": [False]}, "must be a number, not a boolean"),
|
||||
(
|
||||
{"request_latency": [1.0, 2.0], "request_tokens": [16, 8]},
|
||||
r"request_tokens.*must be strictly increasing",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_custom_histogram_buckets_rejects_invalid(buckets, match):
|
||||
"""Each validation rule must name the field, the value, and the rule."""
|
||||
with pytest.raises(ValidationError, match=match):
|
||||
ObservabilityConfig(custom_histogram_buckets=buckets)
|
||||
@@ -108,6 +108,37 @@ def test_mm_prefix_lm_raises_batched_tokens_floor():
|
||||
assert vllm_config.scheduler_config.max_num_batched_tokens >= 2496
|
||||
|
||||
|
||||
def test_custom_histogram_buckets_from_cli():
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
|
||||
# Unset: byte-identical default behavior.
|
||||
args = parser.parse_args([])
|
||||
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
|
||||
assert vllm_config.observability_config.custom_histogram_buckets is None
|
||||
|
||||
# A JSON mapping round-trips into the config verbatim.
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--custom-histogram-buckets",
|
||||
'{"request_latency": [0.01, 0.05, 0.1, 0.5]}',
|
||||
]
|
||||
)
|
||||
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
|
||||
assert vllm_config.observability_config.custom_histogram_buckets == {
|
||||
"request_latency": [0.01, 0.05, 0.1, 0.5]
|
||||
}
|
||||
|
||||
# Malformed JSON is rejected at argument-parsing time.
|
||||
parser.exit_on_error = False
|
||||
with pytest.raises(ArgumentError):
|
||||
parser.parse_args(["--custom-histogram-buckets", "{not json"])
|
||||
|
||||
# An unknown family key parses as JSON but fails config validation.
|
||||
args = parser.parse_args(["--custom-histogram-buckets", '{"bogus": [1.0, 2.0]}'])
|
||||
with pytest.raises(ValueError, match="unknown bucket family"):
|
||||
EngineArgs.from_cli_args(args=args).create_engine_config()
|
||||
|
||||
|
||||
def test_data_parallel_start_rank_zero_infers_hybrid_lb():
|
||||
"""An explicit --data-parallel-start-rank 0 must be treated the same as
|
||||
any other explicit start rank when inferring hybrid LB mode, not as
|
||||
|
||||
@@ -271,3 +271,49 @@ def test_prometheus_logger_default_buckets():
|
||||
assert found[metric_name] == [float(b) for b in expected], metric_name
|
||||
finally:
|
||||
unregister_vllm_metrics()
|
||||
|
||||
|
||||
def test_histogram_buckets_override_precedence():
|
||||
"""A present family key wins verbatim; absent keys keep defaults."""
|
||||
overrides = {
|
||||
"request_latency": [0.5, 1.0],
|
||||
"request_tokens": [16.0, 4096.0],
|
||||
}
|
||||
got = histogram_buckets("request_latency", overrides=overrides)
|
||||
assert got == [0.5, 1.0]
|
||||
assert got is not overrides["request_latency"]
|
||||
# An override replaces the max_model_len-derived series entirely.
|
||||
assert histogram_buckets("request_tokens", overrides=overrides) == [
|
||||
16.0,
|
||||
4096.0,
|
||||
]
|
||||
assert (
|
||||
histogram_buckets("time_to_first_token", overrides=overrides)
|
||||
== DEFAULT_BUCKET_SNAPSHOTS["time_to_first_token"]
|
||||
)
|
||||
|
||||
|
||||
def test_prometheus_logger_applies_overrides():
|
||||
"""Every family must honour an override, with a distinct list each.
|
||||
|
||||
Overriding all of them at once is what catches a histogram wired to the
|
||||
defaults instead of the override, and a family wired to the wrong list.
|
||||
"""
|
||||
overridden: dict[str, list[float]] = {
|
||||
family: [index + 1.0, index + 2.0]
|
||||
for index, family in enumerate(sorted(BUCKET_FAMILY_KEYS))
|
||||
}
|
||||
config = build_logger_config(
|
||||
ObservabilityConfig(
|
||||
kv_cache_metrics=True,
|
||||
custom_histogram_buckets=overridden,
|
||||
)
|
||||
)
|
||||
try:
|
||||
PrometheusStatLogger(config)
|
||||
found = collect_histogram_buckets()
|
||||
assert set(found) == set(METRIC_FAMILIES)
|
||||
for metric_name, family in METRIC_FAMILIES.items():
|
||||
assert found[metric_name] == overridden[family], metric_name
|
||||
finally:
|
||||
unregister_vllm_metrics()
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from functools import cached_property
|
||||
from itertools import pairwise
|
||||
from typing import Any, Literal
|
||||
|
||||
from packaging.version import parse
|
||||
@@ -65,6 +67,18 @@ class ObservabilityConfig:
|
||||
kv_cache_metrics_sample: float = Field(default=0.01, gt=0, le=1)
|
||||
"""Sampling rate for KV cache metrics (0.0, 1.0]. Default 0.01 = 1% of blocks."""
|
||||
|
||||
custom_histogram_buckets: dict[str, list[float]] | None = None
|
||||
"""Custom Prometheus histogram bucket boundaries, as a JSON mapping from
|
||||
bucket-family key to a list of strictly increasing, positive, finite
|
||||
upper bounds. When a family key is present, its list replaces the default
|
||||
buckets for every histogram in that family; families not listed keep
|
||||
their defaults. Known families: `request_latency`, `time_to_first_token`,
|
||||
`inter_token_latency`, `iteration_tokens`, `request_params_n`,
|
||||
`request_num_preemptions`, `request_tokens`, `kv_cache_residency`. Example:
|
||||
`--custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.5]}'`.
|
||||
Note that every extra bucket adds one time series per metric and label
|
||||
set."""
|
||||
|
||||
cudagraph_metrics: bool = False
|
||||
"""Enable CUDA graph metrics (number of padded/unpadded tokens, runtime cudagraph
|
||||
dispatch modes, and their observed frequencies at every logging interval)."""
|
||||
@@ -150,6 +164,55 @@ class ObservabilityConfig:
|
||||
)
|
||||
return value
|
||||
|
||||
@field_validator("custom_histogram_buckets", mode="before")
|
||||
@classmethod
|
||||
def _reject_bool_histogram_bounds(cls, value: object) -> object:
|
||||
"""Reject booleans before pydantic silently coerces them to floats."""
|
||||
if isinstance(value, dict):
|
||||
for family, buckets in value.items():
|
||||
if not isinstance(buckets, list):
|
||||
continue
|
||||
for bound in buckets:
|
||||
if isinstance(bound, bool):
|
||||
raise ValueError(
|
||||
f"custom_histogram_buckets[{family!r}]: bound "
|
||||
f"{bound!r} must be a number, not a boolean"
|
||||
)
|
||||
return value
|
||||
|
||||
@field_validator("custom_histogram_buckets")
|
||||
@classmethod
|
||||
def _validate_custom_histogram_buckets(
|
||||
cls, value: dict[str, list[float]] | None
|
||||
) -> dict[str, list[float]] | None:
|
||||
if value is None:
|
||||
return value
|
||||
from vllm.v1.metrics.buckets import BUCKET_FAMILY_KEYS
|
||||
|
||||
for family, buckets in value.items():
|
||||
if family not in BUCKET_FAMILY_KEYS:
|
||||
raise ValueError(
|
||||
f"custom_histogram_buckets: unknown bucket family "
|
||||
f"{family!r}; known families: {sorted(BUCKET_FAMILY_KEYS)}"
|
||||
)
|
||||
if not buckets:
|
||||
raise ValueError(
|
||||
f"custom_histogram_buckets[{family!r}]: bucket list "
|
||||
"must not be empty"
|
||||
)
|
||||
for bound in buckets:
|
||||
if not math.isfinite(bound) or bound <= 0:
|
||||
raise ValueError(
|
||||
f"custom_histogram_buckets[{family!r}]: bound "
|
||||
f"{bound!r} must be finite and greater than 0"
|
||||
)
|
||||
if any(a >= b for a, b in pairwise(buckets)):
|
||||
raise ValueError(
|
||||
f"custom_histogram_buckets[{family!r}]: bounds {buckets} "
|
||||
"must be strictly increasing"
|
||||
)
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_tracing_config(self):
|
||||
if self.collect_detailed_traces and not self.otlp_traces_endpoint:
|
||||
|
||||
@@ -723,6 +723,9 @@ class EngineArgs:
|
||||
kv_cache_metrics_sample: float = get_field(
|
||||
ObservabilityConfig, "kv_cache_metrics_sample"
|
||||
)
|
||||
custom_histogram_buckets: dict[str, list[float]] | None = (
|
||||
ObservabilityConfig.custom_histogram_buckets
|
||||
)
|
||||
cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics
|
||||
enable_layerwise_nvtx_tracing: bool = (
|
||||
ObservabilityConfig.enable_layerwise_nvtx_tracing
|
||||
@@ -1633,6 +1636,10 @@ class EngineArgs:
|
||||
"--kv-cache-metrics-sample",
|
||||
**observability_kwargs["kv_cache_metrics_sample"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--custom-histogram-buckets",
|
||||
**observability_kwargs["custom_histogram_buckets"],
|
||||
)
|
||||
observability_group.add_argument(
|
||||
"--cudagraph-metrics",
|
||||
**observability_kwargs["cudagraph_metrics"],
|
||||
@@ -2126,6 +2133,7 @@ class EngineArgs:
|
||||
per_request_spec_decode_metrics=self.per_request_spec_decode_metrics,
|
||||
kv_cache_metrics=self.kv_cache_metrics,
|
||||
kv_cache_metrics_sample=self.kv_cache_metrics_sample,
|
||||
custom_histogram_buckets=self.custom_histogram_buckets,
|
||||
cudagraph_metrics=self.cudagraph_metrics,
|
||||
enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing,
|
||||
enable_mfu_metrics=self.enable_mfu_metrics,
|
||||
|
||||
@@ -9,6 +9,7 @@ subsystems, such as the KV connector and offloading metrics, keep their own
|
||||
boundaries.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Literal, get_args
|
||||
|
||||
BucketFamilyKey = Literal[
|
||||
@@ -205,23 +206,31 @@ def build_1_2_5_buckets(max_value: int) -> list[float]:
|
||||
def histogram_buckets(
|
||||
family: BucketFamilyKey,
|
||||
max_model_len: int | None = None,
|
||||
overrides: Mapping[str, list[float]] | None = None,
|
||||
) -> list[float]:
|
||||
"""Return the default bucket boundaries for a histogram family.
|
||||
"""Return the bucket boundaries for a histogram family.
|
||||
|
||||
Args:
|
||||
family: Canonical bucket family key.
|
||||
max_model_len: Cap for the token-count series of the
|
||||
`request_tokens` family; required for that family and ignored
|
||||
otherwise.
|
||||
overrides: Mapping from family key to replacement bucket bounds.
|
||||
A present key replaces the family's defaults verbatim. Values
|
||||
must already be validated; the single validation point is
|
||||
`ObservabilityConfig.custom_histogram_buckets`.
|
||||
|
||||
Returns:
|
||||
A fresh list of bucket upper bounds; callers may mutate it freely.
|
||||
|
||||
Raises:
|
||||
ValueError: If `family` is `request_tokens` and `max_model_len`
|
||||
is None.
|
||||
ValueError: If `family` is `request_tokens`, no override is given,
|
||||
and `max_model_len` is None.
|
||||
|
||||
|
||||
"""
|
||||
if overrides is not None and (custom := overrides.get(family)) is not None:
|
||||
return list(custom)
|
||||
if family == "request_tokens":
|
||||
if max_model_len is None:
|
||||
raise ValueError(
|
||||
|
||||
@@ -739,15 +739,28 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
for idx in engine_indexes
|
||||
}
|
||||
|
||||
custom_buckets = vllm_config.observability_config.custom_histogram_buckets
|
||||
request_tokens_buckets = histogram_buckets(
|
||||
"request_tokens", max_model_len=max_model_len
|
||||
"request_tokens", max_model_len=max_model_len, overrides=custom_buckets
|
||||
)
|
||||
iteration_tokens_buckets = histogram_buckets(
|
||||
"iteration_tokens", overrides=custom_buckets
|
||||
)
|
||||
request_params_n_buckets = histogram_buckets(
|
||||
"request_params_n", overrides=custom_buckets
|
||||
)
|
||||
time_to_first_token_buckets = histogram_buckets(
|
||||
"time_to_first_token", overrides=custom_buckets
|
||||
)
|
||||
inter_token_latency_buckets = histogram_buckets(
|
||||
"inter_token_latency", overrides=custom_buckets
|
||||
)
|
||||
request_latency_buckets = histogram_buckets(
|
||||
"request_latency", overrides=custom_buckets
|
||||
)
|
||||
request_num_preemptions_buckets = histogram_buckets(
|
||||
"request_num_preemptions", overrides=custom_buckets
|
||||
)
|
||||
iteration_tokens_buckets = histogram_buckets("iteration_tokens")
|
||||
request_params_n_buckets = histogram_buckets("request_params_n")
|
||||
time_to_first_token_buckets = histogram_buckets("time_to_first_token")
|
||||
inter_token_latency_buckets = histogram_buckets("inter_token_latency")
|
||||
request_latency_buckets = histogram_buckets("request_latency")
|
||||
request_num_preemptions_buckets = histogram_buckets("request_num_preemptions")
|
||||
|
||||
#
|
||||
# Histograms of counts
|
||||
@@ -925,7 +938,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
# KV Cache residency metrics
|
||||
#
|
||||
if self.kv_cache_metrics_enabled:
|
||||
kv_cache_residency_buckets = histogram_buckets("kv_cache_residency")
|
||||
kv_cache_residency_buckets = histogram_buckets(
|
||||
"kv_cache_residency", overrides=custom_buckets
|
||||
)
|
||||
|
||||
histogram_kv_block_lifetime = self._histogram_cls(
|
||||
name="vllm:kv_block_lifetime_seconds",
|
||||
|
||||
Reference in New Issue
Block a user