[Metrics] Add --custom-histogram-buckets to override histogram bucket families (#48867)

The core engine histograms ship with default bucket boundaries tuned for
typical serving workloads. The `--custom-histogram-buckets` option replaces
the boundaries of one or more bucket families. Use it, for example, to track
sub-300ms latency SLOs with the request-phase histograms, whose smallest
default boundary is 0.3s.

Signed-off-by: Guy Stone <guys@spotify.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Guy Stone
2026-10-01 01:03:39 +00:00
committed by GitHub
co-authored by Claude Fable 5
parent a47e4a5568
commit 37d6174096
8 changed files with 278 additions and 11 deletions
+53
View File
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ObservabilityConfig validation."""
import pytest
from pydantic import ValidationError
from vllm.config.observability import ObservabilityConfig
def test_custom_histogram_buckets_default_none():
assert ObservabilityConfig().custom_histogram_buckets is None
def test_custom_histogram_buckets_valid():
config = ObservabilityConfig(
custom_histogram_buckets={
"request_latency": [0.01, 0.05, 0.1, 0.5],
"time_to_first_token": [1, 2, 5],
}
)
assert config.custom_histogram_buckets == {
"request_latency": [0.01, 0.05, 0.1, 0.5],
"time_to_first_token": [1.0, 2.0, 5.0],
}
@pytest.mark.parametrize(
("buckets", "match"),
[
({"bogus": [1.0, 2.0]}, "unknown bucket family 'bogus'"),
({"request_latency": []}, "must not be empty"),
({"request_latency": [0.0, 1.0]}, "must be finite and greater than 0"),
({"request_latency": [-1.0, 1.0]}, "must be finite and greater than 0"),
(
{"request_latency": [1.0, float("inf")]},
"must be finite and greater than 0",
),
({"request_latency": [float("nan")]}, "must be finite and greater than 0"),
({"request_latency": [1.0, 1.0]}, "must be strictly increasing"),
({"request_latency": [2.0, 1.0]}, "must be strictly increasing"),
({"request_latency": [True, 2.0]}, "must be a number, not a boolean"),
({"request_latency": [False]}, "must be a number, not a boolean"),
(
{"request_latency": [1.0, 2.0], "request_tokens": [16, 8]},
r"request_tokens.*must be strictly increasing",
),
],
)
def test_custom_histogram_buckets_rejects_invalid(buckets, match):
"""Each validation rule must name the field, the value, and the rule."""
with pytest.raises(ValidationError, match=match):
ObservabilityConfig(custom_histogram_buckets=buckets)
+31
View File
@@ -108,6 +108,37 @@ def test_mm_prefix_lm_raises_batched_tokens_floor():
assert vllm_config.scheduler_config.max_num_batched_tokens >= 2496
def test_custom_histogram_buckets_from_cli():
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
# Unset: byte-identical default behavior.
args = parser.parse_args([])
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
assert vllm_config.observability_config.custom_histogram_buckets is None
# A JSON mapping round-trips into the config verbatim.
args = parser.parse_args(
[
"--custom-histogram-buckets",
'{"request_latency": [0.01, 0.05, 0.1, 0.5]}',
]
)
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
assert vllm_config.observability_config.custom_histogram_buckets == {
"request_latency": [0.01, 0.05, 0.1, 0.5]
}
# Malformed JSON is rejected at argument-parsing time.
parser.exit_on_error = False
with pytest.raises(ArgumentError):
parser.parse_args(["--custom-histogram-buckets", "{not json"])
# An unknown family key parses as JSON but fails config validation.
args = parser.parse_args(["--custom-histogram-buckets", '{"bogus": [1.0, 2.0]}'])
with pytest.raises(ValueError, match="unknown bucket family"):
EngineArgs.from_cli_args(args=args).create_engine_config()
def test_data_parallel_start_rank_zero_infers_hybrid_lb():
"""An explicit --data-parallel-start-rank 0 must be treated the same as
any other explicit start rank when inferring hybrid LB mode, not as
@@ -271,3 +271,49 @@ def test_prometheus_logger_default_buckets():
assert found[metric_name] == [float(b) for b in expected], metric_name
finally:
unregister_vllm_metrics()
def test_histogram_buckets_override_precedence():
"""A present family key wins verbatim; absent keys keep defaults."""
overrides = {
"request_latency": [0.5, 1.0],
"request_tokens": [16.0, 4096.0],
}
got = histogram_buckets("request_latency", overrides=overrides)
assert got == [0.5, 1.0]
assert got is not overrides["request_latency"]
# An override replaces the max_model_len-derived series entirely.
assert histogram_buckets("request_tokens", overrides=overrides) == [
16.0,
4096.0,
]
assert (
histogram_buckets("time_to_first_token", overrides=overrides)
== DEFAULT_BUCKET_SNAPSHOTS["time_to_first_token"]
)
def test_prometheus_logger_applies_overrides():
"""Every family must honour an override, with a distinct list each.
Overriding all of them at once is what catches a histogram wired to the
defaults instead of the override, and a family wired to the wrong list.
"""
overridden: dict[str, list[float]] = {
family: [index + 1.0, index + 2.0]
for index, family in enumerate(sorted(BUCKET_FAMILY_KEYS))
}
config = build_logger_config(
ObservabilityConfig(
kv_cache_metrics=True,
custom_histogram_buckets=overridden,
)
)
try:
PrometheusStatLogger(config)
found = collect_histogram_buckets()
assert set(found) == set(METRIC_FAMILIES)
for metric_name, family in METRIC_FAMILIES.items():
assert found[metric_name] == overridden[family], metric_name
finally:
unregister_vllm_metrics()