[Metrics] Add --custom-histogram-buckets to override histogram bucket families (#48867)
The core engine histograms ship with default bucket boundaries tuned for typical serving workloads. The `--custom-histogram-buckets` option replaces the boundaries of one or more bucket families. Use it, for example, to track sub-300ms latency SLOs with the request-phase histograms, whose smallest default boundary is 0.3s. Signed-off-by: Guy Stone <guys@spotify.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
a47e4a5568
commit
37d6174096
@@ -108,6 +108,37 @@ def test_mm_prefix_lm_raises_batched_tokens_floor():
|
||||
assert vllm_config.scheduler_config.max_num_batched_tokens >= 2496
|
||||
|
||||
|
||||
def test_custom_histogram_buckets_from_cli():
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
|
||||
# Unset: byte-identical default behavior.
|
||||
args = parser.parse_args([])
|
||||
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
|
||||
assert vllm_config.observability_config.custom_histogram_buckets is None
|
||||
|
||||
# A JSON mapping round-trips into the config verbatim.
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--custom-histogram-buckets",
|
||||
'{"request_latency": [0.01, 0.05, 0.1, 0.5]}',
|
||||
]
|
||||
)
|
||||
vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config()
|
||||
assert vllm_config.observability_config.custom_histogram_buckets == {
|
||||
"request_latency": [0.01, 0.05, 0.1, 0.5]
|
||||
}
|
||||
|
||||
# Malformed JSON is rejected at argument-parsing time.
|
||||
parser.exit_on_error = False
|
||||
with pytest.raises(ArgumentError):
|
||||
parser.parse_args(["--custom-histogram-buckets", "{not json"])
|
||||
|
||||
# An unknown family key parses as JSON but fails config validation.
|
||||
args = parser.parse_args(["--custom-histogram-buckets", '{"bogus": [1.0, 2.0]}'])
|
||||
with pytest.raises(ValueError, match="unknown bucket family"):
|
||||
EngineArgs.from_cli_args(args=args).create_engine_config()
|
||||
|
||||
|
||||
def test_data_parallel_start_rank_zero_infers_hybrid_lb():
|
||||
"""An explicit --data-parallel-start-rank 0 must be treated the same as
|
||||
any other explicit start rank when inferring hybrid LB mode, not as
|
||||
|
||||
Reference in New Issue
Block a user