[Metrics] Consolidate Prometheus histogram bucket defaults into a single module (#48866)

Signed-off-by: Guy Stone <guys@spotify.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Guy Stone
2026-09-11 10:12:36 -04:00
committed by GitHub
co-authored by Claude Fable 5
parent c191787a68
commit d5a9d0f59b
3 changed files with 524 additions and 147 deletions
+273
View File
@@ -0,0 +1,273 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Snapshot tests locking the default Prometheus histogram buckets.
The expected lists are deliberately hard-coded literals rather than imports:
any change to a default bucket boundary must show up as a failing test here.
"""
import prometheus_client
import pytest
from vllm.config import ModelConfig, VllmConfig
from vllm.config.observability import ObservabilityConfig
from vllm.v1.metrics.buckets import (
BUCKET_FAMILY_KEYS,
build_1_2_5_buckets,
histogram_buckets,
)
from vllm.v1.metrics.loggers import PrometheusStatLogger
from vllm.v1.metrics.prometheus import unregister_vllm_metrics
pytestmark = pytest.mark.cpu_test
DEFAULT_BUCKET_SNAPSHOTS: dict[str, list[float]] = {
"request_latency": [
0.3,
0.5,
0.8,
1.0,
1.5,
2.0,
2.5,
5.0,
10.0,
15.0,
20.0,
30.0,
40.0,
50.0,
60.0,
120.0,
240.0,
480.0,
960.0,
1920.0,
7680.0,
],
"time_to_first_token": [
0.001,
0.005,
0.01,
0.02,
0.04,
0.06,
0.08,
0.1,
0.25,
0.5,
0.75,
1.0,
2.5,
5.0,
7.5,
10.0,
20.0,
40.0,
80.0,
160.0,
640.0,
2560.0,
],
"inter_token_latency": [
0.01,
0.025,
0.05,
0.075,
0.1,
0.15,
0.2,
0.3,
0.4,
0.5,
0.75,
1.0,
2.5,
5.0,
7.5,
10.0,
20.0,
40.0,
80.0,
],
"iteration_tokens": [
1,
8,
16,
32,
64,
128,
256,
512,
1024,
2048,
4096,
8192,
16384,
],
"request_params_n": [1, 2, 5, 10, 20],
"request_num_preemptions": [1, 2, 3, 4, 5, 10, 20],
"kv_cache_residency": [
0.001,
0.002,
0.005,
0.01,
0.02,
0.05,
0.1,
0.2,
0.5,
1,
2,
5,
10,
20,
30,
60,
120,
300,
600,
1200,
1800,
],
}
METRIC_FAMILIES: dict[str, str] = {
"vllm:e2e_request_latency_seconds": "request_latency",
"vllm:request_queue_time_seconds": "request_latency",
"vllm:request_inference_time_seconds": "request_latency",
"vllm:request_prefill_time_seconds": "request_latency",
"vllm:request_decode_time_seconds": "request_latency",
"vllm:time_to_first_token_seconds": "time_to_first_token",
"vllm:inter_token_latency_seconds": "inter_token_latency",
"vllm:request_time_per_output_token_seconds": "inter_token_latency",
"vllm:iteration_tokens_total": "iteration_tokens",
"vllm:request_params_n": "request_params_n",
"vllm:request_num_preemptions": "request_num_preemptions",
"vllm:request_prompt_tokens": "request_tokens",
"vllm:request_generation_tokens": "request_tokens",
"vllm:request_max_num_generation_tokens": "request_tokens",
"vllm:request_params_max_tokens": "request_tokens",
"vllm:request_prefill_kv_computed_tokens": "request_tokens",
"vllm:kv_block_lifetime_seconds": "kv_cache_residency",
"vllm:kv_block_idle_before_evict_seconds": "kv_cache_residency",
"vllm:kv_block_reuse_gap_seconds": "kv_cache_residency",
}
TEST_MODEL = "distilbert/distilgpt2"
TEST_MODEL_MAX_LEN = 1024
REQUEST_TOKENS_SNAPSHOT_FOR_TEST_MODEL: list[float] = [
1,
2,
5,
10,
20,
50,
100,
200,
500,
1000,
]
@pytest.mark.parametrize("family", sorted(DEFAULT_BUCKET_SNAPSHOTS))
def test_default_bucket_snapshots(family):
"""Static families must expose exactly today's default boundaries."""
assert histogram_buckets(family) == DEFAULT_BUCKET_SNAPSHOTS[family]
def test_request_tokens_bucket_snapshot():
"""The 1-2-5 token-count series must be stable for a given model len."""
assert histogram_buckets("request_tokens", max_model_len=32768) == [
1,
2,
5,
10,
20,
50,
100,
200,
500,
1000,
2000,
5000,
10000,
20000,
]
assert build_1_2_5_buckets(100) == [1, 2, 5, 10, 20, 50, 100]
def test_request_tokens_requires_max_model_len():
with pytest.raises(ValueError, match="max_model_len is required"):
histogram_buckets("request_tokens")
def test_buckets_return_fresh_copies():
"""Mutating a returned list must not corrupt the shared defaults."""
first = histogram_buckets("request_latency")
first.append(1e9)
second = histogram_buckets("request_latency")
assert second == DEFAULT_BUCKET_SNAPSHOTS["request_latency"]
assert second is not first
def test_bucket_family_keys():
"""The declared families are exactly these, and each one feeds a histogram."""
assert {
"request_latency",
"time_to_first_token",
"inter_token_latency",
"iteration_tokens",
"request_params_n",
"request_num_preemptions",
"request_tokens",
"kv_cache_residency",
} == BUCKET_FAMILY_KEYS
assert set(METRIC_FAMILIES.values()) == BUCKET_FAMILY_KEYS
def collect_histogram_buckets() -> dict[str, list[float]]:
"""Read back each vllm:* histogram's finite bucket bounds by metric name."""
found: dict[str, list[float]] = {}
for metric in prometheus_client.REGISTRY.collect():
if metric.name.startswith("vllm:") and metric.type == "histogram":
found[metric.name] = sorted(
{
float(sample.labels["le"])
for sample in metric.samples
if sample.name.endswith("_bucket") and sample.labels["le"] != "+Inf"
}
)
return found
def build_logger_config(
observability_config: ObservabilityConfig,
) -> VllmConfig:
return VllmConfig(
model_config=ModelConfig(model=TEST_MODEL),
observability_config=observability_config,
)
def test_prometheus_logger_default_buckets():
"""Every core engine histogram must use its family's defaults.
The config deliberately sets no kv_transfer_config: a KV connector would
register its own histograms, which own their boundaries and are not
covered by METRIC_FAMILIES.
"""
config = build_logger_config(ObservabilityConfig(kv_cache_metrics=True))
assert config.model_config.max_model_len == TEST_MODEL_MAX_LEN
try:
PrometheusStatLogger(config)
found = collect_histogram_buckets()
assert set(found) == set(METRIC_FAMILIES)
for metric_name, family in METRIC_FAMILIES.items():
if family == "request_tokens":
expected = REQUEST_TOKENS_SNAPSHOT_FOR_TEST_MODEL
else:
expected = DEFAULT_BUCKET_SNAPSHOTS[family]
assert found[metric_name] == [float(b) for b in expected], metric_name
finally:
unregister_vllm_metrics()
+228
View File
@@ -0,0 +1,228 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Default bucket boundaries for the Prometheus histograms emitted by vLLM.
Every core engine histogram created by the Prometheus stat logger draws its
bucket boundaries from exactly one of the families defined here, so each
default list has a single source of truth. Histograms owned by other
subsystems, such as the KV connector and offloading metrics, keep their own
boundaries.
"""
from typing import Literal, get_args
BucketFamilyKey = Literal[
"request_latency",
"time_to_first_token",
"inter_token_latency",
"iteration_tokens",
"request_params_n",
"request_num_preemptions",
"request_tokens",
"kv_cache_residency",
]
"""Canonical keys for the histogram bucket families."""
BUCKET_FAMILY_KEYS: frozenset[str] = frozenset(get_args(BucketFamilyKey))
"""All bucket family keys, for membership checks."""
REQUEST_LATENCY_BUCKETS: tuple[float, ...] = (
0.3,
0.5,
0.8,
1.0,
1.5,
2.0,
2.5,
5.0,
10.0,
15.0,
20.0,
30.0,
40.0,
50.0,
60.0,
120.0,
240.0,
480.0,
960.0,
1920.0,
7680.0,
)
"""Sub-second scheduling delays through multi-hour batch requests; shared by
the accumulated request phase-timing histograms (e2e, queue, inference,
prefill, decode)."""
TIME_TO_FIRST_TOKEN_BUCKETS: tuple[float, ...] = (
0.001,
0.005,
0.01,
0.02,
0.04,
0.06,
0.08,
0.1,
0.25,
0.5,
0.75,
1.0,
2.5,
5.0,
7.5,
10.0,
20.0,
40.0,
80.0,
160.0,
640.0,
2560.0,
)
"""Millisecond-scale prefill for tiny prompts up to ~40-minute worst cases,
with the densest resolution around interactive (sub-second) latencies."""
INTER_TOKEN_LATENCY_BUCKETS: tuple[float, ...] = (
0.01,
0.025,
0.05,
0.075,
0.1,
0.15,
0.2,
0.3,
0.4,
0.5,
0.75,
1.0,
2.5,
5.0,
7.5,
10.0,
20.0,
40.0,
80.0,
)
"""Per-decode-step latencies: 10 ms fast decode up to multi-second stalls."""
ITERATION_TOKENS_BUCKETS: tuple[float, ...] = (
1,
8,
16,
32,
64,
128,
256,
512,
1024,
2048,
4096,
8192,
16384,
)
"""Tokens processed per engine step: powers of two up to a typical
max-num-batched-tokens budget."""
REQUEST_PARAMS_N_BUCKETS: tuple[float, ...] = (1, 2, 5, 10, 20)
"""Small integer counts for the ``n`` sampling parameter."""
REQUEST_NUM_PREEMPTIONS_BUCKETS: tuple[float, ...] = (1, 2, 3, 4, 5, 10, 20)
"""Per-request preemption counts, recorded for every finished request: the
first bucket therefore holds never-preempted requests together with
singly-preempted ones, 2 through 5 are separated individually, and the tail
coarsens to 20. Distinct from `REQUEST_PARAMS_N_BUCKETS` because these are the
boundaries the metric shipped with, not a harmonized choice."""
KV_CACHE_RESIDENCY_BUCKETS: tuple[float, ...] = (
0.001,
0.002,
0.005,
0.01,
0.02,
0.05,
0.1,
0.2,
0.5,
1,
2,
5,
10,
20,
30,
60,
120,
300,
600,
1200,
1800,
)
"""KV cache block residency times: millisecond-scale reuse gaps up to
30-minute block lifetimes."""
_STATIC_FAMILY_DEFAULTS: dict[str, tuple[float, ...]] = {
"request_latency": REQUEST_LATENCY_BUCKETS,
"time_to_first_token": TIME_TO_FIRST_TOKEN_BUCKETS,
"inter_token_latency": INTER_TOKEN_LATENCY_BUCKETS,
"iteration_tokens": ITERATION_TOKENS_BUCKETS,
"request_params_n": REQUEST_PARAMS_N_BUCKETS,
"request_num_preemptions": REQUEST_NUM_PREEMPTIONS_BUCKETS,
"kv_cache_residency": KV_CACHE_RESIDENCY_BUCKETS,
}
def build_buckets(mantissa_lst: list[int], max_value: int) -> list[float]:
"""Build buckets from mantissas scaled by increasing powers of 10.
Args:
mantissa_lst: Mantissa values multiplied by each power of 10.
max_value: Largest bucket value to include.
Returns:
Bucket values in increasing order, capped at `max_value`.
"""
exponent = 0
buckets: list[float] = []
while True:
for m in mantissa_lst:
value = m * 10**exponent
if value <= max_value:
buckets.append(value)
else:
return buckets
exponent += 1
def build_1_2_5_buckets(max_value: int) -> list[float]:
"""Build a 1-2-5 series of buckets capped at `max_value`.
Example:
>>> build_1_2_5_buckets(100)
[1, 2, 5, 10, 20, 50, 100]
"""
return build_buckets([1, 2, 5], max_value)
def histogram_buckets(
family: BucketFamilyKey,
max_model_len: int | None = None,
) -> list[float]:
"""Return the default bucket boundaries for a histogram family.
Args:
family: Canonical bucket family key.
max_model_len: Cap for the token-count series of the
`request_tokens` family; required for that family and ignored
otherwise.
Returns:
A fresh list of bucket upper bounds; callers may mutate it freely.
Raises:
ValueError: If `family` is `request_tokens` and `max_model_len`
is None.
"""
if family == "request_tokens":
if max_model_len is None:
raise ValueError(
"max_model_len is required for the 'request_tokens' bucket family"
)
return build_1_2_5_buckets(max_model_len)
return list(_STATIC_FAMILY_DEFAULTS[family])
+23 -147
View File
@@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
from vllm.logger import init_logger
from vllm.plugins import STAT_LOGGER_PLUGINS_GROUP, load_plugins_by_group
from vllm.v1.engine import FinishReason
from vllm.v1.metrics.buckets import histogram_buckets
from vllm.v1.metrics.perf import PerfMetricsLogging, PerfMetricsProm
from vllm.v1.metrics.prometheus import unregister_vllm_metrics
from vllm.v1.metrics.stats import (
@@ -724,13 +725,23 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
for idx in engine_indexes
}
request_tokens_buckets = histogram_buckets(
"request_tokens", max_model_len=max_model_len
)
iteration_tokens_buckets = histogram_buckets("iteration_tokens")
request_params_n_buckets = histogram_buckets("request_params_n")
time_to_first_token_buckets = histogram_buckets("time_to_first_token")
inter_token_latency_buckets = histogram_buckets("inter_token_latency")
request_latency_buckets = histogram_buckets("request_latency")
request_num_preemptions_buckets = histogram_buckets("request_num_preemptions")
#
# Histograms of counts
#
histogram_num_prompt_tokens_request = self._histogram_cls(
name="vllm:request_prompt_tokens",
documentation="Number of prefill tokens processed.",
buckets=build_1_2_5_buckets(max_model_len),
buckets=request_tokens_buckets,
labelnames=labelnames,
)
self.histogram_num_prompt_tokens_request = create_metric_per_engine(
@@ -740,7 +751,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
histogram_num_generation_tokens_request = self._histogram_cls(
name="vllm:request_generation_tokens",
documentation="Number of generation tokens processed.",
buckets=build_1_2_5_buckets(max_model_len),
buckets=request_tokens_buckets,
labelnames=labelnames,
)
self.histogram_num_generation_tokens_request = create_metric_per_engine(
@@ -753,7 +764,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
histogram_iteration_tokens = self._histogram_cls(
name="vllm:iteration_tokens_total",
documentation="Histogram of number of tokens per engine_step.",
buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384],
buckets=iteration_tokens_buckets,
labelnames=labelnames,
)
self.histogram_iteration_tokens = create_metric_per_engine(
@@ -763,7 +774,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
histogram_max_num_generation_tokens_request = self._histogram_cls(
name="vllm:request_max_num_generation_tokens",
documentation="Histogram of maximum number of requested generation tokens.",
buckets=build_1_2_5_buckets(max_model_len),
buckets=request_tokens_buckets,
labelnames=labelnames,
)
self.histogram_max_num_generation_tokens_request = create_metric_per_engine(
@@ -773,7 +784,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
histogram_n_request = self._histogram_cls(
name="vllm:request_params_n",
documentation="Histogram of the n request parameter.",
buckets=[1, 2, 5, 10, 20],
buckets=request_params_n_buckets,
labelnames=labelnames,
)
self.histogram_n_request = create_metric_per_engine(
@@ -783,7 +794,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
histogram_max_tokens_request = self._histogram_cls(
name="vllm:request_params_max_tokens",
documentation="Histogram of the max_tokens request parameter.",
buckets=build_1_2_5_buckets(max_model_len),
buckets=request_tokens_buckets,
labelnames=labelnames,
)
self.histogram_max_tokens_request = create_metric_per_engine(
@@ -796,30 +807,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
histogram_time_to_first_token = self._histogram_cls(
name="vllm:time_to_first_token_seconds",
documentation="Histogram of time to first token in seconds.",
buckets=[
0.001,
0.005,
0.01,
0.02,
0.04,
0.06,
0.08,
0.1,
0.25,
0.5,
0.75,
1.0,
2.5,
5.0,
7.5,
10.0,
20.0,
40.0,
80.0,
160.0,
640.0,
2560.0,
],
buckets=time_to_first_token_buckets,
labelnames=labelnames,
)
self.histogram_time_to_first_token = create_metric_per_engine(
@@ -829,27 +817,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
histogram_inter_token_latency = self._histogram_cls(
name="vllm:inter_token_latency_seconds",
documentation="Histogram of inter-token latency in seconds.",
buckets=[
0.01,
0.025,
0.05,
0.075,
0.1,
0.15,
0.2,
0.3,
0.4,
0.5,
0.75,
1.0,
2.5,
5.0,
7.5,
10.0,
20.0,
40.0,
80.0,
],
buckets=inter_token_latency_buckets,
labelnames=labelnames,
)
self.histogram_inter_token_latency = create_metric_per_engine(
@@ -859,56 +827,13 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
histogram_request_time_per_output_token = self._histogram_cls(
name="vllm:request_time_per_output_token_seconds",
documentation="Histogram of time_per_output_token_seconds per request.",
buckets=[
0.01,
0.025,
0.05,
0.075,
0.1,
0.15,
0.2,
0.3,
0.4,
0.5,
0.75,
1.0,
2.5,
5.0,
7.5,
10.0,
20.0,
40.0,
80.0,
],
buckets=inter_token_latency_buckets,
labelnames=labelnames,
)
self.histogram_request_time_per_output_token = create_metric_per_engine(
histogram_request_time_per_output_token, per_engine_labelvalues
)
request_latency_buckets = [
0.3,
0.5,
0.8,
1.0,
1.5,
2.0,
2.5,
5.0,
10.0,
15.0,
20.0,
30.0,
40.0,
50.0,
60.0,
120.0,
240.0,
480.0,
960.0,
1920.0,
7680.0,
]
histogram_e2e_time_request = self._histogram_cls(
name="vllm:e2e_request_latency_seconds",
documentation="Histogram of e2e request latency in seconds.",
@@ -962,7 +887,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
histogram_request_num_preemptions = self._histogram_cls(
name="vllm:request_num_preemptions",
documentation="Histogram of the number of times a request was preempted.",
buckets=[1, 2, 3, 4, 5, 10, 20],
buckets=request_num_preemptions_buckets,
labelnames=labelnames,
)
self.histogram_request_num_preemptions = create_metric_per_engine(
@@ -975,7 +900,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
"Histogram of new KV tokens computed during prefill "
"(excluding cached tokens)."
),
buckets=build_1_2_5_buckets(max_model_len),
buckets=request_tokens_buckets,
labelnames=labelnames,
)
self.histogram_prefill_kv_computed_request = create_metric_per_engine(
@@ -986,29 +911,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
# KV Cache residency metrics
#
if self.kv_cache_metrics_enabled:
kv_cache_residency_buckets = [
0.001,
0.002,
0.005,
0.01,
0.02,
0.05,
0.1,
0.2,
0.5,
1,
2,
5,
10,
20,
30,
60,
120,
300,
600,
1200,
1800,
]
kv_cache_residency_buckets = histogram_buckets("kv_cache_residency")
histogram_kv_block_lifetime = self._histogram_cls(
name="vllm:kv_block_lifetime_seconds",
@@ -1294,33 +1197,6 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
self.log_metrics_info("cache_config", self.vllm_config.cache_config)
def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:
"""
Builds a list of buckets with increasing powers of 10 multiplied by
mantissa values until the value exceeds the specified maximum.
"""
exponent = 0
buckets: list[int] = []
while True:
for m in mantissa_lst:
value = m * 10**exponent
if value <= max_value:
buckets.append(value)
else:
return buckets
exponent += 1
def build_1_2_5_buckets(max_value: int) -> list[int]:
"""
Example:
>>> build_1_2_5_buckets(100)
[1, 2, 5, 10, 20, 50, 100]
"""
return build_buckets([1, 2, 5], max_value)
class StatLoggerManager:
"""
StatLoggerManager: