[Metrics] Consolidate Prometheus histogram bucket defaults into a single module (#48866)
Signed-off-by: Guy Stone <guys@spotify.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
c191787a68
commit
d5a9d0f59b
@@ -0,0 +1,273 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Snapshot tests locking the default Prometheus histogram buckets.
|
||||
|
||||
The expected lists are deliberately hard-coded literals rather than imports:
|
||||
any change to a default bucket boundary must show up as a failing test here.
|
||||
"""
|
||||
|
||||
import prometheus_client
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config.observability import ObservabilityConfig
|
||||
from vllm.v1.metrics.buckets import (
|
||||
BUCKET_FAMILY_KEYS,
|
||||
build_1_2_5_buckets,
|
||||
histogram_buckets,
|
||||
)
|
||||
from vllm.v1.metrics.loggers import PrometheusStatLogger
|
||||
from vllm.v1.metrics.prometheus import unregister_vllm_metrics
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
DEFAULT_BUCKET_SNAPSHOTS: dict[str, list[float]] = {
|
||||
"request_latency": [
|
||||
0.3,
|
||||
0.5,
|
||||
0.8,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
2.5,
|
||||
5.0,
|
||||
10.0,
|
||||
15.0,
|
||||
20.0,
|
||||
30.0,
|
||||
40.0,
|
||||
50.0,
|
||||
60.0,
|
||||
120.0,
|
||||
240.0,
|
||||
480.0,
|
||||
960.0,
|
||||
1920.0,
|
||||
7680.0,
|
||||
],
|
||||
"time_to_first_token": [
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.02,
|
||||
0.04,
|
||||
0.06,
|
||||
0.08,
|
||||
0.1,
|
||||
0.25,
|
||||
0.5,
|
||||
0.75,
|
||||
1.0,
|
||||
2.5,
|
||||
5.0,
|
||||
7.5,
|
||||
10.0,
|
||||
20.0,
|
||||
40.0,
|
||||
80.0,
|
||||
160.0,
|
||||
640.0,
|
||||
2560.0,
|
||||
],
|
||||
"inter_token_latency": [
|
||||
0.01,
|
||||
0.025,
|
||||
0.05,
|
||||
0.075,
|
||||
0.1,
|
||||
0.15,
|
||||
0.2,
|
||||
0.3,
|
||||
0.4,
|
||||
0.5,
|
||||
0.75,
|
||||
1.0,
|
||||
2.5,
|
||||
5.0,
|
||||
7.5,
|
||||
10.0,
|
||||
20.0,
|
||||
40.0,
|
||||
80.0,
|
||||
],
|
||||
"iteration_tokens": [
|
||||
1,
|
||||
8,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128,
|
||||
256,
|
||||
512,
|
||||
1024,
|
||||
2048,
|
||||
4096,
|
||||
8192,
|
||||
16384,
|
||||
],
|
||||
"request_params_n": [1, 2, 5, 10, 20],
|
||||
"request_num_preemptions": [1, 2, 3, 4, 5, 10, 20],
|
||||
"kv_cache_residency": [
|
||||
0.001,
|
||||
0.002,
|
||||
0.005,
|
||||
0.01,
|
||||
0.02,
|
||||
0.05,
|
||||
0.1,
|
||||
0.2,
|
||||
0.5,
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
10,
|
||||
20,
|
||||
30,
|
||||
60,
|
||||
120,
|
||||
300,
|
||||
600,
|
||||
1200,
|
||||
1800,
|
||||
],
|
||||
}
|
||||
|
||||
METRIC_FAMILIES: dict[str, str] = {
|
||||
"vllm:e2e_request_latency_seconds": "request_latency",
|
||||
"vllm:request_queue_time_seconds": "request_latency",
|
||||
"vllm:request_inference_time_seconds": "request_latency",
|
||||
"vllm:request_prefill_time_seconds": "request_latency",
|
||||
"vllm:request_decode_time_seconds": "request_latency",
|
||||
"vllm:time_to_first_token_seconds": "time_to_first_token",
|
||||
"vllm:inter_token_latency_seconds": "inter_token_latency",
|
||||
"vllm:request_time_per_output_token_seconds": "inter_token_latency",
|
||||
"vllm:iteration_tokens_total": "iteration_tokens",
|
||||
"vllm:request_params_n": "request_params_n",
|
||||
"vllm:request_num_preemptions": "request_num_preemptions",
|
||||
"vllm:request_prompt_tokens": "request_tokens",
|
||||
"vllm:request_generation_tokens": "request_tokens",
|
||||
"vllm:request_max_num_generation_tokens": "request_tokens",
|
||||
"vllm:request_params_max_tokens": "request_tokens",
|
||||
"vllm:request_prefill_kv_computed_tokens": "request_tokens",
|
||||
"vllm:kv_block_lifetime_seconds": "kv_cache_residency",
|
||||
"vllm:kv_block_idle_before_evict_seconds": "kv_cache_residency",
|
||||
"vllm:kv_block_reuse_gap_seconds": "kv_cache_residency",
|
||||
}
|
||||
|
||||
TEST_MODEL = "distilbert/distilgpt2"
|
||||
TEST_MODEL_MAX_LEN = 1024
|
||||
REQUEST_TOKENS_SNAPSHOT_FOR_TEST_MODEL: list[float] = [
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
10,
|
||||
20,
|
||||
50,
|
||||
100,
|
||||
200,
|
||||
500,
|
||||
1000,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("family", sorted(DEFAULT_BUCKET_SNAPSHOTS))
|
||||
def test_default_bucket_snapshots(family):
|
||||
"""Static families must expose exactly today's default boundaries."""
|
||||
assert histogram_buckets(family) == DEFAULT_BUCKET_SNAPSHOTS[family]
|
||||
|
||||
|
||||
def test_request_tokens_bucket_snapshot():
|
||||
"""The 1-2-5 token-count series must be stable for a given model len."""
|
||||
assert histogram_buckets("request_tokens", max_model_len=32768) == [
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
10,
|
||||
20,
|
||||
50,
|
||||
100,
|
||||
200,
|
||||
500,
|
||||
1000,
|
||||
2000,
|
||||
5000,
|
||||
10000,
|
||||
20000,
|
||||
]
|
||||
assert build_1_2_5_buckets(100) == [1, 2, 5, 10, 20, 50, 100]
|
||||
|
||||
|
||||
def test_request_tokens_requires_max_model_len():
|
||||
with pytest.raises(ValueError, match="max_model_len is required"):
|
||||
histogram_buckets("request_tokens")
|
||||
|
||||
|
||||
def test_buckets_return_fresh_copies():
|
||||
"""Mutating a returned list must not corrupt the shared defaults."""
|
||||
first = histogram_buckets("request_latency")
|
||||
first.append(1e9)
|
||||
second = histogram_buckets("request_latency")
|
||||
assert second == DEFAULT_BUCKET_SNAPSHOTS["request_latency"]
|
||||
assert second is not first
|
||||
|
||||
|
||||
def test_bucket_family_keys():
|
||||
"""The declared families are exactly these, and each one feeds a histogram."""
|
||||
assert {
|
||||
"request_latency",
|
||||
"time_to_first_token",
|
||||
"inter_token_latency",
|
||||
"iteration_tokens",
|
||||
"request_params_n",
|
||||
"request_num_preemptions",
|
||||
"request_tokens",
|
||||
"kv_cache_residency",
|
||||
} == BUCKET_FAMILY_KEYS
|
||||
assert set(METRIC_FAMILIES.values()) == BUCKET_FAMILY_KEYS
|
||||
|
||||
|
||||
def collect_histogram_buckets() -> dict[str, list[float]]:
|
||||
"""Read back each vllm:* histogram's finite bucket bounds by metric name."""
|
||||
found: dict[str, list[float]] = {}
|
||||
for metric in prometheus_client.REGISTRY.collect():
|
||||
if metric.name.startswith("vllm:") and metric.type == "histogram":
|
||||
found[metric.name] = sorted(
|
||||
{
|
||||
float(sample.labels["le"])
|
||||
for sample in metric.samples
|
||||
if sample.name.endswith("_bucket") and sample.labels["le"] != "+Inf"
|
||||
}
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
def build_logger_config(
|
||||
observability_config: ObservabilityConfig,
|
||||
) -> VllmConfig:
|
||||
return VllmConfig(
|
||||
model_config=ModelConfig(model=TEST_MODEL),
|
||||
observability_config=observability_config,
|
||||
)
|
||||
|
||||
|
||||
def test_prometheus_logger_default_buckets():
|
||||
"""Every core engine histogram must use its family's defaults.
|
||||
|
||||
The config deliberately sets no kv_transfer_config: a KV connector would
|
||||
register its own histograms, which own their boundaries and are not
|
||||
covered by METRIC_FAMILIES.
|
||||
"""
|
||||
config = build_logger_config(ObservabilityConfig(kv_cache_metrics=True))
|
||||
assert config.model_config.max_model_len == TEST_MODEL_MAX_LEN
|
||||
try:
|
||||
PrometheusStatLogger(config)
|
||||
found = collect_histogram_buckets()
|
||||
assert set(found) == set(METRIC_FAMILIES)
|
||||
for metric_name, family in METRIC_FAMILIES.items():
|
||||
if family == "request_tokens":
|
||||
expected = REQUEST_TOKENS_SNAPSHOT_FOR_TEST_MODEL
|
||||
else:
|
||||
expected = DEFAULT_BUCKET_SNAPSHOTS[family]
|
||||
assert found[metric_name] == [float(b) for b in expected], metric_name
|
||||
finally:
|
||||
unregister_vllm_metrics()
|
||||
@@ -0,0 +1,228 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Default bucket boundaries for the Prometheus histograms emitted by vLLM.
|
||||
|
||||
Every core engine histogram created by the Prometheus stat logger draws its
|
||||
bucket boundaries from exactly one of the families defined here, so each
|
||||
default list has a single source of truth. Histograms owned by other
|
||||
subsystems, such as the KV connector and offloading metrics, keep their own
|
||||
boundaries.
|
||||
"""
|
||||
|
||||
from typing import Literal, get_args
|
||||
|
||||
BucketFamilyKey = Literal[
|
||||
"request_latency",
|
||||
"time_to_first_token",
|
||||
"inter_token_latency",
|
||||
"iteration_tokens",
|
||||
"request_params_n",
|
||||
"request_num_preemptions",
|
||||
"request_tokens",
|
||||
"kv_cache_residency",
|
||||
]
|
||||
"""Canonical keys for the histogram bucket families."""
|
||||
|
||||
BUCKET_FAMILY_KEYS: frozenset[str] = frozenset(get_args(BucketFamilyKey))
|
||||
"""All bucket family keys, for membership checks."""
|
||||
|
||||
REQUEST_LATENCY_BUCKETS: tuple[float, ...] = (
|
||||
0.3,
|
||||
0.5,
|
||||
0.8,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
2.5,
|
||||
5.0,
|
||||
10.0,
|
||||
15.0,
|
||||
20.0,
|
||||
30.0,
|
||||
40.0,
|
||||
50.0,
|
||||
60.0,
|
||||
120.0,
|
||||
240.0,
|
||||
480.0,
|
||||
960.0,
|
||||
1920.0,
|
||||
7680.0,
|
||||
)
|
||||
"""Sub-second scheduling delays through multi-hour batch requests; shared by
|
||||
the accumulated request phase-timing histograms (e2e, queue, inference,
|
||||
prefill, decode)."""
|
||||
|
||||
TIME_TO_FIRST_TOKEN_BUCKETS: tuple[float, ...] = (
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.02,
|
||||
0.04,
|
||||
0.06,
|
||||
0.08,
|
||||
0.1,
|
||||
0.25,
|
||||
0.5,
|
||||
0.75,
|
||||
1.0,
|
||||
2.5,
|
||||
5.0,
|
||||
7.5,
|
||||
10.0,
|
||||
20.0,
|
||||
40.0,
|
||||
80.0,
|
||||
160.0,
|
||||
640.0,
|
||||
2560.0,
|
||||
)
|
||||
"""Millisecond-scale prefill for tiny prompts up to ~40-minute worst cases,
|
||||
with the densest resolution around interactive (sub-second) latencies."""
|
||||
|
||||
INTER_TOKEN_LATENCY_BUCKETS: tuple[float, ...] = (
|
||||
0.01,
|
||||
0.025,
|
||||
0.05,
|
||||
0.075,
|
||||
0.1,
|
||||
0.15,
|
||||
0.2,
|
||||
0.3,
|
||||
0.4,
|
||||
0.5,
|
||||
0.75,
|
||||
1.0,
|
||||
2.5,
|
||||
5.0,
|
||||
7.5,
|
||||
10.0,
|
||||
20.0,
|
||||
40.0,
|
||||
80.0,
|
||||
)
|
||||
"""Per-decode-step latencies: 10 ms fast decode up to multi-second stalls."""
|
||||
|
||||
ITERATION_TOKENS_BUCKETS: tuple[float, ...] = (
|
||||
1,
|
||||
8,
|
||||
16,
|
||||
32,
|
||||
64,
|
||||
128,
|
||||
256,
|
||||
512,
|
||||
1024,
|
||||
2048,
|
||||
4096,
|
||||
8192,
|
||||
16384,
|
||||
)
|
||||
"""Tokens processed per engine step: powers of two up to a typical
|
||||
max-num-batched-tokens budget."""
|
||||
|
||||
REQUEST_PARAMS_N_BUCKETS: tuple[float, ...] = (1, 2, 5, 10, 20)
|
||||
"""Small integer counts for the ``n`` sampling parameter."""
|
||||
|
||||
REQUEST_NUM_PREEMPTIONS_BUCKETS: tuple[float, ...] = (1, 2, 3, 4, 5, 10, 20)
|
||||
"""Per-request preemption counts, recorded for every finished request: the
|
||||
first bucket therefore holds never-preempted requests together with
|
||||
singly-preempted ones, 2 through 5 are separated individually, and the tail
|
||||
coarsens to 20. Distinct from `REQUEST_PARAMS_N_BUCKETS` because these are the
|
||||
boundaries the metric shipped with, not a harmonized choice."""
|
||||
|
||||
KV_CACHE_RESIDENCY_BUCKETS: tuple[float, ...] = (
|
||||
0.001,
|
||||
0.002,
|
||||
0.005,
|
||||
0.01,
|
||||
0.02,
|
||||
0.05,
|
||||
0.1,
|
||||
0.2,
|
||||
0.5,
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
10,
|
||||
20,
|
||||
30,
|
||||
60,
|
||||
120,
|
||||
300,
|
||||
600,
|
||||
1200,
|
||||
1800,
|
||||
)
|
||||
"""KV cache block residency times: millisecond-scale reuse gaps up to
|
||||
30-minute block lifetimes."""
|
||||
|
||||
_STATIC_FAMILY_DEFAULTS: dict[str, tuple[float, ...]] = {
|
||||
"request_latency": REQUEST_LATENCY_BUCKETS,
|
||||
"time_to_first_token": TIME_TO_FIRST_TOKEN_BUCKETS,
|
||||
"inter_token_latency": INTER_TOKEN_LATENCY_BUCKETS,
|
||||
"iteration_tokens": ITERATION_TOKENS_BUCKETS,
|
||||
"request_params_n": REQUEST_PARAMS_N_BUCKETS,
|
||||
"request_num_preemptions": REQUEST_NUM_PREEMPTIONS_BUCKETS,
|
||||
"kv_cache_residency": KV_CACHE_RESIDENCY_BUCKETS,
|
||||
}
|
||||
|
||||
|
||||
def build_buckets(mantissa_lst: list[int], max_value: int) -> list[float]:
|
||||
"""Build buckets from mantissas scaled by increasing powers of 10.
|
||||
|
||||
Args:
|
||||
mantissa_lst: Mantissa values multiplied by each power of 10.
|
||||
max_value: Largest bucket value to include.
|
||||
|
||||
Returns:
|
||||
Bucket values in increasing order, capped at `max_value`.
|
||||
"""
|
||||
exponent = 0
|
||||
buckets: list[float] = []
|
||||
while True:
|
||||
for m in mantissa_lst:
|
||||
value = m * 10**exponent
|
||||
if value <= max_value:
|
||||
buckets.append(value)
|
||||
else:
|
||||
return buckets
|
||||
exponent += 1
|
||||
|
||||
|
||||
def build_1_2_5_buckets(max_value: int) -> list[float]:
|
||||
"""Build a 1-2-5 series of buckets capped at `max_value`.
|
||||
|
||||
Example:
|
||||
>>> build_1_2_5_buckets(100)
|
||||
[1, 2, 5, 10, 20, 50, 100]
|
||||
"""
|
||||
return build_buckets([1, 2, 5], max_value)
|
||||
|
||||
|
||||
def histogram_buckets(
|
||||
family: BucketFamilyKey,
|
||||
max_model_len: int | None = None,
|
||||
) -> list[float]:
|
||||
"""Return the default bucket boundaries for a histogram family.
|
||||
|
||||
Args:
|
||||
family: Canonical bucket family key.
|
||||
max_model_len: Cap for the token-count series of the
|
||||
`request_tokens` family; required for that family and ignored
|
||||
otherwise.
|
||||
|
||||
Returns:
|
||||
A fresh list of bucket upper bounds; callers may mutate it freely.
|
||||
|
||||
Raises:
|
||||
ValueError: If `family` is `request_tokens` and `max_model_len`
|
||||
is None.
|
||||
"""
|
||||
if family == "request_tokens":
|
||||
if max_model_len is None:
|
||||
raise ValueError(
|
||||
"max_model_len is required for the 'request_tokens' bucket family"
|
||||
)
|
||||
return build_1_2_5_buckets(max_model_len)
|
||||
return list(_STATIC_FAMILY_DEFAULTS[family])
|
||||
+23
-147
@@ -18,6 +18,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.plugins import STAT_LOGGER_PLUGINS_GROUP, load_plugins_by_group
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.metrics.buckets import histogram_buckets
|
||||
from vllm.v1.metrics.perf import PerfMetricsLogging, PerfMetricsProm
|
||||
from vllm.v1.metrics.prometheus import unregister_vllm_metrics
|
||||
from vllm.v1.metrics.stats import (
|
||||
@@ -724,13 +725,23 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
for idx in engine_indexes
|
||||
}
|
||||
|
||||
request_tokens_buckets = histogram_buckets(
|
||||
"request_tokens", max_model_len=max_model_len
|
||||
)
|
||||
iteration_tokens_buckets = histogram_buckets("iteration_tokens")
|
||||
request_params_n_buckets = histogram_buckets("request_params_n")
|
||||
time_to_first_token_buckets = histogram_buckets("time_to_first_token")
|
||||
inter_token_latency_buckets = histogram_buckets("inter_token_latency")
|
||||
request_latency_buckets = histogram_buckets("request_latency")
|
||||
request_num_preemptions_buckets = histogram_buckets("request_num_preemptions")
|
||||
|
||||
#
|
||||
# Histograms of counts
|
||||
#
|
||||
histogram_num_prompt_tokens_request = self._histogram_cls(
|
||||
name="vllm:request_prompt_tokens",
|
||||
documentation="Number of prefill tokens processed.",
|
||||
buckets=build_1_2_5_buckets(max_model_len),
|
||||
buckets=request_tokens_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_num_prompt_tokens_request = create_metric_per_engine(
|
||||
@@ -740,7 +751,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
histogram_num_generation_tokens_request = self._histogram_cls(
|
||||
name="vllm:request_generation_tokens",
|
||||
documentation="Number of generation tokens processed.",
|
||||
buckets=build_1_2_5_buckets(max_model_len),
|
||||
buckets=request_tokens_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_num_generation_tokens_request = create_metric_per_engine(
|
||||
@@ -753,7 +764,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
histogram_iteration_tokens = self._histogram_cls(
|
||||
name="vllm:iteration_tokens_total",
|
||||
documentation="Histogram of number of tokens per engine_step.",
|
||||
buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384],
|
||||
buckets=iteration_tokens_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_iteration_tokens = create_metric_per_engine(
|
||||
@@ -763,7 +774,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
histogram_max_num_generation_tokens_request = self._histogram_cls(
|
||||
name="vllm:request_max_num_generation_tokens",
|
||||
documentation="Histogram of maximum number of requested generation tokens.",
|
||||
buckets=build_1_2_5_buckets(max_model_len),
|
||||
buckets=request_tokens_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_max_num_generation_tokens_request = create_metric_per_engine(
|
||||
@@ -773,7 +784,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
histogram_n_request = self._histogram_cls(
|
||||
name="vllm:request_params_n",
|
||||
documentation="Histogram of the n request parameter.",
|
||||
buckets=[1, 2, 5, 10, 20],
|
||||
buckets=request_params_n_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_n_request = create_metric_per_engine(
|
||||
@@ -783,7 +794,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
histogram_max_tokens_request = self._histogram_cls(
|
||||
name="vllm:request_params_max_tokens",
|
||||
documentation="Histogram of the max_tokens request parameter.",
|
||||
buckets=build_1_2_5_buckets(max_model_len),
|
||||
buckets=request_tokens_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_max_tokens_request = create_metric_per_engine(
|
||||
@@ -796,30 +807,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
histogram_time_to_first_token = self._histogram_cls(
|
||||
name="vllm:time_to_first_token_seconds",
|
||||
documentation="Histogram of time to first token in seconds.",
|
||||
buckets=[
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.02,
|
||||
0.04,
|
||||
0.06,
|
||||
0.08,
|
||||
0.1,
|
||||
0.25,
|
||||
0.5,
|
||||
0.75,
|
||||
1.0,
|
||||
2.5,
|
||||
5.0,
|
||||
7.5,
|
||||
10.0,
|
||||
20.0,
|
||||
40.0,
|
||||
80.0,
|
||||
160.0,
|
||||
640.0,
|
||||
2560.0,
|
||||
],
|
||||
buckets=time_to_first_token_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_time_to_first_token = create_metric_per_engine(
|
||||
@@ -829,27 +817,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
histogram_inter_token_latency = self._histogram_cls(
|
||||
name="vllm:inter_token_latency_seconds",
|
||||
documentation="Histogram of inter-token latency in seconds.",
|
||||
buckets=[
|
||||
0.01,
|
||||
0.025,
|
||||
0.05,
|
||||
0.075,
|
||||
0.1,
|
||||
0.15,
|
||||
0.2,
|
||||
0.3,
|
||||
0.4,
|
||||
0.5,
|
||||
0.75,
|
||||
1.0,
|
||||
2.5,
|
||||
5.0,
|
||||
7.5,
|
||||
10.0,
|
||||
20.0,
|
||||
40.0,
|
||||
80.0,
|
||||
],
|
||||
buckets=inter_token_latency_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_inter_token_latency = create_metric_per_engine(
|
||||
@@ -859,56 +827,13 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
histogram_request_time_per_output_token = self._histogram_cls(
|
||||
name="vllm:request_time_per_output_token_seconds",
|
||||
documentation="Histogram of time_per_output_token_seconds per request.",
|
||||
buckets=[
|
||||
0.01,
|
||||
0.025,
|
||||
0.05,
|
||||
0.075,
|
||||
0.1,
|
||||
0.15,
|
||||
0.2,
|
||||
0.3,
|
||||
0.4,
|
||||
0.5,
|
||||
0.75,
|
||||
1.0,
|
||||
2.5,
|
||||
5.0,
|
||||
7.5,
|
||||
10.0,
|
||||
20.0,
|
||||
40.0,
|
||||
80.0,
|
||||
],
|
||||
buckets=inter_token_latency_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_request_time_per_output_token = create_metric_per_engine(
|
||||
histogram_request_time_per_output_token, per_engine_labelvalues
|
||||
)
|
||||
|
||||
request_latency_buckets = [
|
||||
0.3,
|
||||
0.5,
|
||||
0.8,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
2.5,
|
||||
5.0,
|
||||
10.0,
|
||||
15.0,
|
||||
20.0,
|
||||
30.0,
|
||||
40.0,
|
||||
50.0,
|
||||
60.0,
|
||||
120.0,
|
||||
240.0,
|
||||
480.0,
|
||||
960.0,
|
||||
1920.0,
|
||||
7680.0,
|
||||
]
|
||||
histogram_e2e_time_request = self._histogram_cls(
|
||||
name="vllm:e2e_request_latency_seconds",
|
||||
documentation="Histogram of e2e request latency in seconds.",
|
||||
@@ -962,7 +887,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
histogram_request_num_preemptions = self._histogram_cls(
|
||||
name="vllm:request_num_preemptions",
|
||||
documentation="Histogram of the number of times a request was preempted.",
|
||||
buckets=[1, 2, 3, 4, 5, 10, 20],
|
||||
buckets=request_num_preemptions_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_request_num_preemptions = create_metric_per_engine(
|
||||
@@ -975,7 +900,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
"Histogram of new KV tokens computed during prefill "
|
||||
"(excluding cached tokens)."
|
||||
),
|
||||
buckets=build_1_2_5_buckets(max_model_len),
|
||||
buckets=request_tokens_buckets,
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_prefill_kv_computed_request = create_metric_per_engine(
|
||||
@@ -986,29 +911,7 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
# KV Cache residency metrics
|
||||
#
|
||||
if self.kv_cache_metrics_enabled:
|
||||
kv_cache_residency_buckets = [
|
||||
0.001,
|
||||
0.002,
|
||||
0.005,
|
||||
0.01,
|
||||
0.02,
|
||||
0.05,
|
||||
0.1,
|
||||
0.2,
|
||||
0.5,
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
10,
|
||||
20,
|
||||
30,
|
||||
60,
|
||||
120,
|
||||
300,
|
||||
600,
|
||||
1200,
|
||||
1800,
|
||||
]
|
||||
kv_cache_residency_buckets = histogram_buckets("kv_cache_residency")
|
||||
|
||||
histogram_kv_block_lifetime = self._histogram_cls(
|
||||
name="vllm:kv_block_lifetime_seconds",
|
||||
@@ -1294,33 +1197,6 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
self.log_metrics_info("cache_config", self.vllm_config.cache_config)
|
||||
|
||||
|
||||
def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:
|
||||
"""
|
||||
Builds a list of buckets with increasing powers of 10 multiplied by
|
||||
mantissa values until the value exceeds the specified maximum.
|
||||
|
||||
"""
|
||||
exponent = 0
|
||||
buckets: list[int] = []
|
||||
while True:
|
||||
for m in mantissa_lst:
|
||||
value = m * 10**exponent
|
||||
if value <= max_value:
|
||||
buckets.append(value)
|
||||
else:
|
||||
return buckets
|
||||
exponent += 1
|
||||
|
||||
|
||||
def build_1_2_5_buckets(max_value: int) -> list[int]:
|
||||
"""
|
||||
Example:
|
||||
>>> build_1_2_5_buckets(100)
|
||||
[1, 2, 5, 10, 20, 50, 100]
|
||||
"""
|
||||
return build_buckets([1, 2, 5], max_value)
|
||||
|
||||
|
||||
class StatLoggerManager:
|
||||
"""
|
||||
StatLoggerManager:
|
||||
|
||||
Reference in New Issue
Block a user