[Feat] Add request-level preemption count histogram metric (#49984)

This commit is contained in:
Amy (Yu-Hsuan) Lin
2026-08-31 23:04:54 -04:00
committed by GitHub
parent 882ca8d696
commit 923949e6e3
3 changed files with 21 additions and 0 deletions
@@ -85,6 +85,7 @@ def _get_expected_values(num_requests: int, prompt_ids: list[int], max_tokens: i
"vllm:request_inference_time_seconds": [("_count", num_requests)],
"vllm:request_prefill_time_seconds": [("_count", num_requests)],
"vllm:request_decode_time_seconds": [("_count", num_requests)],
"vllm:request_num_preemptions": [("_count", num_requests)],
"vllm:request_prompt_tokens": [
("_sum", num_requests * num_prompt_tokens),
("_count", num_requests),
@@ -225,6 +226,9 @@ EXPECTED_METRICS_V1 = [
"vllm:request_decode_time_seconds_sum",
"vllm:request_decode_time_seconds_bucket",
"vllm:request_decode_time_seconds_count",
"vllm:request_num_preemptions_sum",
"vllm:request_num_preemptions_bucket",
"vllm:request_num_preemptions_count",
]
EXPECTED_METRICS_MM = [
+13
View File
@@ -959,6 +959,16 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
histogram_decode_time_request, per_engine_labelvalues
)
histogram_request_num_preemptions = self._histogram_cls(
name="vllm:request_num_preemptions",
documentation="Histogram of the number of times a request was preempted.",
buckets=[1, 2, 3, 4, 5, 10, 20],
labelnames=labelnames,
)
self.histogram_request_num_preemptions = create_metric_per_engine(
histogram_request_num_preemptions, per_engine_labelvalues
)
histogram_prefill_kv_computed_request = self._histogram_cls(
name="vllm:request_prefill_kv_computed_tokens",
documentation=(
@@ -1237,6 +1247,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
self.histogram_decode_time_request[engine_idx].observe(
finished_request.decode_time
)
self.histogram_request_num_preemptions[engine_idx].observe(
finished_request.num_preemptions
)
# Calculate prefill KV compute (excludes cached tokens)
prefill_kv_computed = finished_request.num_prompt_tokens - max(
finished_request.num_cached_tokens, 0
+4
View File
@@ -219,6 +219,7 @@ class RequestStateStats:
"""Stats that need to be tracked across delta updates."""
num_generation_tokens: int = 0
num_preemptions: int = 0
# This is an engine frontend timestamp (wall-clock)
arrival_time: float = 0.0
@@ -244,6 +245,7 @@ class FinishedRequestStats:
request_id: str | None = None
e2e_latency: float = 0.0
num_prompt_tokens: int = 0
num_preemptions: int = 0
num_generation_tokens: int = 0
max_tokens_param: int | None = None
queued_time: float = 0.0
@@ -523,6 +525,7 @@ class IterationStats:
lora_states.request_running(req_id, lora_name)
elif event.type == EngineCoreEventType.PREEMPTED:
self.num_preempted_reqs += 1
req_stats.num_preemptions += 1
lora_states.request_waiting(req_id, lora_name)
def update_from_finished_request(
@@ -563,6 +566,7 @@ class IterationStats:
request_id=request_id,
e2e_latency=e2e_latency,
num_prompt_tokens=num_prompt_tokens,
num_preemptions=req_stats.num_preemptions,
num_generation_tokens=req_stats.num_generation_tokens,
max_tokens_param=max_tokens_param,
queued_time=queued_time,