[Feat] Add request-level preemption count histogram metric (#49984)
This commit is contained in:
@@ -85,6 +85,7 @@ def _get_expected_values(num_requests: int, prompt_ids: list[int], max_tokens: i
|
||||
"vllm:request_inference_time_seconds": [("_count", num_requests)],
|
||||
"vllm:request_prefill_time_seconds": [("_count", num_requests)],
|
||||
"vllm:request_decode_time_seconds": [("_count", num_requests)],
|
||||
"vllm:request_num_preemptions": [("_count", num_requests)],
|
||||
"vllm:request_prompt_tokens": [
|
||||
("_sum", num_requests * num_prompt_tokens),
|
||||
("_count", num_requests),
|
||||
@@ -225,6 +226,9 @@ EXPECTED_METRICS_V1 = [
|
||||
"vllm:request_decode_time_seconds_sum",
|
||||
"vllm:request_decode_time_seconds_bucket",
|
||||
"vllm:request_decode_time_seconds_count",
|
||||
"vllm:request_num_preemptions_sum",
|
||||
"vllm:request_num_preemptions_bucket",
|
||||
"vllm:request_num_preemptions_count",
|
||||
]
|
||||
|
||||
EXPECTED_METRICS_MM = [
|
||||
|
||||
@@ -959,6 +959,16 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
histogram_decode_time_request, per_engine_labelvalues
|
||||
)
|
||||
|
||||
histogram_request_num_preemptions = self._histogram_cls(
|
||||
name="vllm:request_num_preemptions",
|
||||
documentation="Histogram of the number of times a request was preempted.",
|
||||
buckets=[1, 2, 3, 4, 5, 10, 20],
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.histogram_request_num_preemptions = create_metric_per_engine(
|
||||
histogram_request_num_preemptions, per_engine_labelvalues
|
||||
)
|
||||
|
||||
histogram_prefill_kv_computed_request = self._histogram_cls(
|
||||
name="vllm:request_prefill_kv_computed_tokens",
|
||||
documentation=(
|
||||
@@ -1237,6 +1247,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
|
||||
self.histogram_decode_time_request[engine_idx].observe(
|
||||
finished_request.decode_time
|
||||
)
|
||||
self.histogram_request_num_preemptions[engine_idx].observe(
|
||||
finished_request.num_preemptions
|
||||
)
|
||||
# Calculate prefill KV compute (excludes cached tokens)
|
||||
prefill_kv_computed = finished_request.num_prompt_tokens - max(
|
||||
finished_request.num_cached_tokens, 0
|
||||
|
||||
@@ -219,6 +219,7 @@ class RequestStateStats:
|
||||
"""Stats that need to be tracked across delta updates."""
|
||||
|
||||
num_generation_tokens: int = 0
|
||||
num_preemptions: int = 0
|
||||
|
||||
# This is an engine frontend timestamp (wall-clock)
|
||||
arrival_time: float = 0.0
|
||||
@@ -244,6 +245,7 @@ class FinishedRequestStats:
|
||||
request_id: str | None = None
|
||||
e2e_latency: float = 0.0
|
||||
num_prompt_tokens: int = 0
|
||||
num_preemptions: int = 0
|
||||
num_generation_tokens: int = 0
|
||||
max_tokens_param: int | None = None
|
||||
queued_time: float = 0.0
|
||||
@@ -523,6 +525,7 @@ class IterationStats:
|
||||
lora_states.request_running(req_id, lora_name)
|
||||
elif event.type == EngineCoreEventType.PREEMPTED:
|
||||
self.num_preempted_reqs += 1
|
||||
req_stats.num_preemptions += 1
|
||||
lora_states.request_waiting(req_id, lora_name)
|
||||
|
||||
def update_from_finished_request(
|
||||
@@ -563,6 +566,7 @@ class IterationStats:
|
||||
request_id=request_id,
|
||||
e2e_latency=e2e_latency,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
num_preemptions=req_stats.num_preemptions,
|
||||
num_generation_tokens=req_stats.num_generation_tokens,
|
||||
max_tokens_param=max_tokens_param,
|
||||
queued_time=queued_time,
|
||||
|
||||
Reference in New Issue
Block a user