diff --git a/tests/entrypoints/serve/instrumentator/test_metrics.py b/tests/entrypoints/serve/instrumentator/test_metrics.py index 1fa35feda2..297fe233b5 100644 --- a/tests/entrypoints/serve/instrumentator/test_metrics.py +++ b/tests/entrypoints/serve/instrumentator/test_metrics.py @@ -85,6 +85,7 @@ def _get_expected_values(num_requests: int, prompt_ids: list[int], max_tokens: i "vllm:request_inference_time_seconds": [("_count", num_requests)], "vllm:request_prefill_time_seconds": [("_count", num_requests)], "vllm:request_decode_time_seconds": [("_count", num_requests)], + "vllm:request_num_preemptions": [("_count", num_requests)], "vllm:request_prompt_tokens": [ ("_sum", num_requests * num_prompt_tokens), ("_count", num_requests), @@ -225,6 +226,9 @@ EXPECTED_METRICS_V1 = [ "vllm:request_decode_time_seconds_sum", "vllm:request_decode_time_seconds_bucket", "vllm:request_decode_time_seconds_count", + "vllm:request_num_preemptions_sum", + "vllm:request_num_preemptions_bucket", + "vllm:request_num_preemptions_count", ] EXPECTED_METRICS_MM = [ diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index 692106cf39..4d0b77add7 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -959,6 +959,16 @@ class PrometheusStatLogger(AggregateStatLoggerBase): histogram_decode_time_request, per_engine_labelvalues ) + histogram_request_num_preemptions = self._histogram_cls( + name="vllm:request_num_preemptions", + documentation="Histogram of the number of times a request was preempted.", + buckets=[1, 2, 3, 4, 5, 10, 20], + labelnames=labelnames, + ) + self.histogram_request_num_preemptions = create_metric_per_engine( + histogram_request_num_preemptions, per_engine_labelvalues + ) + histogram_prefill_kv_computed_request = self._histogram_cls( name="vllm:request_prefill_kv_computed_tokens", documentation=( @@ -1237,6 +1247,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase): self.histogram_decode_time_request[engine_idx].observe( finished_request.decode_time ) + self.histogram_request_num_preemptions[engine_idx].observe( + finished_request.num_preemptions + ) # Calculate prefill KV compute (excludes cached tokens) prefill_kv_computed = finished_request.num_prompt_tokens - max( finished_request.num_cached_tokens, 0 diff --git a/vllm/v1/metrics/stats.py b/vllm/v1/metrics/stats.py index 3dbc5206ca..24e07e57cc 100644 --- a/vllm/v1/metrics/stats.py +++ b/vllm/v1/metrics/stats.py @@ -219,6 +219,7 @@ class RequestStateStats: """Stats that need to be tracked across delta updates.""" num_generation_tokens: int = 0 + num_preemptions: int = 0 # This is an engine frontend timestamp (wall-clock) arrival_time: float = 0.0 @@ -244,6 +245,7 @@ class FinishedRequestStats: request_id: str | None = None e2e_latency: float = 0.0 num_prompt_tokens: int = 0 + num_preemptions: int = 0 num_generation_tokens: int = 0 max_tokens_param: int | None = None queued_time: float = 0.0 @@ -523,6 +525,7 @@ class IterationStats: lora_states.request_running(req_id, lora_name) elif event.type == EngineCoreEventType.PREEMPTED: self.num_preempted_reqs += 1 + req_stats.num_preemptions += 1 lora_states.request_waiting(req_id, lora_name) def update_from_finished_request( @@ -563,6 +566,7 @@ class IterationStats: request_id=request_id, e2e_latency=e2e_latency, num_prompt_tokens=num_prompt_tokens, + num_preemptions=req_stats.num_preemptions, num_generation_tokens=req_stats.num_generation_tokens, max_tokens_param=max_tokens_param, queued_time=queued_time,