From 1e4748c66a156af3ea7b72a9dbe3fb686c8750eb Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Tue, 14 Apr 2026 13:00:40 +0200 Subject: [PATCH] [Bugfix] Fix `vllm bench serve` to count multimodal tokens in "total input tokens" (#38654) Signed-off-by: Matthias Gehre --- vllm/benchmarks/lib/endpoint_request_func.py | 4 ++++ vllm/benchmarks/serve.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/vllm/benchmarks/lib/endpoint_request_func.py b/vllm/benchmarks/lib/endpoint_request_func.py index b0ef67889d..61af098f80 100644 --- a/vllm/benchmarks/lib/endpoint_request_func.py +++ b/vllm/benchmarks/lib/endpoint_request_func.py @@ -237,6 +237,8 @@ async def async_request_openai_completions( generated_text += text or "" elif usage := data.get("usage"): output.output_tokens = usage.get("completion_tokens") + if (pt := usage.get("prompt_tokens")) is not None: + output.prompt_len = pt if first_chunk_received: output.success = True else: @@ -358,6 +360,8 @@ async def async_request_openai_chat_completions( generated_text += content or "" elif usage := data.get("usage"): output.output_tokens = usage.get("completion_tokens") + if (pt := usage.get("prompt_tokens")) is not None: + output.prompt_len = pt most_recent_timestamp = timestamp diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 53ae6ca6a8..980ed1d412 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -439,7 +439,7 @@ def calculate_metrics( ).input_ids ) actual_output_lens.append(output_len) - total_input += input_requests[i].prompt_len + total_input += outputs[i].prompt_len tpot = 0 if output_len > 1: latency_minus_ttft = outputs[i].latency - outputs[i].ttft