diff --git a/vllm/benchmarks/lib/endpoint_request_func.py b/vllm/benchmarks/lib/endpoint_request_func.py index b0ef67889d..61af098f80 100644 --- a/vllm/benchmarks/lib/endpoint_request_func.py +++ b/vllm/benchmarks/lib/endpoint_request_func.py @@ -237,6 +237,8 @@ async def async_request_openai_completions( generated_text += text or "" elif usage := data.get("usage"): output.output_tokens = usage.get("completion_tokens") + if (pt := usage.get("prompt_tokens")) is not None: + output.prompt_len = pt if first_chunk_received: output.success = True else: @@ -358,6 +360,8 @@ async def async_request_openai_chat_completions( generated_text += content or "" elif usage := data.get("usage"): output.output_tokens = usage.get("completion_tokens") + if (pt := usage.get("prompt_tokens")) is not None: + output.prompt_len = pt most_recent_timestamp = timestamp diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 53ae6ca6a8..980ed1d412 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -439,7 +439,7 @@ def calculate_metrics( ).input_ids ) actual_output_lens.append(output_len) - total_input += input_requests[i].prompt_len + total_input += outputs[i].prompt_len tpot = 0 if output_len > 1: latency_minus_ttft = outputs[i].latency - outputs[i].ttft