From 64856080b05054fc62754f37234f7b483b753445 Mon Sep 17 00:00:00 2001 From: Taneem Ibrahim Date: Tue, 15 Sep 2026 08:21:05 -0500 Subject: [PATCH] [Bugfix] Measure complete pooling responses (#56760) Signed-off-by: Taneem Ibrahim --- vllm/benchmarks/lib/endpoint_request_func.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/vllm/benchmarks/lib/endpoint_request_func.py b/vllm/benchmarks/lib/endpoint_request_func.py index b55c5f4e16..54c6ce63b0 100644 --- a/vllm/benchmarks/lib/endpoint_request_func.py +++ b/vllm/benchmarks/lib/endpoint_request_func.py @@ -606,6 +606,7 @@ async def _run_pooling_request( headers: dict[str, Any], pbar: tqdm | None = None, num_input_sequences: int = 1, + prompt_len: int = 0, ) -> RequestFuncOutput: output = RequestFuncOutput(num_input_sequences=num_input_sequences) st = time.perf_counter() @@ -613,11 +614,19 @@ async def _run_pooling_request( try: async with session.post(url=api_url, headers=headers, json=payload) as response: if response.status == 200: + encoding_format = payload.get("encoding_format", "float") + if encoding_format in ("bytes", "bytes_only"): + async for _ in response.content.iter_any(): + pass + else: + await response.read() output.ttft = output.latency = time.perf_counter() - st - if payload.get("encoding_format", "float") == "bytes": + if encoding_format == "bytes": metadata = json.loads(response.headers["metadata"]) usage = metadata.get("usage", {}) + elif encoding_format == "bytes_only": + usage = {"prompt_tokens": prompt_len} else: data = await response.json() usage = data.get("usage", {}) @@ -672,6 +681,7 @@ async def async_request_openai_embeddings( headers=headers, pbar=pbar, num_input_sequences=_get_num_input_sequences(request_func_input.prompt), + prompt_len=request_func_input.prompt_len, ) @@ -709,6 +719,7 @@ async def async_request_vllm_rerank( headers=headers, pbar=pbar, num_input_sequences=len(request_func_input.prompt) - 1, + prompt_len=request_func_input.prompt_len, ) @@ -743,6 +754,7 @@ async def async_request_openai_embeddings_chat( payload=payload, headers=headers, pbar=pbar, + prompt_len=request_func_input.prompt_len, ) @@ -847,6 +859,7 @@ async def async_request_infinity_embeddings( headers=headers, pbar=pbar, num_input_sequences=_get_num_input_sequences(request_func_input.prompt), + prompt_len=request_func_input.prompt_len, ) @@ -896,6 +909,7 @@ async def async_request_vllm_pooling( headers=headers, pbar=pbar, num_input_sequences=_get_num_input_sequences(request_func_input.prompt), + prompt_len=request_func_input.prompt_len, )