[Bugfix] Measure complete pooling responses (#56760)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-15 09:21:05 -04:00
committed by GitHub
parent 78eaaad2c7
commit 64856080b0
+15 -1
View File
@@ -606,6 +606,7 @@ async def _run_pooling_request(
headers: dict[str, Any],
pbar: tqdm | None = None,
num_input_sequences: int = 1,
prompt_len: int = 0,
) -> RequestFuncOutput:
output = RequestFuncOutput(num_input_sequences=num_input_sequences)
st = time.perf_counter()
@@ -613,11 +614,19 @@ async def _run_pooling_request(
try:
async with session.post(url=api_url, headers=headers, json=payload) as response:
if response.status == 200:
encoding_format = payload.get("encoding_format", "float")
if encoding_format in ("bytes", "bytes_only"):
async for _ in response.content.iter_any():
pass
else:
await response.read()
output.ttft = output.latency = time.perf_counter() - st
if payload.get("encoding_format", "float") == "bytes":
if encoding_format == "bytes":
metadata = json.loads(response.headers["metadata"])
usage = metadata.get("usage", {})
elif encoding_format == "bytes_only":
usage = {"prompt_tokens": prompt_len}
else:
data = await response.json()
usage = data.get("usage", {})
@@ -672,6 +681,7 @@ async def async_request_openai_embeddings(
headers=headers,
pbar=pbar,
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
prompt_len=request_func_input.prompt_len,
)
@@ -709,6 +719,7 @@ async def async_request_vllm_rerank(
headers=headers,
pbar=pbar,
num_input_sequences=len(request_func_input.prompt) - 1,
prompt_len=request_func_input.prompt_len,
)
@@ -743,6 +754,7 @@ async def async_request_openai_embeddings_chat(
payload=payload,
headers=headers,
pbar=pbar,
prompt_len=request_func_input.prompt_len,
)
@@ -847,6 +859,7 @@ async def async_request_infinity_embeddings(
headers=headers,
pbar=pbar,
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
prompt_len=request_func_input.prompt_len,
)
@@ -896,6 +909,7 @@ async def async_request_vllm_pooling(
headers=headers,
pbar=pbar,
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
prompt_len=request_func_input.prompt_len,
)