[Bugfix] Measure complete pooling responses (#56760)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -606,6 +606,7 @@ async def _run_pooling_request(
|
||||
headers: dict[str, Any],
|
||||
pbar: tqdm | None = None,
|
||||
num_input_sequences: int = 1,
|
||||
prompt_len: int = 0,
|
||||
) -> RequestFuncOutput:
|
||||
output = RequestFuncOutput(num_input_sequences=num_input_sequences)
|
||||
st = time.perf_counter()
|
||||
@@ -613,11 +614,19 @@ async def _run_pooling_request(
|
||||
try:
|
||||
async with session.post(url=api_url, headers=headers, json=payload) as response:
|
||||
if response.status == 200:
|
||||
encoding_format = payload.get("encoding_format", "float")
|
||||
if encoding_format in ("bytes", "bytes_only"):
|
||||
async for _ in response.content.iter_any():
|
||||
pass
|
||||
else:
|
||||
await response.read()
|
||||
output.ttft = output.latency = time.perf_counter() - st
|
||||
|
||||
if payload.get("encoding_format", "float") == "bytes":
|
||||
if encoding_format == "bytes":
|
||||
metadata = json.loads(response.headers["metadata"])
|
||||
usage = metadata.get("usage", {})
|
||||
elif encoding_format == "bytes_only":
|
||||
usage = {"prompt_tokens": prompt_len}
|
||||
else:
|
||||
data = await response.json()
|
||||
usage = data.get("usage", {})
|
||||
@@ -672,6 +681,7 @@ async def async_request_openai_embeddings(
|
||||
headers=headers,
|
||||
pbar=pbar,
|
||||
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
|
||||
prompt_len=request_func_input.prompt_len,
|
||||
)
|
||||
|
||||
|
||||
@@ -709,6 +719,7 @@ async def async_request_vllm_rerank(
|
||||
headers=headers,
|
||||
pbar=pbar,
|
||||
num_input_sequences=len(request_func_input.prompt) - 1,
|
||||
prompt_len=request_func_input.prompt_len,
|
||||
)
|
||||
|
||||
|
||||
@@ -743,6 +754,7 @@ async def async_request_openai_embeddings_chat(
|
||||
payload=payload,
|
||||
headers=headers,
|
||||
pbar=pbar,
|
||||
prompt_len=request_func_input.prompt_len,
|
||||
)
|
||||
|
||||
|
||||
@@ -847,6 +859,7 @@ async def async_request_infinity_embeddings(
|
||||
headers=headers,
|
||||
pbar=pbar,
|
||||
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
|
||||
prompt_len=request_func_input.prompt_len,
|
||||
)
|
||||
|
||||
|
||||
@@ -896,6 +909,7 @@ async def async_request_vllm_pooling(
|
||||
headers=headers,
|
||||
pbar=pbar,
|
||||
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
|
||||
prompt_len=request_func_input.prompt_len,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user