[Pooling] Report input throughput for batched requests (#53213)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-08-22 09:01:40 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 10704541aa
commit 704f12aa1e
12 changed files with 93 additions and 8 deletions
+14 -2
View File
@@ -66,7 +66,7 @@ class StreamedResponseHandler:
class RequestFuncInput:
"""The input for the request function."""
prompt: str | list[str] | list[int] | list[dict[str, Any]]
prompt: str | list[str] | list[int] | list[list[int]] | list[dict[str, Any]]
api_url: str
prompt_len: int
output_len: int
@@ -100,6 +100,7 @@ class RequestFuncOutput:
error: str = ""
start_time: float = 0.0
input_audio_duration: float = 0.0 # in seconds
num_input_sequences: int = 1
class RequestFunc(Protocol):
@@ -586,8 +587,9 @@ async def _run_pooling_request(
payload: dict[str, Any],
headers: dict[str, Any],
pbar: tqdm | None = None,
num_input_sequences: int = 1,
) -> RequestFuncOutput:
output = RequestFuncOutput()
output = RequestFuncOutput(num_input_sequences=num_input_sequences)
st = time.perf_counter()
output.start_time = st
try:
@@ -617,6 +619,12 @@ async def _run_pooling_request(
return output
def _get_num_input_sequences(prompt: Any) -> int:
if prompt and isinstance(prompt, list) and isinstance(prompt[0], (str, list)):
return len(prompt)
return 1
async def async_request_openai_embeddings(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
@@ -645,6 +653,7 @@ async def async_request_openai_embeddings(
payload=payload,
headers=headers,
pbar=pbar,
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
)
@@ -681,6 +690,7 @@ async def async_request_vllm_rerank(
payload=payload,
headers=headers,
pbar=pbar,
num_input_sequences=len(request_func_input.prompt) - 1,
)
@@ -818,6 +828,7 @@ async def async_request_infinity_embeddings(
payload=payload,
headers=headers,
pbar=pbar,
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
)
@@ -866,6 +877,7 @@ async def async_request_vllm_pooling(
payload=payload,
headers=headers,
pbar=pbar,
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
)