[Benchmark] Add Responses API backend to vllm bench serve (#54628)
Signed-off-by: QHarshil <harshil_c@hotmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
Claude
Chauncey
parent
0da126676e
commit
af7f9488c2
@@ -2148,7 +2148,9 @@ def get_samples(
|
||||
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
|
||||
)
|
||||
# For the "sonnet" dataset, formatting depends on the backend.
|
||||
if args.backend == "openai-chat":
|
||||
# Chat-style backends leave templating to the server; completions
|
||||
# backends need the prompt rendered client side.
|
||||
if args.backend in ("openai-chat", "openai-responses"):
|
||||
input_requests = sonnet_dataset.sample(
|
||||
num_requests=args.num_prompts,
|
||||
input_len=args.sonnet_input_len,
|
||||
|
||||
@@ -447,6 +447,178 @@ async def async_request_openai_chat_completions(
|
||||
return output
|
||||
|
||||
|
||||
def _extract_sse_data(message: str) -> str | None:
|
||||
"""Extract the ``data`` payload of a single SSE message.
|
||||
|
||||
The Responses API stream labels every message with an ``event:`` line
|
||||
before its ``data:`` line, so the payload cannot be recovered by
|
||||
stripping a leading ``data: `` the way the Completions and Chat
|
||||
Completions streams allow.
|
||||
|
||||
Args:
|
||||
message: One complete SSE message, without its trailing blank line.
|
||||
|
||||
Returns:
|
||||
The payload, with multi-line payloads joined by newlines as required
|
||||
by the SSE spec, or None if the message carries no ``data:`` line.
|
||||
|
||||
"""
|
||||
# NOTE: split on "\n" rather than using splitlines(), which also breaks on
|
||||
# U+2028, U+2029 and U+0085. Those are legal inside a JSON string and the
|
||||
# server emits them unescaped, so splitlines() would truncate the payload.
|
||||
data_lines = [
|
||||
line.removeprefix("data:").removeprefix(" ")
|
||||
for line in message.split("\n")
|
||||
if line.startswith("data:")
|
||||
]
|
||||
if not data_lines:
|
||||
return None
|
||||
|
||||
return "\n".join(data_lines)
|
||||
|
||||
|
||||
# Responses events that carry generated tokens. Reasoning deltas are included
|
||||
# because the server is already decoding when it emits them, so excluding them
|
||||
# would report the whole reasoning phase as time to first token.
|
||||
_RESPONSES_TOKEN_DELTA_EVENTS = frozenset(
|
||||
{
|
||||
"response.output_text.delta",
|
||||
"response.reasoning_text.delta",
|
||||
}
|
||||
)
|
||||
|
||||
# Responses events that end a request. Truncation at the output length limit
|
||||
# is a normal benchmark outcome: vLLM reports it as `response.completed` with
|
||||
# `status: "incomplete"`, while the spec also defines a dedicated
|
||||
# `response.incomplete` terminal event.
|
||||
_RESPONSES_TERMINAL_EVENTS = frozenset(
|
||||
{
|
||||
"response.completed",
|
||||
"response.incomplete",
|
||||
}
|
||||
)
|
||||
|
||||
# The spec's failure terminals. vLLM does not emit them today, but
|
||||
# `vllm bench serve` can be pointed at other Responses servers.
|
||||
_RESPONSES_ERROR_EVENTS = frozenset({"error", "response.failed"})
|
||||
|
||||
|
||||
async def async_request_openai_responses(
|
||||
request_func_input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
pbar: tqdm | None = None,
|
||||
) -> RequestFuncOutput:
|
||||
api_url = request_func_input.api_url
|
||||
_validate_api_url(api_url, "OpenAI Responses API", "responses")
|
||||
|
||||
payload = {
|
||||
"model": request_func_input.model_name
|
||||
if request_func_input.model_name
|
||||
else request_func_input.model,
|
||||
"input": request_func_input.prompt,
|
||||
"max_output_tokens": request_func_input.output_len,
|
||||
"stream": True,
|
||||
# `store` defaults to true. A server started with
|
||||
# VLLM_ENABLE_RESPONSES_API_STORE=1 keeps every stored response for
|
||||
# the lifetime of the process, which would grow its heap over a long
|
||||
# benchmark run.
|
||||
"store": False,
|
||||
}
|
||||
_update_payload_common(payload, request_func_input)
|
||||
|
||||
headers = _get_headers("application/json")
|
||||
_update_headers_common(headers, request_func_input)
|
||||
|
||||
output = RequestFuncOutput()
|
||||
output.prompt_len = request_func_input.prompt_len
|
||||
|
||||
# Only output text is accumulated. Reasoning text is timed but not
|
||||
# returned, matching how `openai-chat` handles `DeltaMessage.reasoning`.
|
||||
generated_text = ""
|
||||
stream_error: str | None = None
|
||||
saw_terminal_event = False
|
||||
first_chunk_received = False
|
||||
st = time.perf_counter()
|
||||
output.start_time = st
|
||||
most_recent_timestamp = st
|
||||
try:
|
||||
async with session.post(url=api_url, json=payload, headers=headers) as response:
|
||||
if response.status == 200:
|
||||
handler = StreamedResponseHandler()
|
||||
|
||||
async for chunk_bytes in response.content.iter_any():
|
||||
for message in handler.add_chunk(chunk_bytes):
|
||||
# NOTE: `_extract_sse_data` returns None for SSE
|
||||
# comments, which servers may send as keep-alives.
|
||||
chunk = _extract_sse_data(message)
|
||||
if not chunk or chunk == "[DONE]":
|
||||
continue
|
||||
|
||||
timestamp = time.perf_counter()
|
||||
data = json.loads(chunk)
|
||||
event_type = data.get("type")
|
||||
|
||||
if event_type in _RESPONSES_TOKEN_DELTA_EVENTS:
|
||||
# First token
|
||||
if not first_chunk_received:
|
||||
first_chunk_received = True
|
||||
output.ttft = timestamp - st
|
||||
|
||||
# Decoding phase
|
||||
else:
|
||||
output.itl.append(timestamp - most_recent_timestamp)
|
||||
|
||||
if event_type == "response.output_text.delta":
|
||||
generated_text += data.get("delta") or ""
|
||||
|
||||
# Only token events advance the end of the
|
||||
# request, so `latency - ttft` stays equal to
|
||||
# `sum(itl)` and TPOT means what serve.py says.
|
||||
most_recent_timestamp = timestamp
|
||||
elif event_type in _RESPONSES_TERMINAL_EVENTS:
|
||||
saw_terminal_event = True
|
||||
usage = (data.get("response") or {}).get("usage") or {}
|
||||
if (n_output := usage.get("output_tokens")) is not None:
|
||||
output.output_tokens = n_output
|
||||
if (n_input := usage.get("input_tokens")) is not None:
|
||||
output.prompt_len = n_input
|
||||
elif event_type in _RESPONSES_ERROR_EVENTS or data.get("error"):
|
||||
stream_error = json.dumps(data.get("error") or data)
|
||||
|
||||
output.generated_text = generated_text
|
||||
output.latency = most_recent_timestamp - st
|
||||
if stream_error is not None:
|
||||
output.success = False
|
||||
output.error = stream_error
|
||||
elif not first_chunk_received:
|
||||
output.success = False
|
||||
output.error = (
|
||||
"Never received a token delta to calculate TTFT."
|
||||
"This response will be marked as failed!"
|
||||
)
|
||||
elif not saw_terminal_event:
|
||||
# Without the terminal event the response is truncated and
|
||||
# its usage block is missing.
|
||||
output.success = False
|
||||
output.error = (
|
||||
"Stream ended without a terminal event."
|
||||
"This response will be marked as failed!"
|
||||
)
|
||||
else:
|
||||
output.success = True
|
||||
else:
|
||||
output.error = response.reason or ""
|
||||
output.success = False
|
||||
except Exception:
|
||||
output.success = False
|
||||
exc_info = sys.exc_info()
|
||||
output.error = "".join(traceback.format_exception(*exc_info))
|
||||
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
|
||||
async def async_request_openai_audio(
|
||||
request_func_input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
@@ -920,6 +1092,7 @@ ASYNC_REQUEST_FUNCS: dict[str, RequestFunc] = {
|
||||
"vllm": async_request_openai_completions,
|
||||
"openai": async_request_openai_completions,
|
||||
"openai-chat": async_request_openai_chat_completions,
|
||||
"openai-responses": async_request_openai_responses,
|
||||
"openai-audio": async_request_openai_audio,
|
||||
"openai-embeddings": async_request_openai_embeddings,
|
||||
"openai-embeddings-chat": async_request_openai_embeddings_chat,
|
||||
@@ -947,5 +1120,10 @@ POOLING_BACKENDS = {
|
||||
OPENAI_COMPATIBLE_BACKENDS = [
|
||||
k
|
||||
for k, v in ASYNC_REQUEST_FUNCS.items()
|
||||
if v in (async_request_openai_completions, async_request_openai_chat_completions)
|
||||
if v
|
||||
in (
|
||||
async_request_openai_completions,
|
||||
async_request_openai_chat_completions,
|
||||
async_request_openai_responses,
|
||||
)
|
||||
]
|
||||
|
||||
@@ -2216,6 +2216,11 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"Sampling parameters are only supported by openai-compatible backends."
|
||||
)
|
||||
|
||||
# The Responses API accepts every sampling parameter above except
|
||||
# min_p, which it would silently drop as an unknown field.
|
||||
if args.backend == "openai-responses" and "min_p" in sampling_params:
|
||||
raise ValueError("--min-p is not supported by the Responses API.")
|
||||
|
||||
if "temperature" not in sampling_params:
|
||||
print(
|
||||
"WARNING: vllm bench serve no longer sets temperature==0 (greedy) "
|
||||
|
||||
Reference in New Issue
Block a user