[Benchmark] Add Responses API backend to vllm bench serve (#54628)

Signed-off-by: QHarshil <harshil_c@hotmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Harshil
2026-09-28 05:04:11 +00:00
committed by GitHub
co-authored by Claude Chauncey
parent 0da126676e
commit af7f9488c2
5 changed files with 682 additions and 2 deletions
+3 -1
View File
@@ -2148,7 +2148,9 @@ def get_samples(
dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle
)
# For the "sonnet" dataset, formatting depends on the backend.
if args.backend == "openai-chat":
# Chat-style backends leave templating to the server; completions
# backends need the prompt rendered client side.
if args.backend in ("openai-chat", "openai-responses"):
input_requests = sonnet_dataset.sample(
num_requests=args.num_prompts,
input_len=args.sonnet_input_len,
+179 -1
View File
@@ -447,6 +447,178 @@ async def async_request_openai_chat_completions(
return output
def _extract_sse_data(message: str) -> str | None:
"""Extract the ``data`` payload of a single SSE message.
The Responses API stream labels every message with an ``event:`` line
before its ``data:`` line, so the payload cannot be recovered by
stripping a leading ``data: `` the way the Completions and Chat
Completions streams allow.
Args:
message: One complete SSE message, without its trailing blank line.
Returns:
The payload, with multi-line payloads joined by newlines as required
by the SSE spec, or None if the message carries no ``data:`` line.
"""
# NOTE: split on "\n" rather than using splitlines(), which also breaks on
# U+2028, U+2029 and U+0085. Those are legal inside a JSON string and the
# server emits them unescaped, so splitlines() would truncate the payload.
data_lines = [
line.removeprefix("data:").removeprefix(" ")
for line in message.split("\n")
if line.startswith("data:")
]
if not data_lines:
return None
return "\n".join(data_lines)
# Responses events that carry generated tokens. Reasoning deltas are included
# because the server is already decoding when it emits them, so excluding them
# would report the whole reasoning phase as time to first token.
_RESPONSES_TOKEN_DELTA_EVENTS = frozenset(
{
"response.output_text.delta",
"response.reasoning_text.delta",
}
)
# Responses events that end a request. Truncation at the output length limit
# is a normal benchmark outcome: vLLM reports it as `response.completed` with
# `status: "incomplete"`, while the spec also defines a dedicated
# `response.incomplete` terminal event.
_RESPONSES_TERMINAL_EVENTS = frozenset(
{
"response.completed",
"response.incomplete",
}
)
# The spec's failure terminals. vLLM does not emit them today, but
# `vllm bench serve` can be pointed at other Responses servers.
_RESPONSES_ERROR_EVENTS = frozenset({"error", "response.failed"})
async def async_request_openai_responses(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
pbar: tqdm | None = None,
) -> RequestFuncOutput:
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Responses API", "responses")
payload = {
"model": request_func_input.model_name
if request_func_input.model_name
else request_func_input.model,
"input": request_func_input.prompt,
"max_output_tokens": request_func_input.output_len,
"stream": True,
# `store` defaults to true. A server started with
# VLLM_ENABLE_RESPONSES_API_STORE=1 keeps every stored response for
# the lifetime of the process, which would grow its heap over a long
# benchmark run.
"store": False,
}
_update_payload_common(payload, request_func_input)
headers = _get_headers("application/json")
_update_headers_common(headers, request_func_input)
output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
# Only output text is accumulated. Reasoning text is timed but not
# returned, matching how `openai-chat` handles `DeltaMessage.reasoning`.
generated_text = ""
stream_error: str | None = None
saw_terminal_event = False
first_chunk_received = False
st = time.perf_counter()
output.start_time = st
most_recent_timestamp = st
try:
async with session.post(url=api_url, json=payload, headers=headers) as response:
if response.status == 200:
handler = StreamedResponseHandler()
async for chunk_bytes in response.content.iter_any():
for message in handler.add_chunk(chunk_bytes):
# NOTE: `_extract_sse_data` returns None for SSE
# comments, which servers may send as keep-alives.
chunk = _extract_sse_data(message)
if not chunk or chunk == "[DONE]":
continue
timestamp = time.perf_counter()
data = json.loads(chunk)
event_type = data.get("type")
if event_type in _RESPONSES_TOKEN_DELTA_EVENTS:
# First token
if not first_chunk_received:
first_chunk_received = True
output.ttft = timestamp - st
# Decoding phase
else:
output.itl.append(timestamp - most_recent_timestamp)
if event_type == "response.output_text.delta":
generated_text += data.get("delta") or ""
# Only token events advance the end of the
# request, so `latency - ttft` stays equal to
# `sum(itl)` and TPOT means what serve.py says.
most_recent_timestamp = timestamp
elif event_type in _RESPONSES_TERMINAL_EVENTS:
saw_terminal_event = True
usage = (data.get("response") or {}).get("usage") or {}
if (n_output := usage.get("output_tokens")) is not None:
output.output_tokens = n_output
if (n_input := usage.get("input_tokens")) is not None:
output.prompt_len = n_input
elif event_type in _RESPONSES_ERROR_EVENTS or data.get("error"):
stream_error = json.dumps(data.get("error") or data)
output.generated_text = generated_text
output.latency = most_recent_timestamp - st
if stream_error is not None:
output.success = False
output.error = stream_error
elif not first_chunk_received:
output.success = False
output.error = (
"Never received a token delta to calculate TTFT."
"This response will be marked as failed!"
)
elif not saw_terminal_event:
# Without the terminal event the response is truncated and
# its usage block is missing.
output.success = False
output.error = (
"Stream ended without a terminal event."
"This response will be marked as failed!"
)
else:
output.success = True
else:
output.error = response.reason or ""
output.success = False
except Exception:
output.success = False
exc_info = sys.exc_info()
output.error = "".join(traceback.format_exception(*exc_info))
if pbar:
pbar.update(1)
return output
async def async_request_openai_audio(
request_func_input: RequestFuncInput,
session: aiohttp.ClientSession,
@@ -920,6 +1092,7 @@ ASYNC_REQUEST_FUNCS: dict[str, RequestFunc] = {
"vllm": async_request_openai_completions,
"openai": async_request_openai_completions,
"openai-chat": async_request_openai_chat_completions,
"openai-responses": async_request_openai_responses,
"openai-audio": async_request_openai_audio,
"openai-embeddings": async_request_openai_embeddings,
"openai-embeddings-chat": async_request_openai_embeddings_chat,
@@ -947,5 +1120,10 @@ POOLING_BACKENDS = {
OPENAI_COMPATIBLE_BACKENDS = [
k
for k, v in ASYNC_REQUEST_FUNCS.items()
if v in (async_request_openai_completions, async_request_openai_chat_completions)
if v
in (
async_request_openai_completions,
async_request_openai_chat_completions,
async_request_openai_responses,
)
]
+5
View File
@@ -2216,6 +2216,11 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
"Sampling parameters are only supported by openai-compatible backends."
)
# The Responses API accepts every sampling parameter above except
# min_p, which it would silently drop as an unknown field.
if args.backend == "openai-responses" and "min_p" in sampling_params:
raise ValueError("--min-p is not supported by the Responses API.")
if "temperature" not in sampling_params:
print(
"WARNING: vllm bench serve no longer sets temperature==0 (greedy) "