[Bench] benchmark_serving_multi_turn: make non-standard conversation_id payload opt-in (#43756)

Signed-off-by: Change72 <cguo51@asu.edu>
This commit is contained in:
Change72
2026-06-09 19:41:56 -07:00
committed by GitHub
parent 6deb05e0e4
commit 320c52b134
3 changed files with 61 additions and 2 deletions
@@ -65,6 +65,7 @@ class RequestArgs(NamedTuple):
limit_min_tokens: int # Use negative value for no limit
limit_max_tokens: int # Use negative value for no limit
timeout_sec: int
send_conversation_id: bool
headers: dict[str, str]
@@ -453,7 +454,7 @@ async def send_turn(
min_tokens,
max_tokens,
req_args.timeout_sec,
conversation_id=conv_id,
conversation_id=conv_id if req_args.send_conversation_id else None,
headers=req_args.headers,
)
@@ -912,6 +913,7 @@ def get_client_config(
limit_min_tokens=args.limit_min_tokens,
limit_max_tokens=args.limit_max_tokens,
timeout_sec=args.request_timeout_sec,
send_conversation_id=args.send_conversation_id,
headers=headers,
)
@@ -1486,6 +1488,22 @@ async def main() -> None:
help="Disable stream/streaming mode (set 'stream' to False in the API request)",
)
parser.add_argument(
"--send-conversation-id",
default=False,
action="store_true",
help=(
"Inject a `conversation_id` field into each Chat Completions "
"payload. This is a non-standard OpenAI extension consumed by "
"vLLM's disaggregated multi-turn proxy "
"(examples/disaggregated/disaggregated_serving/"
"disagg_proxy_multiturn.py) to key cross-turn KV cache reuse. "
"Leave disabled (default) when targeting strict "
"OpenAI-compatible endpoints; enable when benchmarking the "
"disaggregated proxy."
),
)
parser.add_argument(
"-e",
"--excel-output",