[Bench] benchmark_serving_multi_turn: make non-standard conversation_id payload opt-in (#43756)
Signed-off-by: Change72 <cguo51@asu.edu>
This commit is contained in:
@@ -65,6 +65,7 @@ class RequestArgs(NamedTuple):
|
||||
limit_min_tokens: int # Use negative value for no limit
|
||||
limit_max_tokens: int # Use negative value for no limit
|
||||
timeout_sec: int
|
||||
send_conversation_id: bool
|
||||
headers: dict[str, str]
|
||||
|
||||
|
||||
@@ -453,7 +454,7 @@ async def send_turn(
|
||||
min_tokens,
|
||||
max_tokens,
|
||||
req_args.timeout_sec,
|
||||
conversation_id=conv_id,
|
||||
conversation_id=conv_id if req_args.send_conversation_id else None,
|
||||
headers=req_args.headers,
|
||||
)
|
||||
|
||||
@@ -912,6 +913,7 @@ def get_client_config(
|
||||
limit_min_tokens=args.limit_min_tokens,
|
||||
limit_max_tokens=args.limit_max_tokens,
|
||||
timeout_sec=args.request_timeout_sec,
|
||||
send_conversation_id=args.send_conversation_id,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
@@ -1486,6 +1488,22 @@ async def main() -> None:
|
||||
help="Disable stream/streaming mode (set 'stream' to False in the API request)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--send-conversation-id",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help=(
|
||||
"Inject a `conversation_id` field into each Chat Completions "
|
||||
"payload. This is a non-standard OpenAI extension consumed by "
|
||||
"vLLM's disaggregated multi-turn proxy "
|
||||
"(examples/disaggregated/disaggregated_serving/"
|
||||
"disagg_proxy_multiturn.py) to key cross-turn KV cache reuse. "
|
||||
"Leave disabled (default) when targeting strict "
|
||||
"OpenAI-compatible endpoints; enable when benchmarking the "
|
||||
"disaggregated proxy."
|
||||
),
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-e",
|
||||
"--excel-output",
|
||||
|
||||
Reference in New Issue
Block a user