[Bench] benchmark_serving_multi_turn: make non-standard conversation_id payload opt-in (#43756)

Signed-off-by: Change72 <cguo51@asu.edu>

benchmarks/multi_turn/benchmark_serving_multi_turn.py

@@ -65,6 +65,7 @@ class RequestArgs(NamedTuple):
     limit_min_tokens: int  # Use negative value for no limit
     limit_max_tokens: int  # Use negative value for no limit
     timeout_sec: int
+    send_conversation_id: bool
     headers: dict[str, str]
 
 
@@ -453,7 +454,7 @@ async def send_turn(
         min_tokens,
         max_tokens,
         req_args.timeout_sec,
-        conversation_id=conv_id,
+        conversation_id=conv_id if req_args.send_conversation_id else None,
         headers=req_args.headers,
     )
 
@@ -912,6 +913,7 @@ def get_client_config(
         limit_min_tokens=args.limit_min_tokens,
         limit_max_tokens=args.limit_max_tokens,
         timeout_sec=args.request_timeout_sec,
+        send_conversation_id=args.send_conversation_id,
         headers=headers,
     )
 
@@ -1486,6 +1488,22 @@ async def main() -> None:
         help="Disable stream/streaming mode (set 'stream' to False in the API request)",
     )
 
+    parser.add_argument(
+        "--send-conversation-id",
+        default=False,
+        action="store_true",
+        help=(
+            "Inject a `conversation_id` field into each Chat Completions "
+            "payload. This is a non-standard OpenAI extension consumed by "
+            "vLLM's disaggregated multi-turn proxy "
+            "(examples/disaggregated/disaggregated_serving/"
+            "disagg_proxy_multiturn.py) to key cross-turn KV cache reuse. "
+            "Leave disabled (default) when targeting strict "
+            "OpenAI-compatible endpoints; enable when benchmarking the "
+            "disaggregated proxy."
+        ),
+    )
+
     parser.add_argument(
         "-e",
         "--excel-output",

docs/features/nixl_connector_usage.md

@@ -294,6 +294,21 @@ curl http://localhost:8000/v1/chat/completions \
 !!! note
     The `conversation_id` field is a non-standard extension to the OpenAI API. It is consumed by the proxy and not forwarded to the vLLM engine.
 
+### Benchmarking the multi-turn proxy
+
+[`benchmarks/multi_turn/benchmark_serving_multi_turn.py`](../../benchmarks/multi_turn/benchmark_serving_multi_turn.py) supports targeting the disaggregated multi-turn proxy with the `--send-conversation-id` flag, which injects a per-conversation `conversation_id` into every request payload so the proxy can key cross-turn KV cache reuse.
+
+The flag is **off by default** so the benchmark is compatible with strict OpenAI-compatible frontends that reject unknown top-level fields. When benchmarking the multi-turn proxy you must pass it explicitly — otherwise every turn lands as a cache MISS and the bidirectional KV transfer path is never exercised.
+
+```bash
+python benchmarks/multi_turn/benchmark_serving_multi_turn.py \
+  --model <MODEL> --served-model-name <NAME> \
+  --url http://<proxy_host>:8000 \
+  --input-file benchmarks/multi_turn/generate_multi_turn.json \
+  --num-clients 2 --max-active-conversations 6 \
+  --send-conversation-id
+```
+
 ### Limitations
 
 - Requires a stateful proxy (or equivalent router) to track and forward `kv_transfer_params` between turns.

examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py

@@ -35,12 +35,36 @@ Conversation isolation:
     the JSON body) to scope the KV cache across turns. Without it, the
     proxy cannot link turns and falls back to no-cache behavior.
 
+    ``conversation_id`` is a non-standard extension to the OpenAI Chat
+    Completions schema, consumed by this proxy and not forwarded to the
+    vLLM engine. Strict OpenAI-compatible frontends reject unknown
+    fields, so clients must opt in only when targeting this proxy.
+
 Usage:
     python disagg_proxy_multiturn.py \\
         --host 0.0.0.0 --port 8000 \\
         --prefiller-host 10.0.0.1 --prefiller-port 8100 \\
         --decoder-host 10.0.0.2 --decoder-port 8200
 
+Benchmarking:
+    Use ``benchmarks/multi_turn/benchmark_serving_multi_turn.py`` with
+    the ``--send-conversation-id`` flag to inject a per-conversation
+    ``conversation_id`` into every request so this proxy can key
+    cross-turn KV cache reuse. The flag is *off by default*: without
+    it the benchmark sends OpenAI-schema-compliant payloads and every
+    turn lands as a cache MISS in this proxy.
+
+    Example:
+        python benchmarks/multi_turn/benchmark_serving_multi_turn.py \\
+            --model <MODEL> --served-model-name <NAME> \\
+            --url http://<proxy_host>:8000 \\
+            --input-file generate_multi_turn.json \\
+            --num-clients 2 --max-active-conversations 6 \\
+            --send-conversation-id
+
+    See ``docs/features/nixl_connector_usage.md`` for the broader
+    bidirectional-KV-transfer setup these benchmarks exercise.
+
 Dependencies:
     pip install fastapi uvicorn httpx
 """
@@ -373,7 +397,9 @@ async def _handle_request(api_path: str, request: Request):
         logger.warning(
             "[%s] No conversation_id provided — KV cache reuse disabled "
             "for this request. Add a 'conversation_id' field to enable "
-            "cross-turn KV sharing.",
+            "cross-turn KV sharing. When using "
+            "benchmarks/multi_turn/benchmark_serving_multi_turn.py, pass "
+            "--send-conversation-id (off by default).",
             request_id,
         )