diff --git a/benchmarks/multi_turn/benchmark_serving_multi_turn.py b/benchmarks/multi_turn/benchmark_serving_multi_turn.py index 5f0c194af6..5a60d9c668 100644 --- a/benchmarks/multi_turn/benchmark_serving_multi_turn.py +++ b/benchmarks/multi_turn/benchmark_serving_multi_turn.py @@ -65,6 +65,7 @@ class RequestArgs(NamedTuple): limit_min_tokens: int # Use negative value for no limit limit_max_tokens: int # Use negative value for no limit timeout_sec: int + send_conversation_id: bool headers: dict[str, str] @@ -453,7 +454,7 @@ async def send_turn( min_tokens, max_tokens, req_args.timeout_sec, - conversation_id=conv_id, + conversation_id=conv_id if req_args.send_conversation_id else None, headers=req_args.headers, ) @@ -912,6 +913,7 @@ def get_client_config( limit_min_tokens=args.limit_min_tokens, limit_max_tokens=args.limit_max_tokens, timeout_sec=args.request_timeout_sec, + send_conversation_id=args.send_conversation_id, headers=headers, ) @@ -1486,6 +1488,22 @@ async def main() -> None: help="Disable stream/streaming mode (set 'stream' to False in the API request)", ) + parser.add_argument( + "--send-conversation-id", + default=False, + action="store_true", + help=( + "Inject a `conversation_id` field into each Chat Completions " + "payload. This is a non-standard OpenAI extension consumed by " + "vLLM's disaggregated multi-turn proxy " + "(examples/disaggregated/disaggregated_serving/" + "disagg_proxy_multiturn.py) to key cross-turn KV cache reuse. " + "Leave disabled (default) when targeting strict " + "OpenAI-compatible endpoints; enable when benchmarking the " + "disaggregated proxy." + ), + ) + parser.add_argument( "-e", "--excel-output", diff --git a/docs/features/nixl_connector_usage.md b/docs/features/nixl_connector_usage.md index 0f0cbd5535..8ab29b4388 100644 --- a/docs/features/nixl_connector_usage.md +++ b/docs/features/nixl_connector_usage.md @@ -294,6 +294,21 @@ curl http://localhost:8000/v1/chat/completions \ !!! note The `conversation_id` field is a non-standard extension to the OpenAI API. It is consumed by the proxy and not forwarded to the vLLM engine. +### Benchmarking the multi-turn proxy + +[`benchmarks/multi_turn/benchmark_serving_multi_turn.py`](../../benchmarks/multi_turn/benchmark_serving_multi_turn.py) supports targeting the disaggregated multi-turn proxy with the `--send-conversation-id` flag, which injects a per-conversation `conversation_id` into every request payload so the proxy can key cross-turn KV cache reuse. + +The flag is **off by default** so the benchmark is compatible with strict OpenAI-compatible frontends that reject unknown top-level fields. When benchmarking the multi-turn proxy you must pass it explicitly — otherwise every turn lands as a cache MISS and the bidirectional KV transfer path is never exercised. + +```bash +python benchmarks/multi_turn/benchmark_serving_multi_turn.py \ + --model --served-model-name \ + --url http://:8000 \ + --input-file benchmarks/multi_turn/generate_multi_turn.json \ + --num-clients 2 --max-active-conversations 6 \ + --send-conversation-id +``` + ### Limitations - Requires a stateful proxy (or equivalent router) to track and forward `kv_transfer_params` between turns. diff --git a/examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py b/examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py index 24d90eab02..cc1cc402d2 100644 --- a/examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py +++ b/examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py @@ -35,12 +35,36 @@ Conversation isolation: the JSON body) to scope the KV cache across turns. Without it, the proxy cannot link turns and falls back to no-cache behavior. + ``conversation_id`` is a non-standard extension to the OpenAI Chat + Completions schema, consumed by this proxy and not forwarded to the + vLLM engine. Strict OpenAI-compatible frontends reject unknown + fields, so clients must opt in only when targeting this proxy. + Usage: python disagg_proxy_multiturn.py \\ --host 0.0.0.0 --port 8000 \\ --prefiller-host 10.0.0.1 --prefiller-port 8100 \\ --decoder-host 10.0.0.2 --decoder-port 8200 +Benchmarking: + Use ``benchmarks/multi_turn/benchmark_serving_multi_turn.py`` with + the ``--send-conversation-id`` flag to inject a per-conversation + ``conversation_id`` into every request so this proxy can key + cross-turn KV cache reuse. The flag is *off by default*: without + it the benchmark sends OpenAI-schema-compliant payloads and every + turn lands as a cache MISS in this proxy. + + Example: + python benchmarks/multi_turn/benchmark_serving_multi_turn.py \\ + --model --served-model-name \\ + --url http://:8000 \\ + --input-file generate_multi_turn.json \\ + --num-clients 2 --max-active-conversations 6 \\ + --send-conversation-id + + See ``docs/features/nixl_connector_usage.md`` for the broader + bidirectional-KV-transfer setup these benchmarks exercise. + Dependencies: pip install fastapi uvicorn httpx """ @@ -373,7 +397,9 @@ async def _handle_request(api_path: str, request: Request): logger.warning( "[%s] No conversation_id provided — KV cache reuse disabled " "for this request. Add a 'conversation_id' field to enable " - "cross-turn KV sharing.", + "cross-turn KV sharing. When using " + "benchmarks/multi_turn/benchmark_serving_multi_turn.py, pass " + "--send-conversation-id (off by default).", request_id, )