[Frontend] Reuse prefill token ids on the decode chat path for disaggregated serving (#48145)

Signed-off-by: Seiji Eicher <seiji@anyscale.com>
This commit is contained in:
Seiji Eicher
2026-07-29 10:02:57 +02:00
committed by GitHub
parent 65a1a16594
commit 6370e53f24
6 changed files with 198 additions and 13 deletions
+48 -11
View File
@@ -47,6 +47,19 @@ from vllm.utils.mistral import mt as _mt
logger = init_logger(__name__)
def _reused_prompt_token_ids(request: Any) -> list[int] | None:
"""Pop prompt token ids forwarded for decode-side reuse, if any.
Disaggregated serving carries the prefill stage's ids in
``kv_transfer_params`` so the decode stage can skip re-tokenizing. Removing
the key keeps the id list out of the engine's sampling metadata.
"""
kv = getattr(request, "kv_transfer_params", None)
if not isinstance(kv, dict):
return None
return kv.pop("prompt_token_ids", None) or None
class OnlineRenderer:
def __init__(
self,
@@ -102,6 +115,12 @@ class OnlineRenderer:
Called directly by render_chat_request and delegated to by
OpenAIServingChat.render_chat_request after its engine-aware checks.
Decode-side token reuse (ids forwarded in ``kv_transfer_params``) is
handled deeper, in ``preprocess_chat`` / ``_make_request_with_harmony``,
so it skips only templating and tokenization while tool-choice
validation and ``adjust_request`` still run and the output is
detokenized (text-out).
"""
tokenizer = self.renderer.tokenizer
@@ -186,6 +205,13 @@ class OnlineRenderer:
should_include_tools: bool = True,
):
"""Build Harmony (GPT-OSS) messages and engine prompt from a chat request."""
reuse_ids = _reused_prompt_token_ids(request)
if reuse_ids:
# Decode-side token reuse: feed the forwarded ids straight to the
# engine. Harmony has no adjust_request hook to preserve.
engine_input = tokens_input(reuse_ids, cache_salt=request.cache_salt)
return [], [engine_input]
messages: list[OpenAIMessage] = []
# because of issues with pydantic we need to potentially
@@ -368,17 +394,28 @@ class OnlineRenderer:
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
)
(conversation,), (engine_input,) = await renderer.render_chat_async(
[messages],
chat_params,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
skip_mm_cache=skip_mm_cache,
)
reuse_ids = _reused_prompt_token_ids(request)
if reuse_ids:
# Decode-side token reuse: feed the forwarded ids straight to the
# engine, skipping templating and tokenization. ``messages`` are not
# tokenized, so conversation is empty. The adjust_request tail below
# still runs.
conversation: list[ConversationMessage] = []
engine_input = tokens_input(
reuse_ids, cache_salt=getattr(request, "cache_salt", None)
)
else:
(conversation,), (engine_input,) = await renderer.render_chat_async(
[messages],
chat_params,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
skip_mm_cache=skip_mm_cache,
)
# tool parsing is done only if a tool_parser has been set and if
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser