[Frontend] Reuse prefill token ids on the decode chat path for disaggregated serving (#48145)
Signed-off-by: Seiji Eicher <seiji@anyscale.com>
This commit is contained in:
@@ -47,6 +47,19 @@ from vllm.utils.mistral import mt as _mt
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _reused_prompt_token_ids(request: Any) -> list[int] | None:
|
||||
"""Pop prompt token ids forwarded for decode-side reuse, if any.
|
||||
|
||||
Disaggregated serving carries the prefill stage's ids in
|
||||
``kv_transfer_params`` so the decode stage can skip re-tokenizing. Removing
|
||||
the key keeps the id list out of the engine's sampling metadata.
|
||||
"""
|
||||
kv = getattr(request, "kv_transfer_params", None)
|
||||
if not isinstance(kv, dict):
|
||||
return None
|
||||
return kv.pop("prompt_token_ids", None) or None
|
||||
|
||||
|
||||
class OnlineRenderer:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -102,6 +115,12 @@ class OnlineRenderer:
|
||||
|
||||
Called directly by render_chat_request and delegated to by
|
||||
OpenAIServingChat.render_chat_request after its engine-aware checks.
|
||||
|
||||
Decode-side token reuse (ids forwarded in ``kv_transfer_params``) is
|
||||
handled deeper, in ``preprocess_chat`` / ``_make_request_with_harmony``,
|
||||
so it skips only templating and tokenization while tool-choice
|
||||
validation and ``adjust_request`` still run and the output is
|
||||
detokenized (text-out).
|
||||
"""
|
||||
tokenizer = self.renderer.tokenizer
|
||||
|
||||
@@ -186,6 +205,13 @@ class OnlineRenderer:
|
||||
should_include_tools: bool = True,
|
||||
):
|
||||
"""Build Harmony (GPT-OSS) messages and engine prompt from a chat request."""
|
||||
reuse_ids = _reused_prompt_token_ids(request)
|
||||
if reuse_ids:
|
||||
# Decode-side token reuse: feed the forwarded ids straight to the
|
||||
# engine. Harmony has no adjust_request hook to preserve.
|
||||
engine_input = tokens_input(reuse_ids, cache_salt=request.cache_salt)
|
||||
return [], [engine_input]
|
||||
|
||||
messages: list[OpenAIMessage] = []
|
||||
|
||||
# because of issues with pydantic we need to potentially
|
||||
@@ -368,17 +394,28 @@ class OnlineRenderer:
|
||||
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||
)
|
||||
|
||||
(conversation,), (engine_input,) = await renderer.render_chat_async(
|
||||
[messages],
|
||||
chat_params,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
reuse_ids = _reused_prompt_token_ids(request)
|
||||
if reuse_ids:
|
||||
# Decode-side token reuse: feed the forwarded ids straight to the
|
||||
# engine, skipping templating and tokenization. ``messages`` are not
|
||||
# tokenized, so conversation is empty. The adjust_request tail below
|
||||
# still runs.
|
||||
conversation: list[ConversationMessage] = []
|
||||
engine_input = tokens_input(
|
||||
reuse_ids, cache_salt=getattr(request, "cache_salt", None)
|
||||
)
|
||||
else:
|
||||
(conversation,), (engine_input,) = await renderer.render_chat_async(
|
||||
[messages],
|
||||
chat_params,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
# tool parsing is done only if a tool_parser has been set and if
|
||||
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
|
||||
|
||||
Reference in New Issue
Block a user