[Bugfix][Frontend] Validate reused prompt token ids; render messages for media and echo (#55771)
Signed-off-by: Shijie Lyu <sjlyu@amazon.com> Co-authored-by: Shijie Lyu <sjlyu@amazon.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Shijie Lyu
Claude
parent
0af34418e9
commit
c6843d397c
@@ -107,11 +107,35 @@ def _reused_prompt_token_ids(request: Any) -> list[int] | None:
|
||||
Disaggregated serving carries the prefill stage's ids in
|
||||
``kv_transfer_params`` so the decode stage can skip re-tokenizing. Removing
|
||||
the key keeps the id list out of the engine's sampling metadata.
|
||||
|
||||
Returns None without checking the ids when ``echo`` is set, since echo
|
||||
needs ``messages`` to be rendered. Otherwise raises VLLMValidationError if
|
||||
the ids are malformed.
|
||||
"""
|
||||
kv = getattr(request, "kv_transfer_params", None)
|
||||
if not isinstance(kv, dict):
|
||||
return None
|
||||
return kv.pop("prompt_token_ids", None) or None
|
||||
ids = kv.pop("prompt_token_ids", None)
|
||||
if ids is None:
|
||||
return None
|
||||
if getattr(request, "echo", False):
|
||||
logger.debug(
|
||||
"Ignoring kv_transfer_params['prompt_token_ids']: "
|
||||
"echo is set, so messages are rendered instead."
|
||||
)
|
||||
return None
|
||||
# bool is an int subclass, hence the exact type check.
|
||||
if (
|
||||
not isinstance(ids, list)
|
||||
or not ids
|
||||
or any(type(x) is not int or x < 0 for x in ids)
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"`kv_transfer_params['prompt_token_ids']` must be a non-empty list "
|
||||
"of non-negative integers.",
|
||||
parameter="kv_transfer_params.prompt_token_ids",
|
||||
)
|
||||
return ids
|
||||
|
||||
|
||||
class OnlineRenderer:
|
||||
|
||||
Reference in New Issue
Block a user