[Bugfix][Frontend] Seed derender detokenization from the prompt (#59046)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -40,9 +40,14 @@ from vllm.logger import init_logger
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||
from vllm.tokenizers.detokenizer_utils import (
|
||||
convert_prompt_ids_to_tokens,
|
||||
detokenize_incrementally,
|
||||
get_leading_space_marker,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.v1.engine.detokenizer import uses_fast_detokenizer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -115,17 +120,37 @@ class OnlineDerenderer:
|
||||
self,
|
||||
generate_response: GenerateResponse,
|
||||
chat_request: ChatCompletionRequest | None = None,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> list[ChatCompletionResponseChoice]:
|
||||
return await self._derender_chat_async(generate_response, chat_request)
|
||||
return await self._derender_chat_async(
|
||||
generate_response, chat_request, prompt_token_ids
|
||||
)
|
||||
|
||||
def _derender_chat(
|
||||
self,
|
||||
generate_response: GenerateResponse,
|
||||
chat_request: ChatCompletionRequest | None = None,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> list[ChatCompletionResponseChoice]:
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
choices: list[ChatCompletionResponseChoice] = []
|
||||
|
||||
has_parser = self.parser is not None and chat_request is not None
|
||||
skip_special, spaces_between = _decode_params(
|
||||
tokenizer, chat_request, preserve_special=has_parser
|
||||
)
|
||||
seed_ids = (
|
||||
prompt_token_ids
|
||||
if prompt_token_ids is not None
|
||||
else generate_response.prompt_token_ids
|
||||
)
|
||||
# Choices continue the same prompt, so they share one seeded window.
|
||||
seed_state = _seed_stream_state(
|
||||
tokenizer,
|
||||
seed_ids,
|
||||
skip_special_tokens=skip_special,
|
||||
)
|
||||
|
||||
for choice in generate_response.choices:
|
||||
if not choice.token_ids:
|
||||
raise ValueError(f"choice {choice.index} has empty or null token_ids")
|
||||
@@ -136,12 +161,17 @@ class OnlineDerenderer:
|
||||
else None
|
||||
)
|
||||
|
||||
if self.parser is not None and chat_request is not None:
|
||||
if has_parser:
|
||||
assert self.parser is not None and chat_request is not None
|
||||
# Parser path: decode with special tokens preserved
|
||||
# so the parser can see markers like </think>,
|
||||
# <tool_call>, or Harmony channel tokens.
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=False
|
||||
decoded_text, _ = self._detokenize_delta(
|
||||
tokenizer,
|
||||
choice.token_ids,
|
||||
seed_state,
|
||||
skip_special_tokens=skip_special,
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
|
||||
chat_template_kwargs: dict[str, Any] = {}
|
||||
@@ -201,13 +231,12 @@ class OnlineDerenderer:
|
||||
else:
|
||||
# No parser: plain detokenization honouring the request's
|
||||
# skip_special_tokens (default True when no request was given).
|
||||
skip_special = (
|
||||
chat_request.skip_special_tokens
|
||||
if chat_request is not None
|
||||
else True
|
||||
)
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=skip_special
|
||||
decoded_text, _ = self._detokenize_delta(
|
||||
tokenizer,
|
||||
choice.token_ids,
|
||||
seed_state,
|
||||
skip_special_tokens=skip_special,
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
message = ChatMessage(role="assistant", content=decoded_text)
|
||||
|
||||
@@ -239,7 +268,7 @@ class OnlineDerenderer:
|
||||
(tracked by ``read_offset``). The delta tokens are fed straight onto it.
|
||||
|
||||
The window is bounded. ``detokenize_incrementally`` never reads before
|
||||
``prefix_offset``, so after processing we trim ``prev_tokens`` to that
|
||||
``prefix_offset``, so after each token we trim ``prev_tokens`` to that
|
||||
tail and rebase the offsets to it. State transport therefore stays
|
||||
O(window) per chunk instead of re-sending the full token history.
|
||||
|
||||
@@ -273,18 +302,20 @@ class OnlineDerenderer:
|
||||
skip_special_tokens=skip_special_tokens,
|
||||
spaces_between_special_tokens=spaces_between_special_tokens,
|
||||
)
|
||||
prev_tokens = prev_tokens + new_toks
|
||||
# Trim to the tail still readable by detokenize_incrementally
|
||||
# (everything before prefix_offset is dead) and rebase the
|
||||
# offsets, so the window stays bounded within a long batch
|
||||
# decode as well as across chunks.
|
||||
prev_tokens = (prev_tokens + new_toks)[prefix_offset:]
|
||||
read_offset -= prefix_offset
|
||||
prefix_offset = 0
|
||||
text_parts.append(text)
|
||||
|
||||
# Trim to the tail still readable by detokenize_incrementally
|
||||
# (everything before prefix_offset is dead) and rebase the offsets so
|
||||
# the carried window stays bounded regardless of generation length.
|
||||
trimmed = prev_tokens[prefix_offset:]
|
||||
updated_state = state.model_copy(
|
||||
update={
|
||||
"prev_tokens": trimmed,
|
||||
"prefix_offset": 0,
|
||||
"read_offset": read_offset - prefix_offset,
|
||||
"prev_tokens": prev_tokens,
|
||||
"prefix_offset": prefix_offset,
|
||||
"read_offset": read_offset,
|
||||
}
|
||||
)
|
||||
return "".join(text_parts), updated_state
|
||||
@@ -315,20 +346,16 @@ class OnlineDerenderer:
|
||||
(validated by the caller — see `ServingDerender`) because
|
||||
plain detokenization would leak raw parser markup into `content`.
|
||||
prompt_tokens: Prompt token count for the usage chunk.
|
||||
prompt_token_ids: Prompt token IDs. Parser path only. Lets
|
||||
`parse_delta` settle its initial reasoning state. Required
|
||||
when a reasoning or tool parser is configured (validated by
|
||||
the caller — see ``ServingDerender``). Without it reasoning
|
||||
left open by the prompt would be misclassified as content.
|
||||
prompt_token_ids: Prompt token IDs. Required on the parser path
|
||||
(validated by the caller) and optional otherwise, falling
|
||||
back to ``generate_chunk.prompt_token_ids``. See
|
||||
`DerenderChatStreamRequest.prompt_token_ids`.
|
||||
|
||||
Returns:
|
||||
(chunk, updated_state) — the derendered SSE chunk and the state
|
||||
the client must pass to the next call.
|
||||
|
||||
"""
|
||||
if state is None:
|
||||
state = DerenderStreamState()
|
||||
|
||||
# A single DerenderStreamState is threaded through every choice in
|
||||
# this chunk. Correct only when there is at most one choice per SSE
|
||||
# event (n=1, one call per index), as the streaming derender
|
||||
@@ -353,23 +380,34 @@ class OnlineDerenderer:
|
||||
parser_cls,
|
||||
model,
|
||||
generate_chunk,
|
||||
state,
|
||||
state if state is not None else DerenderStreamState(),
|
||||
chat_request,
|
||||
prompt_tokens,
|
||||
prompt_token_ids,
|
||||
)
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
skip_special = (
|
||||
chat_request.skip_special_tokens if chat_request is not None else True
|
||||
)
|
||||
skip_special, spaces_between = _decode_params(tokenizer, chat_request)
|
||||
# Seed on the first chunk only. A carried state already has the prompt.
|
||||
if state is None:
|
||||
state = _seed_stream_state(
|
||||
tokenizer,
|
||||
prompt_token_ids
|
||||
if prompt_token_ids is not None
|
||||
else generate_chunk.prompt_token_ids,
|
||||
skip_special_tokens=skip_special,
|
||||
)
|
||||
stream_choices: list[ChatCompletionResponseStreamChoice] = []
|
||||
updated_state = state
|
||||
|
||||
for choice in generate_chunk.choices:
|
||||
delta_tids = choice.token_ids or []
|
||||
new_text, updated_state = await self._detokenize_delta_async(
|
||||
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
|
||||
tokenizer,
|
||||
delta_tids,
|
||||
updated_state,
|
||||
skip_special_tokens=skip_special,
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
|
||||
# NOTE: parser-configured servers dispatch to
|
||||
@@ -454,7 +492,8 @@ class OnlineDerenderer:
|
||||
`state.output_chunk_lens`, so the rebuilt parser state matches
|
||||
standard serving, which calls `parse_delta` once per engine step.
|
||||
Text comes from a fresh incremental detokenizer with special tokens
|
||||
preserved (``skip_special_tokens=False``).
|
||||
preserved (``skip_special_tokens=False``), seeded from the prompt
|
||||
tail on every call since replay starts from scratch.
|
||||
"""
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
|
||||
@@ -480,7 +519,12 @@ class OnlineDerenderer:
|
||||
# across both the replay and current chunk phases (via `nonlocal`)
|
||||
# so multi-byte characters split across that boundary still decode
|
||||
# correctly. Discarded once the call returns.
|
||||
detok_state = DerenderStreamState()
|
||||
skip_special, spaces_between = _decode_params(
|
||||
tokenizer, chat_request, preserve_special=True
|
||||
)
|
||||
detok_state = _seed_stream_state(
|
||||
tokenizer, prompt_token_ids, skip_special_tokens=skip_special
|
||||
)
|
||||
|
||||
def _replay(token_ids: list[int], chunk_lens: list[int]) -> None:
|
||||
"""Replay prior chunks through `parse_delta` to rebuild parser
|
||||
@@ -493,7 +537,11 @@ class OnlineDerenderer:
|
||||
chunk = token_ids[start : start + chunk_len]
|
||||
start += chunk_len
|
||||
text, detok_state = self._detokenize_delta(
|
||||
tokenizer, chunk, detok_state, skip_special_tokens=False
|
||||
tokenizer,
|
||||
chunk,
|
||||
detok_state,
|
||||
skip_special_tokens=skip_special,
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
parser.parse_delta(
|
||||
text,
|
||||
@@ -530,7 +578,11 @@ class OnlineDerenderer:
|
||||
# granularity), not one per token. See the granularity note
|
||||
# in this method's docstring.
|
||||
text, detok_state = self._detokenize_delta(
|
||||
tokenizer, delta_tids, detok_state, skip_special_tokens=False
|
||||
tokenizer,
|
||||
delta_tids,
|
||||
detok_state,
|
||||
skip_special_tokens=skip_special,
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
delta_message = parser.parse_delta(
|
||||
text,
|
||||
@@ -633,9 +685,10 @@ class OnlineDerenderer:
|
||||
generate_responses: list[GenerateResponse],
|
||||
prompt_tokens: list[int] | None = None,
|
||||
completion_request: CompletionRequest | None = None,
|
||||
prompt_token_ids: list[list[int] | None] | None = None,
|
||||
) -> tuple[list[CompletionResponseChoice], int, int]:
|
||||
return await self._derender_completion_async(
|
||||
generate_responses, prompt_tokens, completion_request
|
||||
generate_responses, prompt_tokens, completion_request, prompt_token_ids
|
||||
)
|
||||
|
||||
def _derender_completion(
|
||||
@@ -643,24 +696,34 @@ class OnlineDerenderer:
|
||||
generate_responses: list[GenerateResponse],
|
||||
prompt_tokens: list[int] | None = None,
|
||||
completion_request: CompletionRequest | None = None,
|
||||
prompt_token_ids: list[list[int] | None] | None = None,
|
||||
) -> tuple[list[CompletionResponseChoice], int, int]:
|
||||
n = len(generate_responses)
|
||||
prompt_tokens_list: list[int] = (
|
||||
prompt_tokens if prompt_tokens is not None else [0] * n
|
||||
)
|
||||
|
||||
skip_special = (
|
||||
completion_request.skip_special_tokens
|
||||
if completion_request is not None
|
||||
else True
|
||||
prompt_token_ids_list: list[list[int] | None] = (
|
||||
prompt_token_ids if prompt_token_ids is not None else [None] * n
|
||||
)
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
skip_special, spaces_between = _decode_params(tokenizer, completion_request)
|
||||
choices: list[CompletionResponseChoice] = []
|
||||
total_prompt_tokens = 0
|
||||
total_completion_tokens = 0
|
||||
index = 0
|
||||
|
||||
for gen, pt in zip(generate_responses, prompt_tokens_list):
|
||||
for gen, pt, seed_ids_override in zip(
|
||||
generate_responses, prompt_tokens_list, prompt_token_ids_list
|
||||
):
|
||||
seed_ids = (
|
||||
seed_ids_override
|
||||
if seed_ids_override is not None
|
||||
else gen.prompt_token_ids
|
||||
)
|
||||
seed_state = _seed_stream_state(
|
||||
tokenizer, seed_ids, skip_special_tokens=skip_special
|
||||
)
|
||||
for choice in gen.choices:
|
||||
if not choice.token_ids:
|
||||
raise ValueError(
|
||||
@@ -668,8 +731,12 @@ class OnlineDerenderer:
|
||||
"has empty or null token_ids"
|
||||
)
|
||||
|
||||
decoded_text = tokenizer.decode(
|
||||
choice.token_ids, skip_special_tokens=skip_special
|
||||
decoded_text, _ = self._detokenize_delta(
|
||||
tokenizer,
|
||||
choice.token_ids,
|
||||
seed_state,
|
||||
skip_special_tokens=skip_special,
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
completion_logprobs = None
|
||||
if choice.logprobs is not None:
|
||||
@@ -698,6 +765,7 @@ class OnlineDerenderer:
|
||||
state: DerenderStreamState | None = None,
|
||||
prompt_tokens: int | None = None,
|
||||
completion_request: CompletionRequest | None = None,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> tuple[CompletionStreamResponse, DerenderStreamState]:
|
||||
"""Process one GenerateStreamResponse chunk for streaming completions.
|
||||
|
||||
@@ -716,14 +784,13 @@ class OnlineDerenderer:
|
||||
prompt_tokens: Prompt token count for usage (from the render step).
|
||||
completion_request: Original CompletionRequest from ``/render``;
|
||||
supplies ``skip_special_tokens``.
|
||||
prompt_token_ids: Seeds the first chunk's decode. Falls back to
|
||||
``generate_chunk.prompt_token_ids``.
|
||||
|
||||
Returns:
|
||||
(chunk, updated_state) — the derendered chunk and updated state.
|
||||
|
||||
"""
|
||||
if state is None:
|
||||
state = DerenderStreamState()
|
||||
|
||||
# See the equivalent check in derender_chat_stream: a single
|
||||
# DerenderStreamState is threaded through every choice in this
|
||||
# chunk, so more than one choice per chunk would corrupt the
|
||||
@@ -734,18 +801,27 @@ class OnlineDerenderer:
|
||||
)
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
skip_special = (
|
||||
completion_request.skip_special_tokens
|
||||
if completion_request is not None
|
||||
else True
|
||||
)
|
||||
skip_special, spaces_between = _decode_params(tokenizer, completion_request)
|
||||
# Seed on the first chunk only. A carried state already has the prompt.
|
||||
if state is None:
|
||||
state = _seed_stream_state(
|
||||
tokenizer,
|
||||
prompt_token_ids
|
||||
if prompt_token_ids is not None
|
||||
else generate_chunk.prompt_token_ids,
|
||||
skip_special_tokens=skip_special,
|
||||
)
|
||||
stream_choices: list[CompletionResponseStreamChoice] = []
|
||||
updated_state = state
|
||||
|
||||
for choice in generate_chunk.choices:
|
||||
delta_tids = choice.token_ids or []
|
||||
new_text, updated_state = await self._detokenize_delta_async(
|
||||
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
|
||||
tokenizer,
|
||||
delta_tids,
|
||||
updated_state,
|
||||
skip_special_tokens=skip_special,
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
|
||||
completion_logprobs = None
|
||||
@@ -810,6 +886,65 @@ def _logprob_context_tail(
|
||||
return (list(context_token_ids) + list(delta_token_ids))[-_LOGPROB_CONTEXT_WINDOW:]
|
||||
|
||||
|
||||
def _decode_params(
|
||||
tokenizer: TokenizerLike,
|
||||
request: ChatCompletionRequest | CompletionRequest | None,
|
||||
preserve_special: bool = False,
|
||||
) -> tuple[bool, bool]:
|
||||
"""Derive `(skip_special_tokens, spaces_between_special_tokens)` the way
|
||||
the engine does (`IncrementalDetokenizer`).
|
||||
|
||||
Args:
|
||||
tokenizer: Picks the engine's fast or slow detokenizer rules.
|
||||
request: The original request, if the caller supplied one.
|
||||
preserve_special: Keep special tokens so a parser can see markers.
|
||||
The serving side does this via `adjust_request`.
|
||||
|
||||
"""
|
||||
skip_special = (
|
||||
False
|
||||
if preserve_special
|
||||
else (request.skip_special_tokens if request is not None else True)
|
||||
)
|
||||
spaces_between = (
|
||||
request.spaces_between_special_tokens if request is not None else True
|
||||
)
|
||||
# Only the fast detokenizer forces spaces on when skipping special tokens.
|
||||
if skip_special and uses_fast_detokenizer(tokenizer):
|
||||
spaces_between = True
|
||||
return skip_special, spaces_between
|
||||
|
||||
|
||||
def _seed_stream_state(
|
||||
tokenizer: TokenizerLike,
|
||||
prompt_token_ids: list[int] | None,
|
||||
skip_special_tokens: bool,
|
||||
) -> DerenderStreamState:
|
||||
"""Build the initial decode state from the prompt tail, the same way the
|
||||
engine primes its incremental detokenizer. Without it, Metaspace
|
||||
tokenizers drop the first output token's leading space.
|
||||
|
||||
Returns an empty state when `prompt_token_ids` is empty or omitted.
|
||||
"""
|
||||
if not prompt_token_ids:
|
||||
if prompt_token_ids is None and get_leading_space_marker(tokenizer) is not None:
|
||||
logger.warning_once(
|
||||
"derender got no prompt_token_ids, so the first output token "
|
||||
"may lose its leading space compared to the coupled endpoint. "
|
||||
"Pass prompt_token_ids from the render step to avoid this."
|
||||
)
|
||||
return DerenderStreamState()
|
||||
|
||||
prev_tokens, prefix_offset, read_offset = convert_prompt_ids_to_tokens(
|
||||
tokenizer, prompt_token_ids, skip_special_tokens=skip_special_tokens
|
||||
)
|
||||
return DerenderStreamState(
|
||||
prev_tokens=prev_tokens,
|
||||
prefix_offset=prefix_offset,
|
||||
read_offset=read_offset,
|
||||
)
|
||||
|
||||
|
||||
def _parse_token_id_placeholder(token: str) -> int | None:
|
||||
"""Extract token ID from a 'token_id:N' placeholder string."""
|
||||
if not token.startswith("token_id:"):
|
||||
|
||||
Reference in New Issue
Block a user