[Bugfix][Frontend] Seed derender detokenization from the prompt (#59046)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-10-02 03:17:56 +00:00
committed by GitHub
parent 8faacd707f
commit fcfacc3499
9 changed files with 760 additions and 163 deletions
+32 -1
View File
@@ -117,6 +117,20 @@ class ServingDerender(BaseServing):
return None
def _validate_prompt_token_ids(
self,
prompt_token_ids: list[list[int] | None],
) -> ErrorResponse | None:
"""Reject caller-supplied prompt_token_ids longer than max_model_len."""
max_model_len = self.model_config.max_model_len
for ids in prompt_token_ids:
if ids is not None and len(ids) > max_model_len:
return self.create_error_response(
f"prompt_token_ids length ({len(ids)}) exceeds "
f"max_model_len ({max_model_len})."
)
return None
async def derender_chat_response(
self,
request: DerenderChatRequest,
@@ -138,6 +152,10 @@ class ServingDerender(BaseServing):
if bounds_error is not None:
return bounds_error
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
if prompt_ids_error is not None:
return prompt_ids_error
if self.online_derenderer.parser is not None and request.chat_request is None:
return self.create_error_response(
"chat_request is required when a tool or reasoning parser is "
@@ -147,7 +165,9 @@ class ServingDerender(BaseServing):
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response, request.chat_request
request.generate_response,
request.chat_request,
request.prompt_token_ids,
)
except ValueError as exc:
return self.create_error_response(str(exc))
@@ -203,6 +223,11 @@ class ServingDerender(BaseServing):
if bounds_error is not None:
return bounds_error
if request.prompt_token_ids is not None:
prompt_ids_error = self._validate_prompt_token_ids(request.prompt_token_ids)
if prompt_ids_error is not None:
return prompt_ids_error
(
choices,
total_prompt_tokens,
@@ -211,6 +236,7 @@ class ServingDerender(BaseServing):
request.generate_responses,
request.prompt_tokens,
completion_request=request.completion_request,
prompt_token_ids=request.prompt_token_ids,
)
first = request.generate_responses[0]
@@ -367,6 +393,10 @@ class ServingDerender(BaseServing):
if error_check_ret is not None:
return error_check_ret
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
if prompt_ids_error is not None:
return prompt_ids_error
model_name = request.model or self.models.model_name()
try:
(
@@ -378,6 +408,7 @@ class ServingDerender(BaseServing):
state=request.stream_state,
prompt_tokens=request.prompt_tokens,
completion_request=request.completion_request,
prompt_token_ids=request.prompt_token_ids,
)
except ValueError as exc:
return self.create_error_response(str(exc))
@@ -5,6 +5,7 @@ from typing import Annotated, Any, Literal, TypeAlias
from pydantic import (
BaseModel,
Field,
NonNegativeInt,
PrivateAttr,
field_validator,
model_validator,
@@ -362,7 +363,7 @@ class GenerateStreamResponse(BaseModel):
)
choices: list[GenerateResponseStreamChoice]
usage: UsageInfo | None = Field(default=None)
prompt_token_ids: list[int] | None = None
prompt_token_ids: list[NonNegativeInt] | None = None
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
metrics: PerRequestMetrics | None = None
@@ -382,7 +383,7 @@ class GenerateResponse(BaseModel):
usage: UsageInfo | None = Field(default=None)
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
prompt_token_id_logprobs: str | None = None
prompt_token_ids: list[int] | None = None
prompt_token_ids: list[NonNegativeInt] | None = None
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
metrics: PerRequestMetrics | None = None
@@ -421,6 +422,15 @@ class DerenderChatRequest(BaseModel):
len(GenerateRequest.token_ids) from the render step.
"""
prompt_token_ids: list[NonNegativeInt] | None = None
"""Prompt token IDs (`GenerateRequest.token_ids` from /render). Seeds
detokenization from the prompt tail so the first output token keeps its
leading space on SentencePiece tokenizers. Falls back to
`generate_response.prompt_token_ids`, then to unseeded decoding.
Only the last few IDs are read, so a suffix of the prompt is enough.
"""
chat_request: ChatCompletionRequest | None = None
"""The original (post-adjust_request) ChatCompletionRequest from /render.
@@ -455,6 +465,14 @@ class DerenderCompletionRequest(BaseModel):
If provided, len(prompt_tokens) must equal len(generate_responses).
"""
prompt_token_ids: list[list[NonNegativeInt] | None] | None = None
"""One prompt token ID list per response, used to seed detokenization.
See `DerenderChatRequest.prompt_token_ids`.
If provided, len(prompt_token_ids) must equal len(generate_responses).
A `None` entry falls back to `generate_responses[i].prompt_token_ids`.
"""
completion_request: CompletionRequest | None = None
"""The original (post-adjust_request) CompletionRequest from /render.
@@ -472,6 +490,13 @@ class DerenderCompletionRequest(BaseModel):
f"prompt_tokens length ({len(self.prompt_tokens)}) must equal "
f"generate_responses length ({len(self.generate_responses)})"
)
if self.prompt_token_ids is not None and len(self.prompt_token_ids) != len(
self.generate_responses
):
raise ValueError(
f"prompt_token_ids length ({len(self.prompt_token_ids)}) must "
f"equal generate_responses length ({len(self.generate_responses)})"
)
return self
@@ -627,7 +652,7 @@ class DerenderChatStreamRequest(BaseModel):
prompt_tokens: int | None = None
"""Prompt token count for usage. Forwarded from the render step."""
prompt_token_ids: list[int] | None = None
prompt_token_ids: list[NonNegativeInt] | None = None
"""Prompt token IDs. Required by the parser path's `parse_delta` to
settle its initial reasoning state (e.g. chat templates that pre-open
``<think>``). `prompt_tokens` is a usage count and cannot serve this
@@ -636,8 +661,15 @@ class DerenderChatStreamRequest(BaseModel):
Rejected with a 400 (by `ServingDerender`) when a tool or reasoning
parser is configured and this is omitted. Without it, `parse_delta`
cannot tell whether the prompt left reasoning open and would silently
misclassify reasoning content as plain content. Unused on the plain
detokenization path.
misclassify reasoning content as plain content. On all paths it also
seeds detokenization on the first chunk, falling back to
`generate_chunk.prompt_token_ids` when omitted (see
`DerenderChatRequest.prompt_token_ids`).
With a parser configured, send the full list. Parsers look back to the
last reasoning marker which can be anywhere in the prompt, so a suffix
can flip the initial reasoning state. Without a parser, a suffix is
enough.
"""
chat_request: ChatCompletionRequest | None = None
@@ -666,6 +698,12 @@ class DerenderCompletionStreamRequest(BaseModel):
prompt_tokens: int | None = None
"""Prompt token count for usage."""
prompt_token_ids: list[NonNegativeInt] | None = None
"""Prompt token IDs, used on the first chunk to seed detokenization.
Falls back to `generate_chunk.prompt_token_ids`. See
`DerenderChatRequest.prompt_token_ids`.
"""
completion_request: CompletionRequest | None = None
"""The original (post adjust_request) CompletionRequest from /render."""
# --8<-- [end:derender-completion-stream-request]