[Bugfix][Frontend] Seed derender detokenization from the prompt (#59046)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -117,6 +117,20 @@ class ServingDerender(BaseServing):
|
||||
|
||||
return None
|
||||
|
||||
def _validate_prompt_token_ids(
|
||||
self,
|
||||
prompt_token_ids: list[list[int] | None],
|
||||
) -> ErrorResponse | None:
|
||||
"""Reject caller-supplied prompt_token_ids longer than max_model_len."""
|
||||
max_model_len = self.model_config.max_model_len
|
||||
for ids in prompt_token_ids:
|
||||
if ids is not None and len(ids) > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"prompt_token_ids length ({len(ids)}) exceeds "
|
||||
f"max_model_len ({max_model_len})."
|
||||
)
|
||||
return None
|
||||
|
||||
async def derender_chat_response(
|
||||
self,
|
||||
request: DerenderChatRequest,
|
||||
@@ -138,6 +152,10 @@ class ServingDerender(BaseServing):
|
||||
if bounds_error is not None:
|
||||
return bounds_error
|
||||
|
||||
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
|
||||
if prompt_ids_error is not None:
|
||||
return prompt_ids_error
|
||||
|
||||
if self.online_derenderer.parser is not None and request.chat_request is None:
|
||||
return self.create_error_response(
|
||||
"chat_request is required when a tool or reasoning parser is "
|
||||
@@ -147,7 +165,9 @@ class ServingDerender(BaseServing):
|
||||
|
||||
try:
|
||||
choices = await self.online_derenderer.derender_chat(
|
||||
request.generate_response, request.chat_request
|
||||
request.generate_response,
|
||||
request.chat_request,
|
||||
request.prompt_token_ids,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
@@ -203,6 +223,11 @@ class ServingDerender(BaseServing):
|
||||
if bounds_error is not None:
|
||||
return bounds_error
|
||||
|
||||
if request.prompt_token_ids is not None:
|
||||
prompt_ids_error = self._validate_prompt_token_ids(request.prompt_token_ids)
|
||||
if prompt_ids_error is not None:
|
||||
return prompt_ids_error
|
||||
|
||||
(
|
||||
choices,
|
||||
total_prompt_tokens,
|
||||
@@ -211,6 +236,7 @@ class ServingDerender(BaseServing):
|
||||
request.generate_responses,
|
||||
request.prompt_tokens,
|
||||
completion_request=request.completion_request,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
)
|
||||
|
||||
first = request.generate_responses[0]
|
||||
@@ -367,6 +393,10 @@ class ServingDerender(BaseServing):
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids])
|
||||
if prompt_ids_error is not None:
|
||||
return prompt_ids_error
|
||||
|
||||
model_name = request.model or self.models.model_name()
|
||||
try:
|
||||
(
|
||||
@@ -378,6 +408,7 @@ class ServingDerender(BaseServing):
|
||||
state=request.stream_state,
|
||||
prompt_tokens=request.prompt_tokens,
|
||||
completion_request=request.completion_request,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Annotated, Any, Literal, TypeAlias
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
Field,
|
||||
NonNegativeInt,
|
||||
PrivateAttr,
|
||||
field_validator,
|
||||
model_validator,
|
||||
@@ -362,7 +363,7 @@ class GenerateStreamResponse(BaseModel):
|
||||
)
|
||||
choices: list[GenerateResponseStreamChoice]
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_token_ids: list[int] | None = None
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
@@ -382,7 +383,7 @@ class GenerateResponse(BaseModel):
|
||||
usage: UsageInfo | None = Field(default=None)
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_id_logprobs: str | None = None
|
||||
prompt_token_ids: list[int] | None = None
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None
|
||||
metrics: PerRequestMetrics | None = None
|
||||
|
||||
@@ -421,6 +422,15 @@ class DerenderChatRequest(BaseModel):
|
||||
len(GenerateRequest.token_ids) from the render step.
|
||||
"""
|
||||
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
"""Prompt token IDs (`GenerateRequest.token_ids` from /render). Seeds
|
||||
detokenization from the prompt tail so the first output token keeps its
|
||||
leading space on SentencePiece tokenizers. Falls back to
|
||||
`generate_response.prompt_token_ids`, then to unseeded decoding.
|
||||
|
||||
Only the last few IDs are read, so a suffix of the prompt is enough.
|
||||
"""
|
||||
|
||||
chat_request: ChatCompletionRequest | None = None
|
||||
"""The original (post-adjust_request) ChatCompletionRequest from /render.
|
||||
|
||||
@@ -455,6 +465,14 @@ class DerenderCompletionRequest(BaseModel):
|
||||
If provided, len(prompt_tokens) must equal len(generate_responses).
|
||||
"""
|
||||
|
||||
prompt_token_ids: list[list[NonNegativeInt] | None] | None = None
|
||||
"""One prompt token ID list per response, used to seed detokenization.
|
||||
See `DerenderChatRequest.prompt_token_ids`.
|
||||
|
||||
If provided, len(prompt_token_ids) must equal len(generate_responses).
|
||||
A `None` entry falls back to `generate_responses[i].prompt_token_ids`.
|
||||
"""
|
||||
|
||||
completion_request: CompletionRequest | None = None
|
||||
"""The original (post-adjust_request) CompletionRequest from /render.
|
||||
|
||||
@@ -472,6 +490,13 @@ class DerenderCompletionRequest(BaseModel):
|
||||
f"prompt_tokens length ({len(self.prompt_tokens)}) must equal "
|
||||
f"generate_responses length ({len(self.generate_responses)})"
|
||||
)
|
||||
if self.prompt_token_ids is not None and len(self.prompt_token_ids) != len(
|
||||
self.generate_responses
|
||||
):
|
||||
raise ValueError(
|
||||
f"prompt_token_ids length ({len(self.prompt_token_ids)}) must "
|
||||
f"equal generate_responses length ({len(self.generate_responses)})"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
@@ -627,7 +652,7 @@ class DerenderChatStreamRequest(BaseModel):
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage. Forwarded from the render step."""
|
||||
|
||||
prompt_token_ids: list[int] | None = None
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
"""Prompt token IDs. Required by the parser path's `parse_delta` to
|
||||
settle its initial reasoning state (e.g. chat templates that pre-open
|
||||
``<think>``). `prompt_tokens` is a usage count and cannot serve this
|
||||
@@ -636,8 +661,15 @@ class DerenderChatStreamRequest(BaseModel):
|
||||
Rejected with a 400 (by `ServingDerender`) when a tool or reasoning
|
||||
parser is configured and this is omitted. Without it, `parse_delta`
|
||||
cannot tell whether the prompt left reasoning open and would silently
|
||||
misclassify reasoning content as plain content. Unused on the plain
|
||||
detokenization path.
|
||||
misclassify reasoning content as plain content. On all paths it also
|
||||
seeds detokenization on the first chunk, falling back to
|
||||
`generate_chunk.prompt_token_ids` when omitted (see
|
||||
`DerenderChatRequest.prompt_token_ids`).
|
||||
|
||||
With a parser configured, send the full list. Parsers look back to the
|
||||
last reasoning marker which can be anywhere in the prompt, so a suffix
|
||||
can flip the initial reasoning state. Without a parser, a suffix is
|
||||
enough.
|
||||
"""
|
||||
|
||||
chat_request: ChatCompletionRequest | None = None
|
||||
@@ -666,6 +698,12 @@ class DerenderCompletionStreamRequest(BaseModel):
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage."""
|
||||
|
||||
prompt_token_ids: list[NonNegativeInt] | None = None
|
||||
"""Prompt token IDs, used on the first chunk to seed detokenization.
|
||||
Falls back to `generate_chunk.prompt_token_ids`. See
|
||||
`DerenderChatRequest.prompt_token_ids`.
|
||||
"""
|
||||
|
||||
completion_request: CompletionRequest | None = None
|
||||
"""The original (post adjust_request) CompletionRequest from /render."""
|
||||
# --8<-- [end:derender-completion-stream-request]
|
||||
|
||||
Reference in New Issue
Block a user