[Frontend] Add stream reasoning and tool calls from the derender endpoint (#50550)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -138,6 +138,13 @@ class ServingDerender(BaseServing):
|
||||
if bounds_error is not None:
|
||||
return bounds_error
|
||||
|
||||
if self.online_derenderer.parser is not None and request.chat_request is None:
|
||||
return self.create_error_response(
|
||||
"chat_request is required when a tool or reasoning parser is "
|
||||
"configured because plain detokenization would leak raw parser "
|
||||
"markup into content."
|
||||
)
|
||||
|
||||
try:
|
||||
choices = await self.online_derenderer.derender_chat(
|
||||
request.generate_response, request.chat_request
|
||||
@@ -248,14 +255,67 @@ class ServingDerender(BaseServing):
|
||||
|
||||
Processes one ``GenerateStreamResponse`` chunk and returns the
|
||||
derendered chunk together with the updated client carried state.
|
||||
|
||||
``parser is None`` or no ``chat_request`` until reasoning/tool call
|
||||
functionality added in future PR.
|
||||
"""
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
if self.online_derenderer.parser is not None and request.chat_request is None:
|
||||
return self.create_error_response(
|
||||
"chat_request is required when a tool or reasoning parser is "
|
||||
"configured because plain detokenization would leak raw parser "
|
||||
"markup into content."
|
||||
)
|
||||
|
||||
if (
|
||||
self.online_derenderer.parser is not None
|
||||
and request.prompt_token_ids is None
|
||||
):
|
||||
return self.create_error_response(
|
||||
"prompt_token_ids is required when a tool or reasoning "
|
||||
"parser is configured. Without it parse_delta cannot tell "
|
||||
"whether the prompt left reasoning open (e.g. a chat "
|
||||
"template that pre-opens <think>) and would misclassify "
|
||||
"reasoning content as plain content."
|
||||
)
|
||||
|
||||
# Each streamed chunk contains at most one choice per SSE event.
|
||||
# A single DerenderStreamState is threaded
|
||||
# through every choice in the chunk, so >1 would corrupt detok/parser
|
||||
# state across choices. See the matching check in
|
||||
# OnlineDerenderer.derender_chat_stream which this fails ahead of to
|
||||
# avoid touching the tokenizer at all on a malformed chunk.
|
||||
num_choices = len(request.generate_chunk.choices)
|
||||
if num_choices > 1:
|
||||
return self.create_error_response(
|
||||
f"derender_chat_stream expects at most one choice per chunk "
|
||||
f"(got {num_choices})."
|
||||
)
|
||||
|
||||
# prompt_token_ids is folded in here too. It is caller supplied and
|
||||
# otherwise unbounded. A parser configured deployment rescans it
|
||||
# in full on every chunk (is_reasoning_end /
|
||||
# adjust_initial_state_from_prompt, see abstract_parser.parse_delta)
|
||||
# since parser state cannot be carried across calls.
|
||||
prompt_len = (
|
||||
len(request.prompt_token_ids) if request.prompt_token_ids is not None else 0
|
||||
)
|
||||
prior_len = (
|
||||
len(request.stream_state.output_token_ids)
|
||||
if request.stream_state is not None
|
||||
else 0
|
||||
)
|
||||
delta_len = sum(
|
||||
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
|
||||
)
|
||||
max_model_len = self.model_config.max_model_len
|
||||
if prompt_len + prior_len + delta_len > max_model_len:
|
||||
return self.create_error_response(
|
||||
f"prompt_token_ids length ({prompt_len}) plus "
|
||||
f"output_token_ids length ({prior_len}) plus delta "
|
||||
f"({delta_len}) exceeds max_model_len ({max_model_len})."
|
||||
)
|
||||
|
||||
model_name = request.model or self.models.model_name()
|
||||
try:
|
||||
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
|
||||
@@ -264,15 +324,21 @@ class ServingDerender(BaseServing):
|
||||
state=request.stream_state,
|
||||
chat_request=request.chat_request,
|
||||
prompt_tokens=request.prompt_tokens,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
)
|
||||
except NotImplementedError as exc:
|
||||
return self.create_error_response(exc)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(str(exc))
|
||||
except (KeyError, IndexError) as exc:
|
||||
return self.create_error_response(
|
||||
f"invalid stream_state: detokenization failed ({exc!r})"
|
||||
except Exception as exc:
|
||||
# The two ValueErrors derender_chat_stream can raise directly
|
||||
# (missing chat_request, >1 choice per chunk) are already
|
||||
# pre-checked above, so anything reaching here comes from the
|
||||
# parser replaying entirely client controlled state. Hermes
|
||||
# swallows its own streaming exceptions but finalize_generation
|
||||
# and engine based parsers don't, so an unexpected parser
|
||||
# failure on malformed (but well typed) state must surface as
|
||||
# a 400 here rather than an unhandled 500.
|
||||
logger.debug(
|
||||
"derender stream replay failed: %s: %s", type(exc).__name__, exc
|
||||
)
|
||||
return self.create_error_response("invalid stream_state: derender failed")
|
||||
|
||||
logger.debug(
|
||||
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
|
||||
@@ -313,7 +379,7 @@ class ServingDerender(BaseServing):
|
||||
return self.create_error_response(str(exc))
|
||||
except (KeyError, IndexError) as exc:
|
||||
return self.create_error_response(
|
||||
f"invalid stream_state: detokenization failed ({exc!r})"
|
||||
f"invalid stream_state: detokenization failed ({exc})"
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any, Literal, TypeAlias
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
@@ -483,16 +483,32 @@ class DerenderStreamState(BaseModel):
|
||||
streaming derender endpoint. All fields are plain JSON serializable data.
|
||||
No opaque tokenizer or parser internals are stored here.
|
||||
|
||||
Two separate sets of fields support two different streaming modes:
|
||||
|
||||
- For plain streaming (no parser configured including the completions path):
|
||||
`prev_tokens`, `prefix_offset` and `read_offset` maintain a bounded incremental
|
||||
decoding window. This requires O(window) transport and O(delta) computation
|
||||
per chunk.
|
||||
- For parser enabled chat streaming: `output_token_ids`, `output_chunk_lens`,
|
||||
`tools_streamed` and `last_tool_call_ids` are used to replay `parse_delta()`
|
||||
from scratch on every chunk because parser state cannot be serialized. This
|
||||
incurs O(n) transport per chunk (O(n²) per generation) and O(n²)
|
||||
`parse_delta()` calls per generation. Since many parsers re-scan the
|
||||
entire accumulated text on each invocation, `parse_delta()` itself is
|
||||
O(n) yielding a true worst case compute cost of O(n³) per generation.
|
||||
No caching is performed. Work is bounded by `max_model_len`.
|
||||
See `OnlineDerenderer._derender_chat_stream_parsed`.
|
||||
|
||||
The detokenization strategy carries the incremental decode offsets
|
||||
directly rather than re-sending the whole token history each chunk.
|
||||
``detokenize_incrementally`` only ever reads the trailing token window
|
||||
``prev_tokens[prefix_offset:]``, so we carry just that tail plus the two
|
||||
`detokenize_incrementally` only ever reads the trailing token window
|
||||
`prev_tokens[prefix_offset:]`, so we carry just that tail plus the two
|
||||
offsets. Each chunk resumes exactly where the last one stopped, including
|
||||
any partially processed multi-byte character (tracked by ``read_offset``),
|
||||
any partially processed multi-byte character (tracked by `read_offset`),
|
||||
then trims and rebases the window so it never grows with generation length.
|
||||
|
||||
Performance:
|
||||
- Compute per chunk is O(delta). One ``detokenize_incrementally`` call per
|
||||
- Compute per chunk is O(delta). One `detokenize_incrementally` call per
|
||||
new token, independent of how many tokens preceded it.
|
||||
- Transport per chunk is O(window). The carried tail is bounded by the
|
||||
incremental detokenization offset, so cumulative bytes over the wire are
|
||||
@@ -500,18 +516,18 @@ class DerenderStreamState(BaseModel):
|
||||
"""
|
||||
|
||||
prev_tokens: list[str] = Field(default_factory=list)
|
||||
"""Trailing decode window. Token strings from ``prefix_offset`` onward.
|
||||
"""Trailing decode window. Token strings from `prefix_offset` onward.
|
||||
|
||||
Bounded, trimmed and rebased each chunk to the tail
|
||||
``detokenize_incrementally`` still reads, so it does not grow with the
|
||||
`detokenize_incrementally` still reads, so it does not grow with the
|
||||
number of chunks.
|
||||
"""
|
||||
|
||||
prefix_offset: int = Field(default=0, ge=0)
|
||||
"""Prefix offset into ``prev_tokens`` for incremental detokenization."""
|
||||
"""Prefix offset into `prev_tokens` for incremental detokenization."""
|
||||
|
||||
read_offset: int = Field(default=0, ge=0)
|
||||
"""Read offset into ``prev_tokens`` for incremental detokenization."""
|
||||
"""Read offset into `prev_tokens` for incremental detokenization."""
|
||||
|
||||
@field_validator("prev_tokens")
|
||||
@classmethod
|
||||
@@ -525,23 +541,53 @@ class DerenderStreamState(BaseModel):
|
||||
return v
|
||||
|
||||
role_sent: bool = False
|
||||
"""True once the initial ``role: "assistant"`` delta has been emitted.
|
||||
"""True once the initial `role: "assistant"` delta has been emitted.
|
||||
|
||||
Prevents re-emitting the role on subsequent chunks even when the detok
|
||||
window is transiently empty (e.g. usage only final chunk).
|
||||
"""
|
||||
|
||||
# TODO: Properties used in follow on PR for tool call parsing
|
||||
last_content: str | None = None
|
||||
"""Last emitted cumulative assistant content text."""
|
||||
output_token_ids: list[int] = Field(default_factory=list)
|
||||
"""All output tokens seen so far. Parser path only.
|
||||
|
||||
last_reasoning: str | None = None
|
||||
"""Last emitted cumulative reasoning text."""
|
||||
Replay buffer: each chunk rebuilds a fresh parser and replays every
|
||||
token in here through `parse_delta` (discarding the result) before
|
||||
processing the current chunk's tokens since parser internal state
|
||||
cannot be serialized into this stateless model. Unavoidably O(n)
|
||||
bounded by ``max_model_len`` (enforced server side, not by a field
|
||||
validator here since the bound is model dependent).
|
||||
"""
|
||||
|
||||
output_chunk_lens: list[Annotated[int, Field(gt=0)]] = Field(default_factory=list)
|
||||
"""Token count of each chunk in `output_token_ids`. Parser path only.
|
||||
|
||||
Replay uses these to reproduce the original `parse_delta` call
|
||||
boundaries. Must sum to `len(output_token_ids)`.
|
||||
"""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_output_chunk_lens(self) -> "DerenderStreamState":
|
||||
total = sum(self.output_chunk_lens)
|
||||
if total != len(self.output_token_ids):
|
||||
raise ValueError(
|
||||
f"output_chunk_lens must sum to len(output_token_ids) "
|
||||
f"(got sum={total}, len(output_token_ids)="
|
||||
f"{len(self.output_token_ids)})"
|
||||
)
|
||||
return self
|
||||
|
||||
tools_streamed: bool = False
|
||||
"""True once a tool call delta has been emitted. Parser path only.
|
||||
|
||||
Drives the `finish_reason` -> `"tool_calls"` rewrite on the final
|
||||
chunk mirroring the generate streaming path.
|
||||
"""
|
||||
|
||||
last_tool_call_ids: list[str] = Field(default_factory=list)
|
||||
"""Stable tool-call IDs, assigned once when each call first appears.
|
||||
"""Stable tool call IDs, assigned once when each call first appears.
|
||||
|
||||
Prevents ID regeneration across re-parsing.
|
||||
Indexed by tool call index. Parser path only. Prevents ID regeneration
|
||||
when replay reprocesses a tool call that already has a pinned ID.
|
||||
"""
|
||||
|
||||
|
||||
@@ -570,6 +616,19 @@ class DerenderChatStreamRequest(BaseModel):
|
||||
prompt_tokens: int | None = None
|
||||
"""Prompt token count for usage. Forwarded from the render step."""
|
||||
|
||||
prompt_token_ids: list[int] | None = None
|
||||
"""Prompt token IDs. Required by the parser path's `parse_delta` to
|
||||
settle its initial reasoning state (e.g. chat templates that pre-open
|
||||
``<think>``). `prompt_tokens` is a usage count and cannot serve this
|
||||
purpose. Sourced from `GenerateRequest.token_ids` at the render step.
|
||||
|
||||
Rejected with a 400 (by `ServingDerender`) when a tool or reasoning
|
||||
parser is configured and this is omitted. Without it, `parse_delta`
|
||||
cannot tell whether the prompt left reasoning open and would silently
|
||||
misclassify reasoning content as plain content. Unused on the plain
|
||||
detokenization path.
|
||||
"""
|
||||
|
||||
chat_request: ChatCompletionRequest | None = None
|
||||
"""The original (post adjust_request) ChatCompletionRequest from /render."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user