[Frontend] Add stream reasoning and tool calls from the derender endpoint (#50550)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-09-18 16:36:27 +02:00
committed by GitHub
parent 911548247a
commit 70164bdadc
6 changed files with 1658 additions and 104 deletions
+77 -11
View File
@@ -138,6 +138,13 @@ class ServingDerender(BaseServing):
if bounds_error is not None:
return bounds_error
if self.online_derenderer.parser is not None and request.chat_request is None:
return self.create_error_response(
"chat_request is required when a tool or reasoning parser is "
"configured because plain detokenization would leak raw parser "
"markup into content."
)
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response, request.chat_request
@@ -248,14 +255,67 @@ class ServingDerender(BaseServing):
Processes one ``GenerateStreamResponse`` chunk and returns the
derendered chunk together with the updated client carried state.
``parser is None`` or no ``chat_request`` until reasoning/tool call
functionality added in future PR.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if self.online_derenderer.parser is not None and request.chat_request is None:
return self.create_error_response(
"chat_request is required when a tool or reasoning parser is "
"configured because plain detokenization would leak raw parser "
"markup into content."
)
if (
self.online_derenderer.parser is not None
and request.prompt_token_ids is None
):
return self.create_error_response(
"prompt_token_ids is required when a tool or reasoning "
"parser is configured. Without it parse_delta cannot tell "
"whether the prompt left reasoning open (e.g. a chat "
"template that pre-opens <think>) and would misclassify "
"reasoning content as plain content."
)
# Each streamed chunk contains at most one choice per SSE event.
# A single DerenderStreamState is threaded
# through every choice in the chunk, so >1 would corrupt detok/parser
# state across choices. See the matching check in
# OnlineDerenderer.derender_chat_stream which this fails ahead of to
# avoid touching the tokenizer at all on a malformed chunk.
num_choices = len(request.generate_chunk.choices)
if num_choices > 1:
return self.create_error_response(
f"derender_chat_stream expects at most one choice per chunk "
f"(got {num_choices})."
)
# prompt_token_ids is folded in here too. It is caller supplied and
# otherwise unbounded. A parser configured deployment rescans it
# in full on every chunk (is_reasoning_end /
# adjust_initial_state_from_prompt, see abstract_parser.parse_delta)
# since parser state cannot be carried across calls.
prompt_len = (
len(request.prompt_token_ids) if request.prompt_token_ids is not None else 0
)
prior_len = (
len(request.stream_state.output_token_ids)
if request.stream_state is not None
else 0
)
delta_len = sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
)
max_model_len = self.model_config.max_model_len
if prompt_len + prior_len + delta_len > max_model_len:
return self.create_error_response(
f"prompt_token_ids length ({prompt_len}) plus "
f"output_token_ids length ({prior_len}) plus delta "
f"({delta_len}) exceeds max_model_len ({max_model_len})."
)
model_name = request.model or self.models.model_name()
try:
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
@@ -264,15 +324,21 @@ class ServingDerender(BaseServing):
state=request.stream_state,
chat_request=request.chat_request,
prompt_tokens=request.prompt_tokens,
prompt_token_ids=request.prompt_token_ids,
)
except NotImplementedError as exc:
return self.create_error_response(exc)
except ValueError as exc:
return self.create_error_response(str(exc))
except (KeyError, IndexError) as exc:
return self.create_error_response(
f"invalid stream_state: detokenization failed ({exc!r})"
except Exception as exc:
# The two ValueErrors derender_chat_stream can raise directly
# (missing chat_request, >1 choice per chunk) are already
# pre-checked above, so anything reaching here comes from the
# parser replaying entirely client controlled state. Hermes
# swallows its own streaming exceptions but finalize_generation
# and engine based parsers don't, so an unexpected parser
# failure on malformed (but well typed) state must surface as
# a 400 here rather than an unhandled 500.
logger.debug(
"derender stream replay failed: %s: %s", type(exc).__name__, exc
)
return self.create_error_response("invalid stream_state: derender failed")
logger.debug(
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
@@ -313,7 +379,7 @@ class ServingDerender(BaseServing):
return self.create_error_response(str(exc))
except (KeyError, IndexError) as exc:
return self.create_error_response(
f"invalid stream_state: detokenization failed ({exc!r})"
f"invalid stream_state: detokenization failed ({exc})"
)
logger.debug(
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Literal, TypeAlias
from typing import Annotated, Any, Literal, TypeAlias
from pydantic import (
BaseModel,
@@ -483,16 +483,32 @@ class DerenderStreamState(BaseModel):
streaming derender endpoint. All fields are plain JSON serializable data.
No opaque tokenizer or parser internals are stored here.
Two separate sets of fields support two different streaming modes:
- For plain streaming (no parser configured including the completions path):
`prev_tokens`, `prefix_offset` and `read_offset` maintain a bounded incremental
decoding window. This requires O(window) transport and O(delta) computation
per chunk.
- For parser enabled chat streaming: `output_token_ids`, `output_chunk_lens`,
`tools_streamed` and `last_tool_call_ids` are used to replay `parse_delta()`
from scratch on every chunk because parser state cannot be serialized. This
incurs O(n) transport per chunk (O(n²) per generation) and O(n²)
`parse_delta()` calls per generation. Since many parsers re-scan the
entire accumulated text on each invocation, `parse_delta()` itself is
O(n) yielding a true worst case compute cost of O(n³) per generation.
No caching is performed. Work is bounded by `max_model_len`.
See `OnlineDerenderer._derender_chat_stream_parsed`.
The detokenization strategy carries the incremental decode offsets
directly rather than re-sending the whole token history each chunk.
``detokenize_incrementally`` only ever reads the trailing token window
``prev_tokens[prefix_offset:]``, so we carry just that tail plus the two
`detokenize_incrementally` only ever reads the trailing token window
`prev_tokens[prefix_offset:]`, so we carry just that tail plus the two
offsets. Each chunk resumes exactly where the last one stopped, including
any partially processed multi-byte character (tracked by ``read_offset``),
any partially processed multi-byte character (tracked by `read_offset`),
then trims and rebases the window so it never grows with generation length.
Performance:
- Compute per chunk is O(delta). One ``detokenize_incrementally`` call per
- Compute per chunk is O(delta). One `detokenize_incrementally` call per
new token, independent of how many tokens preceded it.
- Transport per chunk is O(window). The carried tail is bounded by the
incremental detokenization offset, so cumulative bytes over the wire are
@@ -500,18 +516,18 @@ class DerenderStreamState(BaseModel):
"""
prev_tokens: list[str] = Field(default_factory=list)
"""Trailing decode window. Token strings from ``prefix_offset`` onward.
"""Trailing decode window. Token strings from `prefix_offset` onward.
Bounded, trimmed and rebased each chunk to the tail
``detokenize_incrementally`` still reads, so it does not grow with the
`detokenize_incrementally` still reads, so it does not grow with the
number of chunks.
"""
prefix_offset: int = Field(default=0, ge=0)
"""Prefix offset into ``prev_tokens`` for incremental detokenization."""
"""Prefix offset into `prev_tokens` for incremental detokenization."""
read_offset: int = Field(default=0, ge=0)
"""Read offset into ``prev_tokens`` for incremental detokenization."""
"""Read offset into `prev_tokens` for incremental detokenization."""
@field_validator("prev_tokens")
@classmethod
@@ -525,23 +541,53 @@ class DerenderStreamState(BaseModel):
return v
role_sent: bool = False
"""True once the initial ``role: "assistant"`` delta has been emitted.
"""True once the initial `role: "assistant"` delta has been emitted.
Prevents re-emitting the role on subsequent chunks even when the detok
window is transiently empty (e.g. usage only final chunk).
"""
# TODO: Properties used in follow on PR for tool call parsing
last_content: str | None = None
"""Last emitted cumulative assistant content text."""
output_token_ids: list[int] = Field(default_factory=list)
"""All output tokens seen so far. Parser path only.
last_reasoning: str | None = None
"""Last emitted cumulative reasoning text."""
Replay buffer: each chunk rebuilds a fresh parser and replays every
token in here through `parse_delta` (discarding the result) before
processing the current chunk's tokens since parser internal state
cannot be serialized into this stateless model. Unavoidably O(n)
bounded by ``max_model_len`` (enforced server side, not by a field
validator here since the bound is model dependent).
"""
output_chunk_lens: list[Annotated[int, Field(gt=0)]] = Field(default_factory=list)
"""Token count of each chunk in `output_token_ids`. Parser path only.
Replay uses these to reproduce the original `parse_delta` call
boundaries. Must sum to `len(output_token_ids)`.
"""
@model_validator(mode="after")
def _validate_output_chunk_lens(self) -> "DerenderStreamState":
total = sum(self.output_chunk_lens)
if total != len(self.output_token_ids):
raise ValueError(
f"output_chunk_lens must sum to len(output_token_ids) "
f"(got sum={total}, len(output_token_ids)="
f"{len(self.output_token_ids)})"
)
return self
tools_streamed: bool = False
"""True once a tool call delta has been emitted. Parser path only.
Drives the `finish_reason` -> `"tool_calls"` rewrite on the final
chunk mirroring the generate streaming path.
"""
last_tool_call_ids: list[str] = Field(default_factory=list)
"""Stable tool-call IDs, assigned once when each call first appears.
"""Stable tool call IDs, assigned once when each call first appears.
Prevents ID regeneration across re-parsing.
Indexed by tool call index. Parser path only. Prevents ID regeneration
when replay reprocesses a tool call that already has a pinned ID.
"""
@@ -570,6 +616,19 @@ class DerenderChatStreamRequest(BaseModel):
prompt_tokens: int | None = None
"""Prompt token count for usage. Forwarded from the render step."""
prompt_token_ids: list[int] | None = None
"""Prompt token IDs. Required by the parser path's `parse_delta` to
settle its initial reasoning state (e.g. chat templates that pre-open
``<think>``). `prompt_tokens` is a usage count and cannot serve this
purpose. Sourced from `GenerateRequest.token_ids` at the render step.
Rejected with a 400 (by `ServingDerender`) when a tool or reasoning
parser is configured and this is omitted. Without it, `parse_delta`
cannot tell whether the prompt left reasoning open and would silently
misclassify reasoning content as plain content. Unused on the plain
detokenization path.
"""
chat_request: ChatCompletionRequest | None = None
"""The original (post adjust_request) ChatCompletionRequest from /render."""
+245 -21
View File
@@ -4,7 +4,10 @@ from typing import Any
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.generate.base.protocol import DeltaMessage, ToolCall
from vllm.entrypoints.generate.base.protocol import (
DeltaMessage,
ToolCall,
)
from vllm.entrypoints.generate.base.serving import resolve_token_id_placeholder
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProbs,
@@ -29,6 +32,9 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
)
from vllm.entrypoints.serve.engine.protocol import UsageInfo
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.entrypoints.serve.utils.tool_calls_utils import (
maybe_filter_parallel_tool_calls,
)
from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer
@@ -95,6 +101,11 @@ class OnlineDerenderer:
self._derender_completion_async = make_async(
self._derender_completion, executor=renderer._executor
)
# Replay is O(n) per chunk (unlike the O(delta) detok-only paths),
# so it must not run on the event loop either.
self._derender_chat_stream_parsed_async = make_async(
self._derender_chat_stream_parsed, executor=renderer._executor
)
async def derender_chat(
self,
@@ -144,6 +155,7 @@ class OnlineDerenderer:
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
model_config=self.model_config,
)
reasoning, content, tool_calls = parser.parse(
decoded_text,
@@ -278,23 +290,30 @@ class OnlineDerenderer:
state: DerenderStreamState | None = None,
chat_request: ChatCompletionRequest | None = None,
prompt_tokens: int | None = None,
prompt_token_ids: list[int] | None = None,
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState]:
"""Process one GenerateStreamResponse chunk for streaming chat derender.
TODO: parse path for reasoning and tool calls is implemented in future PR.
Unlike OpenAI's API, which always emits ``role: "assistant"`` on the
Unlike OpenAI's API, which always emits `role: "assistant"` on the
very first chunk, this emits it on the first chunk with a non empty
``choices`` list. A leading usage only chunk therefore defers the
`choices` list. A leading usage only chunk therefore defers the
role to the following content chunk instead of sending an empty
role only delta.
Args:
model: Model name for the response object.
generate_chunk: One SSE chunk from ``/inference/v1/generate``.
state: Client carried detok state (``None`` for first call).
chat_request: Original ChatCompletionRequest from ``/render``.
generate_chunk: One SSE chunk from `/inference/v1/generate`.
state: Client carried detok state (`None` for first call).
chat_request: Original ChatCompletionRequest from `/render`.
Required when a reasoning or tool parser is configured
(validated by the caller — see `ServingDerender`) because
plain detokenization would leak raw parser markup into `content`.
prompt_tokens: Prompt token count for the usage chunk.
prompt_token_ids: Prompt token IDs. Parser path only. Lets
`parse_delta` settle its initial reasoning state. Required
when a reasoning or tool parser is configured (validated by
the caller — see ``ServingDerender``). Without it reasoning
left open by the prompt would be misclassified as content.
Returns:
(chunk, updated_state) — the derendered SSE chunk and the state
@@ -304,28 +323,36 @@ class OnlineDerenderer:
if state is None:
state = DerenderStreamState()
if self.parser is not None:
# TODO: Follow on PR will implement the parse path. Check on the
# parser alone (fail closed). A parser configured model must never
# fall through to plain detok on the streaming path, even when
# ``chat_request`` is omitted or reasoning/tool markup would leak
# into ``delta.content``.
raise NotImplementedError(
"Streaming chat derender is not yet supported for models with "
"a reasoning or tool parser configured. Use the non-streaming "
"derender endpoint (stream=false) for parsed output."
)
# A single DerenderStreamState is threaded through every choice in
# this chunk. Correct only when there is at most one choice per SSE
# event (n=1, one call per index), as the streaming derender
# protocol assumes. Multiple choices sharing one chunk would corrupt
# each other's detok window.
# each other's detok/parser state.
if len(generate_chunk.choices) > 1:
raise ValueError(
"derender_chat_stream expects at most one choice per chunk"
)
parser_cls = self.parser
if parser_cls is not None:
# Fail (mirrors ServingDerender's pre-check) because a parser
# configured model must never fall through to plain detok or
# reasoning/tool markup would leak into `delta.content`.
if chat_request is None:
raise ValueError(
"chat_request is required for streaming chat derender "
"when a tool or reasoning parser is configured"
)
return await self._derender_chat_stream_parsed_async(
parser_cls,
model,
generate_chunk,
state,
chat_request,
prompt_tokens,
prompt_token_ids,
)
tokenizer = self.renderer.get_tokenizer()
skip_special = (
chat_request.skip_special_tokens if chat_request is not None else True
@@ -374,6 +401,203 @@ class OnlineDerenderer:
)
return chunk, updated_state
def _derender_chat_stream_parsed(
self,
parser_cls: type[Parser],
model: str,
generate_chunk: GenerateStreamResponse,
state: DerenderStreamState,
chat_request: ChatCompletionRequest,
prompt_tokens: int | None,
prompt_token_ids: list[int] | None,
) -> tuple[ChatCompletionStreamResponse, DerenderStreamState]:
"""Parser path for streaming chat derender: replay + `parse_delta`.
Parser internal state (buffered markup, reasoning/tool phase, etc.)
cannot be serialized into `DerenderStreamState`, so each call
builds a fresh parser and replays every prior output token through
`parse_delta` (discarding the result) before processing this
chunk's tokens for real.
Every chunk, replayed or live, goes through one `parse_delta` call
with the tokens it arrived with (more than one under e.g.
speculative decoding). Replay recovers those boundaries from
`state.output_chunk_lens`, so the rebuilt parser state matches
standard serving, which calls `parse_delta` once per engine step.
Text comes from a fresh incremental detokenizer with special tokens
preserved (``skip_special_tokens=False``).
"""
tokenizer = self.renderer.get_tokenizer()
chat_template_kwargs: dict[str, Any] = {}
if not self.use_harmony:
chat_template_kwargs = (
chat_request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)
parser = parser_cls(
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
model_config=self.model_config,
)
# Ephemeral incremental detok window, local to this call. Threaded
# across both the replay and current chunk phases (via `nonlocal`)
# so multi-byte characters split across that boundary still decode
# correctly. Discarded once the call returns.
detok_state = DerenderStreamState()
def _replay(token_ids: list[int], chunk_lens: list[int]) -> None:
"""Replay prior chunks through `parse_delta` to rebuild parser
state, discarding the result. Never `finished` since that only
applies to the current chunk.
"""
nonlocal detok_state
start = 0
for chunk_len in chunk_lens:
chunk = token_ids[start : start + chunk_len]
start += chunk_len
text, detok_state = self._detokenize_delta(
tokenizer, chunk, detok_state, skip_special_tokens=False
)
parser.parse_delta(
text,
chunk,
chat_request,
prompt_token_ids=prompt_token_ids,
finished=False,
)
# Replay history to reconstruct parser state. The result is thrown
# away and only the current chunk's emission goes to the client.
_replay(state.output_token_ids, state.output_chunk_lens)
stream_choices: list[ChatCompletionResponseStreamChoice] = []
output_token_ids = list(state.output_token_ids)
output_chunk_lens = list(state.output_chunk_lens)
role_sent = state.role_sent
tools_streamed = state.tools_streamed
last_tool_call_ids = list(state.last_tool_call_ids)
# At most one choice: the caller (derender_chat_stream) already
# rejects >1 before dispatching here. role_sent/tools_streamed/
# output_token_ids below are updated for that single choice, not
# accumulated across choices. Looping over generate_chunk.choices
# could silently corrupt output if chunks were ever allowed to
# contain multiple choices.
if generate_chunk.choices:
choice = generate_chunk.choices[0]
delta_tids = choice.token_ids or []
is_finished = choice.finish_reason is not None
if delta_tids:
# One parse_delta call for the whole chunk (producer
# granularity), not one per token. See the granularity note
# in this method's docstring.
text, detok_state = self._detokenize_delta(
tokenizer, delta_tids, detok_state, skip_special_tokens=False
)
delta_message = parser.parse_delta(
text,
delta_tids,
chat_request,
prompt_token_ids=prompt_token_ids,
finished=is_finished,
)
elif is_finished:
# Finish only chunk (no new tokens). Still flush any
# buffered tool call arguments.
delta_message = parser.parse_delta(
"",
[],
chat_request,
prompt_token_ids=prompt_token_ids,
finished=True,
)
else:
delta_message = None
if delta_tids:
output_token_ids.extend(delta_tids)
output_chunk_lens.append(len(delta_tids))
if delta_message is None:
delta_message = DeltaMessage()
if delta_message.tool_calls:
tools_streamed = True
for tc in delta_message.tool_calls:
if tc.id is None:
continue
if tc.index < len(last_tool_call_ids):
# Pin: reuse the ID already recorded for this index
# rather than one a from scratch replay regenerated.
# Real trigger not just defensive with
# tool_choice="required",
# extract_required_tool_call_streaming resets
# function_name_returned to False whenever the
# partial JSON transiently fails to parse which
# re-emits id+name for the same index on replay.
tc.id = last_tool_call_ids[tc.index]
else:
last_tool_call_ids.append(tc.id)
if not role_sent:
delta_message.role = "assistant"
role_sent = True
finish_reason = choice.finish_reason
if finish_reason is not None:
is_named_tool_choice = (
type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam
)
if tools_streamed and not is_named_tool_choice:
finish_reason = "tool_calls"
stream_choice = ChatCompletionResponseStreamChoice(
index=choice.index,
delta=delta_message,
finish_reason=finish_reason,
)
stream_choices.append(
maybe_filter_parallel_tool_calls(stream_choice, chat_request)
)
updated_state = state.model_copy(
update={
"output_token_ids": output_token_ids,
"output_chunk_lens": output_chunk_lens,
"role_sent": role_sent,
"tools_streamed": tools_streamed,
"last_tool_call_ids": last_tool_call_ids,
}
)
usage: UsageInfo | None = None
if generate_chunk.usage is not None:
u = generate_chunk.usage
pt = prompt_tokens if prompt_tokens is not None else (u.prompt_tokens or 0)
ct = u.completion_tokens or 0
usage = UsageInfo(
prompt_tokens=pt,
completion_tokens=ct,
total_tokens=pt + ct,
)
chunk = ChatCompletionStreamResponse(
id=generate_chunk.request_id,
model=model,
choices=stream_choices,
usage=usage,
)
return chunk, updated_state
async def derender_completion(
self,
generate_responses: list[GenerateResponse],