[Frontend] Add stream reasoning and tool calls from the derender endpoint (#50550)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-09-18 16:36:27 +02:00
committed by GitHub
parent 911548247a
commit 70164bdadc
6 changed files with 1658 additions and 104 deletions
+77 -11
View File
@@ -138,6 +138,13 @@ class ServingDerender(BaseServing):
if bounds_error is not None:
return bounds_error
if self.online_derenderer.parser is not None and request.chat_request is None:
return self.create_error_response(
"chat_request is required when a tool or reasoning parser is "
"configured because plain detokenization would leak raw parser "
"markup into content."
)
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response, request.chat_request
@@ -248,14 +255,67 @@ class ServingDerender(BaseServing):
Processes one ``GenerateStreamResponse`` chunk and returns the
derendered chunk together with the updated client carried state.
``parser is None`` or no ``chat_request`` until reasoning/tool call
functionality added in future PR.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if self.online_derenderer.parser is not None and request.chat_request is None:
return self.create_error_response(
"chat_request is required when a tool or reasoning parser is "
"configured because plain detokenization would leak raw parser "
"markup into content."
)
if (
self.online_derenderer.parser is not None
and request.prompt_token_ids is None
):
return self.create_error_response(
"prompt_token_ids is required when a tool or reasoning "
"parser is configured. Without it parse_delta cannot tell "
"whether the prompt left reasoning open (e.g. a chat "
"template that pre-opens <think>) and would misclassify "
"reasoning content as plain content."
)
# Each streamed chunk contains at most one choice per SSE event.
# A single DerenderStreamState is threaded
# through every choice in the chunk, so >1 would corrupt detok/parser
# state across choices. See the matching check in
# OnlineDerenderer.derender_chat_stream which this fails ahead of to
# avoid touching the tokenizer at all on a malformed chunk.
num_choices = len(request.generate_chunk.choices)
if num_choices > 1:
return self.create_error_response(
f"derender_chat_stream expects at most one choice per chunk "
f"(got {num_choices})."
)
# prompt_token_ids is folded in here too. It is caller supplied and
# otherwise unbounded. A parser configured deployment rescans it
# in full on every chunk (is_reasoning_end /
# adjust_initial_state_from_prompt, see abstract_parser.parse_delta)
# since parser state cannot be carried across calls.
prompt_len = (
len(request.prompt_token_ids) if request.prompt_token_ids is not None else 0
)
prior_len = (
len(request.stream_state.output_token_ids)
if request.stream_state is not None
else 0
)
delta_len = sum(
len(c.token_ids) for c in request.generate_chunk.choices if c.token_ids
)
max_model_len = self.model_config.max_model_len
if prompt_len + prior_len + delta_len > max_model_len:
return self.create_error_response(
f"prompt_token_ids length ({prompt_len}) plus "
f"output_token_ids length ({prior_len}) plus delta "
f"({delta_len}) exceeds max_model_len ({max_model_len})."
)
model_name = request.model or self.models.model_name()
try:
chunk, updated_state = await self.online_derenderer.derender_chat_stream(
@@ -264,15 +324,21 @@ class ServingDerender(BaseServing):
state=request.stream_state,
chat_request=request.chat_request,
prompt_tokens=request.prompt_tokens,
prompt_token_ids=request.prompt_token_ids,
)
except NotImplementedError as exc:
return self.create_error_response(exc)
except ValueError as exc:
return self.create_error_response(str(exc))
except (KeyError, IndexError) as exc:
return self.create_error_response(
f"invalid stream_state: detokenization failed ({exc!r})"
except Exception as exc:
# The two ValueErrors derender_chat_stream can raise directly
# (missing chat_request, >1 choice per chunk) are already
# pre-checked above, so anything reaching here comes from the
# parser replaying entirely client controlled state. Hermes
# swallows its own streaming exceptions but finalize_generation
# and engine based parsers don't, so an unexpected parser
# failure on malformed (but well typed) state must surface as
# a 400 here rather than an unhandled 500.
logger.debug(
"derender stream replay failed: %s: %s", type(exc).__name__, exc
)
return self.create_error_response("invalid stream_state: derender failed")
logger.debug(
"derender_chat_stream request_id=%s model=%s delta_tokens=%d",
@@ -313,7 +379,7 @@ class ServingDerender(BaseServing):
return self.create_error_response(str(exc))
except (KeyError, IndexError) as exc:
return self.create_error_response(
f"invalid stream_state: detokenization failed ({exc!r})"
f"invalid stream_state: detokenization failed ({exc})"
)
logger.debug(