[Frontend] Extract parse and assembly out of batch chat derender (#60073)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Martin Hickey
2026-10-05 21:57:45 +00:00
committed by GitHub
co-authored by Flora Feng
parent 2e3154aa18
commit 3fb1cd796b
+101 -83
View File
@@ -151,6 +151,12 @@ class OnlineDerenderer:
skip_special_tokens=skip_special,
)
chat_template_kwargs = (
self._resolve_chat_template_kwargs(chat_request)
if has_parser and chat_request is not None
else {}
)
for choice in generate_response.choices:
if not choice.token_ids:
raise ValueError(f"choice {choice.index} has empty or null token_ids")
@@ -161,83 +167,29 @@ class OnlineDerenderer:
else None
)
# With a parser, special tokens are preserved so it can see
# markers like </think>, <tool_call>, or Harmony channel tokens.
# Without one, the request's skip_special_tokens is honoured
# (default True when no request was given).
decoded_text, _ = self._detokenize_delta(
tokenizer,
choice.token_ids,
seed_state,
skip_special_tokens=skip_special,
spaces_between_special_tokens=spaces_between,
)
if has_parser:
assert self.parser is not None and chat_request is not None
# Parser path: decode with special tokens preserved
# so the parser can see markers like </think>,
# <tool_call>, or Harmony channel tokens.
decoded_text, _ = self._detokenize_delta(
assert chat_request is not None
message = self._parse_and_assemble(
tokenizer,
choice.token_ids,
seed_state,
skip_special_tokens=skip_special,
spaces_between_special_tokens=spaces_between,
)
chat_template_kwargs: dict[str, Any] = {}
if not self.use_harmony:
chat_template_kwargs = (
chat_request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)
parser = self.parser(
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
model_config=self.model_config,
)
if generate_response.prompt_token_ids is not None:
parser.set_prompt_token_ids(generate_response.prompt_token_ids)
reasoning, content, tool_calls = parser.parse(
decoded_text,
choice.token_ids,
chat_request,
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=choice.token_ids,
)
if not getattr(chat_request, "include_reasoning", True):
reasoning = None
tc_items = (
[
ToolCall(
id=random_uuid(),
function=tc,
)
for tc in tool_calls
]
if tool_calls
else []
)
is_named_tool_choice = (
type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam
)
is_required_tool_choice = chat_request.tool_choice == "required"
if is_named_tool_choice or is_required_tool_choice:
content = content or ""
message = ChatMessage(
role="assistant",
reasoning=reasoning,
content=content,
tool_calls=tc_items,
chat_template_kwargs,
generate_response.prompt_token_ids,
)
else:
# No parser: plain detokenization honouring the request's
# skip_special_tokens (default True when no request was given).
decoded_text, _ = self._detokenize_delta(
tokenizer,
choice.token_ids,
seed_state,
skip_special_tokens=skip_special,
spaces_between_special_tokens=spaces_between,
)
message = ChatMessage(role="assistant", content=decoded_text)
choices.append(
@@ -251,6 +203,83 @@ class OnlineDerenderer:
return choices
def _resolve_chat_template_kwargs(
self, chat_request: ChatCompletionRequest
) -> dict[str, Any]:
if self.use_harmony:
return {}
return (
chat_request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)
def _parse_and_assemble(
self,
tokenizer: TokenizerLike,
text: str,
token_ids: Sequence[int],
chat_request: ChatCompletionRequest,
chat_template_kwargs: dict[str, Any],
prompt_token_ids: list[int] | None,
) -> ChatMessage:
"""Parse decoded output text into an assistant `ChatMessage`.
Args:
tokenizer: Tokenizer handed to the parser.
text: Decoded output text to parse.
token_ids: Output token IDs the text was decoded from.
chat_request: Request supplying tools, `tool_choice` and
`include_reasoning`.
chat_template_kwargs: Already resolved template kwargs.
prompt_token_ids: Prompt token IDs, if known.
Returns:
The assembled assistant message.
"""
assert self.parser is not None
parser = self.parser(
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
model_config=self.model_config,
)
if prompt_token_ids is not None:
parser.set_prompt_token_ids(prompt_token_ids)
reasoning, content, tool_calls = parser.parse(
text,
chat_request,
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=token_ids,
)
if not getattr(chat_request, "include_reasoning", True):
reasoning = None
tc_items = (
[ToolCall(id=random_uuid(), function=tc) for tc in tool_calls]
if tool_calls
else []
)
is_named_tool_choice = (
type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam
)
is_required_tool_choice = chat_request.tool_choice == "required"
if is_named_tool_choice or is_required_tool_choice:
content = content or ""
return ChatMessage(
role="assistant",
reasoning=reasoning,
content=content,
tool_calls=tc_items,
)
def _detokenize_delta(
self,
tokenizer: TokenizerLike,
@@ -497,21 +526,10 @@ class OnlineDerenderer:
"""
tokenizer = self.renderer.get_tokenizer()
chat_template_kwargs: dict[str, Any] = {}
if not self.use_harmony:
chat_template_kwargs = (
chat_request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)
parser = parser_cls(
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
chat_template_kwargs=self._resolve_chat_template_kwargs(chat_request),
model_config=self.model_config,
)