[Frontend] Extract parse and assembly out of batch chat derender (#60073)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
2e3154aa18
commit
3fb1cd796b
@@ -151,6 +151,12 @@ class OnlineDerenderer:
|
||||
skip_special_tokens=skip_special,
|
||||
)
|
||||
|
||||
chat_template_kwargs = (
|
||||
self._resolve_chat_template_kwargs(chat_request)
|
||||
if has_parser and chat_request is not None
|
||||
else {}
|
||||
)
|
||||
|
||||
for choice in generate_response.choices:
|
||||
if not choice.token_ids:
|
||||
raise ValueError(f"choice {choice.index} has empty or null token_ids")
|
||||
@@ -161,83 +167,29 @@ class OnlineDerenderer:
|
||||
else None
|
||||
)
|
||||
|
||||
# With a parser, special tokens are preserved so it can see
|
||||
# markers like </think>, <tool_call>, or Harmony channel tokens.
|
||||
# Without one, the request's skip_special_tokens is honoured
|
||||
# (default True when no request was given).
|
||||
decoded_text, _ = self._detokenize_delta(
|
||||
tokenizer,
|
||||
choice.token_ids,
|
||||
seed_state,
|
||||
skip_special_tokens=skip_special,
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
|
||||
if has_parser:
|
||||
assert self.parser is not None and chat_request is not None
|
||||
# Parser path: decode with special tokens preserved
|
||||
# so the parser can see markers like </think>,
|
||||
# <tool_call>, or Harmony channel tokens.
|
||||
decoded_text, _ = self._detokenize_delta(
|
||||
assert chat_request is not None
|
||||
message = self._parse_and_assemble(
|
||||
tokenizer,
|
||||
choice.token_ids,
|
||||
seed_state,
|
||||
skip_special_tokens=skip_special,
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
|
||||
chat_template_kwargs: dict[str, Any] = {}
|
||||
if not self.use_harmony:
|
||||
chat_template_kwargs = (
|
||||
chat_request.build_chat_params(
|
||||
self.chat_template,
|
||||
self.chat_template_content_format,
|
||||
)
|
||||
.with_defaults(self.default_chat_template_kwargs)
|
||||
.chat_template_kwargs
|
||||
)
|
||||
|
||||
parser = self.parser(
|
||||
tokenizer,
|
||||
chat_request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
if generate_response.prompt_token_ids is not None:
|
||||
parser.set_prompt_token_ids(generate_response.prompt_token_ids)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
decoded_text,
|
||||
choice.token_ids,
|
||||
chat_request,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
model_output_token_ids=choice.token_ids,
|
||||
)
|
||||
|
||||
if not getattr(chat_request, "include_reasoning", True):
|
||||
reasoning = None
|
||||
|
||||
tc_items = (
|
||||
[
|
||||
ToolCall(
|
||||
id=random_uuid(),
|
||||
function=tc,
|
||||
)
|
||||
for tc in tool_calls
|
||||
]
|
||||
if tool_calls
|
||||
else []
|
||||
)
|
||||
|
||||
is_named_tool_choice = (
|
||||
type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam
|
||||
)
|
||||
is_required_tool_choice = chat_request.tool_choice == "required"
|
||||
if is_named_tool_choice or is_required_tool_choice:
|
||||
content = content or ""
|
||||
|
||||
message = ChatMessage(
|
||||
role="assistant",
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tc_items,
|
||||
chat_template_kwargs,
|
||||
generate_response.prompt_token_ids,
|
||||
)
|
||||
else:
|
||||
# No parser: plain detokenization honouring the request's
|
||||
# skip_special_tokens (default True when no request was given).
|
||||
decoded_text, _ = self._detokenize_delta(
|
||||
tokenizer,
|
||||
choice.token_ids,
|
||||
seed_state,
|
||||
skip_special_tokens=skip_special,
|
||||
spaces_between_special_tokens=spaces_between,
|
||||
)
|
||||
message = ChatMessage(role="assistant", content=decoded_text)
|
||||
|
||||
choices.append(
|
||||
@@ -251,6 +203,83 @@ class OnlineDerenderer:
|
||||
|
||||
return choices
|
||||
|
||||
def _resolve_chat_template_kwargs(
|
||||
self, chat_request: ChatCompletionRequest
|
||||
) -> dict[str, Any]:
|
||||
if self.use_harmony:
|
||||
return {}
|
||||
return (
|
||||
chat_request.build_chat_params(
|
||||
self.chat_template,
|
||||
self.chat_template_content_format,
|
||||
)
|
||||
.with_defaults(self.default_chat_template_kwargs)
|
||||
.chat_template_kwargs
|
||||
)
|
||||
|
||||
def _parse_and_assemble(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
text: str,
|
||||
token_ids: Sequence[int],
|
||||
chat_request: ChatCompletionRequest,
|
||||
chat_template_kwargs: dict[str, Any],
|
||||
prompt_token_ids: list[int] | None,
|
||||
) -> ChatMessage:
|
||||
"""Parse decoded output text into an assistant `ChatMessage`.
|
||||
|
||||
Args:
|
||||
tokenizer: Tokenizer handed to the parser.
|
||||
text: Decoded output text to parse.
|
||||
token_ids: Output token IDs the text was decoded from.
|
||||
chat_request: Request supplying tools, `tool_choice` and
|
||||
`include_reasoning`.
|
||||
chat_template_kwargs: Already resolved template kwargs.
|
||||
prompt_token_ids: Prompt token IDs, if known.
|
||||
|
||||
Returns:
|
||||
The assembled assistant message.
|
||||
|
||||
"""
|
||||
assert self.parser is not None
|
||||
parser = self.parser(
|
||||
tokenizer,
|
||||
chat_request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
if prompt_token_ids is not None:
|
||||
parser.set_prompt_token_ids(prompt_token_ids)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
text,
|
||||
chat_request,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
model_output_token_ids=token_ids,
|
||||
)
|
||||
|
||||
if not getattr(chat_request, "include_reasoning", True):
|
||||
reasoning = None
|
||||
|
||||
tc_items = (
|
||||
[ToolCall(id=random_uuid(), function=tc) for tc in tool_calls]
|
||||
if tool_calls
|
||||
else []
|
||||
)
|
||||
|
||||
is_named_tool_choice = (
|
||||
type(chat_request.tool_choice) is ChatCompletionNamedToolChoiceParam
|
||||
)
|
||||
is_required_tool_choice = chat_request.tool_choice == "required"
|
||||
if is_named_tool_choice or is_required_tool_choice:
|
||||
content = content or ""
|
||||
|
||||
return ChatMessage(
|
||||
role="assistant",
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tc_items,
|
||||
)
|
||||
|
||||
def _detokenize_delta(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
@@ -497,21 +526,10 @@ class OnlineDerenderer:
|
||||
"""
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
|
||||
chat_template_kwargs: dict[str, Any] = {}
|
||||
if not self.use_harmony:
|
||||
chat_template_kwargs = (
|
||||
chat_request.build_chat_params(
|
||||
self.chat_template,
|
||||
self.chat_template_content_format,
|
||||
)
|
||||
.with_defaults(self.default_chat_template_kwargs)
|
||||
.chat_template_kwargs
|
||||
)
|
||||
|
||||
parser = parser_cls(
|
||||
tokenizer,
|
||||
chat_request.tools,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
chat_template_kwargs=self._resolve_chat_template_kwargs(chat_request),
|
||||
model_config=self.model_config,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user