[Frontend] Parse tool calls and reasoning from checkpoint response templates (#58604)

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Yoni Gozlan
2026-10-02 02:33:00 +00:00
committed by GitHub
co-authored by Cursor Bugen Zhao Claude Opus 5.5
parent 5a661487e2
commit 9bd993564f
18 changed files with 2298 additions and 21 deletions
@@ -129,17 +129,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
return render_result
single_requests, all_conversations, engine_prompts = render_result
parser: Parser | None = None
if self.parser_cls is not None:
chat_template_kwargs = self._effective_chat_template_kwargs(
single_requests[0]
)
parser = self.parser_cls(
tokenizer,
None, # tools
chat_template_kwargs=chat_template_kwargs,
)
request_id = (
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
)
@@ -202,7 +191,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
all_conversations,
tokenizer,
request_metadata,
parser,
self.parser_cls,
)
async def chat_completion_full_generator_batch(
@@ -214,7 +203,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
all_conversations: list[list[ConversationMessage]],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
parser: Parser | None = None,
parser_cls: type[Parser] | None = None,
) -> ErrorResponse | ChatCompletionResponse:
"""Handle batched (non-streaming) chat completions.
@@ -273,10 +262,21 @@ class OpenAIServingChatBatch(OpenAIServingChat):
else:
logprobs = None
if parser is not None:
if parser_cls is not None:
single_request = request.to_chat_completion_request(
request.messages[prompt_idx]
)
parser = parser_cls(
tokenizer,
None,
chat_template_kwargs=self._effective_chat_template_kwargs(
single_request
),
)
parser.set_prompt_token_ids(final_res.prompt_token_ids)
reasoning, content, _ = parser.parse(
output.text,
request=request, # type: ignore[arg-type]
request=single_request,
model_output_token_ids=output.token_ids,
)
if not request.include_reasoning: