[Frontend] Parse tool calls and reasoning from checkpoint response templates (#58604)
Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Cursor
Bugen Zhao
Claude Opus 5.5
parent
5a661487e2
commit
9bd993564f
@@ -129,17 +129,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
return render_result
|
||||
single_requests, all_conversations, engine_prompts = render_result
|
||||
|
||||
parser: Parser | None = None
|
||||
if self.parser_cls is not None:
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(
|
||||
single_requests[0]
|
||||
)
|
||||
parser = self.parser_cls(
|
||||
tokenizer,
|
||||
None, # tools
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
request_id = (
|
||||
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||
)
|
||||
@@ -202,7 +191,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
all_conversations,
|
||||
tokenizer,
|
||||
request_metadata,
|
||||
parser,
|
||||
self.parser_cls,
|
||||
)
|
||||
|
||||
async def chat_completion_full_generator_batch(
|
||||
@@ -214,7 +203,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
all_conversations: list[list[ConversationMessage]],
|
||||
tokenizer: TokenizerLike,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
parser: Parser | None = None,
|
||||
parser_cls: type[Parser] | None = None,
|
||||
) -> ErrorResponse | ChatCompletionResponse:
|
||||
"""Handle batched (non-streaming) chat completions.
|
||||
|
||||
@@ -273,10 +262,21 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
if parser is not None:
|
||||
if parser_cls is not None:
|
||||
single_request = request.to_chat_completion_request(
|
||||
request.messages[prompt_idx]
|
||||
)
|
||||
parser = parser_cls(
|
||||
tokenizer,
|
||||
None,
|
||||
chat_template_kwargs=self._effective_chat_template_kwargs(
|
||||
single_request
|
||||
),
|
||||
)
|
||||
parser.set_prompt_token_ids(final_res.prompt_token_ids)
|
||||
reasoning, content, _ = parser.parse(
|
||||
output.text,
|
||||
request=request, # type: ignore[arg-type]
|
||||
request=single_request,
|
||||
model_output_token_ids=output.token_ids,
|
||||
)
|
||||
if not request.include_reasoning:
|
||||
|
||||
@@ -165,6 +165,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
tool_strict_level=tool_strict_level,
|
||||
model_name=self.model_config.model,
|
||||
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||
tokenizer=self.renderer.tokenizer,
|
||||
)
|
||||
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
|
||||
|
||||
@@ -974,6 +975,8 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
parser = self._make_parser(
|
||||
request, tokenizer, self._effective_chat_template_kwargs(request)
|
||||
)
|
||||
if parser is not None and final_res.prompt_token_ids is not None:
|
||||
parser.set_prompt_token_ids(final_res.prompt_token_ids)
|
||||
token_ids = output.token_ids
|
||||
out_logprobs = output.logprobs
|
||||
|
||||
|
||||
Reference in New Issue
Block a user