[Frontend] Parse tool calls and reasoning from checkpoint response templates (#58604)

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Yoni Gozlan
2026-10-02 02:33:00 +00:00
committed by GitHub
co-authored by Cursor Bugen Zhao Claude Opus 5.5
parent 5a661487e2
commit 9bd993564f
18 changed files with 2298 additions and 21 deletions
+11
View File
@@ -22,6 +22,7 @@ from vllm.entrypoints.chat_utils import (
validate_chat_template,
)
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
from vllm.logger import init_logger
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName
from vllm.utils.argparse_utils import FlexibleArgumentParser
@@ -31,6 +32,8 @@ from .utils.constants import (
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
)
logger = init_logger(__name__)
class LoRAParserAction(argparse.Action):
def __call__(
@@ -466,6 +469,14 @@ def validate_parsed_serve_args(args: argparse.Namespace):
# Ensure that the chat template is valid; raises if it likely isn't
validate_chat_template(args.chat_template)
if args.chat_template is not None and "hf" in (
args.tool_call_parser,
getattr(args, "reasoning_parser", None),
):
logger.warning(
"--chat-template is set; the hf parser still expects "
"the checkpoint's output format."
)
# Enable auto tool needs a tool call parser to be valid
if args.enable_auto_tool_choice and not args.tool_call_parser:
@@ -129,17 +129,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
return render_result
single_requests, all_conversations, engine_prompts = render_result
parser: Parser | None = None
if self.parser_cls is not None:
chat_template_kwargs = self._effective_chat_template_kwargs(
single_requests[0]
)
parser = self.parser_cls(
tokenizer,
None, # tools
chat_template_kwargs=chat_template_kwargs,
)
request_id = (
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
)
@@ -202,7 +191,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
all_conversations,
tokenizer,
request_metadata,
parser,
self.parser_cls,
)
async def chat_completion_full_generator_batch(
@@ -214,7 +203,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
all_conversations: list[list[ConversationMessage]],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
parser: Parser | None = None,
parser_cls: type[Parser] | None = None,
) -> ErrorResponse | ChatCompletionResponse:
"""Handle batched (non-streaming) chat completions.
@@ -273,10 +262,21 @@ class OpenAIServingChatBatch(OpenAIServingChat):
else:
logprobs = None
if parser is not None:
if parser_cls is not None:
single_request = request.to_chat_completion_request(
request.messages[prompt_idx]
)
parser = parser_cls(
tokenizer,
None,
chat_template_kwargs=self._effective_chat_template_kwargs(
single_request
),
)
parser.set_prompt_token_ids(final_res.prompt_token_ids)
reasoning, content, _ = parser.parse(
output.text,
request=request, # type: ignore[arg-type]
request=single_request,
model_output_token_ids=output.token_ids,
)
if not request.include_reasoning:
@@ -165,6 +165,7 @@ class OpenAIServingChat(GenerateBaseServing):
tool_strict_level=tool_strict_level,
model_name=self.model_config.model,
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
tokenizer=self.renderer.tokenizer,
)
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
@@ -974,6 +975,8 @@ class OpenAIServingChat(GenerateBaseServing):
parser = self._make_parser(
request, tokenizer, self._effective_chat_template_kwargs(request)
)
if parser is not None and final_res.prompt_token_ids is not None:
parser.set_prompt_token_ids(final_res.prompt_token_ids)
token_ids = output.token_ids
out_logprobs = output.logprobs
+16 -1
View File
@@ -141,6 +141,7 @@ class OpenAIServingResponses(GenerateBaseServing):
tool_strict_level=tool_strict_level,
model_name=self.model_config.model,
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
tokenizer=self.renderer.tokenizer,
)
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_force_include_usage = enable_force_include_usage
@@ -446,6 +447,12 @@ class OpenAIServingResponses(GenerateBaseServing):
response_parser = self._make_response_parser(
request, tokenizer, chat_template_kwargs
)
if response_parser is not None:
prompt_token_ids = self._extract_prompt_components(
engine_input
).token_ids
if prompt_token_ids is not None:
response_parser.set_prompt_token_ids(prompt_token_ids)
context: ConversationContext
function_tool_names = extract_function_tool_names(request.tools)
@@ -703,6 +710,12 @@ class OpenAIServingResponses(GenerateBaseServing):
context.request,
context.response_messages,
)
if context.response_parser is not None:
prompt_token_ids = self._extract_prompt_components(
engine_input
).token_ids
if prompt_token_ids is not None:
context.response_parser.set_prompt_token_ids(prompt_token_ids)
sampling_params.max_tokens = get_max_tokens(
max_model_len,
@@ -1187,7 +1200,9 @@ class OpenAIServingResponses(GenerateBaseServing):
) -> AsyncGenerator[StreamingResponsesResponse, None]:
processor = SimpleStreamingEventProcessor(tools=request.tools)
hide_stream_metadata = not request.include_reasoning and self.parser is not None
hide_stream_metadata = (
not request.include_reasoning and context.response_parser is not None
)
def _get_logprobs(output: CompletionOutput) -> list[Logprob]:
if not request.is_include_output_logprobs():