[Frontend] Parse tool calls and reasoning from checkpoint response templates (#58604)
Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Yoni Gozlan <yonigozlan@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Cursor
Bugen Zhao
Claude Opus 5.5
parent
5a661487e2
commit
9bd993564f
@@ -22,6 +22,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
validate_chat_template,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
@@ -31,6 +32,8 @@ from .utils.constants import (
|
||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LoRAParserAction(argparse.Action):
|
||||
def __call__(
|
||||
@@ -466,6 +469,14 @@ def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
|
||||
# Ensure that the chat template is valid; raises if it likely isn't
|
||||
validate_chat_template(args.chat_template)
|
||||
if args.chat_template is not None and "hf" in (
|
||||
args.tool_call_parser,
|
||||
getattr(args, "reasoning_parser", None),
|
||||
):
|
||||
logger.warning(
|
||||
"--chat-template is set; the hf parser still expects "
|
||||
"the checkpoint's output format."
|
||||
)
|
||||
|
||||
# Enable auto tool needs a tool call parser to be valid
|
||||
if args.enable_auto_tool_choice and not args.tool_call_parser:
|
||||
|
||||
@@ -129,17 +129,6 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
return render_result
|
||||
single_requests, all_conversations, engine_prompts = render_result
|
||||
|
||||
parser: Parser | None = None
|
||||
if self.parser_cls is not None:
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(
|
||||
single_requests[0]
|
||||
)
|
||||
parser = self.parser_cls(
|
||||
tokenizer,
|
||||
None, # tools
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
request_id = (
|
||||
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
||||
)
|
||||
@@ -202,7 +191,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
all_conversations,
|
||||
tokenizer,
|
||||
request_metadata,
|
||||
parser,
|
||||
self.parser_cls,
|
||||
)
|
||||
|
||||
async def chat_completion_full_generator_batch(
|
||||
@@ -214,7 +203,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
all_conversations: list[list[ConversationMessage]],
|
||||
tokenizer: TokenizerLike,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
parser: Parser | None = None,
|
||||
parser_cls: type[Parser] | None = None,
|
||||
) -> ErrorResponse | ChatCompletionResponse:
|
||||
"""Handle batched (non-streaming) chat completions.
|
||||
|
||||
@@ -273,10 +262,21 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
else:
|
||||
logprobs = None
|
||||
|
||||
if parser is not None:
|
||||
if parser_cls is not None:
|
||||
single_request = request.to_chat_completion_request(
|
||||
request.messages[prompt_idx]
|
||||
)
|
||||
parser = parser_cls(
|
||||
tokenizer,
|
||||
None,
|
||||
chat_template_kwargs=self._effective_chat_template_kwargs(
|
||||
single_request
|
||||
),
|
||||
)
|
||||
parser.set_prompt_token_ids(final_res.prompt_token_ids)
|
||||
reasoning, content, _ = parser.parse(
|
||||
output.text,
|
||||
request=request, # type: ignore[arg-type]
|
||||
request=single_request,
|
||||
model_output_token_ids=output.token_ids,
|
||||
)
|
||||
if not request.include_reasoning:
|
||||
|
||||
@@ -165,6 +165,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
tool_strict_level=tool_strict_level,
|
||||
model_name=self.model_config.model,
|
||||
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||
tokenizer=self.renderer.tokenizer,
|
||||
)
|
||||
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
|
||||
|
||||
@@ -974,6 +975,8 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
parser = self._make_parser(
|
||||
request, tokenizer, self._effective_chat_template_kwargs(request)
|
||||
)
|
||||
if parser is not None and final_res.prompt_token_ids is not None:
|
||||
parser.set_prompt_token_ids(final_res.prompt_token_ids)
|
||||
token_ids = output.token_ids
|
||||
out_logprobs = output.logprobs
|
||||
|
||||
|
||||
@@ -141,6 +141,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
tool_strict_level=tool_strict_level,
|
||||
model_name=self.model_config.model,
|
||||
is_harmony=self.model_config.hf_config.model_type == "gpt_oss",
|
||||
tokenizer=self.renderer.tokenizer,
|
||||
)
|
||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||
self.enable_force_include_usage = enable_force_include_usage
|
||||
@@ -446,6 +447,12 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
response_parser = self._make_response_parser(
|
||||
request, tokenizer, chat_template_kwargs
|
||||
)
|
||||
if response_parser is not None:
|
||||
prompt_token_ids = self._extract_prompt_components(
|
||||
engine_input
|
||||
).token_ids
|
||||
if prompt_token_ids is not None:
|
||||
response_parser.set_prompt_token_ids(prompt_token_ids)
|
||||
|
||||
context: ConversationContext
|
||||
function_tool_names = extract_function_tool_names(request.tools)
|
||||
@@ -703,6 +710,12 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
context.request,
|
||||
context.response_messages,
|
||||
)
|
||||
if context.response_parser is not None:
|
||||
prompt_token_ids = self._extract_prompt_components(
|
||||
engine_input
|
||||
).token_ids
|
||||
if prompt_token_ids is not None:
|
||||
context.response_parser.set_prompt_token_ids(prompt_token_ids)
|
||||
|
||||
sampling_params.max_tokens = get_max_tokens(
|
||||
max_model_len,
|
||||
@@ -1187,7 +1200,9 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
) -> AsyncGenerator[StreamingResponsesResponse, None]:
|
||||
processor = SimpleStreamingEventProcessor(tools=request.tools)
|
||||
|
||||
hide_stream_metadata = not request.include_reasoning and self.parser is not None
|
||||
hide_stream_metadata = (
|
||||
not request.include_reasoning and context.response_parser is not None
|
||||
)
|
||||
|
||||
def _get_logprobs(output: CompletionOutput) -> list[Logprob]:
|
||||
if not request.is_include_output_logprobs():
|
||||
|
||||
Reference in New Issue
Block a user