[Frontend] Support strict mode for tool calling (#45003)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
cjackal
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
bd59c913bc
commit
2043258dec
@@ -308,20 +308,6 @@ async def init_app_state(
|
||||
) -> None:
|
||||
vllm_config = engine_client.vllm_config
|
||||
|
||||
# Propagate enable_in_reasoning to the API-server process. The engine core
|
||||
# runs in a separate process, so the contextvar that backs
|
||||
# `get_current_vllm_config_or_none()` is None on this stack. Tool parsers
|
||||
# call `get_enable_structured_outputs_in_reasoning()` during request
|
||||
# handling and need to see the real flag, otherwise they silently fall
|
||||
# back to False and mismatch the engine-side bitmask gating.
|
||||
from vllm.tool_parsers.structural_tag_registry import (
|
||||
set_enable_structured_outputs_in_reasoning,
|
||||
)
|
||||
|
||||
set_enable_structured_outputs_in_reasoning(
|
||||
vllm_config.structured_outputs_config.enable_in_reasoning
|
||||
)
|
||||
|
||||
if args.tool_call_parser is not None:
|
||||
from vllm.parser.metrics import init_parser_metrics
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
|
||||
tool_parser = render.tool_parser
|
||||
parser = render.parser
|
||||
tool_dicts: list[dict] | None = None
|
||||
|
||||
all_conversations: list[list[ConversationMessage]] = []
|
||||
@@ -94,7 +94,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
default_template_content_format=render.chat_template_content_format,
|
||||
default_template_kwargs=render.default_chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=tool_parser,
|
||||
parser=parser,
|
||||
)
|
||||
all_conversations.append(conversation)
|
||||
all_engine_prompts.append(engine_prompts[0])
|
||||
|
||||
@@ -102,10 +102,9 @@ from vllm.logprobs import Logprob as SampleLogprob
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.collection_utils import as_list
|
||||
|
||||
@@ -613,8 +612,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=self.parser.tool_parser_cls if self.parser else None,
|
||||
reasoning_parser=self.parser.reasoning_parser_cls if self.parser else None,
|
||||
parser=self.parser,
|
||||
)
|
||||
return messages, engine_inputs
|
||||
|
||||
@@ -623,7 +621,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
request: ResponsesRequest,
|
||||
messages: list[ResponseInputOutputItem],
|
||||
tool_dicts: list[dict[str, Any]] | None,
|
||||
tool_parser: type[ToolParser] | None,
|
||||
parser: type[Parser] | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
):
|
||||
@@ -638,8 +636,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
default_template_content_format=chat_template_content_format,
|
||||
default_template_kwargs=chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=tool_parser,
|
||||
reasoning_parser=self.parser.reasoning_parser_cls if self.parser else None,
|
||||
parser=parser,
|
||||
)
|
||||
return engine_inputs
|
||||
|
||||
@@ -707,7 +704,7 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
context.request,
|
||||
context.parser.response_messages,
|
||||
context.tool_dicts,
|
||||
context.parser_cls.tool_parser_cls if context.parser_cls else None,
|
||||
context.parser_cls,
|
||||
context.chat_template,
|
||||
context.chat_template_content_format,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user