[Bugfix][Tool Parser] Use LongCat tags in the LongCat structural tag (#60747)
Signed-off-by: hpdkhoa <hpdkhoa2311@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Flora Feng
parent
9824159d94
commit
e31b823bf5
@@ -48,6 +48,7 @@ from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
|||||||
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
|
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
|
||||||
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
|
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
|
||||||
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
|
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
|
||||||
|
from vllm.tool_parsers.longcat_tool_parser import LongcatFlashToolParser
|
||||||
from vllm.tool_parsers.mimo_tool_parser import MiMoToolParser
|
from vllm.tool_parsers.mimo_tool_parser import MiMoToolParser
|
||||||
from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
|
from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
|
||||||
from vllm.tool_parsers.plamo3_engine_tool_parser import Plamo3EngineToolParser
|
from vllm.tool_parsers.plamo3_engine_tool_parser import Plamo3EngineToolParser
|
||||||
@@ -850,6 +851,44 @@ def test_step3p5_forced_tool_choice_round_trips_native_xml(
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"tool_choice",
|
||||||
|
["required", {"type": "function", "function": {"name": "get_weather"}}],
|
||||||
|
)
|
||||||
|
def test_longcat_forced_tool_choice_round_trips_native_tags(
|
||||||
|
sample_tools: list[ChatCompletionToolsParam], tool_choice
|
||||||
|
):
|
||||||
|
"""Forced LongCat calls must use <longcat_tool_call>, not Hermes tags."""
|
||||||
|
|
||||||
|
class LongcatParser(DelegatingParser):
|
||||||
|
tool_parser_cls = LongcatFlashToolParser
|
||||||
|
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
messages=[],
|
||||||
|
model="m",
|
||||||
|
tools=[tool.model_dump(exclude_none=True) for tool in sample_tools],
|
||||||
|
tool_choice=tool_choice,
|
||||||
|
)
|
||||||
|
parser = LongcatParser(MagicMock(), tools=sample_tools)
|
||||||
|
request = parser.adjust_request(request)
|
||||||
|
|
||||||
|
grammar = Grammar.from_structural_tag(request.structured_outputs.structural_tag)
|
||||||
|
# Rendered exactly as the LongCat-Flash chat template renders tool calls.
|
||||||
|
output = (
|
||||||
|
'<longcat_tool_call>\n{"name": "get_weather", '
|
||||||
|
'"arguments": {"city": "Dallas"}}\n</longcat_tool_call>'
|
||||||
|
)
|
||||||
|
assert _is_grammar_accept_string(grammar, output)
|
||||||
|
assert not _is_grammar_accept_string(grammar, output.replace("longcat_", ""))
|
||||||
|
|
||||||
|
_, content, tool_calls = parser.parse(output, request, enable_auto_tools=True)
|
||||||
|
|
||||||
|
assert not content
|
||||||
|
assert [(call.name, json.loads(call.arguments)) for call in tool_calls] == [
|
||||||
|
("get_weather", {"city": "Dallas"})
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_tool_parsers_without_structural_tag_support_required_and_named():
|
def test_tool_parsers_without_structural_tag_support_required_and_named():
|
||||||
class NonStructuralTagToolParser(ToolParser):
|
class NonStructuralTagToolParser(ToolParser):
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
|||||||
|
|
||||||
|
|
||||||
class LongcatFlashToolParser(Hermes2ProToolParser):
|
class LongcatFlashToolParser(Hermes2ProToolParser):
|
||||||
|
structural_tag_model = "longcat"
|
||||||
|
|
||||||
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
|
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
|
||||||
super().__init__(tokenizer, tools)
|
super().__init__(tokenizer, tools)
|
||||||
|
|
||||||
|
|||||||
@@ -79,7 +79,7 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
|
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
|
||||||
{"glm_4_7", "hermes", "hy_v4", "kimi_k3", "plamo3"}
|
{"glm_4_7", "hermes", "hy_v4", "kimi_k3", "longcat", "plamo3"}
|
||||||
)
|
)
|
||||||
SUPPORTED_STRUCTURAL_TAG_MODELS = (
|
SUPPORTED_STRUCTURAL_TAG_MODELS = (
|
||||||
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
|
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
|
||||||
@@ -346,15 +346,19 @@ def get_function_parameters(function) -> dict[str, Any] | bool:
|
|||||||
return function.parameters if function.parameters is not None else True
|
return function.parameters if function.parameters is not None else True
|
||||||
|
|
||||||
|
|
||||||
def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
|
def _hermes_tool_tags(
|
||||||
|
tools: list[FunctionToolParam],
|
||||||
|
start_tag: str,
|
||||||
|
end_tag: str,
|
||||||
|
) -> list[TagFormat]:
|
||||||
arguments_field_prefix = '", "arguments": '
|
arguments_field_prefix = '", "arguments": '
|
||||||
formats = [
|
formats = [
|
||||||
# <tool_call>
|
# <tool_call>
|
||||||
# {"name": "t1", "arguments": {"q": "v"}}
|
# {"name": "t1", "arguments": {"q": "v"}}
|
||||||
# </tool_call>
|
# </tool_call>
|
||||||
('<tool_call>\n{"name": "', "}\n</tool_call>"),
|
(start_tag + '\n{"name": "', "}\n" + end_tag),
|
||||||
# <tool_call>{"name": "t1", "arguments": {"q": "v"}}</tool_call>
|
# <tool_call>{"name": "t1", "arguments": {"q": "v"}}</tool_call>
|
||||||
('<tool_call>{"name": "', "}</tool_call>"),
|
(start_tag + '{"name": "', "}" + end_tag),
|
||||||
]
|
]
|
||||||
|
|
||||||
return [
|
return [
|
||||||
@@ -370,6 +374,36 @@ def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _hermes_style_structural_tag(
|
||||||
|
tools: list[FunctionToolParam],
|
||||||
|
tool_choice: SimplifiedToolChoice,
|
||||||
|
start_tag: str,
|
||||||
|
end_tag: str,
|
||||||
|
) -> StructuralTag:
|
||||||
|
tags = _hermes_tool_tags(tools, start_tag, end_tag)
|
||||||
|
if tool_choice == "auto":
|
||||||
|
suffix_tag = (
|
||||||
|
TriggeredTagsFormat(triggers=[start_tag], tags=tags)
|
||||||
|
if tags
|
||||||
|
else AnyTextFormat()
|
||||||
|
)
|
||||||
|
elif tool_choice == "forced":
|
||||||
|
suffix_tag = TagsWithSeparatorFormat(
|
||||||
|
tags=tags,
|
||||||
|
separator="",
|
||||||
|
at_least_one=True,
|
||||||
|
stop_after_first=True,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
suffix_tag = TagsWithSeparatorFormat(
|
||||||
|
tags=tags,
|
||||||
|
separator="",
|
||||||
|
at_least_one=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
return StructuralTag(format=suffix_tag)
|
||||||
|
|
||||||
|
|
||||||
@register_vllm_structural_tag("hermes")
|
@register_vllm_structural_tag("hermes")
|
||||||
def get_hermes_structural_tag(
|
def get_hermes_structural_tag(
|
||||||
tools: list[FunctionToolParam],
|
tools: list[FunctionToolParam],
|
||||||
@@ -379,31 +413,23 @@ def get_hermes_structural_tag(
|
|||||||
token_suffix: str = "",
|
token_suffix: str = "",
|
||||||
) -> StructuralTag:
|
) -> StructuralTag:
|
||||||
del builtin_tools, reasoning, token_suffix
|
del builtin_tools, reasoning, token_suffix
|
||||||
|
return _hermes_style_structural_tag(
|
||||||
|
tools, tool_choice, "<tool_call>", "</tool_call>"
|
||||||
|
)
|
||||||
|
|
||||||
tool_call_trigger = "<tool_call>"
|
|
||||||
|
|
||||||
if tool_choice == "auto":
|
@register_vllm_structural_tag("longcat")
|
||||||
tags = _hermes_tool_tags(tools)
|
def get_longcat_structural_tag(
|
||||||
suffix_tag = (
|
tools: list[FunctionToolParam],
|
||||||
TriggeredTagsFormat(triggers=[tool_call_trigger], tags=tags)
|
builtin_tools: list[BuiltinToolParam],
|
||||||
if tags
|
tool_choice: SimplifiedToolChoice,
|
||||||
else AnyTextFormat()
|
reasoning: bool,
|
||||||
)
|
token_suffix: str = "",
|
||||||
elif tool_choice == "forced":
|
) -> StructuralTag:
|
||||||
suffix_tag = TagsWithSeparatorFormat(
|
del builtin_tools, reasoning, token_suffix
|
||||||
tags=_hermes_tool_tags(tools),
|
return _hermes_style_structural_tag(
|
||||||
separator="",
|
tools, tool_choice, "<longcat_tool_call>", "</longcat_tool_call>"
|
||||||
at_least_one=True,
|
)
|
||||||
stop_after_first=True,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
suffix_tag = TagsWithSeparatorFormat(
|
|
||||||
tags=_hermes_tool_tags(tools),
|
|
||||||
separator="",
|
|
||||||
at_least_one=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
return StructuralTag(format=suffix_tag)
|
|
||||||
|
|
||||||
|
|
||||||
def _minimax_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
|
def _minimax_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
|
||||||
|
|||||||
Reference in New Issue
Block a user