[Frontend] Add the Beam reasoning and tool parsers (#62052)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> Co-authored-by: neilkg <33635204+neilkg@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Bugen Zhao
Yoni Gozlan
neilkg
Cursor
Claude Opus 5.5
parent
9731f30a03
commit
153a17166e
@@ -0,0 +1,747 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import FunctionTool, ToolChoiceFunction
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.config.reasoning import ReasoningConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponsesRequest,
|
||||
ResponsesResponse,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.beam import BeamParser, BeamReasoningParser, BeamToolParser
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.parser.response_template import ResponseTemplateParser
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
# The `response_template` of the Beam checkpoints.
|
||||
BEAM_RESPONSE_TEMPLATE: dict[str, Any] = {
|
||||
"defaults": {"role": "assistant"},
|
||||
"start_anchor_pattern": r"<\|start\|>assistant",
|
||||
"fields": {
|
||||
"thinking": {
|
||||
"open": "<|channel|>analysis<|message|>",
|
||||
"close": "<|end|><|start|>assistant",
|
||||
"content": "text",
|
||||
},
|
||||
"content": {
|
||||
"open": "<|channel|>final<|message|>",
|
||||
"close": ["<|return|>", "<|end|>", "<|call|>"],
|
||||
"content": "text",
|
||||
},
|
||||
"tool_calls": {
|
||||
"open_pattern": (
|
||||
r" to=(?P<name>[^\s<]+)"
|
||||
r"<\|channel\|>analysis <\|constrain\|>xml<\|message\|>"
|
||||
),
|
||||
"close": ["<|end|><|start|>assistant", "<|call|>"],
|
||||
"repeats": True,
|
||||
"content": "xml-inline",
|
||||
"content_args": {
|
||||
"tag_pattern": (
|
||||
r'<rfl:parameter name="(?P<key>[^"]+)">'
|
||||
r'\n(?P<value>.*?)\n(?:</rfl:parameter>|(?=<rfl:parameter name="))'
|
||||
),
|
||||
"value_parser": {"name": "text", "args": {"strip": False}},
|
||||
},
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "{name}", "arguments": "{content}"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
VOCAB = {
|
||||
"<|start|>": 200,
|
||||
"<|channel|>": 201,
|
||||
"<|message|>": 202,
|
||||
"<|end|>": 203,
|
||||
"<|call|>": 204,
|
||||
"<|return|>": 205,
|
||||
"<|constrain|>": 206,
|
||||
}
|
||||
TEXT_TOKENS = {100: "assistant", 101: "analysis", 102: "final", 103: "plan"}
|
||||
PROMPT_TOKEN_IDS = [200, 100]
|
||||
THOUGHT = "<|channel|>analysis<|message|>plan<|end|><|start|>assistant"
|
||||
FINAL = "<|channel|>final<|message|>"
|
||||
NEXT_MESSAGE = "<|end|><|start|>assistant"
|
||||
|
||||
|
||||
def _tool(name, properties, *, strict=None):
|
||||
return ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": name,
|
||||
"parameters": {"type": "object", "properties": properties},
|
||||
"strict": strict,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
TOOLS = [
|
||||
_tool("get_weather", {"city": {"type": "string"}}),
|
||||
_tool("set_count", {"count": {"type": "integer"}}),
|
||||
]
|
||||
STRICT_TOOLS = [
|
||||
_tool("get_weather", {"city": {"type": "string"}}),
|
||||
_tool("set_count", {"count": {"type": "integer"}}, strict=True),
|
||||
]
|
||||
|
||||
|
||||
class BeamTokenizer:
|
||||
def __init__(self, response_template=BEAM_RESPONSE_TEMPLATE):
|
||||
self.response_template = response_template
|
||||
self.init_kwargs = {}
|
||||
self.name_or_path = ""
|
||||
self.all_special_tokens = list(VOCAB)
|
||||
self.all_special_ids = list(VOCAB.values())
|
||||
|
||||
def decode(self, token_ids, skip_special_tokens=False, **_kwargs):
|
||||
inverse_vocab = {token_id: token for token, token_id in VOCAB.items()}
|
||||
return "".join(
|
||||
TEXT_TOKENS.get(token_id, inverse_vocab.get(token_id, ""))
|
||||
for token_id in token_ids
|
||||
)
|
||||
|
||||
def get_vocab(self):
|
||||
return VOCAB
|
||||
|
||||
def encode(self, text, add_special_tokens=False):
|
||||
del add_special_tokens
|
||||
return list(text.encode())
|
||||
|
||||
|
||||
def request(**overrides):
|
||||
values = dict(
|
||||
tool_choice="auto",
|
||||
include_reasoning=True,
|
||||
tools=TOOLS,
|
||||
parallel_tool_calls=True,
|
||||
)
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def parameter(name, value):
|
||||
return f'<rfl:parameter name="{name}">\n{value}\n</rfl:parameter>'
|
||||
|
||||
|
||||
def call(name, body, *, terminator="<|call|>"):
|
||||
return (
|
||||
f" to={name}<|channel|>analysis <|constrain|>xml<|message|>{body}{terminator}"
|
||||
)
|
||||
|
||||
|
||||
def make_parser(prompt_token_ids=PROMPT_TOKEN_IDS, tools=TOOLS, **kwargs):
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
assert parser_cls is not None
|
||||
parser = parser_cls(BeamTokenizer(), tools, **kwargs)
|
||||
parser.set_prompt_token_ids(prompt_token_ids)
|
||||
return parser
|
||||
|
||||
|
||||
def parse(text, prompt_token_ids=PROMPT_TOKEN_IDS):
|
||||
reasoning, content, calls = make_parser(prompt_token_ids).parse(
|
||||
text, request(), enable_auto_tools=True
|
||||
)
|
||||
return reasoning, content, normalize_calls(calls)
|
||||
|
||||
|
||||
def normalize_calls(calls):
|
||||
return [(call.name, json.loads(call.arguments)) for call in calls or []]
|
||||
|
||||
|
||||
def test_beam_parsers_are_registered():
|
||||
assert ToolParserManager.get_tool_parser("beam") is BeamToolParser
|
||||
assert ReasoningParserManager.get_reasoning_parser("beam") is BeamReasoningParser
|
||||
|
||||
|
||||
def test_parser_manager_selects_beam_parser():
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
|
||||
assert parser_cls is not None
|
||||
assert issubclass(parser_cls, BeamParser)
|
||||
assert parser_cls.reasoning_parser_cls is BeamReasoningParser
|
||||
assert parser_cls.tool_parser_cls is BeamToolParser
|
||||
assert parser_cls._enable_auto_tools
|
||||
|
||||
|
||||
@pytest.mark.parametrize("other", ["hermes", "hf"])
|
||||
def test_beam_parser_rejects_other_parsers(other):
|
||||
with pytest.raises(TypeError, match="cannot be combined"):
|
||||
ParserManager.get_parser(
|
||||
tool_parser_name=other,
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
|
||||
|
||||
def test_beam_parser_requires_checkpoint_metadata_at_startup():
|
||||
with pytest.raises(
|
||||
TypeError, match="--reasoning-parser beam requires `response_template`"
|
||||
):
|
||||
ParserManager.get_parser(
|
||||
reasoning_parser_name="beam",
|
||||
tokenizer=BeamTokenizer(response_template=None),
|
||||
)
|
||||
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
del template["fields"]["tool_calls"]
|
||||
with pytest.raises(TypeError, match="--tool-call-parser beam requires"):
|
||||
ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(template),
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_and_final_answer():
|
||||
assert parse(THOUGHT + FINAL + "Sunny.<|return|>") == ("plan", "Sunny.", [])
|
||||
|
||||
|
||||
def test_prefilled_reasoning_continues_from_prompt():
|
||||
prompt = PROMPT_TOKEN_IDS + [201, 101, 202, 103]
|
||||
text = " more.<|end|><|start|>assistant" + FINAL + "Done.<|return|>"
|
||||
|
||||
assert parse(text, prompt) == (" more.", "Done.", [])
|
||||
|
||||
|
||||
def test_parallel_calls_are_coerced_to_their_schemas():
|
||||
text = (
|
||||
THOUGHT
|
||||
+ call(
|
||||
"get_weather",
|
||||
parameter("city", "Paris"),
|
||||
terminator="<|end|><|start|>assistant",
|
||||
)
|
||||
+ call("set_count", parameter("count", "7"))
|
||||
)
|
||||
|
||||
assert parse(text) == (
|
||||
"plan",
|
||||
None,
|
||||
[("get_weather", {"city": "Paris"}), ("set_count", {"count": 7})],
|
||||
)
|
||||
|
||||
|
||||
def test_call_stop_token_removed_from_the_text_still_ends_the_call():
|
||||
text = THOUGHT + call("get_weather", parameter("city", "Paris"), terminator="")
|
||||
|
||||
assert parse(text) == ("plan", None, [("get_weather", {"city": "Paris"})])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("end", ["<|end|>", "<|end|><|start|>"])
|
||||
def test_output_cut_before_the_next_message_start_ends_its_region(end):
|
||||
"""Output that stops after `<|end|>`, such as at `max_tokens`, has ended
|
||||
its reasoning or call and lacks only the next message's start."""
|
||||
weather = call("get_weather", parameter("city", "Paris"), terminator=end)
|
||||
|
||||
assert parse("<|channel|>analysis<|message|>plan" + end) == ("plan", None, [])
|
||||
assert parse(THOUGHT + weather) == (
|
||||
"plan",
|
||||
None,
|
||||
[("get_weather", {"city": "Paris"})],
|
||||
)
|
||||
|
||||
|
||||
THOUGHT_IDS = [201, 101, 202, 103, 203, 200, 100]
|
||||
FINAL_IDS = [201, 102, 202]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt_token_ids", "token_ids", "count"),
|
||||
[
|
||||
(PROMPT_TOKEN_IDS, [*THOUGHT_IDS, *FINAL_IDS, 205], 1),
|
||||
(PROMPT_TOKEN_IDS + [201, 101, 202], [103, 103, 203, 200, 100, *FINAL_IDS], 2),
|
||||
(PROMPT_TOKEN_IDS, [201, 101, 202, 103, 103, 103], 3),
|
||||
],
|
||||
ids=["reasoning_then_answer", "prefilled_reasoning", "cut_off"],
|
||||
)
|
||||
def test_reasoning_tokens_are_counted(prompt_token_ids, token_ids, count):
|
||||
parser = make_parser(prompt_token_ids)
|
||||
|
||||
assert parser.count_reasoning_tokens(token_ids) == count
|
||||
|
||||
|
||||
def test_reasoning_tokens_are_counted_as_the_generation_grows():
|
||||
token_ids = [*THOUGHT_IDS, *FINAL_IDS, 205]
|
||||
parser = make_parser()
|
||||
|
||||
assert [
|
||||
parser.count_reasoning_tokens(token_ids[:end])
|
||||
for end in range(1, len(token_ids) + 1)
|
||||
] == [
|
||||
make_parser().count_reasoning_tokens(token_ids[:end])
|
||||
for end in range(1, len(token_ids) + 1)
|
||||
]
|
||||
assert parser.count_reasoning_tokens(FINAL_IDS) == 0
|
||||
|
||||
|
||||
def test_call_stop_token_ends_the_answer():
|
||||
assert parse(FINAL + "done<|call|>") == (None, "done", [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 7, 10_000])
|
||||
def test_streaming_matches_non_streaming(chunk_size):
|
||||
text = (
|
||||
THOUGHT
|
||||
+ call(
|
||||
"get_weather",
|
||||
parameter("city", "Paris"),
|
||||
terminator="<|end|><|start|>assistant",
|
||||
)
|
||||
+ call("set_count", parameter("count", "7"))
|
||||
)
|
||||
parser = make_parser()
|
||||
reasoning, calls = [], []
|
||||
for start in range(0, len(text), chunk_size):
|
||||
end = min(start + chunk_size, len(text))
|
||||
delta = parser.parse_delta(
|
||||
text[start:end], [], request(), finished=end == len(text)
|
||||
)
|
||||
if delta is not None:
|
||||
reasoning.append(delta.reasoning or "")
|
||||
calls.extend(delta.tool_calls)
|
||||
|
||||
assert "".join(reasoning) == "plan"
|
||||
assert [
|
||||
(call.function.name, json.loads(call.function.arguments)) for call in calls
|
||||
] == parse(text)[2]
|
||||
|
||||
|
||||
WEATHER_CALL = call("get_weather", parameter("city", "Paris"))
|
||||
WEATHER = [("get_weather", {"city": "Paris"})]
|
||||
|
||||
|
||||
def chat_request(tools=STRICT_TOOLS, **body):
|
||||
return ChatCompletionRequest(
|
||||
model="beam",
|
||||
messages=[{"role": "user", "content": "Weather?"}],
|
||||
tools=tools and [tool.model_dump() for tool in tools],
|
||||
**body,
|
||||
)
|
||||
|
||||
|
||||
def named(name):
|
||||
return {"type": "function", "function": {"name": name}}
|
||||
|
||||
|
||||
def adjust(req, prompt_token_ids=PROMPT_TOKEN_IDS, **kwargs):
|
||||
make_parser(prompt_token_ids, req.tools, **kwargs).adjust_request(req)
|
||||
return req
|
||||
|
||||
|
||||
def accepts(req, text):
|
||||
grammar = Grammar.from_structural_tag(
|
||||
req.extract_structured_outputs().structural_tag
|
||||
)
|
||||
return _is_grammar_accept_string(grammar, text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "text", "calls"),
|
||||
[
|
||||
("required", THOUGHT + WEATHER_CALL, WEATHER),
|
||||
("required", WEATHER_CALL, None),
|
||||
(
|
||||
"required",
|
||||
THOUGHT
|
||||
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
+ call("set_count", parameter("count", "7")),
|
||||
[*WEATHER, ("set_count", {"count": 7})],
|
||||
),
|
||||
("required", THOUGHT + call("set_count", parameter("count", "x")), None),
|
||||
(
|
||||
"required",
|
||||
THOUGHT + call("get_weather", parameter("city", "a<|call|>b")),
|
||||
None,
|
||||
),
|
||||
("required", THOUGHT + FINAL + "Paris<|return|>", None),
|
||||
(named("set_count"), THOUGHT + WEATHER_CALL, None),
|
||||
(named("get_weather"), THOUGHT + WEATHER_CALL, WEATHER),
|
||||
(
|
||||
named("get_weather"),
|
||||
THOUGHT
|
||||
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
+ WEATHER_CALL,
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_forced_tool_calls_follow_the_template_grammar(tool_choice, text, calls):
|
||||
"""From the message start, the grammar opens with the reasoning, and
|
||||
accepts only calls the parser reads back."""
|
||||
req = adjust(chat_request(tool_choice=tool_choice))
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert accepts(req, text) is (calls is not None)
|
||||
if calls is not None:
|
||||
assert parse(text)[2] == calls
|
||||
|
||||
|
||||
def test_reasoning_may_be_skipped_when_thinking_is_disabled():
|
||||
req = adjust(
|
||||
chat_request(tool_choice="required"),
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
|
||||
assert accepts(req, WEATHER_CALL)
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
|
||||
|
||||
def test_parallel_tool_calls_false_allows_a_single_call():
|
||||
req = adjust(chat_request(tool_choice="required", parallel_tool_calls=False))
|
||||
calls = call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
assert not accepts(req, THOUGHT + calls + WEATHER_CALL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("strict", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("separator", "accepted"), [("", True), ("\n", True), ("\n\n", False), (" ", False)]
|
||||
)
|
||||
def test_parameters_are_separated_by_at_most_a_newline(strict, separator, accepted):
|
||||
"""A model that skips a parameter's opener must not write whitespace until
|
||||
`max_tokens`."""
|
||||
write = _tool(
|
||||
"write",
|
||||
{"path": {"type": "string"}, "content": {"type": "string"}},
|
||||
strict=strict,
|
||||
)
|
||||
req = adjust(chat_request([write], tool_choice="required"))
|
||||
body = parameter("path", "a.py") + separator + parameter("content", "x")
|
||||
|
||||
assert accepts(req, THOUGHT + call("write", body)) is accepted
|
||||
assert accepts(req, THOUGHT + call("write", separator + body)) is accepted
|
||||
|
||||
|
||||
def test_forced_calls_in_prefilled_reasoning_start_after_it():
|
||||
req = adjust(
|
||||
chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 101, 202, 103]
|
||||
)
|
||||
|
||||
assert not req._grammar_from_parser
|
||||
assert accepts(req, WEATHER_CALL)
|
||||
|
||||
|
||||
def test_forced_calls_cannot_follow_a_prefilled_answer():
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
adjust(chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 102, 202])
|
||||
|
||||
|
||||
def test_prefilled_answer_can_only_be_finished():
|
||||
req = adjust(chat_request(TOOLS), PROMPT_TOKEN_IDS + [201, 102, 202])
|
||||
|
||||
assert accepts(req, "Sunny.<|return|>")
|
||||
assert not accepts(req, "Sunny." + NEXT_MESSAGE + WEATHER_CALL)
|
||||
|
||||
|
||||
def test_forced_call_with_unencodable_tool_name_is_rejected():
|
||||
tools = [_tool("get weather", {"city": {"type": "string"}})]
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="tool name"):
|
||||
adjust(chat_request(tools, tool_choice="required"))
|
||||
|
||||
|
||||
def test_string_formats_and_patterns_are_held_in_raw_text():
|
||||
"""Like XGrammar, a strict string keeps its format or pattern, unless a
|
||||
match could end the value early."""
|
||||
contact = _tool(
|
||||
"set_contact",
|
||||
{
|
||||
"email": {"type": "string", "format": "email"},
|
||||
"code": {"type": "string", "pattern": "^[A-Z]{3}$"},
|
||||
"n": {"type": "integer"},
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
contact.function.parameters["required"] = ["email", "code", "n"]
|
||||
req = adjust(chat_request([contact], tool_choice="required"))
|
||||
|
||||
def contact_call(email, code, *rest):
|
||||
body = parameter("email", email) + parameter("code", code) + "".join(rest)
|
||||
return THOUGHT + call("set_contact", body)
|
||||
|
||||
text = contact_call("a.b@x.co", "ABC", parameter("n", "1"))
|
||||
calls = make_parser(tools=[contact]).parse(text, req, enable_auto_tools=True)[2]
|
||||
assert accepts(req, text)
|
||||
assert normalize_calls(calls) == [
|
||||
("set_contact", {"email": "a.b@x.co", "code": "ABC", "n": 1})
|
||||
]
|
||||
assert not accepts(req, contact_call("ab", "ABC", parameter("n", "1")))
|
||||
assert not accepts(req, contact_call("a.b@x.co", "ABCD", parameter("n", "1")))
|
||||
assert not accepts(req, contact_call("a.b@x.co", "ABC"))
|
||||
# Line breaks could end the value.
|
||||
dotted = _tool(
|
||||
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
adjust(chat_request([dotted], tool_choice="required"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
(THOUGHT + WEATHER_CALL, True),
|
||||
(THOUGHT + FINAL + "Sunny.<|return|>", True),
|
||||
(THOUGHT + FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
|
||||
(THOUGHT + call("set_count", parameter("count", "x")), False),
|
||||
],
|
||||
)
|
||||
def test_strict_tools_constrain_automatic_tool_choice(text, accepted):
|
||||
assert accepts(adjust(chat_request()), text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
(THOUGHT + FINAL + "Sunny.<|return|>", True),
|
||||
(FINAL + "Sunny.<|return|>", False),
|
||||
(WEATHER_CALL, False),
|
||||
(THOUGHT + WEATHER_CALL, False),
|
||||
(FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
|
||||
],
|
||||
)
|
||||
def test_tool_choice_none_allows_only_an_answer(text, accepted):
|
||||
req = adjust(chat_request(TOOLS, tool_choice="none"))
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert accepts(req, text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("tools", "tool_choice"), [(TOOLS, "auto"), (None, None)])
|
||||
def test_always_constrained_output_follows_the_template_framing(tools, tool_choice):
|
||||
req = adjust(chat_request(tools, tool_choice=tool_choice))
|
||||
count_call = THOUGHT + call("set_count", parameter("count", "3"))
|
||||
|
||||
assert accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, THOUGHT.replace("plan", "") + FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, THOUGHT + "Sunny.")
|
||||
assert accepts(req, count_call) is (tool_choice == "auto")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format", [None, {"type": "json_object"}])
|
||||
@pytest.mark.parametrize("strict", [None, True])
|
||||
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||
def test_automatic_choice_lets_calls_name_undeclared_tools(
|
||||
strict, tool_choice, response_format
|
||||
):
|
||||
"""Unless a forced choice limits calls to the declared tools. The parser
|
||||
passes such calls on, so the harness can tell the model the tool does not
|
||||
exist."""
|
||||
bash = _tool("bash", {"command": {"type": "string"}}, strict=strict)
|
||||
req = adjust(
|
||||
chat_request([bash], tool_choice=tool_choice, response_format=response_format)
|
||||
)
|
||||
undeclared = THOUGHT + call("read_file", parameter("path", "a.py"))
|
||||
parser = make_parser(tools=[bash])
|
||||
|
||||
assert accepts(req, THOUGHT + call("bash", parameter("command", "ls")))
|
||||
assert accepts(req, undeclared) is (tool_choice == "auto")
|
||||
calls = parser.parse(undeclared, req, enable_auto_tools=True)[2]
|
||||
assert normalize_calls(calls) == [("read_file", {"path": "a.py"})]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "body", "accepted"),
|
||||
[
|
||||
("set_count", parameter("count", "1"), True),
|
||||
("set_count", parameter("count", "x"), False),
|
||||
("get_weather", parameter("town", "Paris"), False),
|
||||
# The grammar cannot enforce `minProperties`.
|
||||
("find", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("search_files", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("set_coun", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("set_counts", parameter("x", "1") + parameter("x", "y"), True),
|
||||
],
|
||||
)
|
||||
def test_automatic_calls_enforce_declared_arguments_beside_any_other_name(
|
||||
name, body, accepted
|
||||
):
|
||||
find = _tool("find", {"q": {"type": "string"}})
|
||||
find.function.parameters["minProperties"] = 1
|
||||
req = adjust(chat_request([*TOOLS, find]))
|
||||
|
||||
assert accepts(req, THOUGHT + call(name, body)) is accepted
|
||||
|
||||
|
||||
def test_value_ends_before_an_opener_on_the_next_line():
|
||||
write = _tool("write", {"path": {"type": "string"}, "content": {"type": "string"}})
|
||||
req = adjust(chat_request([write], tool_choice="required"))
|
||||
body = '<rfl:parameter name="content">\nprint(1)\n\n' + parameter("path", "a.py")
|
||||
text = THOUGHT + call("write", body)
|
||||
|
||||
assert accepts(req, text)
|
||||
calls = make_parser(tools=[write]).parse(text, req, enable_auto_tools=True)[2]
|
||||
assert normalize_calls(calls) == [
|
||||
("write", {"content": "print(1)\n", "path": "a.py"})
|
||||
]
|
||||
|
||||
|
||||
def test_always_constrained_output_can_be_disabled(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "0")
|
||||
|
||||
assert adjust(chat_request(TOOLS)).structured_outputs is None
|
||||
|
||||
|
||||
def test_strict_tools_the_template_cannot_hold_leave_automatic_choice_unconstrained():
|
||||
dotted = _tool(
|
||||
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
|
||||
)
|
||||
|
||||
assert adjust(chat_request([dotted])).structured_outputs is None
|
||||
|
||||
|
||||
def test_generic_parser_does_not_derive_the_template_grammar():
|
||||
req = chat_request(TOOLS, tool_choice="required")
|
||||
parser = ResponseTemplateParser(BeamTokenizer(), req.tools, enable_auto_tools=True)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
parser.adjust_request(req)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tools", [None, TOOLS])
|
||||
def test_structured_output_is_framed_as_the_answer(tools):
|
||||
schema = {"type": "object", "properties": {"a": {"type": "integer"}}}
|
||||
req = adjust(
|
||||
chat_request(
|
||||
tools,
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "answer", "schema": schema},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert req.response_format is None
|
||||
assert accepts(req, THOUGHT + FINAL + '{"a": 1}<|return|>')
|
||||
assert not accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL) is (tools is not None)
|
||||
|
||||
|
||||
def test_responses_named_tool_choice_follows_the_template_grammar():
|
||||
req = adjust(
|
||||
ResponsesRequest(
|
||||
input="Weather?",
|
||||
tools=[
|
||||
FunctionTool(
|
||||
type="function",
|
||||
name="get_weather",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
],
|
||||
tool_choice=ToolChoiceFunction(type="function", name="get_weather"),
|
||||
text={"format": {"type": "json_object"}},
|
||||
)
|
||||
)
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert req.text.format.type == "json_object"
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
assert not accepts(req, FINAL + "{}<|return|>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["in_progress", "completed"])
|
||||
@pytest.mark.parametrize("inside_reasoning", [False, True])
|
||||
def test_responses_preserves_text_config_with_template_grammar(
|
||||
status, inside_reasoning
|
||||
):
|
||||
text = {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "answer",
|
||||
"strict": True,
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"a": {"type": "integer"}},
|
||||
"required": ["a"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"verbosity": "low",
|
||||
}
|
||||
req = ResponsesRequest(input="Answer", text=text)
|
||||
expected_text = req.text.model_dump(by_alias=True)
|
||||
prompt = PROMPT_TOKEN_IDS + ([201, 101, 202] if inside_reasoning else [])
|
||||
adjust(req, prompt_token_ids=prompt)
|
||||
params = req.to_sampling_params(default_max_tokens=128)
|
||||
grammar = Grammar.from_structural_tag(params.structured_outputs.structural_tag)
|
||||
prefix = "" if inside_reasoning else THOUGHT
|
||||
assert _is_grammar_accept_string(grammar, prefix + FINAL + '{"a": 1}<|return|>')
|
||||
assert not _is_grammar_accept_string(grammar, prefix + FINAL + "Sunny.<|return|>")
|
||||
response = ResponsesResponse.from_request(req, params, "beam", 0, [], status)
|
||||
assert response.model_dump(by_alias=True)["text"] == expected_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reverse_alternatives", [False, True])
|
||||
def test_reasoning_budget_initializes_from_checkpoint_pattern(
|
||||
monkeypatch, reverse_alternatives
|
||||
):
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
alternatives = [
|
||||
r"^<\|channel\|>analysis<\|message\|>",
|
||||
r"<\|start\|>assistant<\|channel\|>analysis<\|message\|>",
|
||||
]
|
||||
if reverse_alternatives:
|
||||
alternatives.reverse()
|
||||
template["fields"]["thinking"] = {
|
||||
"open_pattern": "(?:" + "|".join(alternatives) + ")",
|
||||
"close": "<|end|>",
|
||||
"content": "text",
|
||||
}
|
||||
tokenizer = BeamTokenizer(template)
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.reasoning.cached_tokenizer_from_config", lambda **_: tokenizer
|
||||
)
|
||||
config = ReasoningConfig(reasoning_parser="beam")
|
||||
config.initialize_token_ids(SimpleNamespace())
|
||||
|
||||
assert config.enabled
|
||||
# The marker must work both after the prompt's assistant header and in
|
||||
# subsequent complete assistant messages, regardless of regex branch order.
|
||||
assert config.reasoning_start_token_ids == tokenizer.encode(
|
||||
"<|channel|>analysis<|message|>"
|
||||
)
|
||||
assert config.reasoning_end_token_ids == tokenizer.encode("<|end|>")
|
||||
assert config.natural_reasoning_end_token_ids == tokenizer.encode("<|end|>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern", [r"analysis.*", r"(?:analysis|thinking)"])
|
||||
def test_reasoning_budget_does_not_guess_a_marker_for_other_patterns(pattern):
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
thinking = template["fields"]["thinking"]
|
||||
del thinking["open"]
|
||||
thinking["open_pattern"] = pattern
|
||||
|
||||
assert BeamReasoningParser(BeamTokenizer(template)).reasoning_start_str is None
|
||||
@@ -0,0 +1,492 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.parser.chat_parsing.response_templates import load_response_template
|
||||
from vllm.parser.response_template_grammar import (
|
||||
answer_rest_structural_tag,
|
||||
tool_call_structural_tag,
|
||||
)
|
||||
|
||||
TEMPLATE: dict[str, Any] = {
|
||||
"start_anchor": "<assistant>",
|
||||
"fields": {
|
||||
"thinking": {"open": "<think>", "close": "</think>", "content": "text"},
|
||||
"content": {"content": "text"},
|
||||
"tool_calls": {
|
||||
"open_pattern": r'<call name="(?P<name>[\w.-]+)">',
|
||||
"close": "</call>",
|
||||
"repeats": True,
|
||||
"content": "xml-inline",
|
||||
"content_args": {
|
||||
"tag_pattern": r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)</arg>'
|
||||
},
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "{name}", "arguments": "{content}"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
ANSWER_TEMPLATE = deepcopy(TEMPLATE)
|
||||
ANSWER_TEMPLATE["fields"]["content"] = {
|
||||
"open": "<answer>",
|
||||
"close": "</answer>",
|
||||
"content": "text",
|
||||
}
|
||||
|
||||
|
||||
def tool(parameters, name="f", strict=True) -> dict[str, Any]:
|
||||
return {"name": name, "parameters": parameters, "strict": strict}
|
||||
|
||||
|
||||
WEATHER = tool(
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}, "days": {"type": "integer"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
name="get.weather",
|
||||
)
|
||||
PING = {"name": "ping"}
|
||||
CALL = '<call name="get.weather"><arg name="city">Paris</arg></call>'
|
||||
|
||||
|
||||
def build(
|
||||
template=TEMPLATE,
|
||||
functions=(WEATHER, PING),
|
||||
*,
|
||||
parallel=True,
|
||||
thinking=True,
|
||||
answer=False,
|
||||
answer_format=None,
|
||||
sorted_arguments=False,
|
||||
):
|
||||
fields = load_response_template(template).fields
|
||||
return tool_call_structural_tag(
|
||||
fields["tool_calls"],
|
||||
list(functions),
|
||||
parallel_tool_calls=parallel,
|
||||
thinking=fields["thinking"] if thinking else None,
|
||||
answer=fields["content"] if answer else None,
|
||||
answer_format=answer_format,
|
||||
sorted_arguments=sorted_arguments,
|
||||
)
|
||||
|
||||
|
||||
def accepts(tag, text):
|
||||
return _is_grammar_accept_string(Grammar.from_structural_tag(json.dumps(tag)), text)
|
||||
|
||||
|
||||
def call(*arguments):
|
||||
body = "".join(f'<arg name="{key}">{value}</arg>' for key, value in arguments)
|
||||
return f'<call name="f">{body}</call>'
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
("<think>plan</think>" + CALL, True),
|
||||
(CALL, True),
|
||||
(CALL.removesuffix("</call>"), True),
|
||||
(
|
||||
'<call name="get.weather"><arg name="city">Paris</arg>\n'
|
||||
'<arg name="days">3</arg></call>',
|
||||
True,
|
||||
),
|
||||
(
|
||||
'<call name="get.weather"><arg name="city">Paris</arg>'
|
||||
'<arg name="days">soon</arg></call>',
|
||||
False,
|
||||
),
|
||||
('<call name="get.weather"><arg name="days">3</arg></call>', False),
|
||||
(CALL + '<call name="ping"><arg name="any">x</arg></call>', True),
|
||||
('<call name="other"></call>', False),
|
||||
("<think>plan</think>The answer.", False),
|
||||
],
|
||||
)
|
||||
def test_forced_calls_follow_the_template_format(text, accepted):
|
||||
assert accepts(build(), text) is accepted
|
||||
|
||||
|
||||
def test_single_call_when_parallel_calls_are_disabled():
|
||||
tag = build(parallel=False)
|
||||
|
||||
assert accepts(tag, CALL)
|
||||
assert not accepts(tag, CALL + CALL)
|
||||
|
||||
|
||||
def test_thinking_is_rejected_when_not_requested():
|
||||
assert not accepts(build(thinking=False), "<think>plan</think>" + CALL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
("<think>plan</think><answer>Sunny.", True),
|
||||
("<answer>Sunny.</answer>", True),
|
||||
("<think>plan</think>" + CALL, True),
|
||||
("<answer>Sunny.</answer>" + CALL, False),
|
||||
('<call name="get.weather"><arg name="days">soon</arg></call>', False),
|
||||
],
|
||||
)
|
||||
def test_answer_field_lets_the_model_reply_instead(text, accepted):
|
||||
assert accepts(build(ANSWER_TEMPLATE, answer=True), text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize("required", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("text", "optional_only"),
|
||||
[
|
||||
("<think>plan</think>" + CALL, False),
|
||||
("<think>\nplan</think><answer>Sunny.", False),
|
||||
("<think> \n</think><answer>Sunny.", False),
|
||||
(CALL, True),
|
||||
("<answer>Sunny.", True),
|
||||
("<think></think><answer>Sunny.", True),
|
||||
("<think><</think><answer>Sunny.", True),
|
||||
],
|
||||
)
|
||||
def test_required_reasoning_opens_the_message_with_nonempty_reasoning(
|
||||
required, text, optional_only
|
||||
):
|
||||
fields = load_response_template(ANSWER_TEMPLATE).fields
|
||||
tag = tool_call_structural_tag(
|
||||
fields["tool_calls"],
|
||||
[WEATHER],
|
||||
parallel_tool_calls=True,
|
||||
thinking=fields["thinking"],
|
||||
answer=fields["content"],
|
||||
message_start=["<assistant>"],
|
||||
requires_reasoning=required,
|
||||
)
|
||||
|
||||
assert accepts(tag, text) is (not required or not optional_only)
|
||||
|
||||
|
||||
def test_answer_rest_finishes_the_open_answer():
|
||||
answer = load_response_template(ANSWER_TEMPLATE).fields["content"]
|
||||
tag = answer_rest_structural_tag(answer)
|
||||
|
||||
assert accepts(tag, "Sunny.</answer>")
|
||||
assert accepts(tag, "Sunny.")
|
||||
assert not accepts(tag, "Sunny.</answer>" + CALL)
|
||||
|
||||
|
||||
def test_answer_without_literal_opener_is_not_derived():
|
||||
assert build(answer=True) is None
|
||||
|
||||
|
||||
def test_answer_holds_the_answer_format():
|
||||
schema = {"type": "json_schema", "json_schema": {"type": "object"}}
|
||||
with_tools = build(ANSWER_TEMPLATE, answer=True, answer_format=schema)
|
||||
answer_only = build(ANSWER_TEMPLATE, (), answer=True, answer_format=schema)
|
||||
|
||||
for tag in (with_tools, answer_only):
|
||||
assert accepts(tag, '<think>plan</think><answer>{"a": 1}</answer>')
|
||||
assert not accepts(tag, "<answer>Sunny.</answer>")
|
||||
assert accepts(with_tools, CALL)
|
||||
assert not accepts(answer_only, CALL)
|
||||
|
||||
|
||||
def test_json_arguments_follow_the_schema():
|
||||
template = deepcopy(TEMPLATE)
|
||||
template["fields"]["tool_calls"].update(content="json", content_args={})
|
||||
tag = build(template, [WEATHER])
|
||||
|
||||
assert accepts(tag, '<call name="get.weather">{"city": "Paris"}</call>')
|
||||
assert not accepts(tag, '<call name="get.weather">{"days": 3}</call>')
|
||||
|
||||
|
||||
END_X = {"tag_pattern": r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)X'}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"content": "json", "content_args": {"unquoted_keys": True}},
|
||||
{
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "fixed", "arguments": "{content}"},
|
||||
}
|
||||
},
|
||||
{"content": "kv-lines", "content_args": {}},
|
||||
# Framing in which argument text can spell a closer, where the parser
|
||||
# would end the call.
|
||||
{"close": ["}X", "</call>"], "content_args": END_X},
|
||||
{
|
||||
"close": ["}X", "</call>"],
|
||||
"content_args": {**END_X, "value_parser": {"name": "json"}},
|
||||
},
|
||||
{"close": ["7X", "</call>"], "content_args": END_X},
|
||||
{"close": [">X", "</call>"]},
|
||||
{"close": ["></call>", "</call>"]},
|
||||
{"close": ["><a", "</call>"]},
|
||||
{"close": ["}</call>", "</call>"], "content": "json", "content_args": {}},
|
||||
],
|
||||
)
|
||||
def test_underivable_tool_formats_are_not_derived(change):
|
||||
template = deepcopy(TEMPLATE)
|
||||
template["fields"]["tool_calls"].update(change)
|
||||
|
||||
assert build(template) is None
|
||||
|
||||
|
||||
def test_non_literal_thinking_delimiters_are_not_derived():
|
||||
template = deepcopy(TEMPLATE)
|
||||
del template["fields"]["thinking"]["open"]
|
||||
template["fields"]["thinking"]["open_pattern"] = r"<think(?: mode=\w+)?>"
|
||||
|
||||
assert build(template) is None
|
||||
assert build(template, thinking=False) is not None
|
||||
|
||||
|
||||
LEVEL = {"type": "object", "properties": {"level": {"$ref": "#/$defs/level"}}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "value", "accepted"),
|
||||
[
|
||||
({"type": "integer", "nullable": True}, "null", True),
|
||||
({"type": "integer", "nullable": True}, "soon", False),
|
||||
({"type": ["integer", "null"]}, "null", True),
|
||||
({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "true", True),
|
||||
({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "soon", False),
|
||||
({"type": ["string", "integer"]}, "soon", True),
|
||||
({"enum": ["celsius", "fahrenheit"]}, "celsius", True),
|
||||
({"enum": ["celsius", "fahrenheit"]}, "kelvin", False),
|
||||
({"type": "string", "const": "on"}, "off", False),
|
||||
(LEVEL, '{"level": 2}', True),
|
||||
(LEVEL, '{"level": "high"}', False),
|
||||
],
|
||||
)
|
||||
def test_argument_values_follow_how_the_parser_reads_them(schema, value, accepted):
|
||||
parameters = {
|
||||
"type": "object",
|
||||
"properties": {"v": schema},
|
||||
"required": ["v"],
|
||||
"$defs": {"level": {"type": "integer"}},
|
||||
}
|
||||
|
||||
assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "strict", "derivable"),
|
||||
[
|
||||
# The parser reads values without a type as text.
|
||||
({"enum": [1, 2]}, True, False),
|
||||
({"$ref": "#/$defs/level"}, True, False),
|
||||
({"$ref": "#/$defs/level"}, False, True),
|
||||
# Raw text cannot hold its closer.
|
||||
({"enum": ["a</arg>b"]}, True, False),
|
||||
({"type": "object", "properties": {"x": {"$ref": "#/x"}}}, True, False),
|
||||
(False, True, False),
|
||||
# A match could write the start of the value's end.
|
||||
({"type": "string", "pattern": "^[a-z<]+$"}, True, False),
|
||||
({"type": "string", "pattern": "^[a-z<]+$"}, False, True),
|
||||
({"type": "string", "pattern": "^a.b$"}, True, False),
|
||||
# XGrammar holds length bounds only in JSON.
|
||||
({"type": "string", "minLength": 3}, True, False),
|
||||
({"type": "string", "format": "ipv6"}, True, False),
|
||||
],
|
||||
)
|
||||
def test_schemas_text_values_cannot_hold_are_not_derived(schema, strict, derivable):
|
||||
parameters = {
|
||||
"type": "object",
|
||||
"properties": {"v": schema},
|
||||
"$defs": {"level": {"type": "integer"}},
|
||||
}
|
||||
|
||||
assert (build(functions=[tool(parameters, strict=strict)]) is not None) is derivable
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("schema", "value", "accepted"),
|
||||
[
|
||||
({"type": "string", "format": "email"}, "a.b@x.co", True),
|
||||
({"type": "string", "format": "email"}, "ab", False),
|
||||
({"type": "string", "format": "date-time"}, "2026-10-06T12:00:00Z", True),
|
||||
({"type": "string", "format": "date-time"}, "2026-13-06T12:00:00Z", False),
|
||||
({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABC", True),
|
||||
({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABCD", False),
|
||||
# Like XGrammar, the format overrides the pattern and the length bounds.
|
||||
(
|
||||
{"type": "string", "format": "uuid", "pattern": "^x$", "maxLength": 1},
|
||||
"123e4567-e89b-12d3-a456-426614174000",
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_text_values_hold_string_formats_and_patterns(schema, value, accepted):
|
||||
parameters = {"type": "object", "properties": {"v": schema}, "required": ["v"]}
|
||||
|
||||
assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted
|
||||
|
||||
|
||||
def test_values_cannot_hold_a_closer():
|
||||
template = deepcopy(TEMPLATE)
|
||||
template["fields"]["tool_calls"]["close"] = ["</call>", "STOP"]
|
||||
text = {"type": "object", "properties": {"v": {"type": "string"}}}
|
||||
strings = {"type": "array", "items": {"type": "string"}}
|
||||
json_text = {"type": "object", "properties": {"v": strings}}
|
||||
|
||||
assert accepts(build(template, [tool(text)]), call(("v", "ab")))
|
||||
assert not accepts(build(template, [tool(text)]), call(("v", "aSTOPb")))
|
||||
assert not accepts(
|
||||
build(template, [tool(None, strict=False)]), call(("v", "aSTOPb"))
|
||||
)
|
||||
assert accepts(build(template, [tool(json_text)]), call(("v", '["ab"]')))
|
||||
assert not accepts(build(template, [tool(json_text)]), call(("v", '["aSTOPb"]')))
|
||||
|
||||
|
||||
def test_extra_arguments_need_additional_properties():
|
||||
parameters = {"type": "object", "properties": {"city": {"type": "string"}}}
|
||||
extra = call(("city", "Paris"), ("zip", "75001"))
|
||||
|
||||
assert not accepts(build(functions=[tool(parameters)]), extra)
|
||||
# Next to declared keys, an extra one could repeat them unchecked.
|
||||
allowed = {**parameters, "additionalProperties": True}
|
||||
assert build(functions=[tool(allowed)]) is None
|
||||
free_form = {"type": "object", "additionalProperties": True}
|
||||
assert accepts(build(functions=[tool(free_form)]), extra)
|
||||
|
||||
|
||||
def test_only_strict_tools_follow_their_schema():
|
||||
off_schema = call(("days", "soon"))
|
||||
parameters = WEATHER["parameters"]
|
||||
|
||||
assert accepts(build(functions=[tool(parameters, strict=False)]), off_schema)
|
||||
assert not accepts(build(functions=[tool(parameters)]), off_schema)
|
||||
assert accepts(build(functions=[tool(None)]), call())
|
||||
assert not accepts(build(functions=[tool(None)]), call(("a", "x")))
|
||||
|
||||
|
||||
def test_annotations_keep_strict_arguments_to_their_schema():
|
||||
parameters = {
|
||||
**WEATHER["parameters"],
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "weather",
|
||||
}
|
||||
tag = build(functions=[tool(parameters)])
|
||||
|
||||
assert accepts(tag, call(("city", "Paris"), ("days", "3")))
|
||||
assert not accepts(tag, call(("city", "Paris"), ("days", "soon")))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("arguments", "accepted", "accepted_sorted"),
|
||||
[
|
||||
((("path", "a"), ("content", "b")), True, True),
|
||||
((("content", "b"), ("path", "a")), False, True),
|
||||
((("path", "a"),), True, True),
|
||||
((("content", "b"),), False, False),
|
||||
((("path", "a"), ("path", "c")), False, False),
|
||||
((("path", "a"), ("mode", "c")), False, False),
|
||||
],
|
||||
)
|
||||
def test_strict_arguments_follow_declared_or_sorted_order(
|
||||
arguments, accepted, accepted_sorted
|
||||
):
|
||||
parameters = {
|
||||
"properties": {"path": {"type": "string"}, "content": {"type": "string"}},
|
||||
"required": ["path"],
|
||||
}
|
||||
tags = [
|
||||
build(functions=[tool(parameters)], sorted_arguments=sorted_arguments)
|
||||
for sorted_arguments in (False, True)
|
||||
]
|
||||
|
||||
assert [accepts(tag, call(*arguments)) for tag in tags] == [
|
||||
accepted,
|
||||
accepted_sorted,
|
||||
]
|
||||
|
||||
|
||||
LOOKAHEAD_TEMPLATE = deepcopy(TEMPLATE)
|
||||
LOOKAHEAD_TEMPLATE["fields"]["tool_calls"]["content_args"]["tag_pattern"] = (
|
||||
r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)\n(?:</arg>|(?=<arg name="))'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "accepted_loose", "accepted_strict"),
|
||||
[
|
||||
('<arg name="city">Paris\n</arg>\n<arg name="days">3\n</arg>', True, True),
|
||||
('<arg name="city">Paris\n<arg name="days">3\n</arg>', True, True),
|
||||
('<arg name="days">3\n<arg name="city">Paris\n</arg>', True, False),
|
||||
('<arg name="city">Paris\n<arg name="city">Lyon\n</arg>', True, False),
|
||||
('<arg name="city">Paris\n<arg name="hours">3\n</arg>', True, False),
|
||||
('<arg name="city">Paris\n<arg name="', False, False),
|
||||
('<arg name="city">Paris\n<arg name="days\n</arg>', False, False),
|
||||
('<arg name="city">a <arg name="b"> c\n</arg>', True, True),
|
||||
],
|
||||
)
|
||||
def test_values_end_where_a_lookahead_end_does(body, accepted_loose, accepted_strict):
|
||||
parameters = WEATHER["parameters"]
|
||||
tags = [
|
||||
build(LOOKAHEAD_TEMPLATE, [tool(parameters, strict=strict)])
|
||||
for strict in (False, True)
|
||||
]
|
||||
text = f'<call name="f">{body}</call>'
|
||||
|
||||
assert [accepts(tag, text) for tag in tags] == [accepted_loose, accepted_strict]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lookahead", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("keys", "accepted"),
|
||||
[
|
||||
("b", True),
|
||||
("ab", True),
|
||||
("bc", True),
|
||||
("bd", True),
|
||||
("be", True),
|
||||
("abde", True),
|
||||
("bcde", True),
|
||||
("", False),
|
||||
("a", False),
|
||||
("acd", False),
|
||||
("bdc", False),
|
||||
("bcc", False),
|
||||
],
|
||||
)
|
||||
def test_last_parameter_follows_every_required_one(keys, accepted, lookahead):
|
||||
parameters = {
|
||||
"properties": {key: {"type": "string"} for key in "abcde"},
|
||||
"required": ["b"],
|
||||
}
|
||||
tag = build(LOOKAHEAD_TEMPLATE, [tool(parameters)])
|
||||
ends = ["\n" if lookahead else "\n</arg>"] * (len(keys) - 1) + ["\n</arg>"]
|
||||
body = "".join(f'<arg name="{k}">x{e}' for k, e in zip(keys, ends))
|
||||
|
||||
assert accepts(tag, f'<call name="f">{body}</call>') is accepted
|
||||
|
||||
|
||||
def test_required_argument_without_schema_is_still_required():
|
||||
parameters = {"type": "object", "properties": {}, "required": ["a"]}
|
||||
tag = build(functions=[tool(parameters)])
|
||||
|
||||
assert accepts(tag, call(("a", "x")))
|
||||
assert not accepts(tag, call())
|
||||
assert build(functions=[tool({**parameters, "required": ["a", 1]})]) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("function", "match"),
|
||||
[
|
||||
(tool({}, name="get weather"), "tool name"),
|
||||
(tool({"properties": {'a"b': {"type": "string"}}}), "parameter name"),
|
||||
],
|
||||
)
|
||||
def test_unencodable_names_are_rejected(function, match):
|
||||
with pytest.raises(ValueError, match=match):
|
||||
build(functions=[function])
|
||||
Reference in New Issue
Block a user