[Frontend] Add the Beam reasoning and tool parsers (#62052)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com>
Co-authored-by: neilkg <33635204+neilkg@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Woosuk Kwon
2026-10-16 14:08:29 -07:00
committed by GitHub
co-authored by Bugen Zhao Yoni Gozlan neilkg Cursor Claude Opus 5.5
parent 9731f30a03
commit 153a17166e
27 changed files with 4464 additions and 92 deletions
+747
View File
@@ -0,0 +1,747 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import pytest
from openai.types.responses import FunctionTool, ToolChoiceFunction
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.config.reasoning import ReasoningConfig
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.entrypoints.openai.responses.protocol import (
ResponsesRequest,
ResponsesResponse,
)
from vllm.exceptions import VLLMValidationError
from vllm.parser.beam import BeamParser, BeamReasoningParser, BeamToolParser
from vllm.parser.parser_manager import ParserManager
from vllm.parser.response_template import ResponseTemplateParser
from vllm.reasoning import ReasoningParserManager
from vllm.tool_parsers import ToolParserManager
# The `response_template` of the Beam checkpoints.
BEAM_RESPONSE_TEMPLATE: dict[str, Any] = {
"defaults": {"role": "assistant"},
"start_anchor_pattern": r"<\|start\|>assistant",
"fields": {
"thinking": {
"open": "<|channel|>analysis<|message|>",
"close": "<|end|><|start|>assistant",
"content": "text",
},
"content": {
"open": "<|channel|>final<|message|>",
"close": ["<|return|>", "<|end|>", "<|call|>"],
"content": "text",
},
"tool_calls": {
"open_pattern": (
r" to=(?P<name>[^\s<]+)"
r"<\|channel\|>analysis <\|constrain\|>xml<\|message\|>"
),
"close": ["<|end|><|start|>assistant", "<|call|>"],
"repeats": True,
"content": "xml-inline",
"content_args": {
"tag_pattern": (
r'<rfl:parameter name="(?P<key>[^"]+)">'
r'\n(?P<value>.*?)\n(?:</rfl:parameter>|(?=<rfl:parameter name="))'
),
"value_parser": {"name": "text", "args": {"strip": False}},
},
"transform": {
"type": "function",
"function": {"name": "{name}", "arguments": "{content}"},
},
},
},
}
VOCAB = {
"<|start|>": 200,
"<|channel|>": 201,
"<|message|>": 202,
"<|end|>": 203,
"<|call|>": 204,
"<|return|>": 205,
"<|constrain|>": 206,
}
TEXT_TOKENS = {100: "assistant", 101: "analysis", 102: "final", 103: "plan"}
PROMPT_TOKEN_IDS = [200, 100]
THOUGHT = "<|channel|>analysis<|message|>plan<|end|><|start|>assistant"
FINAL = "<|channel|>final<|message|>"
NEXT_MESSAGE = "<|end|><|start|>assistant"
def _tool(name, properties, *, strict=None):
return ChatCompletionToolsParam(
type="function",
function={
"name": name,
"parameters": {"type": "object", "properties": properties},
"strict": strict,
},
)
TOOLS = [
_tool("get_weather", {"city": {"type": "string"}}),
_tool("set_count", {"count": {"type": "integer"}}),
]
STRICT_TOOLS = [
_tool("get_weather", {"city": {"type": "string"}}),
_tool("set_count", {"count": {"type": "integer"}}, strict=True),
]
class BeamTokenizer:
def __init__(self, response_template=BEAM_RESPONSE_TEMPLATE):
self.response_template = response_template
self.init_kwargs = {}
self.name_or_path = ""
self.all_special_tokens = list(VOCAB)
self.all_special_ids = list(VOCAB.values())
def decode(self, token_ids, skip_special_tokens=False, **_kwargs):
inverse_vocab = {token_id: token for token, token_id in VOCAB.items()}
return "".join(
TEXT_TOKENS.get(token_id, inverse_vocab.get(token_id, ""))
for token_id in token_ids
)
def get_vocab(self):
return VOCAB
def encode(self, text, add_special_tokens=False):
del add_special_tokens
return list(text.encode())
def request(**overrides):
values = dict(
tool_choice="auto",
include_reasoning=True,
tools=TOOLS,
parallel_tool_calls=True,
)
values.update(overrides)
return SimpleNamespace(**values)
def parameter(name, value):
return f'<rfl:parameter name="{name}">\n{value}\n</rfl:parameter>'
def call(name, body, *, terminator="<|call|>"):
return (
f" to={name}<|channel|>analysis <|constrain|>xml<|message|>{body}{terminator}"
)
def make_parser(prompt_token_ids=PROMPT_TOKEN_IDS, tools=TOOLS, **kwargs):
parser_cls = ParserManager.get_parser(
tool_parser_name="beam",
reasoning_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(),
)
assert parser_cls is not None
parser = parser_cls(BeamTokenizer(), tools, **kwargs)
parser.set_prompt_token_ids(prompt_token_ids)
return parser
def parse(text, prompt_token_ids=PROMPT_TOKEN_IDS):
reasoning, content, calls = make_parser(prompt_token_ids).parse(
text, request(), enable_auto_tools=True
)
return reasoning, content, normalize_calls(calls)
def normalize_calls(calls):
return [(call.name, json.loads(call.arguments)) for call in calls or []]
def test_beam_parsers_are_registered():
assert ToolParserManager.get_tool_parser("beam") is BeamToolParser
assert ReasoningParserManager.get_reasoning_parser("beam") is BeamReasoningParser
def test_parser_manager_selects_beam_parser():
parser_cls = ParserManager.get_parser(
tool_parser_name="beam",
reasoning_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(),
)
assert parser_cls is not None
assert issubclass(parser_cls, BeamParser)
assert parser_cls.reasoning_parser_cls is BeamReasoningParser
assert parser_cls.tool_parser_cls is BeamToolParser
assert parser_cls._enable_auto_tools
@pytest.mark.parametrize("other", ["hermes", "hf"])
def test_beam_parser_rejects_other_parsers(other):
with pytest.raises(TypeError, match="cannot be combined"):
ParserManager.get_parser(
tool_parser_name=other,
reasoning_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(),
)
def test_beam_parser_requires_checkpoint_metadata_at_startup():
with pytest.raises(
TypeError, match="--reasoning-parser beam requires `response_template`"
):
ParserManager.get_parser(
reasoning_parser_name="beam",
tokenizer=BeamTokenizer(response_template=None),
)
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
del template["fields"]["tool_calls"]
with pytest.raises(TypeError, match="--tool-call-parser beam requires"):
ParserManager.get_parser(
tool_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(template),
)
def test_reasoning_and_final_answer():
assert parse(THOUGHT + FINAL + "Sunny.<|return|>") == ("plan", "Sunny.", [])
def test_prefilled_reasoning_continues_from_prompt():
prompt = PROMPT_TOKEN_IDS + [201, 101, 202, 103]
text = " more.<|end|><|start|>assistant" + FINAL + "Done.<|return|>"
assert parse(text, prompt) == (" more.", "Done.", [])
def test_parallel_calls_are_coerced_to_their_schemas():
text = (
THOUGHT
+ call(
"get_weather",
parameter("city", "Paris"),
terminator="<|end|><|start|>assistant",
)
+ call("set_count", parameter("count", "7"))
)
assert parse(text) == (
"plan",
None,
[("get_weather", {"city": "Paris"}), ("set_count", {"count": 7})],
)
def test_call_stop_token_removed_from_the_text_still_ends_the_call():
text = THOUGHT + call("get_weather", parameter("city", "Paris"), terminator="")
assert parse(text) == ("plan", None, [("get_weather", {"city": "Paris"})])
@pytest.mark.parametrize("end", ["<|end|>", "<|end|><|start|>"])
def test_output_cut_before_the_next_message_start_ends_its_region(end):
"""Output that stops after `<|end|>`, such as at `max_tokens`, has ended
its reasoning or call and lacks only the next message's start."""
weather = call("get_weather", parameter("city", "Paris"), terminator=end)
assert parse("<|channel|>analysis<|message|>plan" + end) == ("plan", None, [])
assert parse(THOUGHT + weather) == (
"plan",
None,
[("get_weather", {"city": "Paris"})],
)
THOUGHT_IDS = [201, 101, 202, 103, 203, 200, 100]
FINAL_IDS = [201, 102, 202]
@pytest.mark.parametrize(
("prompt_token_ids", "token_ids", "count"),
[
(PROMPT_TOKEN_IDS, [*THOUGHT_IDS, *FINAL_IDS, 205], 1),
(PROMPT_TOKEN_IDS + [201, 101, 202], [103, 103, 203, 200, 100, *FINAL_IDS], 2),
(PROMPT_TOKEN_IDS, [201, 101, 202, 103, 103, 103], 3),
],
ids=["reasoning_then_answer", "prefilled_reasoning", "cut_off"],
)
def test_reasoning_tokens_are_counted(prompt_token_ids, token_ids, count):
parser = make_parser(prompt_token_ids)
assert parser.count_reasoning_tokens(token_ids) == count
def test_reasoning_tokens_are_counted_as_the_generation_grows():
token_ids = [*THOUGHT_IDS, *FINAL_IDS, 205]
parser = make_parser()
assert [
parser.count_reasoning_tokens(token_ids[:end])
for end in range(1, len(token_ids) + 1)
] == [
make_parser().count_reasoning_tokens(token_ids[:end])
for end in range(1, len(token_ids) + 1)
]
assert parser.count_reasoning_tokens(FINAL_IDS) == 0
def test_call_stop_token_ends_the_answer():
assert parse(FINAL + "done<|call|>") == (None, "done", [])
@pytest.mark.parametrize("chunk_size", [1, 7, 10_000])
def test_streaming_matches_non_streaming(chunk_size):
text = (
THOUGHT
+ call(
"get_weather",
parameter("city", "Paris"),
terminator="<|end|><|start|>assistant",
)
+ call("set_count", parameter("count", "7"))
)
parser = make_parser()
reasoning, calls = [], []
for start in range(0, len(text), chunk_size):
end = min(start + chunk_size, len(text))
delta = parser.parse_delta(
text[start:end], [], request(), finished=end == len(text)
)
if delta is not None:
reasoning.append(delta.reasoning or "")
calls.extend(delta.tool_calls)
assert "".join(reasoning) == "plan"
assert [
(call.function.name, json.loads(call.function.arguments)) for call in calls
] == parse(text)[2]
WEATHER_CALL = call("get_weather", parameter("city", "Paris"))
WEATHER = [("get_weather", {"city": "Paris"})]
def chat_request(tools=STRICT_TOOLS, **body):
return ChatCompletionRequest(
model="beam",
messages=[{"role": "user", "content": "Weather?"}],
tools=tools and [tool.model_dump() for tool in tools],
**body,
)
def named(name):
return {"type": "function", "function": {"name": name}}
def adjust(req, prompt_token_ids=PROMPT_TOKEN_IDS, **kwargs):
make_parser(prompt_token_ids, req.tools, **kwargs).adjust_request(req)
return req
def accepts(req, text):
grammar = Grammar.from_structural_tag(
req.extract_structured_outputs().structural_tag
)
return _is_grammar_accept_string(grammar, text)
@pytest.mark.parametrize(
("tool_choice", "text", "calls"),
[
("required", THOUGHT + WEATHER_CALL, WEATHER),
("required", WEATHER_CALL, None),
(
"required",
THOUGHT
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
+ call("set_count", parameter("count", "7")),
[*WEATHER, ("set_count", {"count": 7})],
),
("required", THOUGHT + call("set_count", parameter("count", "x")), None),
(
"required",
THOUGHT + call("get_weather", parameter("city", "a<|call|>b")),
None,
),
("required", THOUGHT + FINAL + "Paris<|return|>", None),
(named("set_count"), THOUGHT + WEATHER_CALL, None),
(named("get_weather"), THOUGHT + WEATHER_CALL, WEATHER),
(
named("get_weather"),
THOUGHT
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
+ WEATHER_CALL,
None,
),
],
)
def test_forced_tool_calls_follow_the_template_grammar(tool_choice, text, calls):
"""From the message start, the grammar opens with the reasoning, and
accepts only calls the parser reads back."""
req = adjust(chat_request(tool_choice=tool_choice))
assert req._grammar_from_parser
assert accepts(req, text) is (calls is not None)
if calls is not None:
assert parse(text)[2] == calls
def test_reasoning_may_be_skipped_when_thinking_is_disabled():
req = adjust(
chat_request(tool_choice="required"),
chat_template_kwargs={"enable_thinking": False},
)
assert accepts(req, WEATHER_CALL)
assert accepts(req, THOUGHT + WEATHER_CALL)
def test_parallel_tool_calls_false_allows_a_single_call():
req = adjust(chat_request(tool_choice="required", parallel_tool_calls=False))
calls = call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
assert accepts(req, THOUGHT + WEATHER_CALL)
assert not accepts(req, THOUGHT + calls + WEATHER_CALL)
@pytest.mark.parametrize("strict", [False, True])
@pytest.mark.parametrize(
("separator", "accepted"), [("", True), ("\n", True), ("\n\n", False), (" ", False)]
)
def test_parameters_are_separated_by_at_most_a_newline(strict, separator, accepted):
"""A model that skips a parameter's opener must not write whitespace until
`max_tokens`."""
write = _tool(
"write",
{"path": {"type": "string"}, "content": {"type": "string"}},
strict=strict,
)
req = adjust(chat_request([write], tool_choice="required"))
body = parameter("path", "a.py") + separator + parameter("content", "x")
assert accepts(req, THOUGHT + call("write", body)) is accepted
assert accepts(req, THOUGHT + call("write", separator + body)) is accepted
def test_forced_calls_in_prefilled_reasoning_start_after_it():
req = adjust(
chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 101, 202, 103]
)
assert not req._grammar_from_parser
assert accepts(req, WEATHER_CALL)
def test_forced_calls_cannot_follow_a_prefilled_answer():
with pytest.raises(VLLMValidationError, match="required tool choice"):
adjust(chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 102, 202])
def test_prefilled_answer_can_only_be_finished():
req = adjust(chat_request(TOOLS), PROMPT_TOKEN_IDS + [201, 102, 202])
assert accepts(req, "Sunny.<|return|>")
assert not accepts(req, "Sunny." + NEXT_MESSAGE + WEATHER_CALL)
def test_forced_call_with_unencodable_tool_name_is_rejected():
tools = [_tool("get weather", {"city": {"type": "string"}})]
with pytest.raises(VLLMValidationError, match="tool name"):
adjust(chat_request(tools, tool_choice="required"))
def test_string_formats_and_patterns_are_held_in_raw_text():
"""Like XGrammar, a strict string keeps its format or pattern, unless a
match could end the value early."""
contact = _tool(
"set_contact",
{
"email": {"type": "string", "format": "email"},
"code": {"type": "string", "pattern": "^[A-Z]{3}$"},
"n": {"type": "integer"},
},
strict=True,
)
contact.function.parameters["required"] = ["email", "code", "n"]
req = adjust(chat_request([contact], tool_choice="required"))
def contact_call(email, code, *rest):
body = parameter("email", email) + parameter("code", code) + "".join(rest)
return THOUGHT + call("set_contact", body)
text = contact_call("a.b@x.co", "ABC", parameter("n", "1"))
calls = make_parser(tools=[contact]).parse(text, req, enable_auto_tools=True)[2]
assert accepts(req, text)
assert normalize_calls(calls) == [
("set_contact", {"email": "a.b@x.co", "code": "ABC", "n": 1})
]
assert not accepts(req, contact_call("ab", "ABC", parameter("n", "1")))
assert not accepts(req, contact_call("a.b@x.co", "ABCD", parameter("n", "1")))
assert not accepts(req, contact_call("a.b@x.co", "ABC"))
# Line breaks could end the value.
dotted = _tool(
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
)
with pytest.raises(VLLMValidationError, match="required tool choice"):
adjust(chat_request([dotted], tool_choice="required"))
@pytest.mark.parametrize(
("text", "accepted"),
[
(THOUGHT + WEATHER_CALL, True),
(THOUGHT + FINAL + "Sunny.<|return|>", True),
(THOUGHT + FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
(THOUGHT + call("set_count", parameter("count", "x")), False),
],
)
def test_strict_tools_constrain_automatic_tool_choice(text, accepted):
assert accepts(adjust(chat_request()), text) is accepted
@pytest.mark.parametrize(
("text", "accepted"),
[
(THOUGHT + FINAL + "Sunny.<|return|>", True),
(FINAL + "Sunny.<|return|>", False),
(WEATHER_CALL, False),
(THOUGHT + WEATHER_CALL, False),
(FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
],
)
def test_tool_choice_none_allows_only_an_answer(text, accepted):
req = adjust(chat_request(TOOLS, tool_choice="none"))
assert req._grammar_from_parser
assert accepts(req, text) is accepted
@pytest.mark.parametrize(("tools", "tool_choice"), [(TOOLS, "auto"), (None, None)])
def test_always_constrained_output_follows_the_template_framing(tools, tool_choice):
req = adjust(chat_request(tools, tool_choice=tool_choice))
count_call = THOUGHT + call("set_count", parameter("count", "3"))
assert accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
assert not accepts(req, FINAL + "Sunny.<|return|>")
assert not accepts(req, THOUGHT.replace("plan", "") + FINAL + "Sunny.<|return|>")
assert not accepts(req, THOUGHT + "Sunny.")
assert accepts(req, count_call) is (tool_choice == "auto")
@pytest.mark.parametrize("response_format", [None, {"type": "json_object"}])
@pytest.mark.parametrize("strict", [None, True])
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
def test_automatic_choice_lets_calls_name_undeclared_tools(
strict, tool_choice, response_format
):
"""Unless a forced choice limits calls to the declared tools. The parser
passes such calls on, so the harness can tell the model the tool does not
exist."""
bash = _tool("bash", {"command": {"type": "string"}}, strict=strict)
req = adjust(
chat_request([bash], tool_choice=tool_choice, response_format=response_format)
)
undeclared = THOUGHT + call("read_file", parameter("path", "a.py"))
parser = make_parser(tools=[bash])
assert accepts(req, THOUGHT + call("bash", parameter("command", "ls")))
assert accepts(req, undeclared) is (tool_choice == "auto")
calls = parser.parse(undeclared, req, enable_auto_tools=True)[2]
assert normalize_calls(calls) == [("read_file", {"path": "a.py"})]
@pytest.mark.parametrize(
("name", "body", "accepted"),
[
("set_count", parameter("count", "1"), True),
("set_count", parameter("count", "x"), False),
("get_weather", parameter("town", "Paris"), False),
# The grammar cannot enforce `minProperties`.
("find", parameter("x", "1") + parameter("x", "y"), True),
("search_files", parameter("x", "1") + parameter("x", "y"), True),
("set_coun", parameter("x", "1") + parameter("x", "y"), True),
("set_counts", parameter("x", "1") + parameter("x", "y"), True),
],
)
def test_automatic_calls_enforce_declared_arguments_beside_any_other_name(
name, body, accepted
):
find = _tool("find", {"q": {"type": "string"}})
find.function.parameters["minProperties"] = 1
req = adjust(chat_request([*TOOLS, find]))
assert accepts(req, THOUGHT + call(name, body)) is accepted
def test_value_ends_before_an_opener_on_the_next_line():
write = _tool("write", {"path": {"type": "string"}, "content": {"type": "string"}})
req = adjust(chat_request([write], tool_choice="required"))
body = '<rfl:parameter name="content">\nprint(1)\n\n' + parameter("path", "a.py")
text = THOUGHT + call("write", body)
assert accepts(req, text)
calls = make_parser(tools=[write]).parse(text, req, enable_auto_tools=True)[2]
assert normalize_calls(calls) == [
("write", {"content": "print(1)\n", "path": "a.py"})
]
def test_always_constrained_output_can_be_disabled(monkeypatch):
monkeypatch.setenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "0")
assert adjust(chat_request(TOOLS)).structured_outputs is None
def test_strict_tools_the_template_cannot_hold_leave_automatic_choice_unconstrained():
dotted = _tool(
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
)
assert adjust(chat_request([dotted])).structured_outputs is None
def test_generic_parser_does_not_derive_the_template_grammar():
req = chat_request(TOOLS, tool_choice="required")
parser = ResponseTemplateParser(BeamTokenizer(), req.tools, enable_auto_tools=True)
with pytest.raises(VLLMValidationError, match="required tool choice"):
parser.adjust_request(req)
@pytest.mark.parametrize("tools", [None, TOOLS])
def test_structured_output_is_framed_as_the_answer(tools):
schema = {"type": "object", "properties": {"a": {"type": "integer"}}}
req = adjust(
chat_request(
tools,
response_format={
"type": "json_schema",
"json_schema": {"name": "answer", "schema": schema},
},
)
)
assert req.response_format is None
assert accepts(req, THOUGHT + FINAL + '{"a": 1}<|return|>')
assert not accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
assert accepts(req, THOUGHT + WEATHER_CALL) is (tools is not None)
def test_responses_named_tool_choice_follows_the_template_grammar():
req = adjust(
ResponsesRequest(
input="Weather?",
tools=[
FunctionTool(
type="function",
name="get_weather",
parameters={
"type": "object",
"properties": {"city": {"type": "string"}},
},
strict=True,
)
],
tool_choice=ToolChoiceFunction(type="function", name="get_weather"),
text={"format": {"type": "json_object"}},
)
)
assert req._grammar_from_parser
assert req.text.format.type == "json_object"
assert accepts(req, THOUGHT + WEATHER_CALL)
assert not accepts(req, FINAL + "{}<|return|>")
@pytest.mark.parametrize("status", ["in_progress", "completed"])
@pytest.mark.parametrize("inside_reasoning", [False, True])
def test_responses_preserves_text_config_with_template_grammar(
status, inside_reasoning
):
text = {
"format": {
"type": "json_schema",
"name": "answer",
"strict": True,
"schema": {
"type": "object",
"properties": {"a": {"type": "integer"}},
"required": ["a"],
"additionalProperties": False,
},
},
"verbosity": "low",
}
req = ResponsesRequest(input="Answer", text=text)
expected_text = req.text.model_dump(by_alias=True)
prompt = PROMPT_TOKEN_IDS + ([201, 101, 202] if inside_reasoning else [])
adjust(req, prompt_token_ids=prompt)
params = req.to_sampling_params(default_max_tokens=128)
grammar = Grammar.from_structural_tag(params.structured_outputs.structural_tag)
prefix = "" if inside_reasoning else THOUGHT
assert _is_grammar_accept_string(grammar, prefix + FINAL + '{"a": 1}<|return|>')
assert not _is_grammar_accept_string(grammar, prefix + FINAL + "Sunny.<|return|>")
response = ResponsesResponse.from_request(req, params, "beam", 0, [], status)
assert response.model_dump(by_alias=True)["text"] == expected_text
@pytest.mark.parametrize("reverse_alternatives", [False, True])
def test_reasoning_budget_initializes_from_checkpoint_pattern(
monkeypatch, reverse_alternatives
):
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
alternatives = [
r"^<\|channel\|>analysis<\|message\|>",
r"<\|start\|>assistant<\|channel\|>analysis<\|message\|>",
]
if reverse_alternatives:
alternatives.reverse()
template["fields"]["thinking"] = {
"open_pattern": "(?:" + "|".join(alternatives) + ")",
"close": "<|end|>",
"content": "text",
}
tokenizer = BeamTokenizer(template)
monkeypatch.setattr(
"vllm.config.reasoning.cached_tokenizer_from_config", lambda **_: tokenizer
)
config = ReasoningConfig(reasoning_parser="beam")
config.initialize_token_ids(SimpleNamespace())
assert config.enabled
# The marker must work both after the prompt's assistant header and in
# subsequent complete assistant messages, regardless of regex branch order.
assert config.reasoning_start_token_ids == tokenizer.encode(
"<|channel|>analysis<|message|>"
)
assert config.reasoning_end_token_ids == tokenizer.encode("<|end|>")
assert config.natural_reasoning_end_token_ids == tokenizer.encode("<|end|>")
@pytest.mark.parametrize("pattern", [r"analysis.*", r"(?:analysis|thinking)"])
def test_reasoning_budget_does_not_guess_a_marker_for_other_patterns(pattern):
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
thinking = template["fields"]["thinking"]
del thinking["open"]
thinking["open_pattern"] = pattern
assert BeamReasoningParser(BeamTokenizer(template)).reasoning_start_str is None
@@ -0,0 +1,492 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from copy import deepcopy
from typing import Any
import pytest
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.parser.chat_parsing.response_templates import load_response_template
from vllm.parser.response_template_grammar import (
answer_rest_structural_tag,
tool_call_structural_tag,
)
TEMPLATE: dict[str, Any] = {
"start_anchor": "<assistant>",
"fields": {
"thinking": {"open": "<think>", "close": "</think>", "content": "text"},
"content": {"content": "text"},
"tool_calls": {
"open_pattern": r'<call name="(?P<name>[\w.-]+)">',
"close": "</call>",
"repeats": True,
"content": "xml-inline",
"content_args": {
"tag_pattern": r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)</arg>'
},
"transform": {
"type": "function",
"function": {"name": "{name}", "arguments": "{content}"},
},
},
},
}
ANSWER_TEMPLATE = deepcopy(TEMPLATE)
ANSWER_TEMPLATE["fields"]["content"] = {
"open": "<answer>",
"close": "</answer>",
"content": "text",
}
def tool(parameters, name="f", strict=True) -> dict[str, Any]:
return {"name": name, "parameters": parameters, "strict": strict}
WEATHER = tool(
{
"type": "object",
"properties": {"city": {"type": "string"}, "days": {"type": "integer"}},
"required": ["city"],
},
name="get.weather",
)
PING = {"name": "ping"}
CALL = '<call name="get.weather"><arg name="city">Paris</arg></call>'
def build(
template=TEMPLATE,
functions=(WEATHER, PING),
*,
parallel=True,
thinking=True,
answer=False,
answer_format=None,
sorted_arguments=False,
):
fields = load_response_template(template).fields
return tool_call_structural_tag(
fields["tool_calls"],
list(functions),
parallel_tool_calls=parallel,
thinking=fields["thinking"] if thinking else None,
answer=fields["content"] if answer else None,
answer_format=answer_format,
sorted_arguments=sorted_arguments,
)
def accepts(tag, text):
return _is_grammar_accept_string(Grammar.from_structural_tag(json.dumps(tag)), text)
def call(*arguments):
body = "".join(f'<arg name="{key}">{value}</arg>' for key, value in arguments)
return f'<call name="f">{body}</call>'
@pytest.mark.parametrize(
("text", "accepted"),
[
("<think>plan</think>" + CALL, True),
(CALL, True),
(CALL.removesuffix("</call>"), True),
(
'<call name="get.weather"><arg name="city">Paris</arg>\n'
'<arg name="days">3</arg></call>',
True,
),
(
'<call name="get.weather"><arg name="city">Paris</arg>'
'<arg name="days">soon</arg></call>',
False,
),
('<call name="get.weather"><arg name="days">3</arg></call>', False),
(CALL + '<call name="ping"><arg name="any">x</arg></call>', True),
('<call name="other"></call>', False),
("<think>plan</think>The answer.", False),
],
)
def test_forced_calls_follow_the_template_format(text, accepted):
assert accepts(build(), text) is accepted
def test_single_call_when_parallel_calls_are_disabled():
tag = build(parallel=False)
assert accepts(tag, CALL)
assert not accepts(tag, CALL + CALL)
def test_thinking_is_rejected_when_not_requested():
assert not accepts(build(thinking=False), "<think>plan</think>" + CALL)
@pytest.mark.parametrize(
("text", "accepted"),
[
("<think>plan</think><answer>Sunny.", True),
("<answer>Sunny.</answer>", True),
("<think>plan</think>" + CALL, True),
("<answer>Sunny.</answer>" + CALL, False),
('<call name="get.weather"><arg name="days">soon</arg></call>', False),
],
)
def test_answer_field_lets_the_model_reply_instead(text, accepted):
assert accepts(build(ANSWER_TEMPLATE, answer=True), text) is accepted
@pytest.mark.parametrize("required", [False, True])
@pytest.mark.parametrize(
("text", "optional_only"),
[
("<think>plan</think>" + CALL, False),
("<think>\nplan</think><answer>Sunny.", False),
("<think> \n</think><answer>Sunny.", False),
(CALL, True),
("<answer>Sunny.", True),
("<think></think><answer>Sunny.", True),
("<think><</think><answer>Sunny.", True),
],
)
def test_required_reasoning_opens_the_message_with_nonempty_reasoning(
required, text, optional_only
):
fields = load_response_template(ANSWER_TEMPLATE).fields
tag = tool_call_structural_tag(
fields["tool_calls"],
[WEATHER],
parallel_tool_calls=True,
thinking=fields["thinking"],
answer=fields["content"],
message_start=["<assistant>"],
requires_reasoning=required,
)
assert accepts(tag, text) is (not required or not optional_only)
def test_answer_rest_finishes_the_open_answer():
answer = load_response_template(ANSWER_TEMPLATE).fields["content"]
tag = answer_rest_structural_tag(answer)
assert accepts(tag, "Sunny.</answer>")
assert accepts(tag, "Sunny.")
assert not accepts(tag, "Sunny.</answer>" + CALL)
def test_answer_without_literal_opener_is_not_derived():
assert build(answer=True) is None
def test_answer_holds_the_answer_format():
schema = {"type": "json_schema", "json_schema": {"type": "object"}}
with_tools = build(ANSWER_TEMPLATE, answer=True, answer_format=schema)
answer_only = build(ANSWER_TEMPLATE, (), answer=True, answer_format=schema)
for tag in (with_tools, answer_only):
assert accepts(tag, '<think>plan</think><answer>{"a": 1}</answer>')
assert not accepts(tag, "<answer>Sunny.</answer>")
assert accepts(with_tools, CALL)
assert not accepts(answer_only, CALL)
def test_json_arguments_follow_the_schema():
template = deepcopy(TEMPLATE)
template["fields"]["tool_calls"].update(content="json", content_args={})
tag = build(template, [WEATHER])
assert accepts(tag, '<call name="get.weather">{"city": "Paris"}</call>')
assert not accepts(tag, '<call name="get.weather">{"days": 3}</call>')
END_X = {"tag_pattern": r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)X'}
@pytest.mark.parametrize(
"change",
[
{"content": "json", "content_args": {"unquoted_keys": True}},
{
"transform": {
"type": "function",
"function": {"name": "fixed", "arguments": "{content}"},
}
},
{"content": "kv-lines", "content_args": {}},
# Framing in which argument text can spell a closer, where the parser
# would end the call.
{"close": ["}X", "</call>"], "content_args": END_X},
{
"close": ["}X", "</call>"],
"content_args": {**END_X, "value_parser": {"name": "json"}},
},
{"close": ["7X", "</call>"], "content_args": END_X},
{"close": [">X", "</call>"]},
{"close": ["></call>", "</call>"]},
{"close": ["><a", "</call>"]},
{"close": ["}</call>", "</call>"], "content": "json", "content_args": {}},
],
)
def test_underivable_tool_formats_are_not_derived(change):
template = deepcopy(TEMPLATE)
template["fields"]["tool_calls"].update(change)
assert build(template) is None
def test_non_literal_thinking_delimiters_are_not_derived():
template = deepcopy(TEMPLATE)
del template["fields"]["thinking"]["open"]
template["fields"]["thinking"]["open_pattern"] = r"<think(?: mode=\w+)?>"
assert build(template) is None
assert build(template, thinking=False) is not None
LEVEL = {"type": "object", "properties": {"level": {"$ref": "#/$defs/level"}}}
@pytest.mark.parametrize(
("schema", "value", "accepted"),
[
({"type": "integer", "nullable": True}, "null", True),
({"type": "integer", "nullable": True}, "soon", False),
({"type": ["integer", "null"]}, "null", True),
({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "true", True),
({"anyOf": [{"type": "integer"}, {"type": "boolean"}]}, "soon", False),
({"type": ["string", "integer"]}, "soon", True),
({"enum": ["celsius", "fahrenheit"]}, "celsius", True),
({"enum": ["celsius", "fahrenheit"]}, "kelvin", False),
({"type": "string", "const": "on"}, "off", False),
(LEVEL, '{"level": 2}', True),
(LEVEL, '{"level": "high"}', False),
],
)
def test_argument_values_follow_how_the_parser_reads_them(schema, value, accepted):
parameters = {
"type": "object",
"properties": {"v": schema},
"required": ["v"],
"$defs": {"level": {"type": "integer"}},
}
assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted
@pytest.mark.parametrize(
("schema", "strict", "derivable"),
[
# The parser reads values without a type as text.
({"enum": [1, 2]}, True, False),
({"$ref": "#/$defs/level"}, True, False),
({"$ref": "#/$defs/level"}, False, True),
# Raw text cannot hold its closer.
({"enum": ["a</arg>b"]}, True, False),
({"type": "object", "properties": {"x": {"$ref": "#/x"}}}, True, False),
(False, True, False),
# A match could write the start of the value's end.
({"type": "string", "pattern": "^[a-z<]+$"}, True, False),
({"type": "string", "pattern": "^[a-z<]+$"}, False, True),
({"type": "string", "pattern": "^a.b$"}, True, False),
# XGrammar holds length bounds only in JSON.
({"type": "string", "minLength": 3}, True, False),
({"type": "string", "format": "ipv6"}, True, False),
],
)
def test_schemas_text_values_cannot_hold_are_not_derived(schema, strict, derivable):
parameters = {
"type": "object",
"properties": {"v": schema},
"$defs": {"level": {"type": "integer"}},
}
assert (build(functions=[tool(parameters, strict=strict)]) is not None) is derivable
@pytest.mark.parametrize(
("schema", "value", "accepted"),
[
({"type": "string", "format": "email"}, "a.b@x.co", True),
({"type": "string", "format": "email"}, "ab", False),
({"type": "string", "format": "date-time"}, "2026-10-06T12:00:00Z", True),
({"type": "string", "format": "date-time"}, "2026-13-06T12:00:00Z", False),
({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABC", True),
({"type": "string", "pattern": "^[A-Z]{3}$"}, "ABCD", False),
# Like XGrammar, the format overrides the pattern and the length bounds.
(
{"type": "string", "format": "uuid", "pattern": "^x$", "maxLength": 1},
"123e4567-e89b-12d3-a456-426614174000",
True,
),
],
)
def test_text_values_hold_string_formats_and_patterns(schema, value, accepted):
parameters = {"type": "object", "properties": {"v": schema}, "required": ["v"]}
assert accepts(build(functions=[tool(parameters)]), call(("v", value))) is accepted
def test_values_cannot_hold_a_closer():
template = deepcopy(TEMPLATE)
template["fields"]["tool_calls"]["close"] = ["</call>", "STOP"]
text = {"type": "object", "properties": {"v": {"type": "string"}}}
strings = {"type": "array", "items": {"type": "string"}}
json_text = {"type": "object", "properties": {"v": strings}}
assert accepts(build(template, [tool(text)]), call(("v", "ab")))
assert not accepts(build(template, [tool(text)]), call(("v", "aSTOPb")))
assert not accepts(
build(template, [tool(None, strict=False)]), call(("v", "aSTOPb"))
)
assert accepts(build(template, [tool(json_text)]), call(("v", '["ab"]')))
assert not accepts(build(template, [tool(json_text)]), call(("v", '["aSTOPb"]')))
def test_extra_arguments_need_additional_properties():
parameters = {"type": "object", "properties": {"city": {"type": "string"}}}
extra = call(("city", "Paris"), ("zip", "75001"))
assert not accepts(build(functions=[tool(parameters)]), extra)
# Next to declared keys, an extra one could repeat them unchecked.
allowed = {**parameters, "additionalProperties": True}
assert build(functions=[tool(allowed)]) is None
free_form = {"type": "object", "additionalProperties": True}
assert accepts(build(functions=[tool(free_form)]), extra)
def test_only_strict_tools_follow_their_schema():
off_schema = call(("days", "soon"))
parameters = WEATHER["parameters"]
assert accepts(build(functions=[tool(parameters, strict=False)]), off_schema)
assert not accepts(build(functions=[tool(parameters)]), off_schema)
assert accepts(build(functions=[tool(None)]), call())
assert not accepts(build(functions=[tool(None)]), call(("a", "x")))
def test_annotations_keep_strict_arguments_to_their_schema():
parameters = {
**WEATHER["parameters"],
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "weather",
}
tag = build(functions=[tool(parameters)])
assert accepts(tag, call(("city", "Paris"), ("days", "3")))
assert not accepts(tag, call(("city", "Paris"), ("days", "soon")))
@pytest.mark.parametrize(
("arguments", "accepted", "accepted_sorted"),
[
((("path", "a"), ("content", "b")), True, True),
((("content", "b"), ("path", "a")), False, True),
((("path", "a"),), True, True),
((("content", "b"),), False, False),
((("path", "a"), ("path", "c")), False, False),
((("path", "a"), ("mode", "c")), False, False),
],
)
def test_strict_arguments_follow_declared_or_sorted_order(
arguments, accepted, accepted_sorted
):
parameters = {
"properties": {"path": {"type": "string"}, "content": {"type": "string"}},
"required": ["path"],
}
tags = [
build(functions=[tool(parameters)], sorted_arguments=sorted_arguments)
for sorted_arguments in (False, True)
]
assert [accepts(tag, call(*arguments)) for tag in tags] == [
accepted,
accepted_sorted,
]
LOOKAHEAD_TEMPLATE = deepcopy(TEMPLATE)
LOOKAHEAD_TEMPLATE["fields"]["tool_calls"]["content_args"]["tag_pattern"] = (
r'<arg name="(?P<key>[^"]+)">(?P<value>.*?)\n(?:</arg>|(?=<arg name="))'
)
@pytest.mark.parametrize(
("body", "accepted_loose", "accepted_strict"),
[
('<arg name="city">Paris\n</arg>\n<arg name="days">3\n</arg>', True, True),
('<arg name="city">Paris\n<arg name="days">3\n</arg>', True, True),
('<arg name="days">3\n<arg name="city">Paris\n</arg>', True, False),
('<arg name="city">Paris\n<arg name="city">Lyon\n</arg>', True, False),
('<arg name="city">Paris\n<arg name="hours">3\n</arg>', True, False),
('<arg name="city">Paris\n<arg name="', False, False),
('<arg name="city">Paris\n<arg name="days\n</arg>', False, False),
('<arg name="city">a <arg name="b"> c\n</arg>', True, True),
],
)
def test_values_end_where_a_lookahead_end_does(body, accepted_loose, accepted_strict):
parameters = WEATHER["parameters"]
tags = [
build(LOOKAHEAD_TEMPLATE, [tool(parameters, strict=strict)])
for strict in (False, True)
]
text = f'<call name="f">{body}</call>'
assert [accepts(tag, text) for tag in tags] == [accepted_loose, accepted_strict]
@pytest.mark.parametrize("lookahead", [False, True])
@pytest.mark.parametrize(
("keys", "accepted"),
[
("b", True),
("ab", True),
("bc", True),
("bd", True),
("be", True),
("abde", True),
("bcde", True),
("", False),
("a", False),
("acd", False),
("bdc", False),
("bcc", False),
],
)
def test_last_parameter_follows_every_required_one(keys, accepted, lookahead):
parameters = {
"properties": {key: {"type": "string"} for key in "abcde"},
"required": ["b"],
}
tag = build(LOOKAHEAD_TEMPLATE, [tool(parameters)])
ends = ["\n" if lookahead else "\n</arg>"] * (len(keys) - 1) + ["\n</arg>"]
body = "".join(f'<arg name="{k}">x{e}' for k, e in zip(keys, ends))
assert accepts(tag, f'<call name="f">{body}</call>') is accepted
def test_required_argument_without_schema_is_still_required():
parameters = {"type": "object", "properties": {}, "required": ["a"]}
tag = build(functions=[tool(parameters)])
assert accepts(tag, call(("a", "x")))
assert not accepts(tag, call())
assert build(functions=[tool({**parameters, "required": ["a", 1]})]) is None
@pytest.mark.parametrize(
("function", "match"),
[
(tool({}, name="get weather"), "tool name"),
(tool({"properties": {'a"b': {"type": "string"}}}), "parameter name"),
],
)
def test_unencodable_names_are_rejected(function, match):
with pytest.raises(ValueError, match=match):
build(functions=[function])