[Rust Frontend] Replay roundtrip output grammars through XGrammar (#59143)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-09-30 07:29:01 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent fcd317cf33
commit 87228cff01
17 changed files with 4484 additions and 90 deletions
+1
View File
@@ -46,6 +46,7 @@ clap.workspace = true
expect-test.workspace = true
paste.workspace = true
rmp-serde.workspace = true
serde_json = { workspace = true, features = ["raw_value"] }
serial_test.workspace = true
tempfile.workspace = true
tokio.workspace = true
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env -S uv run --script
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# /// script
# requires-python = ">=3.11"
# dependencies = [
# "tiktoken>=0.6.0",
# # xgrammar imports torch unconditionally; only the CPU build is needed.
# "torch",
# "transformers>=5.10.4",
# # Keep in sync with requirements/common.txt.
# "xgrammar==0.2.7",
# ]
#
# [[tool.uv.index]]
# name = "pytorch-cpu"
# url = "https://download.pytorch.org/whl/cpu"
# explicit = true
#
# [tool.uv.sources]
# torch = { index = "pytorch-cpu" }
# ///
"""Replay the grammar cases exported by the roundtrip tests through XGrammar.
Each file holds one model's cases, keyed by tool-choice variant. Every case is
compiled against the model tokenizer the same way as
`vllm/v1/structured_output/backend_xgrammar.py`. Token-zero cases are then
replayed from the first generated token: each token must be allowed by the
bitmask and accepted by the matcher, and the final stop token must terminate it.
"""
import json
import sys
from functools import cache
from pathlib import Path
import xgrammar as xgr
from transformers import AutoTokenizer
CASES_DIR = Path(__file__).parent / "grammar_replay"
@cache
def load_tokenizer(model_id: str, vocab_size: int):
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
tokenizer_info = xgr.TokenizerInfo.from_huggingface(
tokenizer, vocab_size=vocab_size
)
return tokenizer, xgr.GrammarCompiler(tokenizer_info)
def replay(model: dict, case: dict) -> str:
"""Check one case of a model and return a summary of what was checked."""
tokenizer, compiler = load_tokenizer(model["model_id"], model["vocab_size"])
generation = case["generation_token_ids"]
# The case token IDs come from the Rust tokenizer; the runner must load the
# same vocabulary for the grammar to mean the same thing.
encoded = tokenizer.encode(case["completion"], add_special_tokens=False)
if encoded != generation[:-1]:
raise AssertionError(
"tokenizer mismatch: the completion encodes to "
f"{encoded}, but the case records {generation[:-1]}"
)
grammar = case["grammar"]
structural_tag = {"type": "structural_tag", "format": grammar["format"]}
compiled = compiler.compile_structural_tag(json.dumps(structural_tag))
if grammar["coverage"] != "from_token_zero":
return f"compiled only ({grammar['coverage']} is gated by the engine)"
matcher = xgr.GrammarMatcher(
compiled, override_stop_tokens=model["all_stop_token_ids"]
)
bitmask = xgr.allocate_token_bitmask(1, model["vocab_size"])
for index, token_id in enumerate(generation):
matcher.fill_next_token_bitmask(bitmask)
allowed = (bitmask[0, token_id // 32].item() >> (token_id % 32)) & 1
if not allowed or not matcher.accept_token(token_id):
prefix = tokenizer.decode(generation[:index])
raise AssertionError(
f"token {index} ({token_id}, {tokenizer.decode([token_id])!r}) "
f"rejected after {prefix!r}"
)
if not matcher.is_terminated():
raise AssertionError("matcher not terminated after the stop token")
return f"replayed {len(generation)} tokens from token zero"
def main() -> int:
paths = sorted(CASES_DIR.glob("*.json"))
if not paths:
print(f"no grammar replay cases in {CASES_DIR}", file=sys.stderr)
return 1
failures = 0
for path in paths:
model = json.loads(path.read_text())
for name, case in model["cases"].items():
try:
summary = replay(model, case)
except Exception as error:
failures += 1
coverage = case["grammar"]["coverage"]
print(f"FAIL {path.name} {name} [{coverage}]: {error}", file=sys.stderr)
else:
print(f"PASS {path.name} {name}: {summary}")
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main())
+24
View File
@@ -0,0 +1,24 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Replays the grammar cases exported by the roundtrip tests through the real
//! XGrammar, so that parser-built output grammars are checked against the
//! engine's compiler and matcher.
use std::path::PathBuf;
use std::process::Command;
#[test]
fn xgrammar_accepts_roundtrip_generations() {
let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay.py");
let output = Command::new(&script)
.output()
.unwrap_or_else(|error| panic!("failed to execute {:?}: {error}", script));
assert!(
output.status.success(),
"grammar replay failed: status={:?}\nstdout:\n{}\nstderr:\n{}",
output.status.code(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr),
);
}
@@ -0,0 +1,274 @@
{
"model_id": "deepseek-ai/DeepSeek-V3.2-Exp",
"vocab_size": 129280,
"all_stop_token_ids": [1],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "triggered_tags",
"triggers": [
"<|DSML|function_calls>"
],
"tags": [
{
"type": "tag",
"begin": "<|DSML|function_calls>\n",
"content": {
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|DSML|invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "deepseek_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML|invoke>\n"
},
{
"type": "tag",
"begin": "<|DSML|invoke name=\"add\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "deepseek_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML|invoke>\n"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
"end": "</|DSML|function_calls>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>I will call the tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai</|DSML|parameter>\n</|DSML|invoke>\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0</|DSML|parameter>\n<|DSML|parameter name=\"x\" string=\"false\">2</|DSML|parameter>\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</|DSML|parameter>\n</|DSML|invoke>\n</|DSML|function_calls>",
"generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 43, 759, 1926, 270, 6704, 339, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "\n\n<|DSML|function_calls>\n"
},
{
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|DSML|invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "deepseek_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML|invoke>\n"
},
{
"type": "tag",
"begin": "<|DSML|invoke name=\"add\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "deepseek_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML|invoke>\n"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
{
"type": "const_string",
"value": "</|DSML|function_calls>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai</|DSML|parameter>\n</|DSML|invoke>\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0</|DSML|parameter>\n<|DSML|parameter name=\"x\" string=\"false\">2</|DSML|parameter>\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</|DSML|parameter>\n</|DSML|invoke>\n</|DSML|function_calls>",
"generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "\n\n<|DSML|function_calls>\n"
},
{
"type": "tag",
"begin": "<|DSML|invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "deepseek_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML|invoke>\n"
},
{
"type": "const_string",
"value": "</|DSML|function_calls>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai</|DSML|parameter>\n</|DSML|invoke>\n</|DSML|function_calls>",
"generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1]
}
}
}
@@ -0,0 +1,274 @@
{
"model_id": "deepseek-ai/DeepSeek-V4-Flash",
"vocab_size": 129280,
"all_stop_token_ids": [1],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "triggered_tags",
"triggers": [
"<|DSML|tool_calls>"
],
"tags": [
{
"type": "tag",
"begin": "<|DSML|tool_calls>\n",
"content": {
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|DSML|invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "deepseek_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML|invoke>\n"
},
{
"type": "tag",
"begin": "<|DSML|invoke name=\"add\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "deepseek_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML|invoke>\n"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
"end": "</|DSML|tool_calls>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>I will call the tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai</|DSML|parameter>\n</|DSML|invoke>\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0</|DSML|parameter>\n<|DSML|parameter name=\"x\" string=\"false\">2</|DSML|parameter>\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</|DSML|parameter>\n</|DSML|invoke>\n</|DSML|tool_calls>",
"generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "\n\n<|DSML|tool_calls>\n"
},
{
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|DSML|invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "deepseek_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML|invoke>\n"
},
{
"type": "tag",
"begin": "<|DSML|invoke name=\"add\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "deepseek_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML|invoke>\n"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
{
"type": "const_string",
"value": "</|DSML|tool_calls>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai</|DSML|parameter>\n</|DSML|invoke>\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0</|DSML|parameter>\n<|DSML|parameter name=\"x\" string=\"false\">2</|DSML|parameter>\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</|DSML|parameter>\n</|DSML|invoke>\n</|DSML|tool_calls>",
"generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "\n\n<|DSML|tool_calls>\n"
},
{
"type": "tag",
"begin": "<|DSML|invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "deepseek_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML|invoke>\n"
},
{
"type": "const_string",
"value": "</|DSML|tool_calls>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai</|DSML|parameter>\n</|DSML|invoke>\n</|DSML|tool_calls>",
"generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1]
}
}
}
@@ -0,0 +1,274 @@
{
"model_id": "deepseek-ai/DeepSeek-V4.1-Flash",
"vocab_size": 129280,
"all_stop_token_ids": [1],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "triggered_tags",
"triggers": [
"<|DSML| calls>"
],
"tags": [
{
"type": "tag",
"begin": "<|DSML| calls>\n",
"content": {
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|DSML| invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "deepseek_v4_1_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML| invoke>\n"
},
{
"type": "tag",
"begin": "<|DSML| invoke name=\"add\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "deepseek_v4_1_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML| invoke>\n"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
"end": "</|DSML| calls>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>I will call the tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai</|DSML| parameter>\n</|DSML| invoke>\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0</|DSML| parameter>\n<|DSML| parameter name=\"x\" string=\"false\">2</|DSML| parameter>\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls>",
"generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "\n\n<|DSML| calls>\n"
},
{
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|DSML| invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "deepseek_v4_1_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML| invoke>\n"
},
{
"type": "tag",
"begin": "<|DSML| invoke name=\"add\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "deepseek_v4_1_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML| invoke>\n"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
{
"type": "const_string",
"value": "</|DSML| calls>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai</|DSML| parameter>\n</|DSML| invoke>\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0</|DSML| parameter>\n<|DSML| parameter name=\"x\" string=\"false\">2</|DSML| parameter>\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls>",
"generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "\n\n<|DSML| calls>\n"
},
{
"type": "tag",
"begin": "<|DSML| invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "deepseek_v4_1_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</|DSML| invoke>\n"
},
{
"type": "const_string",
"value": "</|DSML| calls>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai</|DSML| parameter>\n</|DSML| invoke>\n</|DSML| calls>",
"generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1]
}
}
}
@@ -0,0 +1,251 @@
{
"model_id": "zai-org/GLM-4.7-Flash",
"vocab_size": 154880,
"all_stop_token_ids": [154820, 154827, 154829],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "triggered_tags",
"triggers": [
"<tool_call>"
],
"tags": [
{
"type": "tag",
"begin": "<tool_call>get_weather",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "glm_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</tool_call>"
},
{
"type": "tag",
"begin": "<tool_call>add",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "glm_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</tool_call>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>",
"</tool_call>",
"<arg_key>",
"</arg_key>",
"<arg_value>",
"</arg_value>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>I will call the tools.<tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call><tool_call>add<arg_key>y</arg_key><arg_value>1.0</arg_value><arg_key>x</arg_key><arg_value>2</arg_value><arg_key>items</arg_key><arg_value>[\"left\", \"right\"]</arg_value></tool_call>",
"generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "triggered_tags",
"triggers": [
"<tool_call>"
],
"tags": [
{
"type": "tag",
"begin": "<tool_call>get_weather",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "glm_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</tool_call>"
},
{
"type": "tag",
"begin": "<tool_call>add",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "glm_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</tool_call>"
}
],
"at_least_one": true,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>",
"</tool_call>",
"<arg_key>",
"</arg_key>",
"<arg_value>",
"</arg_value>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think><tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call><tool_call>add<arg_key>y</arg_key><arg_value>1.0</arg_value><arg_key>x</arg_key><arg_value>2</arg_value><arg_key>items</arg_key><arg_value>[\"left\", \"right\"]</arg_value></tool_call>",
"generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "tag",
"begin": "<tool_call>get_weather",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "glm_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</tool_call>"
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think><tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call>",
"generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820]
}
}
}
@@ -0,0 +1,251 @@
{
"model_id": "zai-org/GLM-5.2-FP8",
"vocab_size": 154880,
"all_stop_token_ids": [154820, 154827, 154829],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "triggered_tags",
"triggers": [
"<tool_call>"
],
"tags": [
{
"type": "tag",
"begin": "<tool_call>get_weather",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "glm_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</tool_call>"
},
{
"type": "tag",
"begin": "<tool_call>add",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "glm_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</tool_call>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>",
"</tool_call>",
"<arg_key>",
"</arg_key>",
"<arg_value>",
"</arg_value>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>I will call the tools.<tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call><tool_call>add<arg_key>y</arg_key><arg_value>1.0</arg_value><arg_key>x</arg_key><arg_value>2</arg_value><arg_key>items</arg_key><arg_value>[\"left\", \"right\"]</arg_value></tool_call>",
"generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "triggered_tags",
"triggers": [
"<tool_call>"
],
"tags": [
{
"type": "tag",
"begin": "<tool_call>get_weather",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "glm_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</tool_call>"
},
{
"type": "tag",
"begin": "<tool_call>add",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "glm_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</tool_call>"
}
],
"at_least_one": true,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>",
"</tool_call>",
"<arg_key>",
"</arg_key>",
"<arg_value>",
"</arg_value>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think><tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call><tool_call>add<arg_key>y</arg_key><arg_value>1.0</arg_value><arg_key>x</arg_key><arg_value>2</arg_value><arg_key>items</arg_key><arg_value>[\"left\", \"right\"]</arg_value></tool_call>",
"generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "tag",
"begin": "<tool_call>get_weather",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "glm_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</tool_call>"
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think><tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call>",
"generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820]
}
}
}
+625
View File
@@ -0,0 +1,625 @@
{
"model_id": "tencent/Hy3",
"vocab_size": 120832,
"all_stop_token_ids": [120025],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think:opensource>"
},
{
"type": "triggered_tags",
"triggers": [
"<tool_calls:opensource>"
],
"tags": [
{
"type": "tag",
"begin": "<tool_calls:opensource>\n",
"content": {
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<tool_call:opensource>get_weather<tool_sep:opensource>\n",
"content": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<arg_key:opensource>"
},
{
"type": "const_string",
"value": "location"
},
{
"type": "const_string",
"value": "</arg_key:opensource>"
},
{
"type": "const_string",
"value": "\n"
},
{
"type": "const_string",
"value": "<arg_value:opensource>"
},
{
"type": "any_text",
"excludes": [
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<tool_sep:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>"
],
"max_tokens": null,
"max_chars": null
},
{
"type": "const_string",
"value": "</arg_value:opensource>"
},
{
"type": "const_string",
"value": "\n"
}
]
}
]
},
"end": "</tool_call:opensource>"
},
{
"type": "tag",
"begin": "<tool_call:opensource>add<tool_sep:opensource>\n",
"content": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<arg_key:opensource>"
},
{
"type": "const_string",
"value": "y"
},
{
"type": "const_string",
"value": "</arg_key:opensource>"
},
{
"type": "const_string",
"value": "\n"
},
{
"type": "const_string",
"value": "<arg_value:opensource>"
},
{
"type": "any_text",
"excludes": [
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<tool_sep:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>"
],
"max_tokens": null,
"max_chars": null
},
{
"type": "const_string",
"value": "</arg_value:opensource>"
},
{
"type": "const_string",
"value": "\n"
}
]
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<arg_key:opensource>"
},
{
"type": "const_string",
"value": "x"
},
{
"type": "const_string",
"value": "</arg_key:opensource>"
},
{
"type": "const_string",
"value": "\n"
},
{
"type": "const_string",
"value": "<arg_value:opensource>"
},
{
"type": "any_text",
"excludes": [
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<tool_sep:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>"
],
"max_tokens": null,
"max_chars": null
},
{
"type": "const_string",
"value": "</arg_value:opensource>"
},
{
"type": "const_string",
"value": "\n"
}
]
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<arg_key:opensource>"
},
{
"type": "const_string",
"value": "items"
},
{
"type": "const_string",
"value": "</arg_key:opensource>"
},
{
"type": "const_string",
"value": "\n"
},
{
"type": "const_string",
"value": "<arg_value:opensource>"
},
{
"type": "any_text",
"excludes": [
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<tool_sep:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>"
],
"max_tokens": null,
"max_chars": null
},
{
"type": "const_string",
"value": "</arg_value:opensource>"
},
{
"type": "const_string",
"value": "\n"
}
]
}
]
},
"end": "</tool_call:opensource>"
}
],
"separator": "\n",
"at_least_one": true,
"stop_after_first": false
},
"end": "\n</tool_calls:opensource>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": []
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think:opensource>I will call the tools.<tool_calls:opensource>\n<tool_call:opensource>get_weather<tool_sep:opensource>\n<arg_key:opensource>location</arg_key:opensource>\n<arg_value:opensource>Shanghai</arg_value:opensource>\n</tool_call:opensource>\n<tool_call:opensource>add<tool_sep:opensource>\n<arg_key:opensource>y</arg_key:opensource>\n<arg_value:opensource>1.0</arg_value:opensource>\n<arg_key:opensource>x</arg_key:opensource>\n<arg_value:opensource>2</arg_value:opensource>\n<arg_key:opensource>items</arg_key:opensource>\n<arg_value:opensource>[\"left\", \"right\"]</arg_value:opensource>\n</tool_call:opensource>\n</tool_calls:opensource>",
"generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 40, 823, 1609, 252, 7106, 13, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think:opensource>"
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<tool_calls:opensource>\n"
},
{
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<tool_call:opensource>get_weather<tool_sep:opensource>\n",
"content": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<arg_key:opensource>"
},
{
"type": "const_string",
"value": "location"
},
{
"type": "const_string",
"value": "</arg_key:opensource>"
},
{
"type": "const_string",
"value": "\n"
},
{
"type": "const_string",
"value": "<arg_value:opensource>"
},
{
"type": "any_text",
"excludes": [
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<tool_sep:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>"
],
"max_tokens": null,
"max_chars": null
},
{
"type": "const_string",
"value": "</arg_value:opensource>"
},
{
"type": "const_string",
"value": "\n"
}
]
}
]
},
"end": "</tool_call:opensource>"
},
{
"type": "tag",
"begin": "<tool_call:opensource>add<tool_sep:opensource>\n",
"content": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<arg_key:opensource>"
},
{
"type": "const_string",
"value": "y"
},
{
"type": "const_string",
"value": "</arg_key:opensource>"
},
{
"type": "const_string",
"value": "\n"
},
{
"type": "const_string",
"value": "<arg_value:opensource>"
},
{
"type": "any_text",
"excludes": [
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<tool_sep:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>"
],
"max_tokens": null,
"max_chars": null
},
{
"type": "const_string",
"value": "</arg_value:opensource>"
},
{
"type": "const_string",
"value": "\n"
}
]
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<arg_key:opensource>"
},
{
"type": "const_string",
"value": "x"
},
{
"type": "const_string",
"value": "</arg_key:opensource>"
},
{
"type": "const_string",
"value": "\n"
},
{
"type": "const_string",
"value": "<arg_value:opensource>"
},
{
"type": "any_text",
"excludes": [
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<tool_sep:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>"
],
"max_tokens": null,
"max_chars": null
},
{
"type": "const_string",
"value": "</arg_value:opensource>"
},
{
"type": "const_string",
"value": "\n"
}
]
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<arg_key:opensource>"
},
{
"type": "const_string",
"value": "items"
},
{
"type": "const_string",
"value": "</arg_key:opensource>"
},
{
"type": "const_string",
"value": "\n"
},
{
"type": "const_string",
"value": "<arg_value:opensource>"
},
{
"type": "any_text",
"excludes": [
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<tool_sep:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>"
],
"max_tokens": null,
"max_chars": null
},
{
"type": "const_string",
"value": "</arg_value:opensource>"
},
{
"type": "const_string",
"value": "\n"
}
]
}
]
},
"end": "</tool_call:opensource>"
}
],
"separator": "\n",
"at_least_one": true,
"stop_after_first": false
},
{
"type": "const_string",
"value": "\n</tool_calls:opensource>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think:opensource><tool_calls:opensource>\n<tool_call:opensource>get_weather<tool_sep:opensource>\n<arg_key:opensource>location</arg_key:opensource>\n<arg_value:opensource>Shanghai</arg_value:opensource>\n</tool_call:opensource>\n<tool_call:opensource>add<tool_sep:opensource>\n<arg_key:opensource>y</arg_key:opensource>\n<arg_value:opensource>1.0</arg_value:opensource>\n<arg_key:opensource>x</arg_key:opensource>\n<arg_value:opensource>2</arg_value:opensource>\n<arg_key:opensource>items</arg_key:opensource>\n<arg_value:opensource>[\"left\", \"right\"]</arg_value:opensource>\n</tool_call:opensource>\n</tool_calls:opensource>",
"generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think:opensource>"
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<tool_calls:opensource>\n"
},
{
"type": "tag",
"begin": "<tool_call:opensource>get_weather<tool_sep:opensource>\n",
"content": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<arg_key:opensource>"
},
{
"type": "const_string",
"value": "location"
},
{
"type": "const_string",
"value": "</arg_key:opensource>"
},
{
"type": "const_string",
"value": "\n"
},
{
"type": "const_string",
"value": "<arg_value:opensource>"
},
{
"type": "any_text",
"excludes": [
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<tool_sep:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>"
],
"max_tokens": null,
"max_chars": null
},
{
"type": "const_string",
"value": "</arg_value:opensource>"
},
{
"type": "const_string",
"value": "\n"
}
]
}
]
},
"end": "</tool_call:opensource>"
},
{
"type": "const_string",
"value": "\n</tool_calls:opensource>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think:opensource><tool_calls:opensource>\n<tool_call:opensource>get_weather<tool_sep:opensource>\n<arg_key:opensource>location</arg_key:opensource>\n<arg_value:opensource>Shanghai</arg_value:opensource>\n</tool_call:opensource>\n</tool_calls:opensource>",
"generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120036, 120025]
}
}
}
@@ -0,0 +1,340 @@
{
"model_id": "moonshotai/Kimi-K2.5",
"vocab_size": 163840,
"all_stop_token_ids": [163585, 163586],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "triggered_tags",
"triggers": [
"<|tool_calls_section_begin|>"
],
"tags": [
{
"type": "tag",
"begin": "<|tool_calls_section_begin|>",
"content": {
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|tool_call_begin|>functions.get_weather:",
"content": {
"type": "sequence",
"elements": [
{
"type": "regex",
"pattern": "\\d+"
},
{
"type": "const_string",
"value": "<|tool_call_argument_begin|>"
},
{
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
}
]
},
"end": "<|tool_call_end|>"
},
{
"type": "tag",
"begin": "<|tool_call_begin|>functions.add:",
"content": {
"type": "sequence",
"elements": [
{
"type": "regex",
"pattern": "\\d+"
},
{
"type": "const_string",
"value": "<|tool_call_argument_begin|>"
},
{
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
}
]
},
"end": "<|tool_call_end|>"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
"end": "<|tool_calls_section_end|>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>",
"<|tool_call_begin|>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think>I will call the tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>",
"generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 40, 748, 2390, 276, 7697, 13, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<|tool_calls_section_begin|>"
},
{
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|tool_call_begin|>functions.get_weather:",
"content": {
"type": "sequence",
"elements": [
{
"type": "regex",
"pattern": "\\d+"
},
{
"type": "const_string",
"value": "<|tool_call_argument_begin|>"
},
{
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
}
]
},
"end": "<|tool_call_end|>"
},
{
"type": "tag",
"begin": "<|tool_call_begin|>functions.add:",
"content": {
"type": "sequence",
"elements": [
{
"type": "regex",
"pattern": "\\d+"
},
{
"type": "const_string",
"value": "<|tool_call_argument_begin|>"
},
{
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
}
]
},
"end": "<|tool_call_end|>"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
{
"type": "const_string",
"value": "<|tool_calls_section_end|>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think><|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>",
"generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "<|tool_calls_section_begin|>"
},
{
"type": "tag",
"begin": "<|tool_call_begin|>functions.get_weather:",
"content": {
"type": "sequence",
"elements": [
{
"type": "regex",
"pattern": "\\d+"
},
{
"type": "const_string",
"value": "<|tool_call_argument_begin|>"
},
{
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
}
]
},
"end": "<|tool_call_end|>"
},
{
"type": "const_string",
"value": "<|tool_calls_section_end|>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.</think><|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_calls_section_end|>",
"generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163596, 163586]
}
}
}
@@ -0,0 +1,587 @@
{
"model_id": "moonshotai/Kimi-K3",
"vocab_size": 163840,
"all_stop_token_ids": [163585, 163586],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "optional",
"content": {
"type": "const_string",
"value": "<|open|>response<|sep|>"
}
},
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [
"<|open|>",
"<|close|>",
"<|end_of_msg|>"
],
"max_tokens": null,
"max_chars": null
},
"end": "<|close|>response<|sep|>"
},
{
"type": "optional",
"content": {
"type": "tag",
"begin": "<|open|>tools<|sep|>",
"content": {
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|open|>call tool=\"get_weather\" index=\"",
"content": {
"type": "sequence",
"elements": [
{
"type": "regex",
"pattern": "[1-9][0-9]*"
},
{
"type": "const_string",
"value": "\"<|sep|>"
},
{
"type": "or",
"elements": [
{
"type": "plus",
"content": {
"type": "tag",
"begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>",
"content": {
"type": "any_text",
"excludes": [
"<|close|>argument<|sep|>",
"<|close|>call<|sep|>"
],
"max_tokens": null,
"max_chars": null
},
"end": "<|close|>argument<|sep|>"
}
},
{
"type": "tag",
"begin": "<|open|>json type=\"object\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>json<|sep|>"
}
]
}
]
},
"end": "<|close|>call<|sep|>"
},
{
"type": "tag",
"begin": "<|open|>call tool=\"add\" index=\"",
"content": {
"type": "sequence",
"elements": [
{
"type": "regex",
"pattern": "[1-9][0-9]*"
},
{
"type": "const_string",
"value": "\"<|sep|>"
},
{
"type": "or",
"elements": [
{
"type": "plus",
"content": {
"type": "or",
"elements": [
{
"type": "tag",
"begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "number"
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>argument<|sep|>"
},
{
"type": "tag",
"begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "number"
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>argument<|sep|>"
},
{
"type": "tag",
"begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "array",
"items": {
"type": "string"
}
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>argument<|sep|>"
}
]
}
},
{
"type": "tag",
"begin": "<|open|>json type=\"object\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>json<|sep|>"
}
]
}
]
},
"end": "<|close|>call<|sep|>"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
"end": "<|close|>tools<|sep|>"
}
},
{
"type": "optional",
"content": {
"type": "const_string",
"value": "<|close|>message<|sep|>"
}
}
]
},
"coverage": "final_output_only"
},
"completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>",
"generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "optional",
"content": {
"type": "const_string",
"value": "<|open|>response<|sep|>"
}
},
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [
"<|open|>",
"<|close|>",
"<|end_of_msg|>"
],
"max_tokens": null,
"max_chars": null
},
"end": "<|close|>response<|sep|>"
},
{
"type": "tag",
"begin": "<|open|>tools<|sep|>",
"content": {
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|open|>call tool=\"get_weather\" index=\"",
"content": {
"type": "sequence",
"elements": [
{
"type": "regex",
"pattern": "[1-9][0-9]*"
},
{
"type": "const_string",
"value": "\"<|sep|>"
},
{
"type": "or",
"elements": [
{
"type": "plus",
"content": {
"type": "tag",
"begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>",
"content": {
"type": "any_text",
"excludes": [
"<|close|>argument<|sep|>",
"<|close|>call<|sep|>"
],
"max_tokens": null,
"max_chars": null
},
"end": "<|close|>argument<|sep|>"
}
},
{
"type": "tag",
"begin": "<|open|>json type=\"object\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>json<|sep|>"
}
]
}
]
},
"end": "<|close|>call<|sep|>"
},
{
"type": "tag",
"begin": "<|open|>call tool=\"add\" index=\"",
"content": {
"type": "sequence",
"elements": [
{
"type": "regex",
"pattern": "[1-9][0-9]*"
},
{
"type": "const_string",
"value": "\"<|sep|>"
},
{
"type": "or",
"elements": [
{
"type": "plus",
"content": {
"type": "or",
"elements": [
{
"type": "tag",
"begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "number"
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>argument<|sep|>"
},
{
"type": "tag",
"begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "number"
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>argument<|sep|>"
},
{
"type": "tag",
"begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "array",
"items": {
"type": "string"
}
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>argument<|sep|>"
}
]
}
},
{
"type": "tag",
"begin": "<|open|>json type=\"object\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>json<|sep|>"
}
]
}
]
},
"end": "<|close|>call<|sep|>"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
"end": "<|close|>tools<|sep|>"
},
{
"type": "optional",
"content": {
"type": "const_string",
"value": "<|close|>message<|sep|>"
}
}
]
},
"coverage": "final_output_only"
},
"completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>",
"generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "optional",
"content": {
"type": "const_string",
"value": "<|open|>response<|sep|>"
}
},
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [
"<|open|>",
"<|close|>",
"<|end_of_msg|>"
],
"max_tokens": null,
"max_chars": null
},
"end": "<|close|>response<|sep|>"
},
{
"type": "tag",
"begin": "<|open|>tools<|sep|>",
"content": {
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<|open|>call tool=\"get_weather\" index=\"",
"content": {
"type": "sequence",
"elements": [
{
"type": "regex",
"pattern": "[1-9][0-9]*"
},
{
"type": "const_string",
"value": "\"<|sep|>"
},
{
"type": "or",
"elements": [
{
"type": "plus",
"content": {
"type": "tag",
"begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>",
"content": {
"type": "any_text",
"excludes": [
"<|close|>argument<|sep|>",
"<|close|>call<|sep|>"
],
"max_tokens": null,
"max_chars": null
},
"end": "<|close|>argument<|sep|>"
}
},
{
"type": "tag",
"begin": "<|open|>json type=\"object\"<|sep|>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "<|close|>json<|sep|>"
}
]
}
]
},
"end": "<|close|>call<|sep|>"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
"end": "<|close|>tools<|sep|>"
},
{
"type": "optional",
"content": {
"type": "const_string",
"value": "<|close|>message<|sep|>"
}
}
]
},
"coverage": "final_output_only"
},
"completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>",
"generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586]
}
}
}
@@ -0,0 +1,254 @@
{
"model_id": "XiaomiMiMo/MiMo-V2.6-Flash-RL",
"vocab_size": 152576,
"all_stop_token_ids": [151643, 151645, 151672],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "optional",
"content": {
"type": "tag",
"begin": "<think>",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
}
},
{
"type": "triggered_tags",
"triggers": [
"<tool_call>"
],
"tags": [
{
"type": "tag",
"begin": "<tool_call><function=get_weather>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "qwen_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</function></tool_call>"
},
{
"type": "tag",
"begin": "<tool_call><function=add>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "qwen_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</function></tool_call>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>",
"</tool_call>",
"<function="
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "<think>Need call the weather and add tools.</think>I will call the tools.<tool_call><function=get_weather><parameter=location>Shanghai</parameter></function></tool_call><tool_call><function=add><parameter=y>1.0</parameter><parameter=x>2</parameter><parameter=items>[\"left\", \"right\"]</parameter></function></tool_call>",
"generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 40, 686, 1618, 279, 7375, 13, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "optional",
"content": {
"type": "tag",
"begin": "<think>",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
}
},
{
"type": "triggered_tags",
"triggers": [
"<tool_call>"
],
"tags": [
{
"type": "tag",
"begin": "<tool_call><function=get_weather>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "qwen_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</function></tool_call>"
},
{
"type": "tag",
"begin": "<tool_call><function=add>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "qwen_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</function></tool_call>"
}
],
"at_least_one": true,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>",
"</tool_call>",
"<function="
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "<think>Need call the weather and add tools.</think><tool_call><function=get_weather><parameter=location>Shanghai</parameter></function></tool_call><tool_call><function=add><parameter=y>1.0</parameter><parameter=x>2</parameter><parameter=items>[\"left\", \"right\"]</parameter></function></tool_call>",
"generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "optional",
"content": {
"type": "tag",
"begin": "<think>",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
}
},
{
"type": "tag",
"begin": "<tool_call><function=get_weather>",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "qwen_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</function></tool_call>"
}
]
},
"coverage": "from_token_zero"
},
"completion": "<think>Need call the weather and add tools.</think><tool_call><function=get_weather><parameter=location>Shanghai</parameter></function></tool_call>",
"generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151645]
}
}
}
@@ -0,0 +1,301 @@
{
"model_id": "MiniMaxAI/MiniMax-M2.5",
"vocab_size": 200064,
"all_stop_token_ids": [200020],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "const_string",
"value": "\n\n"
}
]
},
{
"type": "triggered_tags",
"triggers": [
"<minimax:tool_call>"
],
"tags": [
{
"type": "tag",
"begin": "<minimax:tool_call>\n",
"content": {
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "minimax_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</invoke>\n"
},
{
"type": "tag",
"begin": "<invoke name=\"add\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "minimax_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</invoke>\n"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
"end": "</minimax:tool_call>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.\n</think>\n\nI will call the tools.\n<minimax:tool_call>\n<invoke name=\"get_weather\">\n<parameter name=\"location\">Shanghai</parameter>\n</invoke>\n<invoke name=\"add\">\n<parameter name=\"y\">1.0</parameter>\n<parameter name=\"x\">2</parameter>\n<parameter name=\"items\">[\"left\", \"right\"]</parameter>\n</invoke>\n</minimax:tool_call>",
"generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 73, 715, 1728, 275, 6629, 320, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "const_string",
"value": "\n\n"
}
]
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "\n<minimax:tool_call>\n"
},
{
"type": "tags_with_separator",
"tags": [
{
"type": "tag",
"begin": "<invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "minimax_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</invoke>\n"
},
{
"type": "tag",
"begin": "<invoke name=\"add\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "minimax_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</invoke>\n"
}
],
"separator": "",
"at_least_one": true,
"stop_after_first": false
},
{
"type": "const_string",
"value": "</minimax:tool_call>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.\n</think>\n\n\n<minimax:tool_call>\n<invoke name=\"get_weather\">\n<parameter name=\"location\">Shanghai</parameter>\n</invoke>\n<invoke name=\"add\">\n<parameter name=\"y\">1.0</parameter>\n<parameter name=\"x\">2</parameter>\n<parameter name=\"items\">[\"left\", \"right\"]</parameter>\n</invoke>\n</minimax:tool_call>",
"generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "const_string",
"value": "\n\n"
}
]
},
{
"type": "sequence",
"elements": [
{
"type": "const_string",
"value": "\n<minimax:tool_call>\n"
},
{
"type": "tag",
"begin": "<invoke name=\"get_weather\">\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "minimax_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "</invoke>\n"
},
{
"type": "const_string",
"value": "</minimax:tool_call>"
}
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.\n</think>\n\n\n<minimax:tool_call>\n<invoke name=\"get_weather\">\n<parameter name=\"location\">Shanghai</parameter>\n</invoke>\n</minimax:tool_call>",
"generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020]
}
}
}
@@ -0,0 +1,277 @@
{
"model_id": "Qwen/Qwen3-0.6B",
"vocab_size": 151936,
"all_stop_token_ids": [151643, 151645],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "optional",
"content": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "<think>",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "const_string",
"value": "\n\n"
}
]
}
},
{
"type": "triggered_tags",
"triggers": [
"<tool_call>"
],
"tags": [
{
"type": "tag",
"begin": "<tool_call>\n{\"name\": \"get_weather\", \"arguments\": ",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "}\n</tool_call>"
},
{
"type": "tag",
"begin": "<tool_call>\n{\"name\": \"add\", \"arguments\": ",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "}\n</tool_call>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "<think>\nNeed call the weather and add tools.\n</think>\n\nI will call the tools.\n<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n</tool_call>\n<tool_call>\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n</tool_call>",
"generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 40, 686, 1618, 279, 7375, 624, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "optional",
"content": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "<think>",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "const_string",
"value": "\n\n"
}
]
}
},
{
"type": "triggered_tags",
"triggers": [
"<tool_call>"
],
"tags": [
{
"type": "tag",
"begin": "<tool_call>\n{\"name\": \"get_weather\", \"arguments\": ",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "}\n</tool_call>"
},
{
"type": "tag",
"begin": "<tool_call>\n{\"name\": \"add\", \"arguments\": ",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "}\n</tool_call>"
}
],
"at_least_one": true,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "<think>\nNeed call the weather and add tools.\n</think>\n\n<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n</tool_call>\n<tool_call>\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n</tool_call>",
"generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "optional",
"content": {
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "<think>",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "const_string",
"value": "\n\n"
}
]
}
},
{
"type": "tag",
"begin": "<tool_call>\n{\"name\": \"get_weather\", \"arguments\": ",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "json",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "}\n</tool_call>"
}
]
},
"coverage": "from_token_zero"
},
"completion": "<think>\nNeed call the weather and add tools.\n</think>\n\n<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n</tool_call>",
"generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 151645]
}
}
}
@@ -0,0 +1,268 @@
{
"model_id": "Qwen/Qwen3.5-4B",
"vocab_size": 248320,
"all_stop_token_ids": [248044, 248046],
"cases": {
"auto": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "const_string",
"value": "\n\n"
}
]
},
{
"type": "triggered_tags",
"triggers": [
"<tool_call>\n<function="
],
"tags": [
{
"type": "tag",
"begin": "<tool_call>\n<function=get_weather>\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "qwen_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "\n</function>\n</tool_call>"
},
{
"type": "tag",
"begin": "<tool_call>\n<function=add>\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "qwen_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "\n</function>\n</tool_call>"
}
],
"at_least_one": false,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.\n</think>\n\nI will call the tools.\n\n<tool_call>\n<function=get_weather>\n<parameter=location>\nShanghai\n</parameter>\n</function>\n</tool_call>\n<tool_call>\n<function=add>\n<parameter=y>\n1.0\n</parameter>\n<parameter=x>\n2\n</parameter>\n<parameter=items>\n[\"left\", \"right\"]\n</parameter>\n</function>\n</tool_call>",
"generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 40, 668, 1562, 279, 7141, 13, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046]
},
"required": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "const_string",
"value": "\n\n"
}
]
},
{
"type": "triggered_tags",
"triggers": [
"<tool_call>\n<function="
],
"tags": [
{
"type": "tag",
"begin": "<tool_call>\n<function=get_weather>\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "qwen_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "\n</function>\n</tool_call>"
},
{
"type": "tag",
"begin": "<tool_call>\n<function=add>\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"y": {
"type": "number"
},
"x": {
"type": "number"
},
"items": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"y",
"x",
"items"
]
},
"style": "qwen_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "\n</function>\n</tool_call>"
}
],
"at_least_one": true,
"stop_after_first": false,
"excludes": [
"<think>",
"</think>"
]
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.\n</think>\n\n<tool_call>\n<function=get_weather>\n<parameter=location>\nShanghai\n</parameter>\n</function>\n</tool_call>\n<tool_call>\n<function=add>\n<parameter=y>\n1.0\n</parameter>\n<parameter=x>\n2\n</parameter>\n<parameter=items>\n[\"left\", \"right\"]\n</parameter>\n</function>\n</tool_call>",
"generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046]
},
"named": {
"grammar": {
"format": {
"type": "sequence",
"elements": [
{
"type": "sequence",
"elements": [
{
"type": "tag",
"begin": "",
"content": {
"type": "any_text",
"excludes": [],
"max_tokens": null,
"max_chars": null
},
"end": "</think>"
},
{
"type": "const_string",
"value": "\n\n"
}
]
},
{
"type": "tag",
"begin": "<tool_call>\n<function=get_weather>\n",
"content": {
"type": "json_schema",
"json_schema": {
"type": "object",
"properties": {
"location": {
"type": "string"
}
},
"required": [
"location"
]
},
"style": "qwen_xml",
"any_order": false,
"max_whitespace_cnt": null,
"excludes": []
},
"end": "\n</function>\n</tool_call>"
}
]
},
"coverage": "from_token_zero"
},
"completion": "Need call the weather and add tools.\n</think>\n\n<tool_call>\n<function=get_weather>\n<parameter=location>\nShanghai\n</parameter>\n</function>\n</tool_call>",
"generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046]
}
}
}
+366 -88
View File
@@ -6,20 +6,31 @@
//! The invariant under test is that a structured assistant message rendered as history can be
//! parsed from the generated assistant completion and then rendered back to the exact same
//! assistant completion.
//!
//! The tool-call fixture also snapshots the output grammar built by the initialized parser for each
//! tool-choice variant, with the rendered completion as its generation, as one file per model under
//! `tests/grammar_replay/`. `grammar_replay.rs` replays these cases through the real XGrammar.
use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Debug;
use std::ops::Range;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::Arc;
use anyhow::{Context as _, Result, bail, ensure};
use futures::{Stream, StreamExt as _, stream};
use serde::ser::Error as _;
use serde::{Serialize, Serializer};
use serde_json_fmt::JsonFormat as JsonFmt;
use serial_test::file_serial;
use vllm_chat::{
AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall,
ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, FinishReason, GenerationPromptMode,
LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, RendererSelection,
load_model_backends,
ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, FinishReason,
GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection,
RendererSelection, ToolStrictLevel, load_model_backends,
};
use vllm_parser::output_grammar::BuiltOutputGrammar;
use vllm_text::{DecodedTextEvent, Finished, Prompt};
use vllm_tokenizer::Tokenizer;
@@ -114,6 +125,33 @@ impl ThinkingBehavior {
}
}
/// Tool-choice variant of the tool-call fixture.
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
#[serde(rename_all = "snake_case")]
enum ToolChoiceVariant {
/// `auto`, with content before the tool calls.
Auto,
/// `required`, with tool calls only: builder grammars admit no content
/// before the calls.
Required,
/// Named `get_weather`, with a single call to it and no content.
Named,
}
impl ToolChoiceVariant {
const ALL: [Self; 3] = [Self::Auto, Self::Required, Self::Named];
fn tool_choice(self) -> ChatToolChoice {
match self {
Self::Auto => ChatToolChoice::Auto,
Self::Required => ChatToolChoice::Required,
Self::Named => ChatToolChoice::Function {
name: "get_weather".to_string(),
},
}
}
}
impl RoundtripCase {
/// Qwen3 XML tool-call format with `qwen3` reasoning tags.
fn qwen3() -> Self {
@@ -451,6 +489,7 @@ async fn run_roundtrip_reasoning_and_content_inner(
"roundtrip-reasoning-content",
vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")],
Vec::new(),
None,
thinking,
case.thinking_behavior,
);
@@ -489,11 +528,25 @@ async fn run_roundtrip_reasoning_and_content_inner(
Ok(())
}
/// Run the fixed reasoning+multiple-tools fixture for one model/parser case.
/// Run the fixed reasoning+multiple-tools fixture for one model/parser case,
/// once per tool-choice variant.
async fn run_roundtrip_tool_call_mix(
case: &RoundtripCase,
backends: &vllm_chat::LoadedModelBackends,
) -> Result<()> {
let mut results = Vec::new();
for variant in ToolChoiceVariant::ALL {
let result = run_roundtrip_tool_call_mix_inner(case, backends, variant).await?;
results.push((variant, result));
}
check_grammar_replay_file(case, backends, results)
}
async fn run_roundtrip_tool_call_mix_inner(
case: &RoundtripCase,
backends: &vllm_chat::LoadedModelBackends,
variant: ToolChoiceVariant,
) -> Result<RoundtripResult> {
let request = roundtrip_request(
"roundtrip-reasoning-tools",
vec![ChatMessage::text(
@@ -501,41 +554,40 @@ async fn run_roundtrip_tool_call_mix(
"Check Shanghai weather and add 1.0 plus 2.",
)],
test_tools(),
Some(variant.tool_choice()),
Some(true), // always enable thinking in this fixture
case.thinking_behavior,
);
let expected_reasoning = "Need call the weather and add tools.";
let expected_text = "I will call the tools.";
let (expected_text, tool_call_count) = match variant {
ToolChoiceVariant::Auto => (Some("I will call the tools."), 2),
ToolChoiceVariant::Required => (None, 2),
ToolChoiceVariant::Named => (None, 1),
};
let result = run_roundtrip(
case,
backends,
&request,
AssistantMessage {
content: vec![
AssistantContentBlock::Reasoning {
text: expected_reasoning.to_string(),
},
AssistantContentBlock::Text {
text: expected_text.to_string(),
},
AssistantContentBlock::ToolCall(AssistantToolCall {
id: "functions.get_weather:0".to_string(),
name: "get_weather".to_string(),
arguments: r#"{"location":"Shanghai"}"#.to_string(),
}),
AssistantContentBlock::ToolCall(AssistantToolCall {
id: "functions.add:1".to_string(),
name: "add".to_string(),
// Intentionally use a non-lexical order of keys to verify text-level
// fidelity of the roundtrip where JSON formatting remains stable. The
// `items` key also exercises templates that call `arguments.items()`.
arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(),
}),
],
},
)
.await?;
let mut content = vec![AssistantContentBlock::Reasoning {
text: expected_reasoning.to_string(),
}];
content.extend(expected_text.map(|text| AssistantContentBlock::Text {
text: text.to_string(),
}));
let tool_calls = [
AssistantContentBlock::ToolCall(AssistantToolCall {
id: "functions.get_weather:0".to_string(),
name: "get_weather".to_string(),
arguments: r#"{"location":"Shanghai"}"#.to_string(),
}),
AssistantContentBlock::ToolCall(AssistantToolCall {
id: "functions.add:1".to_string(),
name: "add".to_string(),
// Intentionally use a non-lexical order of keys to verify text-level
// fidelity of the roundtrip where JSON formatting remains stable. The
// `items` key also exercises templates that call `arguments.items()`.
arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(),
}),
];
content.extend(tool_calls.into_iter().take(tool_call_count));
let result = run_roundtrip(case, backends, &request, AssistantMessage { content }).await?;
assert_eq!(
result.parsed_message.reasoning().as_deref(),
@@ -543,12 +595,15 @@ async fn run_roundtrip_tool_call_mix(
"parsed message: {:#?}",
result.parsed_message
);
assert_eq!(result.parsed_message.text(), expected_text);
assert_eq!(
result.parsed_message.text(),
expected_text.unwrap_or_default()
);
let tool_calls = result.parsed_message.tool_calls().collect::<Vec<_>>();
assert_eq!(
tool_calls.len(),
2,
tool_call_count,
"parsed message: {:#?}",
result.parsed_message
);
@@ -557,18 +612,20 @@ async fn run_roundtrip_tool_call_mix(
tool_calls[0].arguments,
expected_arguments(case, r#"{"location": "Shanghai"}"#)?,
);
assert_eq!(tool_calls[1].name, "add");
assert_eq!(
tool_calls[1].arguments,
expected_arguments(case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?,
);
if let Some(add) = tool_calls.get(1) {
assert_eq!(add.name, "add");
assert_eq!(
add.arguments,
expected_arguments(case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?,
);
}
assert_eq!(
result.rerendered_closed_completion,
result.closed_completion
);
Ok(())
Ok(result)
}
/// Compact JSON argument formatting used by JSON-native parsers/renderers.
@@ -645,6 +702,9 @@ struct RoundtripResult {
/// Assistant-completion suffix cut after rendering the parsed assistant
/// back as history.
rerendered_closed_completion: Prompt,
/// Output grammar built by the same initialized parser that parsed the
/// completion.
output_grammar: Option<BuiltOutputGrammar>,
}
/// Render, parse, and rerender one assistant turn through the production
@@ -658,13 +718,17 @@ async fn run_roundtrip(
let renderer = backends.chat_backend.chat_renderer();
let rendered = render_closed_completion(renderer.as_ref(), request, &assistant)?;
let parsed_message = parse_completion(case, backends, request, &rendered).await?;
let ParsedCompletion {
message: parsed_message,
output_grammar,
} = parse_completion(case, backends, request, &rendered).await?;
let rerendered = render_closed_completion(renderer.as_ref(), request, &parsed_message)?;
Ok(RoundtripResult {
parsed_message,
closed_completion: rendered.completion,
rerendered_closed_completion: rerendered.completion,
output_grammar,
})
}
@@ -675,7 +739,7 @@ struct RenderedTurn {
}
/// Render `history` as a production prompt and `history + assistant` as closed
/// history, then return the production prompt and assistant-completion suffix.
/// history, then return the production prompt and the assistant completion.
fn render_closed_completion(
renderer: &dyn vllm_chat::ChatRenderer,
base_request: &ChatRequest,
@@ -692,18 +756,15 @@ fn render_closed_completion(
let completion = match (&prompt, full) {
(Prompt::Text(prompt), Prompt::Text(full)) => {
ensure!(
full.starts_with(prompt),
"full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}"
);
Prompt::Text(full[prompt.len()..].to_string())
let range = completion_range(prompt.as_bytes(), full.as_bytes())
.with_context(|| format!("prompt: {prompt:?}\nfull: {full:?}"))?;
let completion = full.get(range).context("completion splits a UTF-8 character")?;
Prompt::Text(completion.to_string())
}
(Prompt::TokenIds(prompt), Prompt::TokenIds(full)) => {
ensure!(
full.starts_with(prompt),
"full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}"
);
Prompt::TokenIds(full[prompt.len()..].to_vec())
let range = completion_range(prompt, &full)
.with_context(|| format!("prompt: {prompt:?}\nfull: {full:?}"))?;
Prompt::TokenIds(full[range].to_vec())
}
(prompt, full) => bail!("prompt kind changed between renders: {prompt:?} vs {full:?}"),
};
@@ -711,14 +772,81 @@ fn render_closed_completion(
Ok(RenderedTurn { prompt, completion })
}
/// Locate the assistant completion in `full`, the history rendered with the
/// assistant message, given `prompt`, the same history rendered for generation.
///
/// Some templates render request-scoped control messages after the whole
/// history, such as Kimi K3's `tool_choice=required` message. With history `A`,
/// control messages `C`, generation prompt `G`, and completion `B`, the renders
/// are `prompt = A + C + G` and `full = A + G + B + C`. `C` is empty for most
/// templates, which makes `prompt` a prefix of `full`.
fn completion_range<T: PartialEq + Debug>(prompt: &[T], full: &[T]) -> Result<Range<usize>> {
if full.starts_with(prompt) {
return Ok(prompt.len()..full.len());
}
ensure!(
full.len() >= prompt.len(),
"full prompt is shorter than production prompt"
);
// `A` is a common prefix of both renders; try every split of the remaining
// `prompt` into `C + G` and require exactly one to fit `full`.
let common_prefix_len = prompt.iter().zip(full).take_while(|(p, f)| p == f).count();
let mut control_lens = Vec::new();
for history_len in 0..=common_prefix_len {
let rest = &prompt[history_len..];
for control_len in 1..=rest.len() {
let (control, generation) = rest.split_at(control_len);
if full[history_len..].starts_with(generation) && full.ends_with(control) {
control_lens.push(control_len);
}
}
}
let [control_len] = control_lens[..] else {
bail!(
"full prompt must be `A + G + B + C` for production prompt `A + C + G`; found {} splits",
control_lens.len()
);
};
Ok(prompt.len() - control_len..full.len() - control_len)
}
#[test]
fn completion_range_skips_trailing_control_messages() {
let history = "<user>hi</user>";
let control = "<system>call tools</system>";
let generation = "<assistant>";
let completion = "call()</assistant>";
let prompt = format!("{history}{generation}");
let full = format!("{history}{generation}{completion}");
let range = completion_range(prompt.as_bytes(), full.as_bytes()).unwrap();
assert_eq!(&full[range], completion);
let prompt = format!("{history}{control}{generation}");
let full = format!("{history}{generation}{completion}{control}");
let range = completion_range(prompt.as_bytes(), full.as_bytes()).unwrap();
assert_eq!(&full[range], completion);
}
/// Output of one production output processor over a rendered completion.
struct ParsedCompletion {
/// Terminal assistant message.
message: AssistantMessage,
/// Output grammar built by the processor after initialization.
output_grammar: Option<BuiltOutputGrammar>,
}
/// Feed one rendered assistant completion body into the real output processor
/// and collect its terminal assistant message.
/// and collect its terminal assistant message, along with the output grammar
/// the initialized processor builds for the request.
async fn parse_completion(
case: &RoundtripCase,
backends: &vllm_chat::LoadedModelBackends,
base_request: &ChatRequest,
rendered: &RenderedTurn,
) -> Result<AssistantMessage> {
) -> Result<ParsedCompletion> {
let tokenizer = backends.text_backend.tokenizer();
let prompt_token_ids = match &rendered.prompt {
Prompt::Text(prompt) => tokenizer
@@ -731,12 +859,15 @@ async fn parse_completion(
let mut processor = backends.chat_backend.new_chat_output_processor(
&mut request,
NewChatOutputProcessorOptions {
tool_strict_level: vllm_chat::ToolStrictLevel::Auto,
// Pin every tool schema so that builder-backed parsers build an
// output grammar for replay. The strict level does not affect parsing.
tool_strict_level: ToolStrictLevel::Parameter,
tool_call_parser: &case.tool_call_parser,
reasoning_parser: &case.reasoning_parser,
},
)?;
processor.initialize(&prompt_token_ids)?;
let output_grammar = processor.build_output_grammar()?;
let decoded = decoded_completion_stream(
tokenizer.as_ref(),
@@ -748,7 +879,10 @@ async fn parse_completion(
while let Some(event) = events.next().await {
if let ChatEvent::Done { message, .. } = event? {
return Ok(message);
return Ok(ParsedCompletion {
message,
output_grammar,
});
}
}
@@ -772,14 +906,8 @@ fn decoded_completion_stream(
prompt_logprobs: None,
}];
let chunks = match completion {
Prompt::Text(text) => {
let body = text.strip_suffix(assistant_stop_suffix).with_context(|| {
format!(
"closed assistant completion did not end with {:?}: {:?}",
assistant_stop_suffix, text
)
})?;
let chunks = match completion_body(tokenizer, completion, assistant_stop_suffix)? {
CompletionBody::Text(body) => {
// The engine returns token ids, so attribute the rendered text
// through the production decoder, then split it into small char
// chunks to exercise marker splits.
@@ -807,27 +935,12 @@ fn decoded_completion_stream(
})
.collect()
}
Prompt::TokenIds(token_ids) => {
let body = if assistant_stop_suffix.is_empty() {
token_ids.as_slice()
} else {
let stop_token_ids = tokenizer
.encode(assistant_stop_suffix, false)
.context("failed to encode token-id completion stop suffix")?;
token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| {
format!(
"token-id completion did not end with {:?}: {:?}",
assistant_stop_suffix, token_ids
)
})?
};
incremental_decode_chunks(
tokenizer,
&prompt_token_ids,
body,
TOKEN_COMPLETION_CHUNK_TOKENS,
)?
}
CompletionBody::TokenIds(body) => incremental_decode_chunks(
tokenizer,
&prompt_token_ids,
body,
TOKEN_COMPLETION_CHUNK_TOKENS,
)?,
};
if chunks.is_empty() {
@@ -868,6 +981,48 @@ fn decoded_completion_stream(
Ok(stream::iter(events).map(Ok).boxed())
}
/// Generated part of a closed assistant completion, without the rendered stop
/// suffix.
enum CompletionBody<'a> {
Text(&'a str),
TokenIds(&'a [u32]),
}
/// Strip the rendered stop suffix from a closed assistant completion.
fn completion_body<'a>(
tokenizer: &dyn Tokenizer,
completion: &'a Prompt,
assistant_stop_suffix: &str,
) -> Result<CompletionBody<'a>> {
match completion {
Prompt::Text(text) => {
let body = text.strip_suffix(assistant_stop_suffix).with_context(|| {
format!(
"closed assistant completion did not end with {:?}: {:?}",
assistant_stop_suffix, text
)
})?;
Ok(CompletionBody::Text(body))
}
Prompt::TokenIds(token_ids) => {
let body = if assistant_stop_suffix.is_empty() {
token_ids.as_slice()
} else {
let stop_token_ids = tokenizer
.encode(assistant_stop_suffix, false)
.context("failed to encode token-id completion stop suffix")?;
token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| {
format!(
"token-id completion did not end with {:?}: {:?}",
assistant_stop_suffix, token_ids
)
})?
};
Ok(CompletionBody::TokenIds(body))
}
}
}
/// One decoded completion chunk fed into the output processor.
struct DecodedCompletionChunk {
/// Attributed for token-id completions, as the production decoder emits it.
@@ -941,15 +1096,138 @@ fn incremental_decode_chunks(
Ok(chunks)
}
/// Grammar replay cases of one model, replayed through the real XGrammar by
/// `grammar_replay.py`.
#[derive(Serialize)]
struct GrammarReplayFile {
/// Hugging Face model id whose tokenizer the replay loads.
model_id: &'static str,
/// Model vocabulary size, which sizes the XGrammar token mask.
vocab_size: usize,
/// Request stop set, passed to the matcher as `override_stop_tokens`.
#[serde(serialize_with = "serialize_one_line")]
all_stop_token_ids: BTreeSet<u32>,
/// Cases keyed by tool-choice variant.
cases: BTreeMap<ToolChoiceVariant, GrammarReplayCase>,
}
/// One parser-built output grammar with the generation it must accept.
#[derive(Serialize)]
struct GrammarReplayCase {
/// Parser-built grammar; only token-zero grammars are replayed from the
/// first generated token.
grammar: BuiltOutputGrammar,
/// Generated completion text, without the stop token.
completion: String,
/// Generated token IDs presented to the matcher, ending with a stop token.
#[serde(serialize_with = "serialize_one_line")]
generation_token_ids: Vec<u32>,
}
/// Serialize a token ID list on one line inside the pretty-printed file.
fn serialize_one_line<S: Serializer>(
token_ids: &impl Serialize,
serializer: S,
) -> std::result::Result<S::Ok, S::Error> {
let json = spaced_json_fmt().format_to_string(token_ids).map_err(S::Error::custom)?;
serde_json::value::RawValue::from_string(json)
.map_err(S::Error::custom)?
.serialize(serializer)
}
/// Directory of checked-in grammar replay files.
fn grammar_replay_dir() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay")
}
/// Snapshot the output grammars of one model's roundtrips, keyed by variant,
/// as its grammar replay file, or check that no stale file remains when the
/// parser builds none.
fn check_grammar_replay_file(
case: &RoundtripCase,
backends: &vllm_chat::LoadedModelBackends,
results: Vec<(ToolChoiceVariant, RoundtripResult)>,
) -> Result<()> {
let model_name = case.model_id.rsplit_once('/').map_or(case.model_id, |(_, name)| name);
let path = grammar_replay_dir().join(format!("{model_name}.json"));
let results = results
.into_iter()
.filter_map(|(variant, result)| {
Some((variant, result.output_grammar?, result.closed_completion))
})
.collect::<Vec<_>>();
if results.is_empty() {
ensure!(
!path.exists(),
"{} builds no output grammar, but grammar replay file {path:?} exists",
case.model_id
);
return Ok(());
}
let tokenizer = backends.text_backend.tokenizer();
let hints = backends.text_backend.sampling_hints()?;
// Lowered `_all_stop_token_ids` of a request without user stop tokens.
let mut all_stop_token_ids = hints.extra_eos_token_ids;
all_stop_token_ids.extend(hints.primary_eos_token_id);
// The generation ends with the first token of the rendered stop suffix, or
// with the primary EOS for templates that render none.
let stop_token_id = if case.assistant_stop_suffix.is_empty() {
hints.primary_eos_token_id.context("model has no primary EOS token")?
} else {
tokenizer.encode(case.assistant_stop_suffix, false)?[0]
};
ensure!(
all_stop_token_ids.contains(&stop_token_id),
"stop token {stop_token_id} of {:?} is not in the stop set {all_stop_token_ids:?}",
case.assistant_stop_suffix
);
let mut cases = BTreeMap::new();
for (variant, grammar, closed_completion) in results {
let (completion, mut generation_token_ids) = match completion_body(
tokenizer.as_ref(),
&closed_completion,
case.assistant_stop_suffix,
)? {
CompletionBody::Text(body) => (body.to_string(), tokenizer.encode(body, false)?),
CompletionBody::TokenIds(body) => (tokenizer.decode(body, false)?, body.to_vec()),
};
generation_token_ids.push(stop_token_id);
cases.insert(
variant,
GrammarReplayCase {
grammar,
completion,
generation_token_ids,
},
);
}
let file = GrammarReplayFile {
model_id: case.model_id,
vocab_size: backends.text_backend.model_vocab_size(),
all_stop_token_ids,
cases,
};
let mut json = serde_json::to_string_pretty(&file)?;
json.push('\n');
expect_test::expect_file![path].assert_eq(&json);
Ok(())
}
/// Build a chat request fixture with parser-enabling tool-choice semantics.
fn roundtrip_request(
request_id: impl Into<String>,
messages: Vec<ChatMessage>,
tools: Vec<ChatTool>,
tool_choice: Option<ChatToolChoice>,
thinking: Option<bool>,
thinking_behavior: ThinkingBehavior,
) -> ChatRequest {
let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true)
let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, tool_choice, true)
.expect("tool context should resolve");
let mut request = ChatRequest {
request_id: request_id.into(),
+4 -2
View File
@@ -5,6 +5,7 @@
use std::{fmt, str::FromStr};
use serde::Serialize;
use serde_with::{DeserializeFromStr, SerializeDisplay};
use thiserror::Error;
use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagOptions};
@@ -20,7 +21,8 @@ use crate::tool::Tool;
pub type Result<T> = std::result::Result<T, OutputGrammarError>;
/// How much of the generated stream an output grammar covers.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum GrammarCoverage {
/// The grammar covers reasoning and advances from the first generated token.
FromTokenZero,
@@ -29,7 +31,7 @@ pub enum GrammarCoverage {
}
/// One parser-built output grammar and its generated-stream coverage.
#[derive(Debug, Clone, PartialEq)]
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct BuiltOutputGrammar {
/// Typed XGrammar structural-tag format.
pub format: Format,