From 87228cff01d7a56988e572c9a689444728f26568 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Wed, 30 Sep 2026 15:29:01 +0800 Subject: [PATCH] [Rust Frontend] Replay roundtrip output grammars through XGrammar (#59143) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- rust/src/chat/Cargo.toml | 1 + rust/src/chat/tests/grammar_replay.py | 113 ++++ rust/src/chat/tests/grammar_replay.rs | 24 + .../grammar_replay/DeepSeek-V3.2-Exp.json | 274 ++++++++ .../grammar_replay/DeepSeek-V4-Flash.json | 274 ++++++++ .../grammar_replay/DeepSeek-V4.1-Flash.json | 274 ++++++++ .../tests/grammar_replay/GLM-4.7-Flash.json | 251 +++++++ .../tests/grammar_replay/GLM-5.2-FP8.json | 251 +++++++ rust/src/chat/tests/grammar_replay/Hy3.json | 625 ++++++++++++++++++ .../chat/tests/grammar_replay/Kimi-K2.5.json | 340 ++++++++++ .../chat/tests/grammar_replay/Kimi-K3.json | 587 ++++++++++++++++ .../grammar_replay/MiMo-V2.6-Flash-RL.json | 254 +++++++ .../tests/grammar_replay/MiniMax-M2.5.json | 301 +++++++++ .../chat/tests/grammar_replay/Qwen3-0.6B.json | 277 ++++++++ .../chat/tests/grammar_replay/Qwen3.5-4B.json | 268 ++++++++ rust/src/chat/tests/roundtrip.rs | 454 ++++++++++--- rust/src/parser/src/output_grammar.rs | 6 +- 17 files changed, 4484 insertions(+), 90 deletions(-) create mode 100755 rust/src/chat/tests/grammar_replay.py create mode 100644 rust/src/chat/tests/grammar_replay.rs create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json create mode 100644 rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json create mode 100644 rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json create mode 100644 rust/src/chat/tests/grammar_replay/Hy3.json create mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K2.5.json create mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K3.json create mode 100644 rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json create mode 100644 rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json create mode 100644 rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json create mode 100644 rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index a110020ec1..d145fc237a 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -46,6 +46,7 @@ clap.workspace = true expect-test.workspace = true paste.workspace = true rmp-serde.workspace = true +serde_json = { workspace = true, features = ["raw_value"] } serial_test.workspace = true tempfile.workspace = true tokio.workspace = true diff --git a/rust/src/chat/tests/grammar_replay.py b/rust/src/chat/tests/grammar_replay.py new file mode 100755 index 0000000000..d34df627ba --- /dev/null +++ b/rust/src/chat/tests/grammar_replay.py @@ -0,0 +1,113 @@ +#!/usr/bin/env -S uv run --script +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "tiktoken>=0.6.0", +# # xgrammar imports torch unconditionally; only the CPU build is needed. +# "torch", +# "transformers>=5.10.4", +# # Keep in sync with requirements/common.txt. +# "xgrammar==0.2.7", +# ] +# +# [[tool.uv.index]] +# name = "pytorch-cpu" +# url = "https://download.pytorch.org/whl/cpu" +# explicit = true +# +# [tool.uv.sources] +# torch = { index = "pytorch-cpu" } +# /// +"""Replay the grammar cases exported by the roundtrip tests through XGrammar. + +Each file holds one model's cases, keyed by tool-choice variant. Every case is +compiled against the model tokenizer the same way as +`vllm/v1/structured_output/backend_xgrammar.py`. Token-zero cases are then +replayed from the first generated token: each token must be allowed by the +bitmask and accepted by the matcher, and the final stop token must terminate it. +""" + +import json +import sys +from functools import cache +from pathlib import Path + +import xgrammar as xgr +from transformers import AutoTokenizer + +CASES_DIR = Path(__file__).parent / "grammar_replay" + + +@cache +def load_tokenizer(model_id: str, vocab_size: int): + tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) + tokenizer_info = xgr.TokenizerInfo.from_huggingface( + tokenizer, vocab_size=vocab_size + ) + return tokenizer, xgr.GrammarCompiler(tokenizer_info) + + +def replay(model: dict, case: dict) -> str: + """Check one case of a model and return a summary of what was checked.""" + tokenizer, compiler = load_tokenizer(model["model_id"], model["vocab_size"]) + generation = case["generation_token_ids"] + + # The case token IDs come from the Rust tokenizer; the runner must load the + # same vocabulary for the grammar to mean the same thing. + encoded = tokenizer.encode(case["completion"], add_special_tokens=False) + if encoded != generation[:-1]: + raise AssertionError( + "tokenizer mismatch: the completion encodes to " + f"{encoded}, but the case records {generation[:-1]}" + ) + + grammar = case["grammar"] + structural_tag = {"type": "structural_tag", "format": grammar["format"]} + compiled = compiler.compile_structural_tag(json.dumps(structural_tag)) + if grammar["coverage"] != "from_token_zero": + return f"compiled only ({grammar['coverage']} is gated by the engine)" + + matcher = xgr.GrammarMatcher( + compiled, override_stop_tokens=model["all_stop_token_ids"] + ) + bitmask = xgr.allocate_token_bitmask(1, model["vocab_size"]) + for index, token_id in enumerate(generation): + matcher.fill_next_token_bitmask(bitmask) + allowed = (bitmask[0, token_id // 32].item() >> (token_id % 32)) & 1 + if not allowed or not matcher.accept_token(token_id): + prefix = tokenizer.decode(generation[:index]) + raise AssertionError( + f"token {index} ({token_id}, {tokenizer.decode([token_id])!r}) " + f"rejected after {prefix!r}" + ) + if not matcher.is_terminated(): + raise AssertionError("matcher not terminated after the stop token") + return f"replayed {len(generation)} tokens from token zero" + + +def main() -> int: + paths = sorted(CASES_DIR.glob("*.json")) + if not paths: + print(f"no grammar replay cases in {CASES_DIR}", file=sys.stderr) + return 1 + + failures = 0 + for path in paths: + model = json.loads(path.read_text()) + for name, case in model["cases"].items(): + try: + summary = replay(model, case) + except Exception as error: + failures += 1 + coverage = case["grammar"]["coverage"] + print(f"FAIL {path.name} {name} [{coverage}]: {error}", file=sys.stderr) + else: + print(f"PASS {path.name} {name}: {summary}") + + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/rust/src/chat/tests/grammar_replay.rs b/rust/src/chat/tests/grammar_replay.rs new file mode 100644 index 0000000000..4ddfa6bf24 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Replays the grammar cases exported by the roundtrip tests through the real +//! XGrammar, so that parser-built output grammars are checked against the +//! engine's compiler and matcher. + +use std::path::PathBuf; +use std::process::Command; + +#[test] +fn xgrammar_accepts_roundtrip_generations() { + let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay.py"); + let output = Command::new(&script) + .output() + .unwrap_or_else(|error| panic!("failed to execute {:?}: {error}", script)); + assert!( + output.status.success(), + "grammar replay failed: status={:?}\nstdout:\n{}\nstderr:\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json new file mode 100644 index 0000000000..4903b5900c --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json @@ -0,0 +1,274 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V3.2-Exp", + "vocab_size": 129280, + "all_stop_token_ids": [1], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<|DSML|function_calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<|DSML|function_calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 43, 759, 1926, 270, 6704, 339, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML|function_calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML|function_calls>\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json new file mode 100644 index 0000000000..1a286d8acd --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json @@ -0,0 +1,274 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V4-Flash", + "vocab_size": 129280, + "all_stop_token_ids": [1], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<|DSML|tool_calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<|DSML|tool_calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML|tool_calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML|tool_calls>\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json new file mode 100644 index 0000000000..a21d8b90e5 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json @@ -0,0 +1,274 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V4.1-Flash", + "vocab_size": 129280, + "all_stop_token_ids": [1], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<|DSML| calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<|DSML| calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML| invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML| invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0\n<|DSML| parameter name=\"x\" string=\"false\">2\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML| calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML| invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML| invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0\n<|DSML| parameter name=\"x\" string=\"false\">2\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML| calls>\n" + }, + { + "type": "tag", + "begin": "<|DSML| invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json new file mode 100644 index 0000000000..b289ec62e6 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json @@ -0,0 +1,251 @@ +{ + "model_id": "zai-org/GLM-4.7-Flash", + "vocab_size": 154880, + "all_stop_token_ids": [154820, 154827, 154829], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghai", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json new file mode 100644 index 0000000000..404667c30a --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json @@ -0,0 +1,251 @@ +{ + "model_id": "zai-org/GLM-5.2-FP8", + "vocab_size": 154880, + "all_stop_token_ids": [154820, 154827, 154829], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghai", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Hy3.json b/rust/src/chat/tests/grammar_replay/Hy3.json new file mode 100644 index 0000000000..f6ae7b0969 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Hy3.json @@ -0,0 +1,625 @@ +{ + "model_id": "tencent/Hy3", + "vocab_size": 120832, + "all_stop_token_ids": [120025], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "y" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "x" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "items" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + } + ], + "separator": "\n", + "at_least_one": true, + "stop_after_first": false + }, + "end": "\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 40, 823, 1609, 252, 7106, 13, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "y" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "x" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "items" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + } + ], + "separator": "\n", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n" + }, + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120036, 120025] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json new file mode 100644 index 0000000000..8ac6e689f7 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json @@ -0,0 +1,340 @@ +{ + "model_id": "moonshotai/Kimi-K2.5", + "vocab_size": 163840, + "all_stop_token_ids": [163585, 163586], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<|tool_calls_section_begin|>" + ], + "tags": [ + { + "type": "tag", + "begin": "<|tool_calls_section_begin|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.add:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|tool_calls_section_end|>" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "<|tool_call_begin|>" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 40, 748, 2390, 276, 7697, 13, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "<|tool_calls_section_begin|>" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.add:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "<|tool_calls_section_end|>" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "<|tool_calls_section_begin|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "const_string", + "value": "<|tool_calls_section_end|>" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163596, 163586] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.json b/rust/src/chat/tests/grammar_replay/Kimi-K3.json new file mode 100644 index 0000000000..ffe5061192 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.json @@ -0,0 +1,587 @@ +{ + "model_id": "moonshotai/Kimi-K3", + "vocab_size": 163840, + "all_stop_token_ids": [163585, 163586], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + } + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "final_output_only" + }, + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "final_output_only" + }, + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "final_output_only" + }, + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json new file mode 100644 index 0000000000..582f29a1b9 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json @@ -0,0 +1,254 @@ +{ + "model_id": "XiaomiMiMo/MiMo-V2.6-Flash-RL", + "vocab_size": 152576, + "all_stop_token_ids": [151643, 151645, 151672], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "Need call the weather and add tools.I will call the tools.Shanghai1.02[\"left\", \"right\"]", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 40, 686, 1618, 279, 7375, 13, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "Need call the weather and add tools.Shanghai1.02[\"left\", \"right\"]", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.Shanghai", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151645] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json new file mode 100644 index 0000000000..64f65e2f5a --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json @@ -0,0 +1,301 @@ +{ + "model_id": "MiniMaxAI/MiniMax-M2.5", + "vocab_size": 200064, + "all_stop_token_ids": [200020], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 73, 715, 1728, 275, 6629, 320, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json new file mode 100644 index 0000000000..5d171fc5f0 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json @@ -0,0 +1,277 @@ +{ + "model_id": "Qwen/Qwen3-0.6B", + "vocab_size": 151936, + "all_stop_token_ids": [151643, 151645], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + }, + { + "type": "tag", + "begin": "\n{\"name\": \"add\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "\nNeed call the weather and add tools.\n\n\nI will call the tools.\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 40, 686, 1618, 279, 7375, 624, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + }, + { + "type": "tag", + "begin": "\n{\"name\": \"add\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 151645] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json new file mode 100644 index 0000000000..8bd4c901e5 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json @@ -0,0 +1,268 @@ +{ + "model_id": "Qwen/Qwen3.5-4B", + "vocab_size": 248320, + "all_stop_token_ids": [248044, 248046], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "\n\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 40, 668, 1562, 279, 7141, 13, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "\n\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + } + } +} diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index a0ccbe59ab..6925ae1b02 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -6,20 +6,31 @@ //! The invariant under test is that a structured assistant message rendered as history can be //! parsed from the generated assistant completion and then rendered back to the exact same //! assistant completion. +//! +//! The tool-call fixture also snapshots the output grammar built by the initialized parser for each +//! tool-choice variant, with the rendered completion as its generation, as one file per model under +//! `tests/grammar_replay/`. `grammar_replay.rs` replays these cases through the real XGrammar. +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Debug; +use std::ops::Range; +use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use anyhow::{Context as _, Result, bail, ensure}; use futures::{Stream, StreamExt as _, stream}; +use serde::ser::Error as _; +use serde::{Serialize, Serializer}; use serde_json_fmt::JsonFormat as JsonFmt; use serial_test::file_serial; use vllm_chat::{ AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall, - ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, FinishReason, GenerationPromptMode, - LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, RendererSelection, - load_model_backends, + ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, FinishReason, + GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, + RendererSelection, ToolStrictLevel, load_model_backends, }; +use vllm_parser::output_grammar::BuiltOutputGrammar; use vllm_text::{DecodedTextEvent, Finished, Prompt}; use vllm_tokenizer::Tokenizer; @@ -114,6 +125,33 @@ impl ThinkingBehavior { } } +/// Tool-choice variant of the tool-call fixture. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +enum ToolChoiceVariant { + /// `auto`, with content before the tool calls. + Auto, + /// `required`, with tool calls only: builder grammars admit no content + /// before the calls. + Required, + /// Named `get_weather`, with a single call to it and no content. + Named, +} + +impl ToolChoiceVariant { + const ALL: [Self; 3] = [Self::Auto, Self::Required, Self::Named]; + + fn tool_choice(self) -> ChatToolChoice { + match self { + Self::Auto => ChatToolChoice::Auto, + Self::Required => ChatToolChoice::Required, + Self::Named => ChatToolChoice::Function { + name: "get_weather".to_string(), + }, + } + } +} + impl RoundtripCase { /// Qwen3 XML tool-call format with `qwen3` reasoning tags. fn qwen3() -> Self { @@ -451,6 +489,7 @@ async fn run_roundtrip_reasoning_and_content_inner( "roundtrip-reasoning-content", vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")], Vec::new(), + None, thinking, case.thinking_behavior, ); @@ -489,11 +528,25 @@ async fn run_roundtrip_reasoning_and_content_inner( Ok(()) } -/// Run the fixed reasoning+multiple-tools fixture for one model/parser case. +/// Run the fixed reasoning+multiple-tools fixture for one model/parser case, +/// once per tool-choice variant. async fn run_roundtrip_tool_call_mix( case: &RoundtripCase, backends: &vllm_chat::LoadedModelBackends, ) -> Result<()> { + let mut results = Vec::new(); + for variant in ToolChoiceVariant::ALL { + let result = run_roundtrip_tool_call_mix_inner(case, backends, variant).await?; + results.push((variant, result)); + } + check_grammar_replay_file(case, backends, results) +} + +async fn run_roundtrip_tool_call_mix_inner( + case: &RoundtripCase, + backends: &vllm_chat::LoadedModelBackends, + variant: ToolChoiceVariant, +) -> Result { let request = roundtrip_request( "roundtrip-reasoning-tools", vec![ChatMessage::text( @@ -501,41 +554,40 @@ async fn run_roundtrip_tool_call_mix( "Check Shanghai weather and add 1.0 plus 2.", )], test_tools(), + Some(variant.tool_choice()), Some(true), // always enable thinking in this fixture case.thinking_behavior, ); let expected_reasoning = "Need call the weather and add tools."; - let expected_text = "I will call the tools."; + let (expected_text, tool_call_count) = match variant { + ToolChoiceVariant::Auto => (Some("I will call the tools."), 2), + ToolChoiceVariant::Required => (None, 2), + ToolChoiceVariant::Named => (None, 1), + }; - let result = run_roundtrip( - case, - backends, - &request, - AssistantMessage { - content: vec![ - AssistantContentBlock::Reasoning { - text: expected_reasoning.to_string(), - }, - AssistantContentBlock::Text { - text: expected_text.to_string(), - }, - AssistantContentBlock::ToolCall(AssistantToolCall { - id: "functions.get_weather:0".to_string(), - name: "get_weather".to_string(), - arguments: r#"{"location":"Shanghai"}"#.to_string(), - }), - AssistantContentBlock::ToolCall(AssistantToolCall { - id: "functions.add:1".to_string(), - name: "add".to_string(), - // Intentionally use a non-lexical order of keys to verify text-level - // fidelity of the roundtrip where JSON formatting remains stable. The - // `items` key also exercises templates that call `arguments.items()`. - arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(), - }), - ], - }, - ) - .await?; + let mut content = vec![AssistantContentBlock::Reasoning { + text: expected_reasoning.to_string(), + }]; + content.extend(expected_text.map(|text| AssistantContentBlock::Text { + text: text.to_string(), + })); + let tool_calls = [ + AssistantContentBlock::ToolCall(AssistantToolCall { + id: "functions.get_weather:0".to_string(), + name: "get_weather".to_string(), + arguments: r#"{"location":"Shanghai"}"#.to_string(), + }), + AssistantContentBlock::ToolCall(AssistantToolCall { + id: "functions.add:1".to_string(), + name: "add".to_string(), + // Intentionally use a non-lexical order of keys to verify text-level + // fidelity of the roundtrip where JSON formatting remains stable. The + // `items` key also exercises templates that call `arguments.items()`. + arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(), + }), + ]; + content.extend(tool_calls.into_iter().take(tool_call_count)); + let result = run_roundtrip(case, backends, &request, AssistantMessage { content }).await?; assert_eq!( result.parsed_message.reasoning().as_deref(), @@ -543,12 +595,15 @@ async fn run_roundtrip_tool_call_mix( "parsed message: {:#?}", result.parsed_message ); - assert_eq!(result.parsed_message.text(), expected_text); + assert_eq!( + result.parsed_message.text(), + expected_text.unwrap_or_default() + ); let tool_calls = result.parsed_message.tool_calls().collect::>(); assert_eq!( tool_calls.len(), - 2, + tool_call_count, "parsed message: {:#?}", result.parsed_message ); @@ -557,18 +612,20 @@ async fn run_roundtrip_tool_call_mix( tool_calls[0].arguments, expected_arguments(case, r#"{"location": "Shanghai"}"#)?, ); - assert_eq!(tool_calls[1].name, "add"); - assert_eq!( - tool_calls[1].arguments, - expected_arguments(case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?, - ); + if let Some(add) = tool_calls.get(1) { + assert_eq!(add.name, "add"); + assert_eq!( + add.arguments, + expected_arguments(case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?, + ); + } assert_eq!( result.rerendered_closed_completion, result.closed_completion ); - Ok(()) + Ok(result) } /// Compact JSON argument formatting used by JSON-native parsers/renderers. @@ -645,6 +702,9 @@ struct RoundtripResult { /// Assistant-completion suffix cut after rendering the parsed assistant /// back as history. rerendered_closed_completion: Prompt, + /// Output grammar built by the same initialized parser that parsed the + /// completion. + output_grammar: Option, } /// Render, parse, and rerender one assistant turn through the production @@ -658,13 +718,17 @@ async fn run_roundtrip( let renderer = backends.chat_backend.chat_renderer(); let rendered = render_closed_completion(renderer.as_ref(), request, &assistant)?; - let parsed_message = parse_completion(case, backends, request, &rendered).await?; + let ParsedCompletion { + message: parsed_message, + output_grammar, + } = parse_completion(case, backends, request, &rendered).await?; let rerendered = render_closed_completion(renderer.as_ref(), request, &parsed_message)?; Ok(RoundtripResult { parsed_message, closed_completion: rendered.completion, rerendered_closed_completion: rerendered.completion, + output_grammar, }) } @@ -675,7 +739,7 @@ struct RenderedTurn { } /// Render `history` as a production prompt and `history + assistant` as closed -/// history, then return the production prompt and assistant-completion suffix. +/// history, then return the production prompt and the assistant completion. fn render_closed_completion( renderer: &dyn vllm_chat::ChatRenderer, base_request: &ChatRequest, @@ -692,18 +756,15 @@ fn render_closed_completion( let completion = match (&prompt, full) { (Prompt::Text(prompt), Prompt::Text(full)) => { - ensure!( - full.starts_with(prompt), - "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}" - ); - Prompt::Text(full[prompt.len()..].to_string()) + let range = completion_range(prompt.as_bytes(), full.as_bytes()) + .with_context(|| format!("prompt: {prompt:?}\nfull: {full:?}"))?; + let completion = full.get(range).context("completion splits a UTF-8 character")?; + Prompt::Text(completion.to_string()) } (Prompt::TokenIds(prompt), Prompt::TokenIds(full)) => { - ensure!( - full.starts_with(prompt), - "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}" - ); - Prompt::TokenIds(full[prompt.len()..].to_vec()) + let range = completion_range(prompt, &full) + .with_context(|| format!("prompt: {prompt:?}\nfull: {full:?}"))?; + Prompt::TokenIds(full[range].to_vec()) } (prompt, full) => bail!("prompt kind changed between renders: {prompt:?} vs {full:?}"), }; @@ -711,14 +772,81 @@ fn render_closed_completion( Ok(RenderedTurn { prompt, completion }) } +/// Locate the assistant completion in `full`, the history rendered with the +/// assistant message, given `prompt`, the same history rendered for generation. +/// +/// Some templates render request-scoped control messages after the whole +/// history, such as Kimi K3's `tool_choice=required` message. With history `A`, +/// control messages `C`, generation prompt `G`, and completion `B`, the renders +/// are `prompt = A + C + G` and `full = A + G + B + C`. `C` is empty for most +/// templates, which makes `prompt` a prefix of `full`. +fn completion_range(prompt: &[T], full: &[T]) -> Result> { + if full.starts_with(prompt) { + return Ok(prompt.len()..full.len()); + } + ensure!( + full.len() >= prompt.len(), + "full prompt is shorter than production prompt" + ); + + // `A` is a common prefix of both renders; try every split of the remaining + // `prompt` into `C + G` and require exactly one to fit `full`. + let common_prefix_len = prompt.iter().zip(full).take_while(|(p, f)| p == f).count(); + let mut control_lens = Vec::new(); + for history_len in 0..=common_prefix_len { + let rest = &prompt[history_len..]; + for control_len in 1..=rest.len() { + let (control, generation) = rest.split_at(control_len); + if full[history_len..].starts_with(generation) && full.ends_with(control) { + control_lens.push(control_len); + } + } + } + let [control_len] = control_lens[..] else { + bail!( + "full prompt must be `A + G + B + C` for production prompt `A + C + G`; found {} splits", + control_lens.len() + ); + }; + + Ok(prompt.len() - control_len..full.len() - control_len) +} + +#[test] +fn completion_range_skips_trailing_control_messages() { + let history = "hi"; + let control = "call tools"; + let generation = ""; + let completion = "call()"; + + let prompt = format!("{history}{generation}"); + let full = format!("{history}{generation}{completion}"); + let range = completion_range(prompt.as_bytes(), full.as_bytes()).unwrap(); + assert_eq!(&full[range], completion); + + let prompt = format!("{history}{control}{generation}"); + let full = format!("{history}{generation}{completion}{control}"); + let range = completion_range(prompt.as_bytes(), full.as_bytes()).unwrap(); + assert_eq!(&full[range], completion); +} + +/// Output of one production output processor over a rendered completion. +struct ParsedCompletion { + /// Terminal assistant message. + message: AssistantMessage, + /// Output grammar built by the processor after initialization. + output_grammar: Option, +} + /// Feed one rendered assistant completion body into the real output processor -/// and collect its terminal assistant message. +/// and collect its terminal assistant message, along with the output grammar +/// the initialized processor builds for the request. async fn parse_completion( case: &RoundtripCase, backends: &vllm_chat::LoadedModelBackends, base_request: &ChatRequest, rendered: &RenderedTurn, -) -> Result { +) -> Result { let tokenizer = backends.text_backend.tokenizer(); let prompt_token_ids = match &rendered.prompt { Prompt::Text(prompt) => tokenizer @@ -731,12 +859,15 @@ async fn parse_completion( let mut processor = backends.chat_backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { - tool_strict_level: vllm_chat::ToolStrictLevel::Auto, + // Pin every tool schema so that builder-backed parsers build an + // output grammar for replay. The strict level does not affect parsing. + tool_strict_level: ToolStrictLevel::Parameter, tool_call_parser: &case.tool_call_parser, reasoning_parser: &case.reasoning_parser, }, )?; processor.initialize(&prompt_token_ids)?; + let output_grammar = processor.build_output_grammar()?; let decoded = decoded_completion_stream( tokenizer.as_ref(), @@ -748,7 +879,10 @@ async fn parse_completion( while let Some(event) = events.next().await { if let ChatEvent::Done { message, .. } = event? { - return Ok(message); + return Ok(ParsedCompletion { + message, + output_grammar, + }); } } @@ -772,14 +906,8 @@ fn decoded_completion_stream( prompt_logprobs: None, }]; - let chunks = match completion { - Prompt::Text(text) => { - let body = text.strip_suffix(assistant_stop_suffix).with_context(|| { - format!( - "closed assistant completion did not end with {:?}: {:?}", - assistant_stop_suffix, text - ) - })?; + let chunks = match completion_body(tokenizer, completion, assistant_stop_suffix)? { + CompletionBody::Text(body) => { // The engine returns token ids, so attribute the rendered text // through the production decoder, then split it into small char // chunks to exercise marker splits. @@ -807,27 +935,12 @@ fn decoded_completion_stream( }) .collect() } - Prompt::TokenIds(token_ids) => { - let body = if assistant_stop_suffix.is_empty() { - token_ids.as_slice() - } else { - let stop_token_ids = tokenizer - .encode(assistant_stop_suffix, false) - .context("failed to encode token-id completion stop suffix")?; - token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| { - format!( - "token-id completion did not end with {:?}: {:?}", - assistant_stop_suffix, token_ids - ) - })? - }; - incremental_decode_chunks( - tokenizer, - &prompt_token_ids, - body, - TOKEN_COMPLETION_CHUNK_TOKENS, - )? - } + CompletionBody::TokenIds(body) => incremental_decode_chunks( + tokenizer, + &prompt_token_ids, + body, + TOKEN_COMPLETION_CHUNK_TOKENS, + )?, }; if chunks.is_empty() { @@ -868,6 +981,48 @@ fn decoded_completion_stream( Ok(stream::iter(events).map(Ok).boxed()) } +/// Generated part of a closed assistant completion, without the rendered stop +/// suffix. +enum CompletionBody<'a> { + Text(&'a str), + TokenIds(&'a [u32]), +} + +/// Strip the rendered stop suffix from a closed assistant completion. +fn completion_body<'a>( + tokenizer: &dyn Tokenizer, + completion: &'a Prompt, + assistant_stop_suffix: &str, +) -> Result> { + match completion { + Prompt::Text(text) => { + let body = text.strip_suffix(assistant_stop_suffix).with_context(|| { + format!( + "closed assistant completion did not end with {:?}: {:?}", + assistant_stop_suffix, text + ) + })?; + Ok(CompletionBody::Text(body)) + } + Prompt::TokenIds(token_ids) => { + let body = if assistant_stop_suffix.is_empty() { + token_ids.as_slice() + } else { + let stop_token_ids = tokenizer + .encode(assistant_stop_suffix, false) + .context("failed to encode token-id completion stop suffix")?; + token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| { + format!( + "token-id completion did not end with {:?}: {:?}", + assistant_stop_suffix, token_ids + ) + })? + }; + Ok(CompletionBody::TokenIds(body)) + } + } +} + /// One decoded completion chunk fed into the output processor. struct DecodedCompletionChunk { /// Attributed for token-id completions, as the production decoder emits it. @@ -941,15 +1096,138 @@ fn incremental_decode_chunks( Ok(chunks) } +/// Grammar replay cases of one model, replayed through the real XGrammar by +/// `grammar_replay.py`. +#[derive(Serialize)] +struct GrammarReplayFile { + /// Hugging Face model id whose tokenizer the replay loads. + model_id: &'static str, + /// Model vocabulary size, which sizes the XGrammar token mask. + vocab_size: usize, + /// Request stop set, passed to the matcher as `override_stop_tokens`. + #[serde(serialize_with = "serialize_one_line")] + all_stop_token_ids: BTreeSet, + /// Cases keyed by tool-choice variant. + cases: BTreeMap, +} + +/// One parser-built output grammar with the generation it must accept. +#[derive(Serialize)] +struct GrammarReplayCase { + /// Parser-built grammar; only token-zero grammars are replayed from the + /// first generated token. + grammar: BuiltOutputGrammar, + /// Generated completion text, without the stop token. + completion: String, + /// Generated token IDs presented to the matcher, ending with a stop token. + #[serde(serialize_with = "serialize_one_line")] + generation_token_ids: Vec, +} + +/// Serialize a token ID list on one line inside the pretty-printed file. +fn serialize_one_line( + token_ids: &impl Serialize, + serializer: S, +) -> std::result::Result { + let json = spaced_json_fmt().format_to_string(token_ids).map_err(S::Error::custom)?; + serde_json::value::RawValue::from_string(json) + .map_err(S::Error::custom)? + .serialize(serializer) +} + +/// Directory of checked-in grammar replay files. +fn grammar_replay_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay") +} + +/// Snapshot the output grammars of one model's roundtrips, keyed by variant, +/// as its grammar replay file, or check that no stale file remains when the +/// parser builds none. +fn check_grammar_replay_file( + case: &RoundtripCase, + backends: &vllm_chat::LoadedModelBackends, + results: Vec<(ToolChoiceVariant, RoundtripResult)>, +) -> Result<()> { + let model_name = case.model_id.rsplit_once('/').map_or(case.model_id, |(_, name)| name); + let path = grammar_replay_dir().join(format!("{model_name}.json")); + let results = results + .into_iter() + .filter_map(|(variant, result)| { + Some((variant, result.output_grammar?, result.closed_completion)) + }) + .collect::>(); + if results.is_empty() { + ensure!( + !path.exists(), + "{} builds no output grammar, but grammar replay file {path:?} exists", + case.model_id + ); + return Ok(()); + } + + let tokenizer = backends.text_backend.tokenizer(); + let hints = backends.text_backend.sampling_hints()?; + // Lowered `_all_stop_token_ids` of a request without user stop tokens. + let mut all_stop_token_ids = hints.extra_eos_token_ids; + all_stop_token_ids.extend(hints.primary_eos_token_id); + + // The generation ends with the first token of the rendered stop suffix, or + // with the primary EOS for templates that render none. + let stop_token_id = if case.assistant_stop_suffix.is_empty() { + hints.primary_eos_token_id.context("model has no primary EOS token")? + } else { + tokenizer.encode(case.assistant_stop_suffix, false)?[0] + }; + ensure!( + all_stop_token_ids.contains(&stop_token_id), + "stop token {stop_token_id} of {:?} is not in the stop set {all_stop_token_ids:?}", + case.assistant_stop_suffix + ); + + let mut cases = BTreeMap::new(); + for (variant, grammar, closed_completion) in results { + let (completion, mut generation_token_ids) = match completion_body( + tokenizer.as_ref(), + &closed_completion, + case.assistant_stop_suffix, + )? { + CompletionBody::Text(body) => (body.to_string(), tokenizer.encode(body, false)?), + CompletionBody::TokenIds(body) => (tokenizer.decode(body, false)?, body.to_vec()), + }; + generation_token_ids.push(stop_token_id); + cases.insert( + variant, + GrammarReplayCase { + grammar, + completion, + generation_token_ids, + }, + ); + } + + let file = GrammarReplayFile { + model_id: case.model_id, + vocab_size: backends.text_backend.model_vocab_size(), + all_stop_token_ids, + cases, + }; + let mut json = serde_json::to_string_pretty(&file)?; + json.push('\n'); + expect_test::expect_file![path].assert_eq(&json); + + Ok(()) +} + /// Build a chat request fixture with parser-enabling tool-choice semantics. fn roundtrip_request( request_id: impl Into, messages: Vec, tools: Vec, + tool_choice: Option, thinking: Option, thinking_behavior: ThinkingBehavior, ) -> ChatRequest { - let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true) + let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, tool_choice, true) .expect("tool context should resolve"); let mut request = ChatRequest { request_id: request_id.into(), diff --git a/rust/src/parser/src/output_grammar.rs b/rust/src/parser/src/output_grammar.rs index f19254d364..db56f97708 100644 --- a/rust/src/parser/src/output_grammar.rs +++ b/rust/src/parser/src/output_grammar.rs @@ -5,6 +5,7 @@ use std::{fmt, str::FromStr}; +use serde::Serialize; use serde_with::{DeserializeFromStr, SerializeDisplay}; use thiserror::Error; use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagOptions}; @@ -20,7 +21,8 @@ use crate::tool::Tool; pub type Result = std::result::Result; /// How much of the generated stream an output grammar covers. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] pub enum GrammarCoverage { /// The grammar covers reasoning and advances from the first generated token. FromTokenZero, @@ -29,7 +31,7 @@ pub enum GrammarCoverage { } /// One parser-built output grammar and its generated-stream coverage. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize)] pub struct BuiltOutputGrammar { /// Typed XGrammar structural-tag format. pub format: Format,