From 5a0906263209da2c1165ce82b9fe8e767aa89669 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Mon, 12 Oct 2026 17:43:48 +0800 Subject: [PATCH] [Rust Frontend] Compose answer constraints into parser-owned output grammars (#60781) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- rust/src/chat/src/output/default/mod.rs | 55 ++ rust/src/chat/src/output/structural_tag.rs | 165 +++++- rust/src/chat/tests/grammar_replay.py | 93 ++- .../grammar_replay/DeepSeek-V3.2-Exp.json | 211 ++++++- .../grammar_replay/DeepSeek-V3.2-Exp.txt | 17 + .../grammar_replay/DeepSeek-V4-Flash.json | 211 ++++++- .../grammar_replay/DeepSeek-V4-Flash.txt | 17 + .../grammar_replay/DeepSeek-V4.1-Flash.json | 211 ++++++- .../grammar_replay/DeepSeek-V4.1-Flash.txt | 17 + .../chat/tests/grammar_replay/GLM-4.5.json | 68 +++ .../src/chat/tests/grammar_replay/GLM-4.5.txt | 7 + .../tests/grammar_replay/GLM-4.7-Flash.json | 209 ++++++- .../tests/grammar_replay/GLM-4.7-Flash.txt | 14 + .../tests/grammar_replay/GLM-5.2-FP8.json | 209 ++++++- .../chat/tests/grammar_replay/GLM-5.2-FP8.txt | 14 + rust/src/chat/tests/grammar_replay/Hy3.json | 321 +++++++++- rust/src/chat/tests/grammar_replay/Hy3.txt | 50 ++ .../chat/tests/grammar_replay/Kimi-K2.5.json | 188 +++++- .../chat/tests/grammar_replay/Kimi-K2.5.txt | 20 + .../chat/tests/grammar_replay/Kimi-K3.json | 549 +++++++++++++++++- .../src/chat/tests/grammar_replay/Kimi-K3.txt | 55 ++ .../grammar_replay/MiMo-V2.6-Flash-RL.json | 212 ++++++- .../grammar_replay/MiMo-V2.6-Flash-RL.txt | 14 + .../tests/grammar_replay/MiniMax-M2.5.json | 229 +++++++- .../tests/grammar_replay/MiniMax-M2.5.txt | 21 + .../chat/tests/grammar_replay/MiniMax-M3.json | 40 ++ .../chat/tests/grammar_replay/MiniMax-M3.txt | 2 + .../chat/tests/grammar_replay/Qwen3-0.6B.json | 228 +++++++- .../chat/tests/grammar_replay/Qwen3-0.6B.txt | 20 + .../chat/tests/grammar_replay/Qwen3.5-4B.json | 222 ++++++- .../chat/tests/grammar_replay/Qwen3.5-4B.txt | 18 + .../grammar_replay/Seed-OSS-36B-Instruct.json | 40 ++ .../grammar_replay/Seed-OSS-36B-Instruct.txt | 2 + rust/src/chat/tests/roundtrip.rs | 188 ++++-- rust/src/parser/src/output_grammar.rs | 9 + rust/src/parser/src/tool/minimax_m2.rs | 2 +- rust/src/parser/src/tool/mod.rs | 69 ++- rust/src/parser/src/unified/combined.rs | 114 +++- rust/src/parser/src/unified/hy/hy_v3.rs | 2 + rust/src/parser/src/unified/hy/hy_v4.rs | 1 + .../src/parser/src/unified/kimi_k3/grammar.rs | 254 ++++++-- 41 files changed, 4133 insertions(+), 255 deletions(-) create mode 100644 rust/src/chat/tests/grammar_replay/GLM-4.5.json create mode 100644 rust/src/chat/tests/grammar_replay/GLM-4.5.txt create mode 100644 rust/src/chat/tests/grammar_replay/MiniMax-M3.json create mode 100644 rust/src/chat/tests/grammar_replay/MiniMax-M3.txt create mode 100644 rust/src/chat/tests/grammar_replay/Seed-OSS-36B-Instruct.json create mode 100644 rust/src/chat/tests/grammar_replay/Seed-OSS-36B-Instruct.txt diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index b31c5e9e4c..981173dc74 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -13,8 +13,10 @@ use vllm_parser::output_grammar::{BuiltOutputGrammar, OutputGrammarContext}; use vllm_parser::unified::{CombinedParser, UnifiedParser}; use vllm_text::tokenizer::DynTokenizer; use xgrammar_structural_tag::ToolChoice; +use xgrammar_structural_tag::format::Format; use self::unified::unified_event_stream; +use super::structural_tag::answer_format; use super::structured::structured_chat_event_stream; use crate::error::Result; use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream}; @@ -48,6 +50,10 @@ struct GrammarInputs { tools: Vec, tool_choice: ToolChoice, tool_strict_level: ToolStrictLevel, + /// The request's structured-output constraint, normalized for composition + /// into the output grammar. The original stays on the request until an + /// output grammar replaces it. + answer: Option, } impl DefaultChatOutputProcessor { @@ -137,6 +143,7 @@ impl DefaultChatOutputProcessor { tools: request.tools().to_vec(), tool_choice: request.tool_choice().into(), tool_strict_level, + answer: request.sampling_params.structured_outputs.as_ref().and_then(answer_format), }), }) } @@ -258,6 +265,7 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor { tool_choice: &inputs.tool_choice, tool_strict_level: inputs.tool_strict_level, parallel_tool_calls: self.parallel_tool_calls, + answer: inputs.answer.as_ref(), }) .map_err(|error| Error::OutputGrammar { error: Box::new(error), @@ -283,7 +291,9 @@ mod tests { use std::sync::Arc; use thiserror_ext::AsReport as _; + use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams; use vllm_tokenizer::test_utils::TestTokenizer; + use xgrammar_structural_tag::format::Format; use super::DefaultChatOutputProcessor; use crate::output::ChatOutputProcessor; @@ -339,6 +349,51 @@ mod tests { assert!(build(ToolStrictLevel::Parameter, None, ChatToolChoice::None).is_none()); } + #[test] + fn answer_constraint_keeps_optional_tool_calls() { + let tools = vec![ChatTool { + name: "search".to_string(), + description: None, + parameters: serde_json::json!({"type": "object"}), + strict: None, + defer_loading: None, + }]; + let schema = serde_json::json!({"type": "object"}); + let build = |structured_outputs| { + let mut request = ChatRequest { + tool_context: ResolvedToolContext::new( + &[], + tools.clone(), + Some(ChatToolChoice::Auto), + true, + ) + .unwrap(), + ..ChatRequest::for_test() + }; + request.sampling_params.structured_outputs = structured_outputs; + let mut processor = DefaultChatOutputProcessor::new( + &mut request, + "other-model", + tokenizer(), + &ParserSelection::Explicit("qwen3_coder".to_string()), + &ParserSelection::None, + ToolStrictLevel::Auto, + ) + .unwrap(); + processor.initialize(&[]).unwrap(); + processor.build_output_grammar().unwrap() + }; + + // Non-strict `auto` alone needs no grammar; with an answer constraint + // the grammar holds the answer or a call. + assert!(build(None).is_none()); + let built = build(Some(StructuredOutputsParams::json(schema.clone()))).unwrap(); + let Format::Or(branches) = built.format else { + panic!("expected the calls or the answer, got {:?}", built.format); + }; + assert_eq!(branches.elements.last(), Some(&Format::json_schema(schema))); + } + #[test] fn tool_grammar_honors_parallel_call_policy() { for parallel in [false, true] { diff --git a/rust/src/chat/src/output/structural_tag.rs b/rust/src/chat/src/output/structural_tag.rs index dba6b6d1b7..b99a32d5bd 100644 --- a/rust/src/chat/src/output/structural_tag.rs +++ b/rust/src/chat/src/output/structural_tag.rs @@ -3,19 +3,91 @@ //! Applies parser-built xgrammar structural-tag constraints. +use serde::Deserialize; +use serde_json::{Value, json}; use vllm_engine_core_client::protocol::structured_outputs::{ - StructuredOutputBackend, StructuredOutputsParams, + StructuredOutputBackend, StructuredOutputConstraint, StructuredOutputsParams, }; use vllm_parser::output_grammar::{BuiltOutputGrammar, GrammarCoverage}; use vllm_text::TextRequest; use xgrammar_structural_tag::StructuralTag; +use xgrammar_structural_tag::format::{Format, GrammarFormat, TagFormat}; use crate::{Error, Result}; +/// Normalize the user's structured-output constraint to the answer format a +/// parser may compose into its output grammar. +/// +/// Like Python's `structured_outputs_to_format`, this ignores the request's +/// options: no backend reads them per request, only from the engine config. +/// +/// Returns `None` when the constraint must stay on its own wire field, so the +/// request keeps today's behavior: a grammar that is not XGrammar EBNF (such +/// as Lark), or a structural tag that does not parse. +pub(crate) fn answer_format(params: &StructuredOutputsParams) -> Option { + Some(match ¶ms.constraint { + StructuredOutputConstraint::Json(Value::String(schema)) => { + Format::json_schema(serde_json::from_str(schema).ok()?) + } + StructuredOutputConstraint::Json(schema) => Format::json_schema(schema.clone()), + StructuredOutputConstraint::JsonObject => Format::json_schema(json!({ "type": "object" })), + StructuredOutputConstraint::Regex(pattern) => Format::regex(pattern), + StructuredOutputConstraint::Choice(choices) => { + Format::or(choices.iter().map(Format::const_string).collect()) + } + // Lark rules use `:`; XGrammar EBNF rules use `::=`. + StructuredOutputConstraint::Grammar(grammar) if grammar.contains("::=") => { + Format::Grammar(GrammarFormat { + grammar: grammar.clone(), + }) + } + StructuredOutputConstraint::Grammar(_) => return None, + StructuredOutputConstraint::StructuralTag(tag) => structural_tag_format(tag)?, + }) +} + +/// The legacy structural tag shape: free text with JSON tags behind triggers. +#[derive(Deserialize)] +struct LegacyStructuralTag { + structures: Vec, + triggers: Vec, +} + +#[derive(Deserialize)] +struct LegacyStructure { + begin: String, + schema: Value, + end: String, +} + +/// Read a structural tag in its current (`format`) or legacy +/// (`structures`/`triggers`) shape. +fn structural_tag_format(tag: &str) -> Option { + let value: Value = serde_json::from_str(tag).ok()?; + if value.get("structures").is_none() { + return Some(serde_json::from_value::(value).ok()?.format); + } + let legacy: LegacyStructuralTag = serde_json::from_value(value).ok()?; + let triggers: Vec<&str> = legacy.triggers.iter().map(String::as_str).collect(); + let tags = legacy + .structures + .into_iter() + .map(|structure| { + TagFormat::new( + structure.begin, + Format::json_schema(structure.schema), + structure.end, + ) + }) + .collect(); + Some(Format::triggered_tags(&triggers, tags)) +} + /// Apply one parser-built output grammar to the prepared text request. /// -/// A parser-owned grammar replaces any structured output constraint the user -/// supplied, matching the Python frontend. `None` leaves the request untouched. +/// The parser has already composed the user's answer constraint into the +/// grammar where its protocol allows, so the grammar replaces the request's +/// structured outputs. `None` leaves the request untouched. pub(crate) fn apply_output_grammar( request: &mut TextRequest, built: Option, @@ -45,13 +117,92 @@ pub(crate) fn apply_output_grammar( #[cfg(test)] mod tests { use serde_json::json; - use vllm_engine_core_client::protocol::structured_outputs::{ - StructuredOutputBackend, StructuredOutputsParams, - }; - use xgrammar_structural_tag::format::Format; + use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputOptions; use super::*; + #[test] + fn answer_format_normalizes_composable_constraints() { + let schema = json!({ "type": "object", "properties": { "a": { "type": "integer" } } }); + let cases = [ + ( + StructuredOutputsParams::json(schema.clone()), + Format::json_schema(schema.clone()), + ), + ( + StructuredOutputsParams::json(Value::String(schema.to_string())), + Format::json_schema(schema), + ), + ( + StructuredOutputsParams::json_object(), + Format::json_schema(json!({ "type": "object" })), + ), + ( + StructuredOutputsParams::regex("[a-z]+"), + Format::regex("[a-z]+"), + ), + ( + StructuredOutputsParams::choice(vec!["yes".to_string(), "no".to_string()]), + Format::or(vec![ + Format::const_string("yes"), + Format::const_string("no"), + ]), + ), + ( + StructuredOutputsParams::grammar("root ::= \"a\""), + Format::Grammar(GrammarFormat { + grammar: "root ::= \"a\"".to_string(), + }), + ), + ( + StructuredOutputsParams::structural_tag( + r#"{"type": "structural_tag", "format": {"type": "const_string", "value": "x"}}"#, + ), + Format::const_string("x"), + ), + ( + StructuredOutputsParams::structural_tag( + r#"{"type": "structural_tag", "structures": [{"begin": "", "schema": {"type": "object"}, "end": ""}], "triggers": ["", + Format::json_schema(json!({ "type": "object" })), + "", + )], + ), + ), + ( + StructuredOutputsParams { + options: StructuredOutputOptions { + disable_any_whitespace: true, + disable_additional_properties: true, + whitespace_pattern: Some(" ?".to_string()), + }, + ..StructuredOutputsParams::json_object() + }, + Format::json_schema(json!({ "type": "object" })), + ), + ]; + for (params, expected) in cases { + assert_eq!(answer_format(¶ms), Some(expected), "{params:?}"); + } + } + + #[test] + fn answer_format_leaves_other_constraints_on_their_wire_field() { + let cases = [ + StructuredOutputsParams::json(Value::String("{".to_string())), + StructuredOutputsParams::grammar("start: \"a\""), + StructuredOutputsParams::structural_tag(r#"{"structures": [{"begin": ""}]}"#), + StructuredOutputsParams::structural_tag(r#"{"format": {"type": "unknown"}}"#), + ]; + for params in cases { + assert_eq!(answer_format(¶ms), None, "{params:?}"); + } + } + #[test] fn output_grammar_overwrites_answer_constraint() { let mut request = TextRequest::for_test(); diff --git a/rust/src/chat/tests/grammar_replay.py b/rust/src/chat/tests/grammar_replay.py index d34df627ba..056ab1d352 100755 --- a/rust/src/chat/tests/grammar_replay.py +++ b/rust/src/chat/tests/grammar_replay.py @@ -22,14 +22,17 @@ # /// """Replay the grammar cases exported by the roundtrip tests through XGrammar. -Each file holds one model's cases, keyed by tool-choice variant. Every case is -compiled against the model tokenizer the same way as -`vllm/v1/structured_output/backend_xgrammar.py`. Token-zero cases are then -replayed from the first generated token: each token must be allowed by the -bitmask and accepted by the matcher, and the final stop token must terminate it. +Each file holds one model's distinct output grammars, each with the generations +of the fixture variants that build it. Every grammar is compiled against the +model tokenizer the same way as `vllm/v1/structured_output/backend_xgrammar.py`. +The generations of token-zero grammars are then replayed from the first +generated token: each token must be allowed by the bitmask and accepted by the +matcher, and the final stop token must terminate it. """ import json +import multiprocessing +import os import sys from functools import cache from pathlib import Path @@ -49,23 +52,29 @@ def load_tokenizer(model_id: str, vocab_size: int): return tokenizer, xgr.GrammarCompiler(tokenizer_info) -def replay(model: dict, case: dict) -> str: - """Check one case of a model and return a summary of what was checked.""" - tokenizer, compiler = load_tokenizer(model["model_id"], model["vocab_size"]) - generation = case["generation_token_ids"] +def compile_grammar(model: dict, grammar: dict) -> xgr.CompiledGrammar: + """Compile one output grammar of a model.""" + _, compiler = load_tokenizer(model["model_id"], model["vocab_size"]) + structural_tag = {"type": "structural_tag", "format": grammar["format"]} + return compiler.compile_structural_tag(json.dumps(structural_tag)) - # The case token IDs come from the Rust tokenizer; the runner must load the + +def replay( + model: dict, grammar: dict, compiled: xgr.CompiledGrammar, generation: dict +) -> str: + """Check one generation of a grammar and return a summary of what was checked.""" + tokenizer, _ = load_tokenizer(model["model_id"], model["vocab_size"]) + token_ids = generation["generation_token_ids"] + + # The recorded token IDs come from the Rust tokenizer; the runner must load the # same vocabulary for the grammar to mean the same thing. - encoded = tokenizer.encode(case["completion"], add_special_tokens=False) - if encoded != generation[:-1]: + encoded = tokenizer.encode(generation["completion"], add_special_tokens=False) + if encoded != token_ids[:-1]: raise AssertionError( "tokenizer mismatch: the completion encodes to " - f"{encoded}, but the case records {generation[:-1]}" + f"{encoded}, but the generation records {token_ids[:-1]}" ) - grammar = case["grammar"] - structural_tag = {"type": "structural_tag", "format": grammar["format"]} - compiled = compiler.compile_structural_tag(json.dumps(structural_tag)) if grammar["coverage"] != "from_token_zero": return f"compiled only ({grammar['coverage']} is gated by the engine)" @@ -73,18 +82,41 @@ def replay(model: dict, case: dict) -> str: compiled, override_stop_tokens=model["all_stop_token_ids"] ) bitmask = xgr.allocate_token_bitmask(1, model["vocab_size"]) - for index, token_id in enumerate(generation): + for index, token_id in enumerate(token_ids): matcher.fill_next_token_bitmask(bitmask) allowed = (bitmask[0, token_id // 32].item() >> (token_id % 32)) & 1 if not allowed or not matcher.accept_token(token_id): - prefix = tokenizer.decode(generation[:index]) + prefix = tokenizer.decode(token_ids[:index]) raise AssertionError( f"token {index} ({token_id}, {tokenizer.decode([token_id])!r}) " f"rejected after {prefix!r}" ) if not matcher.is_terminated(): raise AssertionError("matcher not terminated after the stop token") - return f"replayed {len(generation)} tokens from token zero" + return f"replayed {len(token_ids)} tokens from token zero" + + +def check_model(path: Path) -> tuple[list[str], list[str]]: + """Check every grammar of one model file and return its stdout and stderr lines.""" + model = json.loads(path.read_text()) + out, err = [], [] + for case in model["grammars"]: + grammar = case["grammar"] + coverage = grammar["coverage"] + try: + compiled = compile_grammar(model, grammar) + except Exception as error: + names = ", ".join(case["generations"]) + err.append(f"FAIL {path.name} {names} [{coverage}]: {error}") + continue + for name, generation in case["generations"].items(): + try: + summary = replay(model, grammar, compiled, generation) + except Exception as error: + err.append(f"FAIL {path.name} {name} [{coverage}]: {error}") + else: + out.append(f"PASS {path.name} {name}: {summary}") + return out, err def main() -> int: @@ -93,19 +125,18 @@ def main() -> int: print(f"no grammar replay cases in {CASES_DIR}", file=sys.stderr) return 1 - failures = 0 - for path in paths: - model = json.loads(path.read_text()) - for name, case in model["cases"].items(): - try: - summary = replay(model, case) - except Exception as error: - failures += 1 - coverage = case["grammar"]["coverage"] - print(f"FAIL {path.name} {name} [{coverage}]: {error}", file=sys.stderr) - else: - print(f"PASS {path.name} {name}: {summary}") + # The cost is dominated by loading each model's tokenizer, which no other model + # shares, so check the model files in parallel. Fork after the imports above so + # that the workers share them instead of paying for them again. + workers = min(len(paths), os.cpu_count() or 1) + with multiprocessing.get_context("fork").Pool(workers) as pool: + results = pool.map(check_model, paths, chunksize=1) + failures = 0 + for out, err in results: + print("\n".join(out)) + print("\n".join(err), file=sys.stderr) + failures += len(err) return 1 if failures else 0 diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json index 4903b5900c..f08b418bf0 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json @@ -2,8 +2,8 @@ "model_id": "deepseek-ai/DeepSeek-V3.2-Exp", "vocab_size": 129280, "all_stop_token_ids": [1], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -107,10 +107,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 43, 759, 1926, 270, 6704, 339, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 43, 759, 1926, 270, 6704, 339, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -208,10 +212,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + "generations": { + "required": { + "completion": "Need call the weather and add tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -267,8 +275,189 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + "generations": { + "named": { + "completion": "Need call the weather and add tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 24313, 19995, 16562, 69958, 2928, 62773, 1] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML|function_calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_content": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 24313, 19995, 16562, 69958, 2928, 62773, 1] + }, + "auto_schema_calls": { + "completion": "Need call the weather and add tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.txt b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.txt index 5cf40f3218..25b572b8a2 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.txt +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.txt @@ -24,3 +24,20 @@ sequence `\n\n<|DSML|function_calls>\n` tag `<|DSML|invoke name="get_weather">\n` deepseek_xml({ location: string }) `\n` `` + +# none_schema (from_token_zero) +sequence + tag `` text `` + json({ answer: string }) + +# auto_schema_content, auto_schema_calls (from_token_zero) +sequence + tag `` text `` + or + sequence + `\n\n<|DSML|function_calls>\n` + tags_with_separator `` at_least_one + tag `<|DSML|invoke name="get_weather">\n` deepseek_xml({ location: string }) `\n` + tag `<|DSML|invoke name="add">\n` deepseek_xml({ y: number, x: number, items: string[] }) `\n` + `` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json index 1a286d8acd..8ab51cbbf9 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json @@ -2,8 +2,8 @@ "model_id": "deepseek-ai/DeepSeek-V4-Flash", "vocab_size": 129280, "all_stop_token_ids": [1], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -107,10 +107,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -208,10 +212,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + "generations": { + "required": { + "completion": "Need call the weather and add tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -267,8 +275,189 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + "generations": { + "named": { + "completion": "Need call the weather and add tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 24313, 19995, 16562, 69958, 2928, 62773, 1] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML|tool_calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_content": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 24313, 19995, 16562, 69958, 2928, 62773, 1] + }, + "auto_schema_calls": { + "completion": "Need call the weather and add tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.txt b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.txt index 5e216e8ffc..535da9ba28 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.txt +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.txt @@ -24,3 +24,20 @@ sequence `\n\n<|DSML|tool_calls>\n` tag `<|DSML|invoke name="get_weather">\n` deepseek_xml({ location: string }) `\n` `` + +# none_schema (from_token_zero) +sequence + tag `` text `` + json({ answer: string }) + +# auto_schema_content, auto_schema_calls (from_token_zero) +sequence + tag `` text `` + or + sequence + `\n\n<|DSML|tool_calls>\n` + tags_with_separator `` at_least_one + tag `<|DSML|invoke name="get_weather">\n` deepseek_xml({ location: string }) `\n` + tag `<|DSML|invoke name="add">\n` deepseek_xml({ y: number, x: number, items: string[] }) `\n` + `` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json index a21d8b90e5..a47ce49c2f 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json @@ -2,8 +2,8 @@ "model_id": "deepseek-ai/DeepSeek-V4.1-Flash", "vocab_size": 129280, "all_stop_token_ids": [1], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -107,10 +107,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0\n<|DSML| parameter name=\"x\" string=\"false\">2\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0\n<|DSML| parameter name=\"x\" string=\"false\">2\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -208,10 +212,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0\n<|DSML| parameter name=\"x\" string=\"false\">2\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + "generations": { + "required": { + "completion": "Need call the weather and add tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0\n<|DSML| parameter name=\"x\" string=\"false\">2\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -267,8 +275,189 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + "generations": { + "named": { + "completion": "Need call the weather and add tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 24313, 19995, 16562, 69958, 2928, 62773, 1] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML| calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML| invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML| invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_content": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 24313, 19995, 16562, 69958, 2928, 62773, 1] + }, + "auto_schema_calls": { + "completion": "Need call the weather and add tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0\n<|DSML| parameter name=\"x\" string=\"false\">2\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.txt b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.txt index 8c92c3a5a3..cb2c646545 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.txt +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.txt @@ -24,3 +24,20 @@ sequence `\n\n<|DSML| calls>\n` tag `<|DSML| invoke name="get_weather">\n` deepseek_v4_1_xml({ location: string }) `\n` `` + +# none_schema (from_token_zero) +sequence + tag `` text `` + json({ answer: string }) + +# auto_schema_content, auto_schema_calls (from_token_zero) +sequence + tag `` text `` + or + sequence + `\n\n<|DSML| calls>\n` + tags_with_separator `` at_least_one + tag `<|DSML| invoke name="get_weather">\n` deepseek_v4_1_xml({ location: string }) `\n` + tag `<|DSML| invoke name="add">\n` deepseek_v4_1_xml({ y: number, x: number, items: string[] }) `\n` + `` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.5.json b/rust/src/chat/tests/grammar_replay/GLM-4.5.json new file mode 100644 index 0000000000..e00f05757b --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-4.5.json @@ -0,0 +1,68 @@ +{ + "model_id": "zai-org/GLM-4.5", + "vocab_size": 151552, + "all_stop_token_ids": [151329, 151336, 151338], + "grammars": [ + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "\nNeed call the weather and add tools.\n{\"answer\":\"sunny\"}", + "generation_token_ids": [198, 151350, 23591, 1618, 279, 9101, 323, 912, 7375, 13, 151351, 198, 4913, 9214, 3252, 82, 27208, 9204, 151329] + }, + "auto_schema_content": { + "completion": "\nNeed call the weather and add tools.\n{\"answer\":\"sunny\"}", + "generation_token_ids": [198, 151350, 23591, 1618, 279, 9101, 323, 912, 7375, 13, 151351, 198, 4913, 9214, 3252, 82, 27208, 9204, 151329] + } + } + } + ] +} diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.5.txt b/rust/src/chat/tests/grammar_replay/GLM-4.5.txt new file mode 100644 index 0000000000..8a37a06dbd --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-4.5.txt @@ -0,0 +1,7 @@ +# none_schema, auto_schema_content (from_token_zero) +sequence + optional + sequence + tag `\n` text `` + `\n` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json index b289ec62e6..6fd2a440e4 100644 --- a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json +++ b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json @@ -2,8 +2,8 @@ "model_id": "zai-org/GLM-4.7-Flash", "vocab_size": 154880, "all_stop_token_ids": [154820, 154827, 154829], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -99,10 +99,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", - "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -198,10 +202,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", - "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + "generations": { + "required": { + "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -244,8 +252,187 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.get_weatherlocationShanghai", - "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820] + "generations": { + "named": { + "completion": "Need call the weather and add tools.get_weatherlocationShanghai", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 4913, 9214, 3252, 82, 27208, 9204, 154820] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "or", + "elements": [ + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_content": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 4913, 9214, 3252, 82, 27208, 9204, 154820] + }, + "auto_schema_calls": { + "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.txt b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.txt index 58b4bef2da..bc58a47fc9 100644 --- a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.txt +++ b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.txt @@ -16,3 +16,17 @@ sequence sequence tag `` text `` tag `get_weather` glm_xml({ location: string }) `` + +# none_schema (from_token_zero) +sequence + tag `` text `` + json({ answer: string }) + +# auto_schema_content, auto_schema_calls (from_token_zero) +sequence + tag `` text `` + or + triggered_tags [``] excluding [``, ``, ``, ``, ``, ``, ``] at_least_one + tag `get_weather` glm_xml({ location: string }) `` + tag `add` glm_xml({ y: number, x: number, items: string[] }) `` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json index 404667c30a..ecf8880406 100644 --- a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json +++ b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json @@ -2,8 +2,8 @@ "model_id": "zai-org/GLM-5.2-FP8", "vocab_size": 154880, "all_stop_token_ids": [154820, 154827, 154829], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -99,10 +99,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", - "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -198,10 +202,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", - "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + "generations": { + "required": { + "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -244,8 +252,187 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.get_weatherlocationShanghai", - "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820] + "generations": { + "named": { + "completion": "Need call the weather and add tools.get_weatherlocationShanghai", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 4913, 9214, 3252, 82, 27208, 9204, 154820] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "or", + "elements": [ + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_content": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 4913, 9214, 3252, 82, 27208, 9204, 154820] + }, + "auto_schema_calls": { + "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.txt b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.txt index 58b4bef2da..bc58a47fc9 100644 --- a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.txt +++ b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.txt @@ -16,3 +16,17 @@ sequence sequence tag `` text `` tag `get_weather` glm_xml({ location: string }) `` + +# none_schema (from_token_zero) +sequence + tag `` text `` + json({ answer: string }) + +# auto_schema_content, auto_schema_calls (from_token_zero) +sequence + tag `` text `` + or + triggered_tags [``] excluding [``, ``, ``, ``, ``, ``, ``] at_least_one + tag `get_weather` glm_xml({ location: string }) `` + tag `add` glm_xml({ y: number, x: number, items: string[] }) `` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/Hy3.json b/rust/src/chat/tests/grammar_replay/Hy3.json index f6ae7b0969..da08961630 100644 --- a/rust/src/chat/tests/grammar_replay/Hy3.json +++ b/rust/src/chat/tests/grammar_replay/Hy3.json @@ -2,8 +2,8 @@ "model_id": "tencent/Hy3", "vocab_size": 120832, "all_stop_token_ids": [120025], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -263,10 +263,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.I will call the tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", - "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 40, 823, 1609, 252, 7106, 13, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.I will call the tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 40, 823, 1609, 252, 7106, 13, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -523,10 +527,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", - "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + "generations": { + "required": { + "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -618,8 +626,299 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\n", - "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120036, 120025] + "generations": { + "named": { + "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120036, 120025] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "y" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "x" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "items" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + } + ], + "separator": "\n", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_calls": { + "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/Hy3.txt b/rust/src/chat/tests/grammar_replay/Hy3.txt index dcd6bb08ec..5642b36970 100644 --- a/rust/src/chat/tests/grammar_replay/Hy3.txt +++ b/rust/src/chat/tests/grammar_replay/Hy3.txt @@ -110,3 +110,53 @@ sequence `` `\n` `\n` + +# auto_schema_calls (from_token_zero) +sequence + tag `` text `` + or + sequence + `\n` + tags_with_separator `\n` at_least_one + tag `get_weather\n` .. `` + sequence + sequence + `` + `location` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + tag `add\n` .. `` + sequence + sequence + `` + `y` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + sequence + `` + `x` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + sequence + `` + `items` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + `\n` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json index 8ac6e689f7..3fb22cc3b8 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json +++ b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json @@ -2,8 +2,8 @@ "model_id": "moonshotai/Kimi-K2.5", "vocab_size": 163840, "all_stop_token_ids": [163585, 163586], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -134,10 +134,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.I will call the tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", - "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 40, 748, 2390, 276, 7697, 13, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.I will call the tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 40, 748, 2390, 276, 7697, 13, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -261,10 +265,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", - "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + "generations": { + "required": { + "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -333,8 +341,166 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_calls_section_end|>", - "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163596, 163586] + "generations": { + "named": { + "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163596, 163586] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "<|tool_calls_section_begin|>" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.add:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "<|tool_calls_section_end|>" + } + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_calls": { + "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.txt b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.txt index 553c1ce6f4..ca3bab8061 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.txt +++ b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.txt @@ -44,3 +44,23 @@ sequence `<|tool_call_argument_begin|>` json({ location: string }) `<|tool_calls_section_end|>` + +# auto_schema_calls (from_token_zero) +sequence + tag `` text `` + or + sequence + `<|tool_calls_section_begin|>` + tags_with_separator `` at_least_one + tag `<|tool_call_begin|>functions.get_weather:` .. `<|tool_call_end|>` + sequence + /\d+/ + `<|tool_call_argument_begin|>` + json({ location: string }) + tag `<|tool_call_begin|>functions.add:` .. `<|tool_call_end|>` + sequence + /\d+/ + `<|tool_call_argument_begin|>` + json({ y: number, x: number, items: string[] }) + `<|tool_calls_section_end|>` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.json b/rust/src/chat/tests/grammar_replay/Kimi-K3.json index 5306b93660..3e1832e805 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K3.json +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.json @@ -2,8 +2,8 @@ "model_id": "moonshotai/Kimi-K3", "vocab_size": 163840, "all_stop_token_ids": [163585, 163586], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -427,10 +427,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", - "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -666,10 +670,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", - "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + "generations": { + "required": { + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -795,8 +803,527 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", - "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + "generations": { + "named": { + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>think<|sep|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>think<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>response<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>{\"answer\":\"sunny\"}<|close|>response<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 8264, 26921, 7471, 82, 38975, 16934, 163588, 12092, 163589, 163588, 2778, 163589, 163586] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>think<|sep|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>think<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "<|open|>response<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + } + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|><|close|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + } + ] + } + ] + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_content": { + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>{\"answer\":\"sunny\"}<|close|>response<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 8264, 26921, 7471, 82, 38975, 16934, 163588, 12092, 163589, 163588, 2778, 163589, 163586] + }, + "auto_schema_calls": { + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.txt b/rust/src/chat/tests/grammar_replay/Kimi-K3.txt index 6a1c814c13..d7357e275c 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K3.txt +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.txt @@ -86,3 +86,58 @@ sequence tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` optional `<|close|>message<|sep|>` + +# none_schema (from_token_zero) +sequence + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + tag `<|open|>response<|sep|>` json({ answer: string }) `<|close|>response<|sep|>` + optional `<|close|>message<|sep|>` + +# auto_schema_content, auto_schema_calls (from_token_zero) +sequence + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + or + sequence + tag `<|open|>response<|sep|>` json({ answer: string }) `<|close|>response<|sep|>` + optional + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` + tag `<|open|>call tool="add" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + sequence + tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ y: number, x: number, items: string[] }) `<|close|>json<|sep|>` + sequence + optional `<|open|>response<|sep|><|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` + tag `<|open|>call tool="add" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + sequence + tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ y: number, x: number, items: string[] }) `<|close|>json<|sep|>` + optional `<|close|>message<|sep|>` diff --git a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json index 582f29a1b9..451255dd8c 100644 --- a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json +++ b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json @@ -2,8 +2,8 @@ "model_id": "XiaomiMiMo/MiMo-V2.6-Flash-RL", "vocab_size": 152576, "all_stop_token_ids": [151643, 151645, 151672], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -99,10 +99,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.I will call the tools.Shanghai1.02[\"left\", \"right\"]", - "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 40, 686, 1618, 279, 7375, 13, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.I will call the tools.Shanghai1.02[\"left\", \"right\"]", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 40, 686, 1618, 279, 7375, 13, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -198,10 +202,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.Shanghai1.02[\"left\", \"right\"]", - "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + "generations": { + "required": { + "completion": "Need call the weather and add tools.Shanghai1.02[\"left\", \"right\"]", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -247,8 +255,190 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.Shanghai", - "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151645] + "generations": { + "named": { + "completion": "Need call the weather and add tools.Shanghai", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151645] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 4913, 9217, 3252, 82, 27297, 9207, 151645] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "or", + "elements": [ + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 4913, 9217, 3252, 82, 27297, 9207, 151645] + }, + "auto_schema_calls": { + "completion": "Need call the weather and add tools.Shanghai1.02[\"left\", \"right\"]", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.txt b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.txt index 1d12560eb0..7cc9aa0e59 100644 --- a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.txt +++ b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.txt @@ -16,3 +16,17 @@ sequence sequence optional tag `` text `` tag `` qwen_xml({ location: string }) `` + +# none_schema (from_token_zero) +sequence + optional tag `` text `` + json({ answer: string }) + +# auto_schema_content, auto_schema_calls (from_token_zero) +sequence + optional tag `` text `` + or + triggered_tags [``] excluding [``, ``, ``, `` qwen_xml({ location: string }) `` + tag `` qwen_xml({ y: number, x: number, items: string[] }) `` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json index 64f65e2f5a..d4c2e8e789 100644 --- a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json @@ -2,8 +2,8 @@ "model_id": "MiniMaxAI/MiniMax-M2.5", "vocab_size": 200064, "all_stop_token_ids": [200020], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -116,10 +116,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", - "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 73, 715, 1728, 275, 6629, 320, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 73, 715, 1728, 275, 6629, 320, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -226,10 +230,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", - "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + "generations": { + "required": { + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -294,8 +302,207 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n", - "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + "generations": { + "named": { + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.\n\n\n{\"answer\":\"sunny\"}", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 16673, 33591, 15368, 78697, 7582, 22089, 200020] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_content": { + "completion": "Need call the weather and add tools.\n\n\n{\"answer\":\"sunny\"}", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 16673, 33591, 15368, 78697, 7582, 22089, 200020] + }, + "auto_schema_calls": { + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.txt b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.txt index 431238009a..fae13b8445 100644 --- a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.txt +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.txt @@ -30,3 +30,24 @@ sequence `\n\n` tag `\n` minimax_xml({ location: string }) `\n` `` + +# none_schema (from_token_zero) +sequence + sequence + tag `` text `` + `\n\n` + json({ answer: string }) + +# auto_schema_content, auto_schema_calls (from_token_zero) +sequence + sequence + tag `` text `` + `\n\n` + or + sequence + `\n\n` + tags_with_separator `` at_least_one + tag `\n` minimax_xml({ location: string }) `\n` + tag `\n` minimax_xml({ y: number, x: number, items: string[] }) `\n` + `` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M3.json b/rust/src/chat/tests/grammar_replay/MiniMax-M3.json new file mode 100644 index 0000000000..73679eff6a --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M3.json @@ -0,0 +1,40 @@ +{ + "model_id": "MiniMaxAI/MiniMax-M3", + "vocab_size": 200064, + "all_stop_token_ids": [200020], + "grammars": [ + { + "grammar": { + "format": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "coverage": "final_output_only" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [200059, 42478, 1728, 275, 12562, 306, 1187, 6629, 46, 200060, 16673, 33591, 15368, 78697, 7582, 22089, 200020] + }, + "auto_schema_content": { + "completion": "Need call the weather and add tools.{\"answer\":\"sunny\"}", + "generation_token_ids": [200059, 42478, 1728, 275, 12562, 306, 1187, 6629, 46, 200060, 16673, 33591, 15368, 78697, 7582, 22089, 200020] + } + } + } + ] +} diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M3.txt b/rust/src/chat/tests/grammar_replay/MiniMax-M3.txt new file mode 100644 index 0000000000..f54e72d4c8 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M3.txt @@ -0,0 +1,2 @@ +# none_schema, auto_schema_content (final_output_only) +json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json index 5d171fc5f0..8206b8b40c 100644 --- a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json +++ b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json @@ -2,8 +2,8 @@ "model_id": "Qwen/Qwen3-0.6B", "vocab_size": 151936, "all_stop_token_ids": [151643, 151645], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -106,10 +106,14 @@ }, "coverage": "from_token_zero" }, - "completion": "\nNeed call the weather and add tools.\n\n\nI will call the tools.\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", - "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 40, 686, 1618, 279, 7375, 624, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + "generations": { + "auto": { + "completion": "\nNeed call the weather and add tools.\n\n\nI will call the tools.\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 40, 686, 1618, 279, 7375, 624, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -212,10 +216,14 @@ }, "coverage": "from_token_zero" }, - "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", - "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + "generations": { + "required": { + "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -270,8 +278,206 @@ }, "coverage": "from_token_zero" }, - "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n", - "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 151645] + "generations": { + "named": { + "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 151645] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "\nNeed call the weather and add tools.\n\n\n{\"answer\":\"sunny\"}", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 4913, 9217, 3252, 82, 27297, 9207, 151645] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "or", + "elements": [ + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + }, + { + "type": "tag", + "begin": "\n{\"name\": \"add\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_content": { + "completion": "\nNeed call the weather and add tools.\n\n\n{\"answer\":\"sunny\"}", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 4913, 9217, 3252, 82, 27297, 9207, 151645] + }, + "auto_schema_calls": { + "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.txt b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.txt index fe38f5e9ff..865d1b5430 100644 --- a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.txt +++ b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.txt @@ -25,3 +25,23 @@ sequence tag `` text `` `\n\n` tag `\n{"name": "get_weather", "arguments": ` json({ location: string }) `}\n` + +# none_schema (from_token_zero) +sequence + optional + sequence + tag `` text `` + `\n\n` + json({ answer: string }) + +# auto_schema_content, auto_schema_calls (from_token_zero) +sequence + optional + sequence + tag `` text `` + `\n\n` + or + triggered_tags [``] excluding [``, ``] at_least_one + tag `\n{"name": "get_weather", "arguments": ` json({ location: string }) `}\n` + tag `\n{"name": "add", "arguments": ` json({ y: number, x: number, items: string[] }) `}\n` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json index 8bd4c901e5..9cc7b53ff9 100644 --- a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json +++ b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json @@ -2,8 +2,8 @@ "model_id": "Qwen/Qwen3.5-4B", "vocab_size": 248320, "all_stop_token_ids": [248044, 248046], - "cases": { - "auto": { + "grammars": [ + { "grammar": { "format": { "type": "sequence", @@ -103,10 +103,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", - "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 40, 668, 1562, 279, 7141, 13, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + "generations": { + "auto": { + "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 40, 668, 1562, 279, 7141, 13, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + } + } }, - "required": { + { "grammar": { "format": { "type": "sequence", @@ -206,10 +210,14 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", - "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + "generations": { + "required": { + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + } + } }, - "named": { + { "grammar": { "format": { "type": "sequence", @@ -261,8 +269,200 @@ }, "coverage": "from_token_zero" }, - "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n", - "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + "generations": { + "named": { + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.\n\n\n{\"answer\":\"sunny\"}", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 4754, 8944, 3147, 82, 26415, 8934, 248046] + } + } + }, + { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "or", + "elements": [ + { + "type": "triggered_tags", + "triggers": [ + "\n\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "generations": { + "auto_schema_content": { + "completion": "Need call the weather and add tools.\n\n\n{\"answer\":\"sunny\"}", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 4754, 8944, 3147, 82, 26415, 8934, 248046] + }, + "auto_schema_calls": { + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + } + } } - } + ] } diff --git a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.txt b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.txt index b057780649..56831217d8 100644 --- a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.txt +++ b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.txt @@ -22,3 +22,21 @@ sequence tag `` text `` `\n\n` tag `\n\n` qwen_xml({ location: string }) `\n\n` + +# none_schema (from_token_zero) +sequence + sequence + tag `` text `` + `\n\n` + json({ answer: string }) + +# auto_schema_content, auto_schema_calls (from_token_zero) +sequence + sequence + tag `` text `` + `\n\n` + or + triggered_tags [`\n`, ``] at_least_one + tag `\n\n` qwen_xml({ location: string }) `\n\n` + tag `\n\n` qwen_xml({ y: number, x: number, items: string[] }) `\n\n` + json({ answer: string }) diff --git a/rust/src/chat/tests/grammar_replay/Seed-OSS-36B-Instruct.json b/rust/src/chat/tests/grammar_replay/Seed-OSS-36B-Instruct.json new file mode 100644 index 0000000000..e933ae4abb --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Seed-OSS-36B-Instruct.json @@ -0,0 +1,40 @@ +{ + "model_id": "ByteDance-Seed/Seed-OSS-36B-Instruct", + "vocab_size": 155136, + "all_stop_token_ids": [2], + "grammars": [ + { + "grammar": { + "format": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "answer": { + "type": "string" + } + }, + "required": [ + "answer" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "coverage": "final_output_only" + }, + "generations": { + "none_schema": { + "completion": "Need call the weather and add tools.\n{\"answer\":\"sunny\"}", + "generation_token_ids": [3, 40143, 2434, 405, 11785, 440, 1731, 9633, 141, 4, 326, 13059, 35733, 15301, 40841, 2313, 16320, 2] + }, + "auto_schema_content": { + "completion": "Need call the weather and add tools.\n{\"answer\":\"sunny\"}", + "generation_token_ids": [3, 40143, 2434, 405, 11785, 440, 1731, 9633, 141, 4, 326, 13059, 35733, 15301, 40841, 2313, 16320, 2] + } + } + } + ] +} diff --git a/rust/src/chat/tests/grammar_replay/Seed-OSS-36B-Instruct.txt b/rust/src/chat/tests/grammar_replay/Seed-OSS-36B-Instruct.txt new file mode 100644 index 0000000000..f54e72d4c8 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Seed-OSS-36B-Instruct.txt @@ -0,0 +1,2 @@ +# none_schema, auto_schema_content (final_output_only) +json({ answer: string }) diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 079ce3e1a7..c5080a59c5 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -7,9 +7,10 @@ //! parsed from the generated assistant completion and then rendered back to the exact same //! assistant completion. //! -//! The tool-call fixture also snapshots the output grammar built by the initialized parser for each -//! tool-choice variant, with the rendered completion as its generation, as one file per model under -//! `tests/grammar_replay/`. `grammar_replay.rs` replays these cases through the real XGrammar. +//! The tool-call fixture also snapshots the output grammars built by the initialized parser, one +//! entry per distinct grammar with the rendered completion of every variant that builds it as a +//! generation, as one file per model under `tests/grammar_replay/`. `grammar_replay.rs` replays +//! these generations through the real XGrammar. use std::collections::{BTreeMap, BTreeSet}; use std::fmt::Debug; @@ -30,6 +31,7 @@ use vllm_chat::{ GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, RendererSelection, ToolStrictLevel, load_model_backends, }; +use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams; use vllm_parser::output_grammar::BuiltOutputGrammar; use vllm_text::{DecodedTextEvent, Finished, Prompt}; use vllm_tokenizer::Tokenizer; @@ -57,6 +59,13 @@ struct RoundtripCase { json_fmt: JsonFmt, /// Whether the template renders tool-call argument object keys in sorted order. sort_json_keys: bool, + /// Which tool-call fixture variants with a JSON schema this case runs; + /// every case runs the variants without one. The variants answered with + /// content need a template that renders a final turn without tool calls + /// back to its completion (some templates drop its reasoning or its stop + /// suffix); the variant answered with calls needs a tool grammar, which + /// holds the calls next to the schema. + with_schema: &'static [ToolCallMixVariant], } #[derive(Clone, Copy)] @@ -125,10 +134,11 @@ impl ThinkingBehavior { } } -/// Tool-choice variant of the tool-call fixture. +/// Variant of the tool-call fixture: the request's tool choice and optional +/// JSON schema, and the completion that answers it. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] #[serde(rename_all = "snake_case")] -enum ToolChoiceVariant { +enum ToolCallMixVariant { /// `auto`, with content before the tool calls. Auto, /// `required`, with tool calls only: builder grammars admit no content @@ -136,18 +146,59 @@ enum ToolChoiceVariant { Required, /// Named `get_weather`, with a single call to it and no content. Named, + + /// `none` with a JSON schema, answered with schema-valid content. + NoneSchema, + /// `auto` with a JSON schema, answered with schema-valid content. + AutoSchemaContent, + /// `auto` with a JSON schema, answered with tool calls only: builder + /// grammars hold either schema-valid content or the calls. + AutoSchemaCalls, } -impl ToolChoiceVariant { - const ALL: [Self; 3] = [Self::Auto, Self::Required, Self::Named]; +/// The schema-valid content of the schema variants answered with content. +const SCHEMA_CONTENT: &str = r#"{"answer":"sunny"}"#; + +impl ToolCallMixVariant { + const WITHOUT_SCHEMA: &[Self] = &[Self::Auto, Self::Required, Self::Named]; + const WITH_SCHEMA: &[Self] = &[ + Self::NoneSchema, + Self::AutoSchemaContent, + Self::AutoSchemaCalls, + ]; fn tool_choice(self) -> ChatToolChoice { match self { - Self::Auto => ChatToolChoice::Auto, + Self::Auto | Self::AutoSchemaContent | Self::AutoSchemaCalls => ChatToolChoice::Auto, Self::Required => ChatToolChoice::Required, Self::Named => ChatToolChoice::Function { name: "get_weather".to_string(), }, + Self::NoneSchema => ChatToolChoice::None, + } + } + + fn structured_outputs(self) -> Option { + matches!( + self, + Self::NoneSchema | Self::AutoSchemaContent | Self::AutoSchemaCalls + ) + .then(|| { + StructuredOutputsParams::json(serde_json::json!({ + "type": "object", + "properties": { "answer": { "type": "string" } }, + "required": ["answer"] + })) + }) + } + + /// The expected text content and number of tool calls. + fn expected(self) -> (Option<&'static str>, usize) { + match self { + Self::Auto => (Some("I will call the tools."), 2), + Self::Required | Self::AutoSchemaCalls => (None, 2), + Self::Named => (None, 1), + Self::NoneSchema | Self::AutoSchemaContent => (Some(SCHEMA_CONTENT), 0), } } } @@ -163,6 +214,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: spaced_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -176,6 +228,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -189,6 +242,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -202,6 +256,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -215,6 +270,12 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + // The MiniMax M3 tool parser builds no tool grammar, so the schema + // excludes the calls. + with_schema: &[ + ToolCallMixVariant::NoneSchema, + ToolCallMixVariant::AutoSchemaContent, + ], } } @@ -228,6 +289,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: false }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -241,6 +303,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -254,6 +317,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: false }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -267,6 +331,12 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + // The GLM-4.5 tool parser builds no tool grammar, so the schema + // excludes the calls. + with_schema: &[ + ToolCallMixVariant::NoneSchema, + ToolCallMixVariant::AutoSchemaContent, + ], } } @@ -280,6 +350,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -293,6 +364,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -306,6 +378,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: true, + with_schema: &[ToolCallMixVariant::AutoSchemaCalls], } } @@ -329,6 +402,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: spaced_json_fmt(), sort_json_keys: false, + with_schema: &[ToolCallMixVariant::AutoSchemaCalls], } } @@ -346,6 +420,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -363,6 +438,7 @@ impl RoundtripCase { }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: &[ToolCallMixVariant::AutoSchemaCalls], } } @@ -376,6 +452,12 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + // The Seed-OSS tool parser builds no tool grammar, so the schema + // excludes the calls. + with_schema: &[ + ToolCallMixVariant::NoneSchema, + ToolCallMixVariant::AutoSchemaContent, + ], } } @@ -389,6 +471,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -402,6 +485,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } @@ -415,6 +499,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + with_schema: &[ToolCallMixVariant::AutoSchemaCalls], } } @@ -428,6 +513,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: true, + with_schema: ToolCallMixVariant::WITH_SCHEMA, } } } @@ -545,8 +631,10 @@ async fn run_roundtrip_tool_call_mix( backends: &vllm_chat::LoadedModelBackends, ) -> Result<()> { let mut results = Vec::new(); - for variant in ToolChoiceVariant::ALL { - let result = run_roundtrip_tool_call_mix_inner(case, backends, variant).await?; + for &variant in ToolCallMixVariant::WITHOUT_SCHEMA.iter().chain(case.with_schema) { + let result = run_roundtrip_tool_call_mix_inner(case, backends, variant) + .await + .with_context(|| format!("variant {}", serde_json::to_value(variant).unwrap()))?; results.push((variant, result)); } check_grammar_replay_file(case, backends, results) @@ -555,9 +643,9 @@ async fn run_roundtrip_tool_call_mix( async fn run_roundtrip_tool_call_mix_inner( case: &RoundtripCase, backends: &vllm_chat::LoadedModelBackends, - variant: ToolChoiceVariant, + variant: ToolCallMixVariant, ) -> Result { - let request = roundtrip_request( + let mut request = roundtrip_request( "roundtrip-reasoning-tools", vec![ChatMessage::text( ChatRole::User, @@ -568,12 +656,9 @@ async fn run_roundtrip_tool_call_mix_inner( Some(true), // always enable thinking in this fixture case.thinking_behavior, ); + request.sampling_params.structured_outputs = variant.structured_outputs(); let expected_reasoning = "Need call the weather and add tools."; - let (expected_text, tool_call_count) = match variant { - ToolChoiceVariant::Auto => (Some("I will call the tools."), 2), - ToolChoiceVariant::Required => (None, 2), - ToolChoiceVariant::Named => (None, 1), - }; + let (expected_text, tool_call_count) = variant.expected(); let mut content = vec![AssistantContentBlock::Reasoning { text: expected_reasoning.to_string(), @@ -617,11 +702,13 @@ async fn run_roundtrip_tool_call_mix_inner( "parsed message: {:#?}", result.parsed_message ); - assert_eq!(tool_calls[0].name, "get_weather"); - assert_eq!( - tool_calls[0].arguments, - expected_arguments(case, r#"{"location": "Shanghai"}"#)?, - ); + if let Some(weather) = tool_calls.first() { + assert_eq!(weather.name, "get_weather"); + assert_eq!( + weather.arguments, + expected_arguments(case, r#"{"location": "Shanghai"}"#)?, + ); + } if let Some(add) = tool_calls.get(1) { assert_eq!(add.name, "add"); assert_eq!( @@ -1117,16 +1204,23 @@ struct GrammarReplayFile { /// Request stop set, passed to the matcher as `override_stop_tokens`. #[serde(serialize_with = "serialize_one_line")] all_stop_token_ids: BTreeSet, - /// Cases keyed by tool-choice variant. - cases: BTreeMap, + /// Distinct output grammars, in the order of the first variant building each. + grammars: Vec, } -/// One parser-built output grammar with the generation it must accept. +/// One parser-built output grammar with the generations it must accept. #[derive(Serialize)] struct GrammarReplayCase { /// Parser-built grammar; only token-zero grammars are replayed from the /// first generated token. grammar: BuiltOutputGrammar, + /// Generations keyed by the fixture variants whose requests build this grammar. + generations: BTreeMap, +} + +/// One rendered completion replayed against its output grammar. +#[derive(Serialize)] +struct GrammarReplayGeneration { /// Generated completion text, without the stop token. completion: String, /// Generated token IDs presented to the matcher, ending with a stop token. @@ -1150,13 +1244,13 @@ fn grammar_replay_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay") } -/// Snapshot the output grammars of one model's roundtrips, keyed by variant, -/// as its grammar replay file, or check that no stale file remains when the +/// Snapshot the output grammars of one model's roundtrips, grouping the +/// variants that build the same grammar, as its grammar replay file, or check that no stale file remains when the /// parser builds none. fn check_grammar_replay_file( case: &RoundtripCase, backends: &vllm_chat::LoadedModelBackends, - results: Vec<(ToolChoiceVariant, RoundtripResult)>, + results: Vec<(ToolCallMixVariant, RoundtripResult)>, ) -> Result<()> { let model_name = case.model_id.rsplit_once('/').map_or(case.model_id, |(_, name)| name); let path = grammar_replay_dir().join(format!("{model_name}.json")); @@ -1197,7 +1291,7 @@ fn check_grammar_replay_file( case.assistant_stop_suffix ); - let mut cases = BTreeMap::new(); + let mut grammars = Vec::::new(); for (variant, grammar, closed_completion) in results { let (completion, mut generation_token_ids) = match completion_body( tokenizer.as_ref(), @@ -1208,27 +1302,41 @@ fn check_grammar_replay_file( CompletionBody::TokenIds(body) => (tokenizer.decode(body, false)?, body.to_vec()), }; generation_token_ids.push(stop_token_id); - cases.insert( - variant, - GrammarReplayCase { + let generation = GrammarReplayGeneration { + completion, + generation_token_ids, + }; + match grammars.iter_mut().find(|case| case.grammar == grammar) { + Some(case) => { + case.generations.insert(variant, generation); + } + None => grammars.push(GrammarReplayCase { grammar, - completion, - generation_token_ids, - }, - ); + generations: BTreeMap::from([(variant, generation)]), + }), + } } // A readable outline of the same grammars, for review only. let mut outline = String::new(); - for (variant, case) in &cases { + for case in &grammars { if !outline.is_empty() { outline.push('\n'); } - let variant = serde_json::to_value(variant)?; + let variants = case + .generations + .keys() + .map(|variant| { + serde_json::to_value(variant)? + .as_str() + .map(str::to_owned) + .context("variant name") + }) + .collect::>>()?; let coverage = serde_json::to_value(case.grammar.coverage)?; outline.push_str(&format!( "# {} ({})\n", - variant.as_str().context("variant name")?, + variants.join(", "), coverage.as_str().context("coverage name")?, )); outline.push_str(&vllm_parser::output_grammar::test_utils::outline( @@ -1241,7 +1349,7 @@ fn check_grammar_replay_file( model_id: case.model_id, vocab_size: backends.text_backend.model_vocab_size(), all_stop_token_ids, - cases, + grammars, }; let mut json = serde_json::to_string_pretty(&file)?; json.push('\n'); diff --git a/rust/src/parser/src/output_grammar.rs b/rust/src/parser/src/output_grammar.rs index 0108c3193e..ba9b4968d5 100644 --- a/rust/src/parser/src/output_grammar.rs +++ b/rust/src/parser/src/output_grammar.rs @@ -132,6 +132,10 @@ pub struct OutputGrammarContext<'a> { pub tool_strict_level: ToolStrictLevel, /// Whether the request permits more than one tool call. pub parallel_tool_calls: bool, + /// The user's constraint on the answer text, such as a `response_format` + /// JSON schema, normalized to a format. A builder that composes it inserts + /// it where the model writes its answer; `None` leaves the answer free. + pub answer: Option<&'a Format>, } /// Errors produced while building an output grammar. @@ -243,9 +247,12 @@ fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool { match ctx.tool_choice { ToolChoice::Value(ToolChoiceValue::None) => false, + // With an answer constraint, the grammar must hold the calls `auto` + // allows, or the answer would exclude them. ToolChoice::Value(ToolChoiceValue::Auto) => { ctx.tool_strict_level >= ToolStrictLevel::Function || ctx.tools.iter().any(|tool| tool.strict == Some(true)) + || ctx.answer.is_some() } ToolChoice::Value(ToolChoiceValue::Required) | ToolChoice::NamedFunction(_) @@ -284,6 +291,7 @@ mod tests { tool_choice, tool_strict_level: ToolStrictLevel::Auto, parallel_tool_calls: true, + answer: None, }, ) .unwrap() @@ -310,6 +318,7 @@ mod tests { tool_choice: &tool_choice, tool_strict_level: level, parallel_tool_calls: true, + answer: None, }, ) .unwrap() diff --git a/rust/src/parser/src/tool/minimax_m2.rs b/rust/src/parser/src/tool/minimax_m2.rs index 3b868013bb..33d316579f 100644 --- a/rust/src/parser/src/tool/minimax_m2.rs +++ b/rust/src/parser/src/tool/minimax_m2.rs @@ -122,7 +122,7 @@ impl ToolParser for MinimaxM2ToolParser { Some(xgrammar_structural_tag::Model::Minimax.builder()) } - fn build_visible_format( + fn build_call_format( &self, ctx: &OutputGrammarContext<'_>, ) -> output_grammar::Result> { diff --git a/rust/src/parser/src/tool/mod.rs b/rust/src/parser/src/tool/mod.rs index e4afbda3e9..974d6c758a 100644 --- a/rust/src/parser/src/tool/mod.rs +++ b/rust/src/parser/src/tool/mod.rs @@ -20,6 +20,7 @@ mod seed_oss; #[cfg(any(test, feature = "test-util"))] pub mod test_utils; use std::collections::{BTreeMap, btree_map}; +use std::sync::Once; pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser, DeepSeekV41ToolParser}; pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser}; @@ -39,8 +40,11 @@ pub use qwen_coder::Qwen3CoderToolParser; pub use seed_oss::SeedOssToolParser; use serde::{Deserialize, Serialize}; use serde_json::Value; +use tracing::warn; +use xgrammar_structural_tag::ToolChoice; pub use xgrammar_structural_tag::builders::StructuralTagBuilder; use xgrammar_structural_tag::format::Format; +use xgrammar_structural_tag::tool::{AllowedToolsMode, ToolChoiceValue}; use crate::output_grammar::{self, OutputGrammarContext}; use crate::utils; @@ -196,21 +200,54 @@ pub trait ToolParser: Send { } /// Build the language of everything the model may emit after reasoning - /// ends: visible text plus tool calls. `None` means no tool grammar applies. + /// ends for the context's tool choice: visible text plus tool calls. `None` + /// means no tool grammar applies. /// /// The default forwards to [`Self::structural_tag_builder`] with reasoning /// disabled, which is exactly today's strict-tool-calling grammar. - fn build_visible_format( + fn build_call_format( &self, ctx: &OutputGrammarContext<'_>, ) -> output_grammar::Result> { output_grammar::visible_format_from_builder(self.structural_tag_builder(), ctx) } + /// Build the visible language with the request's answer constraint + /// composed in, as the Python frontend does: an optional call becomes + /// `Or[calls built as required, answer]`, and a tool choice that obliges a + /// call ignores the answer. Parsers override [`Self::build_call_format`] + /// instead. + fn build_visible_format( + &self, + ctx: &OutputGrammarContext<'_>, + ) -> output_grammar::Result> { + let Some(answer) = ctx.answer else { + return self.build_call_format(ctx); + }; + match call_required(ctx.tool_choice) { + // Calls built from the optional choice would also admit free text + // in place of the answer. + Some(tool_choice) => { + let calls = self.build_call_format(&OutputGrammarContext { + tool_choice: &tool_choice, + ..*ctx + })?; + Ok(calls.map(|calls| Format::or(vec![calls, answer.clone()]))) + } + None => { + let calls = self.build_call_format(ctx)?; + if calls.is_some() { + warn_answer_ignored_once(); + } + Ok(calls) + } + } + } + /// Return the xgrammar structural-tag builder used for strict tool calling. /// /// To be deprecated: this only exists to back the default - /// [`Self::build_visible_format`]. New parsers should override that method + /// [`Self::build_call_format`]. New parsers should override that method /// directly instead of exposing a builder. fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> { None @@ -243,6 +280,32 @@ pub trait ToolParser: Send { fn reset(&mut self) -> String; } +/// The tool choice with an optional call made obligatory, or `None` when the +/// choice forbids or already obliges a call. +fn call_required(tool_choice: &ToolChoice) -> Option { + let mut required = tool_choice.clone(); + match &mut required { + ToolChoice::Value(value @ ToolChoiceValue::Auto) => *value = ToolChoiceValue::Required, + ToolChoice::AllowedTools(choice) if choice.allowed_tools.mode == AllowedToolsMode::Auto => { + choice.allowed_tools.mode = AllowedToolsMode::Required; + } + ToolChoice::FlatAllowedTools(choice) if choice.mode == AllowedToolsMode::Auto => { + choice.mode = AllowedToolsMode::Required; + } + _ => return None, + } + Some(required) +} + +/// Warn that a request's answer constraint is ignored because its tool +/// choice obliges a call, as the Python frontend does. +fn warn_answer_ignored_once() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + warn!("structured outputs are ignored because tool_choice forces a tool call"); + }); +} + /// Extension methods for easily testing `ToolParser` implementations. /// /// These helpers do not handle partial parsing or error recovery, so they are diff --git a/rust/src/parser/src/unified/combined.rs b/rust/src/parser/src/unified/combined.rs index ba44e7fb96..b1f36e8855 100644 --- a/rust/src/parser/src/unified/combined.rs +++ b/rust/src/parser/src/unified/combined.rs @@ -87,10 +87,19 @@ impl UnifiedParser for CombinedParser { &self, ctx: &OutputGrammarContext<'_>, ) -> output_grammar::Result> { - let Some(tool) = self.tool.as_ref() else { + // Without an active reasoning or tool parser, the request's own + // constraint applies unchanged. + if self.reasoning.is_none() && self.tool.is_none() { return Ok(None); + } + + // The tool parser composes the answer constraint into its calls where + // it can; without a tool grammar, the answer is the visible language. + let calls = match self.tool.as_ref() { + Some(tool) => tool.build_visible_format(ctx)?, + None => None, }; - let Some(visible) = tool.build_visible_format(ctx)? else { + let Some(visible) = calls.or_else(|| ctx.answer.cloned()) else { return Ok(None); }; // The reasoning parser may wrap that language with the reasoning phase @@ -165,7 +174,8 @@ mod tests { use super::CombinedParser; use crate::output_grammar::{ - GrammarCoverage, OutputGrammarContext, full_format_from_builder_for_test, + BuiltOutputGrammar, GrammarCoverage, OutputGrammarContext, + full_format_from_builder_for_test, }; use crate::reasoning::{ DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser, Glm45ReasoningParser, @@ -235,6 +245,7 @@ mod tests { tool_choice: &tool_choice, tool_strict_level: Default::default(), parallel_tool_calls: true, + answer: None, }; let tool = T::create(&tools).unwrap(); let expected = full_format_from_builder_for_test( @@ -524,6 +535,7 @@ mod tests { tool_choice: &tool_choice, tool_strict_level: Default::default(), parallel_tool_calls: true, + answer: None, }; let reasoning = Qwen3ReasoningParser::create(Arc::new(tokenizer())).unwrap(); let tool = Qwen3XmlToolParser::create(&tools).unwrap(); @@ -553,6 +565,7 @@ mod tests { tool_choice: &tool_choice, tool_strict_level: Default::default(), parallel_tool_calls: true, + answer: None, }; let visible = Format::const_string("answer"); @@ -586,6 +599,7 @@ mod tests { tool_choice: &tool_choice, tool_strict_level: Default::default(), parallel_tool_calls: true, + answer: None, }; let visible = Format::const_string("visible"); @@ -622,6 +636,7 @@ mod tests { tool_choice: &tool_choice, tool_strict_level: Default::default(), parallel_tool_calls: true, + answer: None, }; let tokenizer = Arc::new(tokenizer()); let reasoning = Qwen3ReasoningParser::create(tokenizer.clone()).unwrap(); @@ -650,4 +665,97 @@ mod tests { ); } } + + fn grammar_ctx<'a>( + tools: &'a [Tool], + tool_choice: &'a ToolChoice, + answer: Option<&'a Format>, + ) -> OutputGrammarContext<'a> { + OutputGrammarContext { + tools, + tool_choice, + tool_strict_level: Default::default(), + parallel_tool_calls: true, + answer, + } + } + + fn qwen3_parser(tools: &[Tool], with_tool: bool) -> CombinedParser { + let reasoning = Qwen3ReasoningParser::create(Arc::new(tokenizer())).unwrap(); + let tool = with_tool.then(|| Qwen3XmlToolParser::create(tools).unwrap()); + let mut parser = CombinedParser::new(Some(reasoning), tool); + parser.initialize(&[]).unwrap(); + parser + } + + #[test] + fn answer_constraint_composes_with_the_tool_choice() { + let tools = test_tools(); + let answer = Format::json_schema(serde_json::json!({ "type": "object" })); + let reasoning = |visible| { + Format::sequence(vec![ + Format::optional(Format::sequence(vec![ + Format::tag("", Format::any_text(), ""), + Format::const_string("\n\n"), + ])), + visible, + ]) + }; + let required = ToolChoice::required(); + let required_calls = Qwen3XmlToolParser::create(&tools) + .unwrap() + .build_visible_format(&grammar_ctx(&tools, &required, None)) + .unwrap() + .unwrap(); + + // No callable tool: the answer alone, after the reasoning phase. + for (tool_choice, with_tool) in [(ToolChoice::none(), true), (ToolChoice::auto(), false)] { + let parser = qwen3_parser(&tools, with_tool); + let ctx = grammar_ctx(&tools, &tool_choice, Some(&answer)); + let actual = parser.build_output_grammar(&ctx).unwrap().unwrap(); + assert_eq!(actual.coverage, GrammarCoverage::FromTokenZero); + assert_eq!(actual.format, reasoning(answer.clone()), "{tool_choice:?}"); + } + + // `auto`, strict or not, keeps tool calls next to the answer. + let parser = qwen3_parser(&tools, true); + let auto = ToolChoice::auto(); + let actual = parser.build_output_grammar(&grammar_ctx(&tools, &auto, Some(&answer))); + let expected = Format::or(vec![required_calls, answer.clone()]); + assert_eq!(actual.unwrap().unwrap().format, reasoning(expected)); + + // A required call ignores the answer constraint. + for tool_choice in [ToolChoice::required(), ToolChoice::function("get_weather")] { + let with_answer = grammar_ctx(&tools, &tool_choice, Some(&answer)); + let without = grammar_ctx(&tools, &tool_choice, None); + assert_eq!( + parser.build_output_grammar(&with_answer).unwrap(), + parser.build_output_grammar(&without).unwrap(), + "{tool_choice:?}" + ); + } + } + + #[test] + fn answer_without_a_reasoning_phase_covers_the_final_output() { + let tools = test_tools(); + let answer = Format::regex("[a-z]+"); + let none = ToolChoice::none(); + let auto = ToolChoice::auto(); + + // Without an active parser, the request keeps its own constraint. + let parser = CombinedParser::new(None, None); + let ctx = grammar_ctx(&tools, &auto, Some(&answer)); + assert!(parser.build_output_grammar(&ctx).unwrap().is_none()); + + let parser = CombinedParser::new(None, Some(Qwen3XmlToolParser::create(&tools).unwrap())); + let ctx = grammar_ctx(&tools, &none, Some(&answer)); + let alone = parser.build_output_grammar(&ctx).unwrap().unwrap(); + assert_eq!(alone, BuiltOutputGrammar::final_output_only(answer.clone())); + + let ctx = grammar_ctx(&tools, &auto, Some(&answer)); + let composed = parser.build_output_grammar(&ctx).unwrap().unwrap(); + assert_eq!(composed.coverage, GrammarCoverage::FinalOutputOnly); + assert!(matches!(composed.format, Format::Or(_))); + } } diff --git a/rust/src/parser/src/unified/hy/hy_v3.rs b/rust/src/parser/src/unified/hy/hy_v3.rs index 3599ce2cb9..18a34954b3 100644 --- a/rust/src/parser/src/unified/hy/hy_v3.rs +++ b/rust/src/parser/src/unified/hy/hy_v3.rs @@ -164,6 +164,7 @@ mod tests { tools: &tools, tool_strict_level: Default::default(), parallel_tool_calls: true, + answer: None, tool_choice: &ToolChoice::required(), }) .unwrap() @@ -194,6 +195,7 @@ mod tests { tool_choice: &ToolChoice::required(), tool_strict_level: Default::default(), parallel_tool_calls: true, + answer: None, }) .unwrap() .unwrap(); diff --git a/rust/src/parser/src/unified/hy/hy_v4.rs b/rust/src/parser/src/unified/hy/hy_v4.rs index a5d81e0f2a..f10fb662ec 100644 --- a/rust/src/parser/src/unified/hy/hy_v4.rs +++ b/rust/src/parser/src/unified/hy/hy_v4.rs @@ -178,6 +178,7 @@ mod tests { tools: &tools, tool_strict_level: Default::default(), parallel_tool_calls: true, + answer: None, tool_choice: &ToolChoice::required(), }) .unwrap() diff --git a/rust/src/parser/src/unified/kimi_k3/grammar.rs b/rust/src/parser/src/unified/kimi_k3/grammar.rs index 166fbafb0f..e6ef746fa2 100644 --- a/rust/src/parser/src/unified/kimi_k3/grammar.rs +++ b/rust/src/parser/src/unified/kimi_k3/grammar.rs @@ -16,6 +16,10 @@ //! //! Every path may end with the message close, and a stop token is legal only //! after the last channel closes. +//! +//! The user's answer constraint, if any, replaces the free text of the +//! `response` channel. Without a callable tool, the `response` channel then +//! follows `think` alone. mod arguments; @@ -33,38 +37,44 @@ use crate::output_grammar::{ }; /// Build the grammar of everything generated after a prompt that left the -/// parser in `mode`, or `None` when the request asks for no tool grammar. +/// parser in `mode`, or `None` when the request asks for no tool grammar and +/// has no answer constraint. pub(super) fn build_output_grammar( mode: &KimiK3Mode, ctx: &OutputGrammarContext<'_>, ) -> output_grammar::Result> { - let Some(NormalizedToolChoice { - function_tools, - choice, - .. - }) = normalize_tool_choice(ctx)? - else { + let normalized = normalize_tool_choice(ctx)?; + if normalized.is_none() && ctx.answer.is_none() { return Ok(None); - }; - - let single_call = choice == BuilderToolChoice::Forced || !ctx.parallel_tool_calls; - let tools = Format::tag( - TOOLS_OPEN, - Format::tags_with_separator( - function_tools.iter().map(call_tag).collect(), - "", - true, - single_call, - ), - TOOLS_CLOSE, + } + let calls = normalized.map( + |NormalizedToolChoice { + function_tools, + choice, + .. + }| { + let single_call = choice == BuilderToolChoice::Forced || !ctx.parallel_tool_calls; + let tools = Format::tag( + TOOLS_OPEN, + Format::tags_with_separator( + function_tools.iter().map(call_tag).collect(), + "", + true, + single_call, + ), + TOOLS_CLOSE, + ); + (choice, tools) + }, ); + let answer = ctx.answer; let mut elements = match mode { - KimiK3Mode::Reasoning => vec![think_body(), after_reasoning(choice, tools)], - KimiK3Mode::Response => vec![response_body(), after_response(choice, tools)], + KimiK3Mode::Reasoning => vec![think_body(), after_reasoning(calls, answer)], + KimiK3Mode::Response => rest_of_response(calls, answer), KimiK3Mode::Idle => vec![ Format::optional(Format::tag(THINK_OPEN, think_text(), THINK_CLOSE)), - after_reasoning(choice, tools), + after_reasoning(calls, answer), ], KimiK3Mode::Epilogue | KimiK3Mode::Tools | KimiK3Mode::Call { .. } | KimiK3Mode::Done => { unreachable!("initialization leaves the parser idle or inside `think` or `response`") @@ -77,38 +87,73 @@ pub(super) fn build_output_grammar( )))) } -/// Channels that may follow the closed `think` channel. -fn after_reasoning(tool_choice: BuilderToolChoice, tools: Format) -> Format { - let response = Format::tag(RESPONSE_OPEN, response_text(), RESPONSE_CLOSE); - match tool_choice { - BuilderToolChoice::Auto => Format::or(vec![ +/// Channels that may follow the closed `think` channel, given the tool +/// choice and `tools` channel when a tool is callable, and the user's answer +/// constraint for the `response` channel. +fn after_reasoning(calls: Option<(BuilderToolChoice, Format)>, answer: Option<&Format>) -> Format { + let response = Format::tag(RESPONSE_OPEN, response_content(answer), RESPONSE_CLOSE); + let Some((choice, tools)) = calls else { + return response; + }; + let empty = || Format::const_string(format!("{RESPONSE_OPEN}{RESPONSE_CLOSE}")); + match (choice, answer) { + (BuilderToolChoice::Auto, None) => Format::or(vec![ Format::sequence(vec![response, Format::optional(tools.clone())]), tools, ]), - BuilderToolChoice::Required | BuilderToolChoice::Forced => { + (BuilderToolChoice::Auto, Some(_)) => Format::or(vec![ + Format::sequence(vec![response, Format::optional(tools.clone())]), + Format::sequence(vec![Format::optional(empty()), tools]), + ]), + (BuilderToolChoice::Required | BuilderToolChoice::Forced, None) => { Format::sequence(vec![Format::optional(response), tools]) } + (BuilderToolChoice::Required | BuilderToolChoice::Forced, Some(_)) => { + Format::sequence(vec![ + Format::optional(Format::or(vec![response, empty()])), + tools, + ]) + } } } -/// Channels that may follow the closed `response` channel. -fn after_response(tool_choice: BuilderToolChoice, tools: Format) -> Format { - match tool_choice { - BuilderToolChoice::Auto => Format::optional(tools), - BuilderToolChoice::Required | BuilderToolChoice::Forced => tools, +/// The rest of the `response` channel the prompt opened, then the channels +/// that may follow it. +fn rest_of_response( + calls: Option<(BuilderToolChoice, Format)>, + answer: Option<&Format>, +) -> Vec { + let response = Format::tag("", response_content(answer), RESPONSE_CLOSE); + let Some((choice, tools)) = calls else { + return vec![response]; + }; + let empty = || Format::const_string(RESPONSE_CLOSE); + match (choice, answer) { + (BuilderToolChoice::Auto, None) => vec![response, Format::optional(tools)], + (BuilderToolChoice::Auto, Some(_)) => vec![Format::or(vec![ + Format::sequence(vec![response, Format::optional(tools.clone())]), + Format::sequence(vec![empty(), tools]), + ])], + (BuilderToolChoice::Required | BuilderToolChoice::Forced, None) => vec![response, tools], + (BuilderToolChoice::Required | BuilderToolChoice::Forced, Some(_)) => { + vec![Format::or(vec![response, empty()]), tools] + } } } +/// The text of the `response` channel: the user's answer constraint, or free +/// text. A constrained answer rules out the empty `response` channel the chat +/// template writes before tool calls, so callers allow that channel where +/// calls follow. +fn response_content(answer: Option<&Format>) -> Format { + answer.cloned().unwrap_or_else(response_text) +} + /// The rest of the `think` channel the prompt opened. fn think_body() -> Format { Format::tag("", think_text(), THINK_CLOSE) } -/// The rest of the `response` channel the prompt opened. -fn response_body() -> Format { - Format::tag("", response_text(), RESPONSE_CLOSE) -} - fn think_text() -> Format { Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]) } @@ -158,6 +203,28 @@ mod tests { tool_choice: &tool_choice, tool_strict_level: ToolStrictLevel::Function, parallel_tool_calls, + answer: None, + }, + ) + .unwrap() + .unwrap(); + assert_eq!(grammar.coverage, GrammarCoverage::FromTokenZero); + let mut format = grammar.format; + elide_call_bodies(&mut format); + expected.assert_eq(&outline(&format)); + } + + /// Like [`check`], for non-strict tools and a JSON answer constraint. + fn check_answer(mode: KimiK3Mode, tools: &[Tool], tool_choice: ToolChoice, expected: Expect) { + let answer = Format::json_schema(json!({ "type": "object" })); + let grammar = build_output_grammar( + &mode, + &OutputGrammarContext { + tools, + tool_choice: &tool_choice, + tool_strict_level: ToolStrictLevel::Auto, + parallel_tool_calls: true, + answer: Some(&answer), }, ) .unwrap() @@ -348,9 +415,120 @@ mod tests { tool_choice: &ToolChoice::auto(), tool_strict_level: ToolStrictLevel::Auto, parallel_tool_calls: true, + answer: None, }, ) .unwrap(); assert_eq!(grammar, None); } + + #[test] + fn answer_constraint_fills_the_response_channel() { + // Non-strict `auto` keeps the calls next to the constrained answer. + // Calls may follow the empty `response` channel the chat template + // writes before them. + check_answer( + KimiK3Mode::Reasoning, + &tools(), + ToolChoice::auto(), + expect![[r#" + sequence + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + or + sequence + tag `<|open|>response<|sep|>` json(object) `<|close|>response<|sep|>` + optional + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + sequence + optional `<|open|>response<|sep|><|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + // A required call may follow a constrained answer. + check_answer( + KimiK3Mode::Reasoning, + &tools(), + ToolChoice::required(), + expect![[r#" + sequence + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + sequence + optional + or + tag `<|open|>response<|sep|>` json(object) `<|close|>response<|sep|>` + `<|open|>response<|sep|><|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + check_answer( + KimiK3Mode::Response, + &tools(), + ToolChoice::auto(), + expect![[r#" + sequence + or + sequence + tag `` json(object) `<|close|>response<|sep|>` + optional + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + sequence + `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + } + + #[test] + fn answer_constraint_without_a_callable_tool_requires_the_response() { + check_answer( + KimiK3Mode::Idle, + &tools(), + ToolChoice::none(), + expect![[r#" + sequence + optional tag `<|open|>think<|sep|>` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + tag `<|open|>response<|sep|>` json(object) `<|close|>response<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + check_answer( + KimiK3Mode::Reasoning, + &[], + ToolChoice::auto(), + expect![[r#" + sequence + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + tag `<|open|>response<|sep|>` json(object) `<|close|>response<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + check_answer( + KimiK3Mode::Response, + &[], + ToolChoice::auto(), + expect![[r#" + sequence + tag `` json(object) `<|close|>response<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + } }