From 0466bd36ea9f6300acbc15c705b1ebe61590c691 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Mon, 12 Oct 2026 17:43:47 +0800 Subject: [PATCH] [Rust Frontend] Build the Kimi K3 full-output grammar from its parser mode (#59190) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../chat/tests/grammar_replay/Kimi-K3.json | 1042 ++++++++++------- .../src/chat/tests/grammar_replay/Kimi-K3.txt | 99 +- rust/src/parser/src/output_grammar.rs | 19 +- rust/src/parser/src/unified/kimi_k3.rs | 14 +- .../src/parser/src/unified/kimi_k3/grammar.rs | 356 ++++++ .../src/unified/kimi_k3/grammar/arguments.rs | 184 +++ .../src/unified/kimi_k3/structural_tag.rs | 343 ------ rust/src/parser/src/unified/mod.rs | 2 +- 8 files changed, 1259 insertions(+), 800 deletions(-) create mode 100644 rust/src/parser/src/unified/kimi_k3/grammar.rs create mode 100644 rust/src/parser/src/unified/kimi_k3/grammar/arguments.rs delete mode 100644 rust/src/parser/src/unified/kimi_k3/structural_tag.rs diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.json b/rust/src/chat/tests/grammar_replay/Kimi-K3.json index 7fe53b0ab2..5306b93660 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K3.json +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.json @@ -8,213 +8,413 @@ "format": { "type": "sequence", "elements": [ - { - "type": "optional", - "content": { - "type": "const_string", - "value": "<|open|>response<|sep|>" - } - }, { "type": "tag", "begin": "", "content": { "type": "any_text", "excludes": [ - "<|open|>", - "<|close|>", + "<|close|>think<|sep|>", "<|end_of_msg|>" ], "max_tokens": null, "max_chars": null }, - "end": "<|close|>response<|sep|>" + "end": "<|close|>think<|sep|>" }, { - "type": "optional", - "content": { - "type": "tag", - "begin": "<|open|>tools<|sep|>", - "content": { - "type": "tags_with_separator", - "tags": [ + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ { "type": "tag", - "begin": "<|open|>call tool=\"get_weather\" index=\"", + "begin": "<|open|>response<|sep|>", "content": { - "type": "sequence", - "elements": [ - { - "type": "regex", - "pattern": "[1-9][0-9]*" - }, - { - "type": "const_string", - "value": "\"<|sep|>" - }, - { - "type": "or", - "elements": [ - { - "type": "tag", - "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", - "content": { - "type": "any_text", - "excludes": [ - "<|close|>argument<|sep|>", - "<|close|>call<|sep|>" - ], - "max_tokens": null, - "max_chars": null - }, - "end": "<|close|>argument<|sep|>" - }, - { - "type": "tag", - "begin": "<|open|>json type=\"object\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] - }, - "end": "<|close|>json<|sep|>" - } - ] - } - ] + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null }, - "end": "<|close|>call<|sep|>" + "end": "<|close|>response<|sep|>" }, { - "type": "tag", - "begin": "<|open|>call tool=\"add\" index=\"", + "type": "optional", "content": { - "type": "sequence", - "elements": [ - { - "type": "regex", - "pattern": "[1-9][0-9]*" - }, - { - "type": "const_string", - "value": "\"<|sep|>" - }, - { - "type": "or", - "elements": [ - { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { "type": "sequence", "elements": [ { - "type": "tag", - "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "number" - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] - }, - "end": "<|close|>argument<|sep|>" + "type": "regex", + "pattern": "[1-9][0-9]*" }, { - "type": "tag", - "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "number" - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] - }, - "end": "<|close|>argument<|sep|>" + "type": "const_string", + "value": "\"<|sep|>" }, { - "type": "tag", - "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "array", - "items": { - "type": "string" - } + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] - }, - "end": "<|close|>argument<|sep|>" + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] } ] }, - { - "type": "tag", - "begin": "<|open|>json type=\"object\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + } + } + ] + }, + { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { "type": "string" } - } + }, + "required": [ + "location" + ] }, - "required": [ - "y", - "x", - "items" - ] + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] - }, - "end": "<|close|>json<|sep|>" - } - ] - } - ] + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" }, - "end": "<|close|>call<|sep|>" - } - ], - "separator": "", - "at_least_one": true, - "stop_after_first": false - }, - "end": "<|close|>tools<|sep|>" - } + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + } + ] }, { "type": "optional", @@ -225,7 +425,7 @@ } ] }, - "coverage": "final_output_only" + "coverage": "from_token_zero" }, "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] @@ -235,210 +435,225 @@ "format": { "type": "sequence", "elements": [ - { - "type": "optional", - "content": { - "type": "const_string", - "value": "<|open|>response<|sep|>" - } - }, { "type": "tag", "begin": "", "content": { "type": "any_text", "excludes": [ - "<|open|>", - "<|close|>", + "<|close|>think<|sep|>", "<|end_of_msg|>" ], "max_tokens": null, "max_chars": null }, - "end": "<|close|>response<|sep|>" + "end": "<|close|>think<|sep|>" }, { - "type": "tag", - "begin": "<|open|>tools<|sep|>", - "content": { - "type": "tags_with_separator", - "tags": [ - { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { "type": "tag", - "begin": "<|open|>call tool=\"get_weather\" index=\"", + "begin": "<|open|>response<|sep|>", "content": { - "type": "sequence", - "elements": [ - { - "type": "regex", - "pattern": "[1-9][0-9]*" - }, - { - "type": "const_string", - "value": "\"<|sep|>" - }, - { - "type": "or", + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", "elements": [ { - "type": "tag", - "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", - "content": { - "type": "any_text", - "excludes": [ - "<|close|>argument<|sep|>", - "<|close|>call<|sep|>" - ], - "max_tokens": null, - "max_chars": null - }, - "end": "<|close|>argument<|sep|>" + "type": "regex", + "pattern": "[1-9][0-9]*" }, { - "type": "tag", - "begin": "<|open|>json type=\"object\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] - }, - "end": "<|close|>json<|sep|>" - } - ] - } - ] - }, - "end": "<|close|>call<|sep|>" - }, - { - "type": "tag", - "begin": "<|open|>call tool=\"add\" index=\"", - "content": { - "type": "sequence", - "elements": [ - { - "type": "regex", - "pattern": "[1-9][0-9]*" - }, - { - "type": "const_string", - "value": "\"<|sep|>" - }, - { - "type": "or", - "elements": [ + "type": "const_string", + "value": "\"<|sep|>" + }, { - "type": "sequence", + "type": "or", "elements": [ { "type": "tag", - "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", "content": { - "type": "json_schema", - "json_schema": { - "type": "number" - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null }, "end": "<|close|>argument<|sep|>" }, { "type": "tag", - "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "begin": "<|open|>json type=\"object\"<|sep|>", "content": { "type": "json_schema", "json_schema": { - "type": "number" + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] }, "style": "json", "any_order": false, "max_whitespace_cnt": null, "excludes": [] }, - "end": "<|close|>argument<|sep|>" - }, - { - "type": "tag", - "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "array", - "items": { - "type": "string" - } - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] - }, - "end": "<|close|>argument<|sep|>" + "end": "<|close|>json<|sep|>" } ] - }, - { - "type": "tag", - "begin": "<|open|>json type=\"object\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] - }, - "end": "<|close|>json<|sep|>" } ] - } - ] - }, - "end": "<|close|>call<|sep|>" - } - ], - "separator": "", - "at_least_one": true, - "stop_after_first": false - }, - "end": "<|close|>tools<|sep|>" + }, + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + } + ] }, { "type": "optional", @@ -449,7 +664,7 @@ } ] }, - "coverage": "final_output_only" + "coverage": "from_token_zero" }, "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] @@ -459,100 +674,115 @@ "format": { "type": "sequence", "elements": [ - { - "type": "optional", - "content": { - "type": "const_string", - "value": "<|open|>response<|sep|>" - } - }, { "type": "tag", "begin": "", "content": { "type": "any_text", "excludes": [ - "<|open|>", - "<|close|>", + "<|close|>think<|sep|>", "<|end_of_msg|>" ], "max_tokens": null, "max_chars": null }, - "end": "<|close|>response<|sep|>" + "end": "<|close|>think<|sep|>" }, { - "type": "tag", - "begin": "<|open|>tools<|sep|>", - "content": { - "type": "tags_with_separator", - "tags": [ - { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { "type": "tag", - "begin": "<|open|>call tool=\"get_weather\" index=\"", + "begin": "<|open|>response<|sep|>", "content": { - "type": "sequence", - "elements": [ - { - "type": "regex", - "pattern": "[1-9][0-9]*" - }, - { - "type": "const_string", - "value": "\"<|sep|>" - }, - { - "type": "or", + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", "elements": [ { - "type": "tag", - "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", - "content": { - "type": "any_text", - "excludes": [ - "<|close|>argument<|sep|>", - "<|close|>call<|sep|>" - ], - "max_tokens": null, - "max_chars": null - }, - "end": "<|close|>argument<|sep|>" + "type": "regex", + "pattern": "[1-9][0-9]*" }, { - "type": "tag", - "begin": "<|open|>json type=\"object\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null }, - "required": [ - "location" - ] + "end": "<|close|>argument<|sep|>" }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] - }, - "end": "<|close|>json<|sep|>" + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] } ] - } - ] - }, - "end": "<|close|>call<|sep|>" - } - ], - "separator": "", - "at_least_one": true, - "stop_after_first": false - }, - "end": "<|close|>tools<|sep|>" + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": true + }, + "end": "<|close|>tools<|sep|>" + } + ] }, { "type": "optional", @@ -563,7 +793,7 @@ } ] }, - "coverage": "final_output_only" + "coverage": "from_token_zero" }, "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.txt b/rust/src/chat/tests/grammar_replay/Kimi-K3.txt index 3aeb612642..6a1c814c13 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K3.txt +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.txt @@ -1,8 +1,29 @@ -# auto (final_output_only) +# auto (from_token_zero) sequence - optional `<|open|>response<|sep|>` - tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` - optional + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + or + sequence + tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + optional + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` + tag `<|open|>call tool="add" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + sequence + tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ y: number, x: number, items: string[] }) `<|close|>json<|sep|>` tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` tags_with_separator `` at_least_one tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` @@ -24,42 +45,44 @@ sequence tag `<|open|>json type="object"<|sep|>` json({ y: number, x: number, items: string[] }) `<|close|>json<|sep|>` optional `<|close|>message<|sep|>` -# required (final_output_only) +# required (from_token_zero) sequence - optional `<|open|>response<|sep|>` - tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` - tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` - tags_with_separator `` at_least_one - tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` - sequence - /[1-9][0-9]*/ - `"<|sep|>` - or - tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` - tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` - tag `<|open|>call tool="add" index="` .. `<|close|>call<|sep|>` - sequence - /[1-9][0-9]*/ - `"<|sep|>` - or - sequence - tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` - tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` - tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` - tag `<|open|>json type="object"<|sep|>` json({ y: number, x: number, items: string[] }) `<|close|>json<|sep|>` + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + sequence + optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` + tag `<|open|>call tool="add" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + sequence + tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ y: number, x: number, items: string[] }) `<|close|>json<|sep|>` optional `<|close|>message<|sep|>` -# named (final_output_only) +# named (from_token_zero) sequence - optional `<|open|>response<|sep|>` - tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` - tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` - tags_with_separator `` at_least_one - tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` - sequence - /[1-9][0-9]*/ - `"<|sep|>` - or - tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` - tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + sequence + optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one stop_after_first + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` optional `<|close|>message<|sep|>` diff --git a/rust/src/parser/src/output_grammar.rs b/rust/src/parser/src/output_grammar.rs index 53c52538ee..0108c3193e 100644 --- a/rust/src/parser/src/output_grammar.rs +++ b/rust/src/parser/src/output_grammar.rs @@ -12,7 +12,8 @@ use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagOptio use xgrammar_structural_tag::format::Format; use xgrammar_structural_tag::tool::ToolChoiceValue; use xgrammar_structural_tag::{ - FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag, + FunctionDefinition, FunctionToolParam, NormalizedToolChoice, ToolChoice, ToolParam, + build_structural_tag, }; use crate::tool::Tool; @@ -218,6 +219,22 @@ fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec .collect() } +/// Normalize the request tools the way `build_structural_tag` does before +/// calling a crate builder, for model-owned grammar builders. Returns `None` +/// when the request asks for no tool grammar. +pub(crate) fn normalize_tool_choice( + ctx: &OutputGrammarContext<'_>, +) -> Result> { + if !tool_grammar_applies(ctx) { + return Ok(None); + } + + let tools = tool_params(ctx.tools, ctx.tool_strict_level); + let normalized = + xgrammar_structural_tag::normalize_tool_choice(&tools, ctx.tool_choice.clone())?; + Ok(Some(normalized)) +} + /// Whether the request asks for a tool grammar at all. fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool { if ctx.tools.is_empty() { diff --git a/rust/src/parser/src/unified/kimi_k3.rs b/rust/src/parser/src/unified/kimi_k3.rs index d6e32463c1..13d6fa42eb 100644 --- a/rust/src/parser/src/unified/kimi_k3.rs +++ b/rust/src/parser/src/unified/kimi_k3.rs @@ -46,9 +46,7 @@ //! now only applies to `argument` and `json` blocks inside a call body, which //! are still parsed from text. -mod structural_tag; - -pub use structural_tag::KimiK3StructuralTagBuilder; +mod grammar; use serde_json::{Map, Value}; use vllm_tokenizer::{DecodedText, DynTokenizer, Tokenizer}; @@ -58,11 +56,8 @@ use winnow::error::{ContextError, ErrMode, ModalResult, StrContext}; use winnow::prelude::*; use winnow::token::{literal, rest, take_till, take_until, take_while}; -use self::structural_tag::KIMI_K3_STRUCTURAL_TAG_BUILDER; use super::{Result, UnifiedParser, UnifiedParserOutput, special_token}; -use crate::output_grammar::{ - self, BuiltOutputGrammar, OutputGrammarContext, visible_format_from_builder, -}; +use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext}; use crate::tool::{Tool, ToolCallDelta}; use crate::unified::parsing_failed; use crate::utils::{ @@ -347,10 +342,7 @@ impl UnifiedParser for KimiK3UnifiedParser { &self, ctx: &OutputGrammarContext<'_>, ) -> output_grammar::Result> { - Ok( - visible_format_from_builder(Some(&KIMI_K3_STRUCTURAL_TAG_BUILDER), ctx)? - .map(BuiltOutputGrammar::final_output_only), - ) + grammar::build_output_grammar(&self.mode, ctx) } fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> { diff --git a/rust/src/parser/src/unified/kimi_k3/grammar.rs b/rust/src/parser/src/unified/kimi_k3/grammar.rs new file mode 100644 index 0000000000..166fbafb0f --- /dev/null +++ b/rust/src/parser/src/unified/kimi_k3/grammar.rs @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Full-output grammar for Kimi K3 XTML generations. +//! +//! The grammar starts at the channel the prompt left open, which is the +//! parser's initialized [`KimiK3Mode`], and follows the same channel +//! transitions as the parser. A prompt that opened no channel may first emit a +//! whole `think` channel. +//! +//! | Tool choice | After `think` closes | After `response` closes | +//! | ----------- | ------------------------------ | ----------------------- | +//! | `auto` | `response [tools]`, or `tools` | `[tools]` | +//! | `required` | `[response] tools` | `tools` | +//! | named | `[response] tools`, one call | `tools`, one call | +//! +//! Every path may end with the message close, and a stop token is legal only +//! after the last channel closes. + +mod arguments; + +use xgrammar_structural_tag::NormalizedToolChoice; +use xgrammar_structural_tag::format::Format; +use xgrammar_structural_tag::tool::BuilderToolChoice; + +use self::arguments::call_tag; +use super::{ + CLOSE, END_OF_MSG, KimiK3Mode, MESSAGE_CLOSE, OPEN, RESPONSE_CLOSE, RESPONSE_OPEN, THINK_CLOSE, + THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN, +}; +use crate::output_grammar::{ + self, BuiltOutputGrammar, OutputGrammarContext, normalize_tool_choice, +}; + +/// Build the grammar of everything generated after a prompt that left the +/// parser in `mode`, or `None` when the request asks for no tool grammar. +pub(super) fn build_output_grammar( + mode: &KimiK3Mode, + ctx: &OutputGrammarContext<'_>, +) -> output_grammar::Result> { + let Some(NormalizedToolChoice { + function_tools, + choice, + .. + }) = normalize_tool_choice(ctx)? + else { + return Ok(None); + }; + + let single_call = choice == BuilderToolChoice::Forced || !ctx.parallel_tool_calls; + let tools = Format::tag( + TOOLS_OPEN, + Format::tags_with_separator( + function_tools.iter().map(call_tag).collect(), + "", + true, + single_call, + ), + TOOLS_CLOSE, + ); + + let mut elements = match mode { + KimiK3Mode::Reasoning => vec![think_body(), after_reasoning(choice, tools)], + KimiK3Mode::Response => vec![response_body(), after_response(choice, tools)], + KimiK3Mode::Idle => vec![ + Format::optional(Format::tag(THINK_OPEN, think_text(), THINK_CLOSE)), + after_reasoning(choice, tools), + ], + KimiK3Mode::Epilogue | KimiK3Mode::Tools | KimiK3Mode::Call { .. } | KimiK3Mode::Done => { + unreachable!("initialization leaves the parser idle or inside `think` or `response`") + } + }; + elements.push(Format::optional(Format::const_string(MESSAGE_CLOSE))); + + Ok(Some(BuiltOutputGrammar::from_token_zero(Format::sequence( + elements, + )))) +} + +/// Channels that may follow the closed `think` channel. +fn after_reasoning(tool_choice: BuilderToolChoice, tools: Format) -> Format { + let response = Format::tag(RESPONSE_OPEN, response_text(), RESPONSE_CLOSE); + match tool_choice { + BuilderToolChoice::Auto => Format::or(vec![ + Format::sequence(vec![response, Format::optional(tools.clone())]), + tools, + ]), + BuilderToolChoice::Required | BuilderToolChoice::Forced => { + Format::sequence(vec![Format::optional(response), tools]) + } + } +} + +/// Channels that may follow the closed `response` channel. +fn after_response(tool_choice: BuilderToolChoice, tools: Format) -> Format { + match tool_choice { + BuilderToolChoice::Auto => Format::optional(tools), + BuilderToolChoice::Required | BuilderToolChoice::Forced => tools, + } +} + +/// The rest of the `think` channel the prompt opened. +fn think_body() -> Format { + Format::tag("", think_text(), THINK_CLOSE) +} + +/// The rest of the `response` channel the prompt opened. +fn response_body() -> Format { + Format::tag("", response_text(), RESPONSE_CLOSE) +} + +fn think_text() -> Format { + Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]) +} + +fn response_text() -> Format { + // Keep marker-looking prefixes out of response text so the grammar must + // resolve them through a valid channel boundary. + Format::any_text_excluding(&[OPEN, CLOSE, END_OF_MSG]) +} + +#[cfg(test)] +mod tests { + use expect_test::{Expect, expect}; + use serde_json::json; + use xgrammar_structural_tag::ToolChoice; + use xgrammar_structural_tag::format::{EndBoundary, TagFormat}; + + use super::super::CALL_CLOSE; + use super::*; + use crate::output_grammar::test_utils::outline; + use crate::output_grammar::{GrammarCoverage, ToolStrictLevel}; + use crate::tool::Tool; + + fn tools() -> Vec { + ["get_weather", "add"] + .map(|name| Tool { + name: name.to_string(), + description: None, + parameters: json!({ "type": "object", "properties": {} }), + strict: None, + defer_loading: None, + }) + .into() + } + + fn check( + mode: KimiK3Mode, + tool_choice: ToolChoice, + parallel_tool_calls: bool, + expected: Expect, + ) { + let tools = tools(); + let grammar = build_output_grammar( + &mode, + &OutputGrammarContext { + tools: &tools, + tool_choice: &tool_choice, + tool_strict_level: ToolStrictLevel::Function, + parallel_tool_calls, + }, + ) + .unwrap() + .unwrap(); + assert_eq!(grammar.coverage, GrammarCoverage::FromTokenZero); + let mut format = grammar.format; + elide_call_bodies(&mut format); + expected.assert_eq(&outline(&format)); + } + + /// Replace each call body with `..`, so that outlines show the channel + /// structure. The `arguments` tests cover call bodies. + fn elide_call_bodies(format: &mut Format) { + match format { + Format::Sequence(format) => format.elements.iter_mut().for_each(elide_call_bodies), + Format::Or(format) => format.elements.iter_mut().for_each(elide_call_bodies), + Format::Optional(format) => elide_call_bodies(&mut format.content), + Format::Tag(tag) => elide_tag(tag), + Format::TagsWithSeparator(format) => format.tags.iter_mut().for_each(elide_tag), + _ => {} + } + } + + fn elide_tag(tag: &mut TagFormat) { + if matches!(&tag.end, EndBoundary::Text(end) if end == CALL_CLOSE) { + *tag.content = Format::const_string(".."); + } else { + elide_call_bodies(&mut tag.content); + } + } + + #[test] + fn prompt_opened_think_channel() { + check( + KimiK3Mode::Reasoning, + ToolChoice::auto(), + true, + expect![[r#" + sequence + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + or + sequence + tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + optional + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + check( + KimiK3Mode::Reasoning, + ToolChoice::required(), + true, + expect![[r#" + sequence + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + sequence + optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + check( + KimiK3Mode::Reasoning, + ToolChoice::function("add"), + true, + expect![[r#" + sequence + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + sequence + optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one stop_after_first + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + } + + #[test] + fn prompt_opened_response_channel() { + check( + KimiK3Mode::Response, + ToolChoice::auto(), + true, + expect![[r#" + sequence + tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + optional + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + check( + KimiK3Mode::Response, + ToolChoice::required(), + true, + expect![[r#" + sequence + tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + check( + KimiK3Mode::Response, + ToolChoice::function("add"), + true, + expect![[r#" + sequence + tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one stop_after_first + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + } + + #[test] + fn no_prompt_opened_channel() { + check( + KimiK3Mode::Idle, + ToolChoice::auto(), + true, + expect![[r#" + sequence + optional tag `<|open|>think<|sep|>` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + or + sequence + tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + optional + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + } + + #[test] + fn serial_tool_calls_stop_after_one_call() { + check( + KimiK3Mode::Reasoning, + ToolChoice::required(), + false, + expect![[r#" + sequence + tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>` + sequence + optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one stop_after_first + tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>` + tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>` + optional `<|close|>message<|sep|>` + "#]], + ); + } + + #[test] + fn non_strict_auto_builds_no_grammar() { + let tools = tools(); + let grammar = build_output_grammar( + &KimiK3Mode::Reasoning, + &OutputGrammarContext { + tools: &tools, + tool_choice: &ToolChoice::auto(), + tool_strict_level: ToolStrictLevel::Auto, + parallel_tool_calls: true, + }, + ) + .unwrap(); + assert_eq!(grammar, None); + } +} diff --git a/rust/src/parser/src/unified/kimi_k3/grammar/arguments.rs b/rust/src/parser/src/unified/kimi_k3/grammar/arguments.rs new file mode 100644 index 0000000000..49587535aa --- /dev/null +++ b/rust/src/parser/src/unified/kimi_k3/grammar/arguments.rs @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Kimi K3 XTML tool-call grammar: call tags and their typed arguments. + +use serde_json::Value; +use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, TagFormat}; +use xgrammar_structural_tag::tool::{FunctionToolParam, function_parameters}; + +use super::super::{ARG_CLOSE, ARG_OPEN, CALL_CLOSE, JSON_CLOSE, JSON_OPEN, OPEN, SEP}; +use crate::output_grammar::arguments::{ + self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, ValueOption, group_by, one_of, +}; + +/// XTML arguments: +/// `<|open|>argument key="KEY" type="TYPE"<|sep|>VALUE<|close|>argument<|sep|>`, +/// adjacent to each other. +/// +/// The parser decodes each value by its `type` attribute, so every value option +/// sits under the attribute of its own JSON type. String values are raw text +/// and every other value is JSON. +struct XtmlArguments; + +impl ArgumentSyntax for XtmlArguments { + fn separator(&self) -> Option { + None + } + + fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option { + let escaped; + let key = match key { + ParameterKey::Declared(key) => { + escaped = escape_attr_value(key); + ParameterKey::Declared(&escaped) + } + ParameterKey::Free => ParameterKey::Free, + }; + let arguments = group_by(options, |option| xtml_type(option.ty)) + .into_iter() + .filter_map(|(type_name, options)| { + let values = options + .into_iter() + .filter_map(|option| match option.ty { + // The parser ends the call at its close marker, so a + // string value may contain neither close marker. + JsonType::String => option.raw_string(&[ARG_CLOSE, CALL_CLOSE]), + _ => Some(option.json()), + }) + .collect::>(); + (!values.is_empty()).then(|| { + let suffix = format!("\" type=\"{type_name}\"{SEP}"); + key.tag( + &format!("{ARG_OPEN} key=\""), + &suffix, + one_of(values), + ARG_CLOSE, + ) + }) + }) + .collect::>(); + (!arguments.is_empty()).then(|| one_of(arguments)) + } +} + +/// The `type` attribute K3 renders for a value of type `ty`, as +/// `_xtml_type` in the checkpoint's `encoding_k3.py`: integers and floats are +/// both `number`. +fn xtml_type(ty: JsonType) -> &'static str { + match ty { + JsonType::String => "string", + JsonType::Integer | JsonType::Number => "number", + JsonType::Boolean => "boolean", + JsonType::Null => "null", + JsonType::Object => "object", + JsonType::Array => "array", + } +} + +/// Build the tag for one call to `tool`. +pub(super) fn call_tag(tool: &FunctionToolParam) -> TagFormat { + let parameters = function_parameters(&tool.function); + let typed_arguments = + arguments::arguments(¶meters, &XtmlArguments, &ArgumentOptions::default()); + let call_body = Format::or(vec![typed_arguments, raw_json_arguments(¶meters)]); + + TagFormat::new( + format!( + "{OPEN}call tool=\"{}\" index=\"", + escape_attr_value(&tool.function.name) + ), + Format::sequence(vec![ + Format::regex("[1-9][0-9]*"), + Format::const_string(format!("\"{SEP}")), + call_body, + ]), + CALL_CLOSE, + ) +} + +fn raw_json_arguments(parameters: &Value) -> Format { + Format::tag( + format!("{JSON_OPEN} type=\"object\"{SEP}"), + Format::JsonSchema(JsonSchemaFormat::new(parameters.clone())), + JSON_CLOSE, + ) +} + +fn escape_attr_value(value: &str) -> String { + value.replace('&', "&").replace('"', """) +} + +#[cfg(test)] +mod tests { + use expect_test::{Expect, expect}; + use serde_json::json; + use xgrammar_structural_tag::FunctionDefinition; + use xgrammar_structural_tag::format::Format; + use xgrammar_structural_tag::tool::FunctionToolParam; + + use crate::output_grammar::test_utils::outline; + + fn check(parameters: serde_json::Value, expected: Expect) { + let tool = FunctionToolParam::new( + FunctionDefinition::new("get_weather").with_parameters(parameters), + ); + expected.assert_eq(&outline(&Format::Tag(super::call_tag(&tool)))); + } + + #[test] + fn call_tag_matches_xtml_arguments() { + check( + json!({ + "$defs": { + "place": { "type": "object", "properties": { "city": { "type": "string" } } } + }, + "type": "object", + "properties": { + "unit": { "type": "string", "enum": ["celsius", "fahrenheit"] }, + "place": { "$ref": "#/$defs/place", "type": "object" } + }, + "required": ["place"] + }), + expect![[r#" + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + sequence + optional + tag `<|open|>argument key="unit" type="string"<|sep|>` .. `<|close|>argument<|sep|>` + or + `celsius` + `fahrenheit` + tag `<|open|>argument key="place" type="object"<|sep|>` json({ city?: string } where place = { city?: string }) `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ unit?: "celsius" | "fahrenheit", place: place & object } where place = { city?: string }) `<|close|>json<|sep|>` + "#]], + ); + } + + #[test] + fn union_argument_content_matches_its_xtml_type() { + check( + json!({ + "type": "object", + "properties": { + "count": { "type": ["integer", "null"] } + } + }), + expect![[r#" + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + optional + or + tag `<|open|>argument key="count" type="number"<|sep|>` json(integer) `<|close|>argument<|sep|>` + tag `<|open|>argument key="count" type="null"<|sep|>` json(null) `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ count?: integer | null }) `<|close|>json<|sep|>` + "#]], + ); + } +} diff --git a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs deleted file mode 100644 index 4754726c1b..0000000000 --- a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs +++ /dev/null @@ -1,343 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -//! Structural-tag grammar for Kimi K3 XTML tool calls. - -use serde_json::Value; -use xgrammar_structural_tag::Result; -use xgrammar_structural_tag::builders::{ - ReasoningMode, StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, -}; -use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat}; -use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters}; - -use crate::output_grammar::arguments::{ - self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, ValueOption, group_by, one_of, -}; - -use super::{ - ARG_CLOSE, ARG_OPEN, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN, - RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN, -}; - -pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder = - KimiK3StructuralTagBuilder; - -/// XTML arguments: -/// `<|open|>argument key="KEY" type="TYPE"<|sep|>VALUE<|close|>argument<|sep|>`, -/// adjacent to each other. -/// -/// The parser decodes each value by its `type` attribute, so every value option -/// sits under the attribute of its own JSON type. String values are raw text -/// and every other value is JSON. -struct XtmlArguments; - -impl ArgumentSyntax for XtmlArguments { - fn separator(&self) -> Option { - None - } - - fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option { - let escaped; - let key = match key { - ParameterKey::Declared(key) => { - escaped = escape_attr_value(key); - ParameterKey::Declared(&escaped) - } - ParameterKey::Free => ParameterKey::Free, - }; - let arguments = group_by(options, |option| xtml_type(option.ty)) - .into_iter() - .filter_map(|(type_name, options)| { - let values = options - .into_iter() - .filter_map(|option| match option.ty { - // The parser ends the call at its close marker, so a - // string value may contain neither close marker. - JsonType::String => option.raw_string(&[ARG_CLOSE, CALL_CLOSE]), - _ => Some(option.json()), - }) - .collect::>(); - (!values.is_empty()).then(|| { - let suffix = format!("\" type=\"{type_name}\"{SEP}"); - key.tag( - &format!("{ARG_OPEN} key=\""), - &suffix, - one_of(values), - ARG_CLOSE, - ) - }) - }) - .collect::>(); - (!arguments.is_empty()).then(|| one_of(arguments)) - } -} - -/// The `type` attribute K3 renders for a value of type `ty`, as -/// `_xtml_type` in the checkpoint's `encoding_k3.py`: integers and floats are -/// both `number`. -fn xtml_type(ty: JsonType) -> &'static str { - match ty { - JsonType::String => "string", - JsonType::Integer | JsonType::Number => "number", - JsonType::Boolean => "boolean", - JsonType::Null => "null", - JsonType::Object => "object", - JsonType::Array => "array", - } -} - -/// Kimi K3 XTML structural-tag builder. -#[derive(Debug, Clone, Copy, Default)] -pub struct KimiK3StructuralTagBuilder; - -impl StructuralTagBuilder for KimiK3StructuralTagBuilder { - fn build(&self, ctx: StructuralTagContext<'_>) -> Result { - let mut elements = response_prefix(ctx.options.reasoning); - - let tools = match ctx.tool_choice { - // Serving lowering filters empty tools before calling the builder, - // while direct `build_structural_tag(..., auto, ...)` calls do not. - BuilderToolChoice::Auto if ctx.function_tools.is_empty() => None, - BuilderToolChoice::Auto => Some(Format::optional(tools_channel( - ctx.function_tools, - ctx.options, - ))), - BuilderToolChoice::Forced | BuilderToolChoice::Required => { - Some(tools_channel(ctx.function_tools, ctx.options)) - } - }; - if let Some(tools) = tools { - elements.push(tools); - } - elements.push(Format::optional(Format::const_string(MESSAGE_CLOSE))); - - Ok(StructuralTag::new(Format::sequence(elements))) - } -} - -fn response_prefix(reasoning: ReasoningMode) -> Vec { - let mut elements = Vec::new(); - if reasoning != ReasoningMode::Disabled { - let prefix = Format::tag( - if reasoning == ReasoningMode::Auto { - THINK_OPEN - } else { - "" - }, - Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]), - THINK_CLOSE, - ); - elements.push(if reasoning == ReasoningMode::Auto { - Format::optional(prefix) - } else { - prefix - }); - elements.push(Format::const_string(RESPONSE_OPEN)); - } else { - elements.push(Format::optional(Format::const_string(RESPONSE_OPEN))); - } - elements.push(Format::tag( - "", - // Keep marker-looking prefixes out of response text so the grammar - // must resolve them through a valid channel boundary. - Format::any_text_excluding(&[OPEN, CLOSE, END_OF_MSG]), - RESPONSE_CLOSE, - )); - elements -} - -fn tools_channel(tools: &[FunctionToolParam], options: StructuralTagOptions) -> Format { - let calls = tools.iter().map(|tool| call_tag(tool, options)).collect(); - Format::tag( - TOOLS_OPEN, - Format::tags_with_separator(calls, "", true, false), - TOOLS_CLOSE, - ) -} - -fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat { - let parameters = function_parameters(&tool.function); - let typed_arguments = arguments::arguments( - ¶meters, - &XtmlArguments, - &ArgumentOptions { - any_order: options.any_order, - max_whitespace_cnt: options.max_whitespace_cnt, - excludes: vec![], - }, - ); - let call_body = Format::or(vec![ - typed_arguments, - raw_json_arguments(¶meters, options), - ]); - - TagFormat::new( - format!( - "{OPEN}call tool=\"{}\" index=\"", - escape_attr_value(&tool.function.name) - ), - Format::sequence(vec![ - Format::regex("[1-9][0-9]*"), - Format::const_string(format!("\"{SEP}")), - call_body, - ]), - CALL_CLOSE, - ) -} - -fn raw_json_arguments(parameters: &Value, options: StructuralTagOptions) -> Format { - Format::tag( - format!("{JSON_OPEN} type=\"object\"{SEP}"), - json_schema(parameters.clone(), options), - JSON_CLOSE, - ) -} - -fn json_schema(schema: Value, options: StructuralTagOptions) -> Format { - Format::JsonSchema( - JsonSchemaFormat::new(schema) - .with_any_order(options.any_order) - .with_max_whitespace_cnt(options.max_whitespace_cnt), - ) -} - -fn escape_attr_value(value: &str) -> String { - value.replace('&', "&").replace('"', """) -} - -#[cfg(test)] -mod tests { - use expect_test::expect; - use serde_json::json; - use xgrammar_structural_tag::builders::StructuralTagOptions; - use xgrammar_structural_tag::{ - FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag, - }; - - use super::KimiK3StructuralTagBuilder; - use crate::output_grammar::test_utils::outline; - - fn tool(name: &str, parameters: serde_json::Value) -> ToolParam { - ToolParam::Function(FunctionToolParam::new( - FunctionDefinition::new(name).with_parameters(parameters), - )) - } - - #[test] - fn required_structural_tag_matches_xtml_channels() { - let tools = vec![tool( - "get_weather", - json!({ - "$defs": { - "place": { "type": "object", "properties": { "city": { "type": "string" } } } - }, - "type": "object", - "properties": { - "unit": { "type": "string", "enum": ["celsius", "fahrenheit"] }, - "place": { "$ref": "#/$defs/place", "type": "object" } - }, - "required": ["place"] - }), - )]; - let tag = build_structural_tag( - KimiK3StructuralTagBuilder, - &tools, - ToolChoice::required(), - StructuralTagOptions::default().with_reasoning(false), - ) - .unwrap(); - - expect![[r#" - sequence - optional `<|open|>response<|sep|>` - tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` - tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` - tags_with_separator `` at_least_one - tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` - sequence - /[1-9][0-9]*/ - `"<|sep|>` - or - sequence - optional - tag `<|open|>argument key="unit" type="string"<|sep|>` .. `<|close|>argument<|sep|>` - or - `celsius` - `fahrenheit` - tag `<|open|>argument key="place" type="object"<|sep|>` json({ city?: string } where place = { city?: string }) `<|close|>argument<|sep|>` - tag `<|open|>json type="object"<|sep|>` json({ unit?: "celsius" | "fahrenheit", place: place & object } where place = { city?: string }) `<|close|>json<|sep|>` - optional `<|close|>message<|sep|>` - "#]].assert_eq(&outline(&tag.format)); - } - - #[test] - fn reasoning_grammar_starts_inside_prefilled_think_channel() { - let tools = vec![tool("ping", json!({ "type": "object", "properties": {} }))]; - let tag = build_structural_tag( - KimiK3StructuralTagBuilder, - &tools, - ToolChoice::auto(), - StructuralTagOptions::default().with_reasoning(true), - ) - .unwrap(); - let value = serde_json::to_value(tag).unwrap(); - - assert_eq!( - value["format"]["elements"][0]["end"], - "<|close|>think<|sep|>" - ); - assert_eq!( - value["format"]["elements"][1]["value"], - "<|open|>response<|sep|>" - ); - assert_eq!(value["format"]["elements"][3]["type"], "optional"); - } - - #[test] - fn forced_choice_keeps_only_the_named_tool() { - let tools = vec![ - tool("search", json!({ "type": "object" })), - tool("lookup", json!({ "type": "object" })), - ]; - let tag = build_structural_tag( - KimiK3StructuralTagBuilder, - &tools, - ToolChoice::function("lookup"), - StructuralTagOptions::default().with_reasoning(false), - ) - .unwrap() - .to_json_string() - .unwrap(); - - assert!(tag.contains("lookup")); - assert!(!tag.contains("search")); - } - - #[test] - fn union_argument_content_matches_its_xtml_type() { - let tools = vec![tool( - "set_count", - json!({ - "type": "object", - "properties": { - "count": { "type": ["integer", "null"] } - } - }), - )]; - let tag = build_structural_tag( - KimiK3StructuralTagBuilder, - &tools, - ToolChoice::required(), - StructuralTagOptions::default(), - ) - .unwrap() - .to_json_string() - .unwrap(); - - assert!(tag.contains(r#"type=\"number\""#)); - assert!(tag.contains(r#""json_schema":{"type":"integer"}"#), "{tag}"); - assert!(tag.contains(r#"type=\"null\""#)); - assert!(tag.contains(r#""json_schema":{"type":"null"}"#), "{tag}"); - } -} diff --git a/rust/src/parser/src/unified/mod.rs b/rust/src/parser/src/unified/mod.rs index aa0e92e6e1..493b7b3f1b 100644 --- a/rust/src/parser/src/unified/mod.rs +++ b/rust/src/parser/src/unified/mod.rs @@ -15,7 +15,7 @@ pub use gemma4::Gemma4UnifiedParser; pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate}; pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser}; pub use inkling::InklingUnifiedParser; -pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser}; +pub use kimi_k3::KimiK3UnifiedParser; use thiserror::Error; use thiserror_ext::Macro; use vllm_tokenizer::{DecodedText, DynTokenizer};