[Rust Frontend] Build the Kimi K3 full-output grammar from its parser mode (#59190)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Bugen Zhao
2026-10-12 09:43:47 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 83773ecdf3
commit 0466bd36ea
8 changed files with 1259 additions and 800 deletions
+18 -1
View File
@@ -12,7 +12,8 @@ use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagOptio
use xgrammar_structural_tag::format::Format;
use xgrammar_structural_tag::tool::ToolChoiceValue;
use xgrammar_structural_tag::{
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
FunctionDefinition, FunctionToolParam, NormalizedToolChoice, ToolChoice, ToolParam,
build_structural_tag,
};
use crate::tool::Tool;
@@ -218,6 +219,22 @@ fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec<ToolParam>
.collect()
}
/// Normalize the request tools the way `build_structural_tag` does before
/// calling a crate builder, for model-owned grammar builders. Returns `None`
/// when the request asks for no tool grammar.
pub(crate) fn normalize_tool_choice(
ctx: &OutputGrammarContext<'_>,
) -> Result<Option<NormalizedToolChoice>> {
if !tool_grammar_applies(ctx) {
return Ok(None);
}
let tools = tool_params(ctx.tools, ctx.tool_strict_level);
let normalized =
xgrammar_structural_tag::normalize_tool_choice(&tools, ctx.tool_choice.clone())?;
Ok(Some(normalized))
}
/// Whether the request asks for a tool grammar at all.
fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
if ctx.tools.is_empty() {
+3 -11
View File
@@ -46,9 +46,7 @@
//! now only applies to `argument` and `json` blocks inside a call body, which
//! are still parsed from text.
mod structural_tag;
pub use structural_tag::KimiK3StructuralTagBuilder;
mod grammar;
use serde_json::{Map, Value};
use vllm_tokenizer::{DecodedText, DynTokenizer, Tokenizer};
@@ -58,11 +56,8 @@ use winnow::error::{ContextError, ErrMode, ModalResult, StrContext};
use winnow::prelude::*;
use winnow::token::{literal, rest, take_till, take_until, take_while};
use self::structural_tag::KIMI_K3_STRUCTURAL_TAG_BUILDER;
use super::{Result, UnifiedParser, UnifiedParserOutput, special_token};
use crate::output_grammar::{
self, BuiltOutputGrammar, OutputGrammarContext, visible_format_from_builder,
};
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
use crate::tool::{Tool, ToolCallDelta};
use crate::unified::parsing_failed;
use crate::utils::{
@@ -347,10 +342,7 @@ impl UnifiedParser for KimiK3UnifiedParser {
&self,
ctx: &OutputGrammarContext<'_>,
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
Ok(
visible_format_from_builder(Some(&KIMI_K3_STRUCTURAL_TAG_BUILDER), ctx)?
.map(BuiltOutputGrammar::final_output_only),
)
grammar::build_output_grammar(&self.mode, ctx)
}
fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> {
@@ -0,0 +1,356 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Full-output grammar for Kimi K3 XTML generations.
//!
//! The grammar starts at the channel the prompt left open, which is the
//! parser's initialized [`KimiK3Mode`], and follows the same channel
//! transitions as the parser. A prompt that opened no channel may first emit a
//! whole `think` channel.
//!
//! | Tool choice | After `think` closes | After `response` closes |
//! | ----------- | ------------------------------ | ----------------------- |
//! | `auto` | `response [tools]`, or `tools` | `[tools]` |
//! | `required` | `[response] tools` | `tools` |
//! | named | `[response] tools`, one call | `tools`, one call |
//!
//! Every path may end with the message close, and a stop token is legal only
//! after the last channel closes.
mod arguments;
use xgrammar_structural_tag::NormalizedToolChoice;
use xgrammar_structural_tag::format::Format;
use xgrammar_structural_tag::tool::BuilderToolChoice;
use self::arguments::call_tag;
use super::{
CLOSE, END_OF_MSG, KimiK3Mode, MESSAGE_CLOSE, OPEN, RESPONSE_CLOSE, RESPONSE_OPEN, THINK_CLOSE,
THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN,
};
use crate::output_grammar::{
self, BuiltOutputGrammar, OutputGrammarContext, normalize_tool_choice,
};
/// Build the grammar of everything generated after a prompt that left the
/// parser in `mode`, or `None` when the request asks for no tool grammar.
pub(super) fn build_output_grammar(
mode: &KimiK3Mode,
ctx: &OutputGrammarContext<'_>,
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
let Some(NormalizedToolChoice {
function_tools,
choice,
..
}) = normalize_tool_choice(ctx)?
else {
return Ok(None);
};
let single_call = choice == BuilderToolChoice::Forced || !ctx.parallel_tool_calls;
let tools = Format::tag(
TOOLS_OPEN,
Format::tags_with_separator(
function_tools.iter().map(call_tag).collect(),
"",
true,
single_call,
),
TOOLS_CLOSE,
);
let mut elements = match mode {
KimiK3Mode::Reasoning => vec![think_body(), after_reasoning(choice, tools)],
KimiK3Mode::Response => vec![response_body(), after_response(choice, tools)],
KimiK3Mode::Idle => vec![
Format::optional(Format::tag(THINK_OPEN, think_text(), THINK_CLOSE)),
after_reasoning(choice, tools),
],
KimiK3Mode::Epilogue | KimiK3Mode::Tools | KimiK3Mode::Call { .. } | KimiK3Mode::Done => {
unreachable!("initialization leaves the parser idle or inside `think` or `response`")
}
};
elements.push(Format::optional(Format::const_string(MESSAGE_CLOSE)));
Ok(Some(BuiltOutputGrammar::from_token_zero(Format::sequence(
elements,
))))
}
/// Channels that may follow the closed `think` channel.
fn after_reasoning(tool_choice: BuilderToolChoice, tools: Format) -> Format {
let response = Format::tag(RESPONSE_OPEN, response_text(), RESPONSE_CLOSE);
match tool_choice {
BuilderToolChoice::Auto => Format::or(vec![
Format::sequence(vec![response, Format::optional(tools.clone())]),
tools,
]),
BuilderToolChoice::Required | BuilderToolChoice::Forced => {
Format::sequence(vec![Format::optional(response), tools])
}
}
}
/// Channels that may follow the closed `response` channel.
fn after_response(tool_choice: BuilderToolChoice, tools: Format) -> Format {
match tool_choice {
BuilderToolChoice::Auto => Format::optional(tools),
BuilderToolChoice::Required | BuilderToolChoice::Forced => tools,
}
}
/// The rest of the `think` channel the prompt opened.
fn think_body() -> Format {
Format::tag("", think_text(), THINK_CLOSE)
}
/// The rest of the `response` channel the prompt opened.
fn response_body() -> Format {
Format::tag("", response_text(), RESPONSE_CLOSE)
}
fn think_text() -> Format {
Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG])
}
fn response_text() -> Format {
// Keep marker-looking prefixes out of response text so the grammar must
// resolve them through a valid channel boundary.
Format::any_text_excluding(&[OPEN, CLOSE, END_OF_MSG])
}
#[cfg(test)]
mod tests {
use expect_test::{Expect, expect};
use serde_json::json;
use xgrammar_structural_tag::ToolChoice;
use xgrammar_structural_tag::format::{EndBoundary, TagFormat};
use super::super::CALL_CLOSE;
use super::*;
use crate::output_grammar::test_utils::outline;
use crate::output_grammar::{GrammarCoverage, ToolStrictLevel};
use crate::tool::Tool;
fn tools() -> Vec<Tool> {
["get_weather", "add"]
.map(|name| Tool {
name: name.to_string(),
description: None,
parameters: json!({ "type": "object", "properties": {} }),
strict: None,
defer_loading: None,
})
.into()
}
fn check(
mode: KimiK3Mode,
tool_choice: ToolChoice,
parallel_tool_calls: bool,
expected: Expect,
) {
let tools = tools();
let grammar = build_output_grammar(
&mode,
&OutputGrammarContext {
tools: &tools,
tool_choice: &tool_choice,
tool_strict_level: ToolStrictLevel::Function,
parallel_tool_calls,
},
)
.unwrap()
.unwrap();
assert_eq!(grammar.coverage, GrammarCoverage::FromTokenZero);
let mut format = grammar.format;
elide_call_bodies(&mut format);
expected.assert_eq(&outline(&format));
}
/// Replace each call body with `..`, so that outlines show the channel
/// structure. The `arguments` tests cover call bodies.
fn elide_call_bodies(format: &mut Format) {
match format {
Format::Sequence(format) => format.elements.iter_mut().for_each(elide_call_bodies),
Format::Or(format) => format.elements.iter_mut().for_each(elide_call_bodies),
Format::Optional(format) => elide_call_bodies(&mut format.content),
Format::Tag(tag) => elide_tag(tag),
Format::TagsWithSeparator(format) => format.tags.iter_mut().for_each(elide_tag),
_ => {}
}
}
fn elide_tag(tag: &mut TagFormat) {
if matches!(&tag.end, EndBoundary::Text(end) if end == CALL_CLOSE) {
*tag.content = Format::const_string("..");
} else {
elide_call_bodies(&mut tag.content);
}
}
#[test]
fn prompt_opened_think_channel() {
check(
KimiK3Mode::Reasoning,
ToolChoice::auto(),
true,
expect![[r#"
sequence
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
or
sequence
tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
optional
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
optional `<|close|>message<|sep|>`
"#]],
);
check(
KimiK3Mode::Reasoning,
ToolChoice::required(),
true,
expect![[r#"
sequence
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
sequence
optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
optional `<|close|>message<|sep|>`
"#]],
);
check(
KimiK3Mode::Reasoning,
ToolChoice::function("add"),
true,
expect![[r#"
sequence
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
sequence
optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one stop_after_first
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
optional `<|close|>message<|sep|>`
"#]],
);
}
#[test]
fn prompt_opened_response_channel() {
check(
KimiK3Mode::Response,
ToolChoice::auto(),
true,
expect![[r#"
sequence
tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
optional
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
optional `<|close|>message<|sep|>`
"#]],
);
check(
KimiK3Mode::Response,
ToolChoice::required(),
true,
expect![[r#"
sequence
tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
optional `<|close|>message<|sep|>`
"#]],
);
check(
KimiK3Mode::Response,
ToolChoice::function("add"),
true,
expect![[r#"
sequence
tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one stop_after_first
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
optional `<|close|>message<|sep|>`
"#]],
);
}
#[test]
fn no_prompt_opened_channel() {
check(
KimiK3Mode::Idle,
ToolChoice::auto(),
true,
expect![[r#"
sequence
optional tag `<|open|>think<|sep|>` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
or
sequence
tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
optional
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
optional `<|close|>message<|sep|>`
"#]],
);
}
#[test]
fn serial_tool_calls_stop_after_one_call() {
check(
KimiK3Mode::Reasoning,
ToolChoice::required(),
false,
expect![[r#"
sequence
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
sequence
optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one stop_after_first
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
optional `<|close|>message<|sep|>`
"#]],
);
}
#[test]
fn non_strict_auto_builds_no_grammar() {
let tools = tools();
let grammar = build_output_grammar(
&KimiK3Mode::Reasoning,
&OutputGrammarContext {
tools: &tools,
tool_choice: &ToolChoice::auto(),
tool_strict_level: ToolStrictLevel::Auto,
parallel_tool_calls: true,
},
)
.unwrap();
assert_eq!(grammar, None);
}
}
@@ -0,0 +1,184 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Kimi K3 XTML tool-call grammar: call tags and their typed arguments.
use serde_json::Value;
use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, TagFormat};
use xgrammar_structural_tag::tool::{FunctionToolParam, function_parameters};
use super::super::{ARG_CLOSE, ARG_OPEN, CALL_CLOSE, JSON_CLOSE, JSON_OPEN, OPEN, SEP};
use crate::output_grammar::arguments::{
self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, ValueOption, group_by, one_of,
};
/// XTML arguments:
/// `<|open|>argument key="KEY" type="TYPE"<|sep|>VALUE<|close|>argument<|sep|>`,
/// adjacent to each other.
///
/// The parser decodes each value by its `type` attribute, so every value option
/// sits under the attribute of its own JSON type. String values are raw text
/// and every other value is JSON.
struct XtmlArguments;
impl ArgumentSyntax for XtmlArguments {
fn separator(&self) -> Option<Format> {
None
}
fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option<Format> {
let escaped;
let key = match key {
ParameterKey::Declared(key) => {
escaped = escape_attr_value(key);
ParameterKey::Declared(&escaped)
}
ParameterKey::Free => ParameterKey::Free,
};
let arguments = group_by(options, |option| xtml_type(option.ty))
.into_iter()
.filter_map(|(type_name, options)| {
let values = options
.into_iter()
.filter_map(|option| match option.ty {
// The parser ends the call at its close marker, so a
// string value may contain neither close marker.
JsonType::String => option.raw_string(&[ARG_CLOSE, CALL_CLOSE]),
_ => Some(option.json()),
})
.collect::<Vec<_>>();
(!values.is_empty()).then(|| {
let suffix = format!("\" type=\"{type_name}\"{SEP}");
key.tag(
&format!("{ARG_OPEN} key=\""),
&suffix,
one_of(values),
ARG_CLOSE,
)
})
})
.collect::<Vec<_>>();
(!arguments.is_empty()).then(|| one_of(arguments))
}
}
/// The `type` attribute K3 renders for a value of type `ty`, as
/// `_xtml_type` in the checkpoint's `encoding_k3.py`: integers and floats are
/// both `number`.
fn xtml_type(ty: JsonType) -> &'static str {
match ty {
JsonType::String => "string",
JsonType::Integer | JsonType::Number => "number",
JsonType::Boolean => "boolean",
JsonType::Null => "null",
JsonType::Object => "object",
JsonType::Array => "array",
}
}
/// Build the tag for one call to `tool`.
pub(super) fn call_tag(tool: &FunctionToolParam) -> TagFormat {
let parameters = function_parameters(&tool.function);
let typed_arguments =
arguments::arguments(&parameters, &XtmlArguments, &ArgumentOptions::default());
let call_body = Format::or(vec![typed_arguments, raw_json_arguments(&parameters)]);
TagFormat::new(
format!(
"{OPEN}call tool=\"{}\" index=\"",
escape_attr_value(&tool.function.name)
),
Format::sequence(vec![
Format::regex("[1-9][0-9]*"),
Format::const_string(format!("\"{SEP}")),
call_body,
]),
CALL_CLOSE,
)
}
fn raw_json_arguments(parameters: &Value) -> Format {
Format::tag(
format!("{JSON_OPEN} type=\"object\"{SEP}"),
Format::JsonSchema(JsonSchemaFormat::new(parameters.clone())),
JSON_CLOSE,
)
}
fn escape_attr_value(value: &str) -> String {
value.replace('&', "&amp;").replace('"', "&quot;")
}
#[cfg(test)]
mod tests {
use expect_test::{Expect, expect};
use serde_json::json;
use xgrammar_structural_tag::FunctionDefinition;
use xgrammar_structural_tag::format::Format;
use xgrammar_structural_tag::tool::FunctionToolParam;
use crate::output_grammar::test_utils::outline;
fn check(parameters: serde_json::Value, expected: Expect) {
let tool = FunctionToolParam::new(
FunctionDefinition::new("get_weather").with_parameters(parameters),
);
expected.assert_eq(&outline(&Format::Tag(super::call_tag(&tool))));
}
#[test]
fn call_tag_matches_xtml_arguments() {
check(
json!({
"$defs": {
"place": { "type": "object", "properties": { "city": { "type": "string" } } }
},
"type": "object",
"properties": {
"unit": { "type": "string", "enum": ["celsius", "fahrenheit"] },
"place": { "$ref": "#/$defs/place", "type": "object" }
},
"required": ["place"]
}),
expect![[r#"
tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>`
sequence
/[1-9][0-9]*/
`"<|sep|>`
or
sequence
optional
tag `<|open|>argument key="unit" type="string"<|sep|>` .. `<|close|>argument<|sep|>`
or
`celsius`
`fahrenheit`
tag `<|open|>argument key="place" type="object"<|sep|>` json({ city?: string } where place = { city?: string }) `<|close|>argument<|sep|>`
tag `<|open|>json type="object"<|sep|>` json({ unit?: "celsius" | "fahrenheit", place: place & object } where place = { city?: string }) `<|close|>json<|sep|>`
"#]],
);
}
#[test]
fn union_argument_content_matches_its_xtml_type() {
check(
json!({
"type": "object",
"properties": {
"count": { "type": ["integer", "null"] }
}
}),
expect![[r#"
tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>`
sequence
/[1-9][0-9]*/
`"<|sep|>`
or
optional
or
tag `<|open|>argument key="count" type="number"<|sep|>` json(integer) `<|close|>argument<|sep|>`
tag `<|open|>argument key="count" type="null"<|sep|>` json(null) `<|close|>argument<|sep|>`
tag `<|open|>json type="object"<|sep|>` json({ count?: integer | null }) `<|close|>json<|sep|>`
"#]],
);
}
}
@@ -1,343 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Structural-tag grammar for Kimi K3 XTML tool calls.
use serde_json::Value;
use xgrammar_structural_tag::Result;
use xgrammar_structural_tag::builders::{
ReasoningMode, StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
};
use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat};
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters};
use crate::output_grammar::arguments::{
self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, ValueOption, group_by, one_of,
};
use super::{
ARG_CLOSE, ARG_OPEN, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN,
RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN,
};
pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder =
KimiK3StructuralTagBuilder;
/// XTML arguments:
/// `<|open|>argument key="KEY" type="TYPE"<|sep|>VALUE<|close|>argument<|sep|>`,
/// adjacent to each other.
///
/// The parser decodes each value by its `type` attribute, so every value option
/// sits under the attribute of its own JSON type. String values are raw text
/// and every other value is JSON.
struct XtmlArguments;
impl ArgumentSyntax for XtmlArguments {
fn separator(&self) -> Option<Format> {
None
}
fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option<Format> {
let escaped;
let key = match key {
ParameterKey::Declared(key) => {
escaped = escape_attr_value(key);
ParameterKey::Declared(&escaped)
}
ParameterKey::Free => ParameterKey::Free,
};
let arguments = group_by(options, |option| xtml_type(option.ty))
.into_iter()
.filter_map(|(type_name, options)| {
let values = options
.into_iter()
.filter_map(|option| match option.ty {
// The parser ends the call at its close marker, so a
// string value may contain neither close marker.
JsonType::String => option.raw_string(&[ARG_CLOSE, CALL_CLOSE]),
_ => Some(option.json()),
})
.collect::<Vec<_>>();
(!values.is_empty()).then(|| {
let suffix = format!("\" type=\"{type_name}\"{SEP}");
key.tag(
&format!("{ARG_OPEN} key=\""),
&suffix,
one_of(values),
ARG_CLOSE,
)
})
})
.collect::<Vec<_>>();
(!arguments.is_empty()).then(|| one_of(arguments))
}
}
/// The `type` attribute K3 renders for a value of type `ty`, as
/// `_xtml_type` in the checkpoint's `encoding_k3.py`: integers and floats are
/// both `number`.
fn xtml_type(ty: JsonType) -> &'static str {
match ty {
JsonType::String => "string",
JsonType::Integer | JsonType::Number => "number",
JsonType::Boolean => "boolean",
JsonType::Null => "null",
JsonType::Object => "object",
JsonType::Array => "array",
}
}
/// Kimi K3 XTML structural-tag builder.
#[derive(Debug, Clone, Copy, Default)]
pub struct KimiK3StructuralTagBuilder;
impl StructuralTagBuilder for KimiK3StructuralTagBuilder {
fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
let mut elements = response_prefix(ctx.options.reasoning);
let tools = match ctx.tool_choice {
// Serving lowering filters empty tools before calling the builder,
// while direct `build_structural_tag(..., auto, ...)` calls do not.
BuilderToolChoice::Auto if ctx.function_tools.is_empty() => None,
BuilderToolChoice::Auto => Some(Format::optional(tools_channel(
ctx.function_tools,
ctx.options,
))),
BuilderToolChoice::Forced | BuilderToolChoice::Required => {
Some(tools_channel(ctx.function_tools, ctx.options))
}
};
if let Some(tools) = tools {
elements.push(tools);
}
elements.push(Format::optional(Format::const_string(MESSAGE_CLOSE)));
Ok(StructuralTag::new(Format::sequence(elements)))
}
}
fn response_prefix(reasoning: ReasoningMode) -> Vec<Format> {
let mut elements = Vec::new();
if reasoning != ReasoningMode::Disabled {
let prefix = Format::tag(
if reasoning == ReasoningMode::Auto {
THINK_OPEN
} else {
""
},
Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]),
THINK_CLOSE,
);
elements.push(if reasoning == ReasoningMode::Auto {
Format::optional(prefix)
} else {
prefix
});
elements.push(Format::const_string(RESPONSE_OPEN));
} else {
elements.push(Format::optional(Format::const_string(RESPONSE_OPEN)));
}
elements.push(Format::tag(
"",
// Keep marker-looking prefixes out of response text so the grammar
// must resolve them through a valid channel boundary.
Format::any_text_excluding(&[OPEN, CLOSE, END_OF_MSG]),
RESPONSE_CLOSE,
));
elements
}
fn tools_channel(tools: &[FunctionToolParam], options: StructuralTagOptions) -> Format {
let calls = tools.iter().map(|tool| call_tag(tool, options)).collect();
Format::tag(
TOOLS_OPEN,
Format::tags_with_separator(calls, "", true, false),
TOOLS_CLOSE,
)
}
fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat {
let parameters = function_parameters(&tool.function);
let typed_arguments = arguments::arguments(
&parameters,
&XtmlArguments,
&ArgumentOptions {
any_order: options.any_order,
max_whitespace_cnt: options.max_whitespace_cnt,
excludes: vec![],
},
);
let call_body = Format::or(vec![
typed_arguments,
raw_json_arguments(&parameters, options),
]);
TagFormat::new(
format!(
"{OPEN}call tool=\"{}\" index=\"",
escape_attr_value(&tool.function.name)
),
Format::sequence(vec![
Format::regex("[1-9][0-9]*"),
Format::const_string(format!("\"{SEP}")),
call_body,
]),
CALL_CLOSE,
)
}
fn raw_json_arguments(parameters: &Value, options: StructuralTagOptions) -> Format {
Format::tag(
format!("{JSON_OPEN} type=\"object\"{SEP}"),
json_schema(parameters.clone(), options),
JSON_CLOSE,
)
}
fn json_schema(schema: Value, options: StructuralTagOptions) -> Format {
Format::JsonSchema(
JsonSchemaFormat::new(schema)
.with_any_order(options.any_order)
.with_max_whitespace_cnt(options.max_whitespace_cnt),
)
}
fn escape_attr_value(value: &str) -> String {
value.replace('&', "&amp;").replace('"', "&quot;")
}
#[cfg(test)]
mod tests {
use expect_test::expect;
use serde_json::json;
use xgrammar_structural_tag::builders::StructuralTagOptions;
use xgrammar_structural_tag::{
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
};
use super::KimiK3StructuralTagBuilder;
use crate::output_grammar::test_utils::outline;
fn tool(name: &str, parameters: serde_json::Value) -> ToolParam {
ToolParam::Function(FunctionToolParam::new(
FunctionDefinition::new(name).with_parameters(parameters),
))
}
#[test]
fn required_structural_tag_matches_xtml_channels() {
let tools = vec![tool(
"get_weather",
json!({
"$defs": {
"place": { "type": "object", "properties": { "city": { "type": "string" } } }
},
"type": "object",
"properties": {
"unit": { "type": "string", "enum": ["celsius", "fahrenheit"] },
"place": { "$ref": "#/$defs/place", "type": "object" }
},
"required": ["place"]
}),
)];
let tag = build_structural_tag(
KimiK3StructuralTagBuilder,
&tools,
ToolChoice::required(),
StructuralTagOptions::default().with_reasoning(false),
)
.unwrap();
expect![[r#"
sequence
optional `<|open|>response<|sep|>`
tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
tags_with_separator `` at_least_one
tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>`
sequence
/[1-9][0-9]*/
`"<|sep|>`
or
sequence
optional
tag `<|open|>argument key="unit" type="string"<|sep|>` .. `<|close|>argument<|sep|>`
or
`celsius`
`fahrenheit`
tag `<|open|>argument key="place" type="object"<|sep|>` json({ city?: string } where place = { city?: string }) `<|close|>argument<|sep|>`
tag `<|open|>json type="object"<|sep|>` json({ unit?: "celsius" | "fahrenheit", place: place & object } where place = { city?: string }) `<|close|>json<|sep|>`
optional `<|close|>message<|sep|>`
"#]].assert_eq(&outline(&tag.format));
}
#[test]
fn reasoning_grammar_starts_inside_prefilled_think_channel() {
let tools = vec![tool("ping", json!({ "type": "object", "properties": {} }))];
let tag = build_structural_tag(
KimiK3StructuralTagBuilder,
&tools,
ToolChoice::auto(),
StructuralTagOptions::default().with_reasoning(true),
)
.unwrap();
let value = serde_json::to_value(tag).unwrap();
assert_eq!(
value["format"]["elements"][0]["end"],
"<|close|>think<|sep|>"
);
assert_eq!(
value["format"]["elements"][1]["value"],
"<|open|>response<|sep|>"
);
assert_eq!(value["format"]["elements"][3]["type"], "optional");
}
#[test]
fn forced_choice_keeps_only_the_named_tool() {
let tools = vec![
tool("search", json!({ "type": "object" })),
tool("lookup", json!({ "type": "object" })),
];
let tag = build_structural_tag(
KimiK3StructuralTagBuilder,
&tools,
ToolChoice::function("lookup"),
StructuralTagOptions::default().with_reasoning(false),
)
.unwrap()
.to_json_string()
.unwrap();
assert!(tag.contains("lookup"));
assert!(!tag.contains("search"));
}
#[test]
fn union_argument_content_matches_its_xtml_type() {
let tools = vec![tool(
"set_count",
json!({
"type": "object",
"properties": {
"count": { "type": ["integer", "null"] }
}
}),
)];
let tag = build_structural_tag(
KimiK3StructuralTagBuilder,
&tools,
ToolChoice::required(),
StructuralTagOptions::default(),
)
.unwrap()
.to_json_string()
.unwrap();
assert!(tag.contains(r#"type=\"number\""#));
assert!(tag.contains(r#""json_schema":{"type":"integer"}"#), "{tag}");
assert!(tag.contains(r#"type=\"null\""#));
assert!(tag.contains(r#""json_schema":{"type":"null"}"#), "{tag}");
}
}
+1 -1
View File
@@ -15,7 +15,7 @@ pub use gemma4::Gemma4UnifiedParser;
pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate};
pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser};
pub use inkling::InklingUnifiedParser;
pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser};
pub use kimi_k3::KimiK3UnifiedParser;
use thiserror::Error;
use thiserror_ext::Macro;
use vllm_tokenizer::{DecodedText, DynTokenizer};