[Rust Frontend] Build the Kimi K3 full-output grammar from its parser mode (#59190)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
83773ecdf3
commit
0466bd36ea
@@ -12,7 +12,8 @@ use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagOptio
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
use xgrammar_structural_tag::tool::ToolChoiceValue;
|
||||
use xgrammar_structural_tag::{
|
||||
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
|
||||
FunctionDefinition, FunctionToolParam, NormalizedToolChoice, ToolChoice, ToolParam,
|
||||
build_structural_tag,
|
||||
};
|
||||
|
||||
use crate::tool::Tool;
|
||||
@@ -218,6 +219,22 @@ fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec<ToolParam>
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Normalize the request tools the way `build_structural_tag` does before
|
||||
/// calling a crate builder, for model-owned grammar builders. Returns `None`
|
||||
/// when the request asks for no tool grammar.
|
||||
pub(crate) fn normalize_tool_choice(
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> Result<Option<NormalizedToolChoice>> {
|
||||
if !tool_grammar_applies(ctx) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let tools = tool_params(ctx.tools, ctx.tool_strict_level);
|
||||
let normalized =
|
||||
xgrammar_structural_tag::normalize_tool_choice(&tools, ctx.tool_choice.clone())?;
|
||||
Ok(Some(normalized))
|
||||
}
|
||||
|
||||
/// Whether the request asks for a tool grammar at all.
|
||||
fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
|
||||
if ctx.tools.is_empty() {
|
||||
|
||||
@@ -46,9 +46,7 @@
|
||||
//! now only applies to `argument` and `json` blocks inside a call body, which
|
||||
//! are still parsed from text.
|
||||
|
||||
mod structural_tag;
|
||||
|
||||
pub use structural_tag::KimiK3StructuralTagBuilder;
|
||||
mod grammar;
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer, Tokenizer};
|
||||
@@ -58,11 +56,8 @@ use winnow::error::{ContextError, ErrMode, ModalResult, StrContext};
|
||||
use winnow::prelude::*;
|
||||
use winnow::token::{literal, rest, take_till, take_until, take_while};
|
||||
|
||||
use self::structural_tag::KIMI_K3_STRUCTURAL_TAG_BUILDER;
|
||||
use super::{Result, UnifiedParser, UnifiedParserOutput, special_token};
|
||||
use crate::output_grammar::{
|
||||
self, BuiltOutputGrammar, OutputGrammarContext, visible_format_from_builder,
|
||||
};
|
||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
|
||||
use crate::tool::{Tool, ToolCallDelta};
|
||||
use crate::unified::parsing_failed;
|
||||
use crate::utils::{
|
||||
@@ -347,10 +342,7 @@ impl UnifiedParser for KimiK3UnifiedParser {
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
Ok(
|
||||
visible_format_from_builder(Some(&KIMI_K3_STRUCTURAL_TAG_BUILDER), ctx)?
|
||||
.map(BuiltOutputGrammar::final_output_only),
|
||||
)
|
||||
grammar::build_output_grammar(&self.mode, ctx)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> {
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Full-output grammar for Kimi K3 XTML generations.
|
||||
//!
|
||||
//! The grammar starts at the channel the prompt left open, which is the
|
||||
//! parser's initialized [`KimiK3Mode`], and follows the same channel
|
||||
//! transitions as the parser. A prompt that opened no channel may first emit a
|
||||
//! whole `think` channel.
|
||||
//!
|
||||
//! | Tool choice | After `think` closes | After `response` closes |
|
||||
//! | ----------- | ------------------------------ | ----------------------- |
|
||||
//! | `auto` | `response [tools]`, or `tools` | `[tools]` |
|
||||
//! | `required` | `[response] tools` | `tools` |
|
||||
//! | named | `[response] tools`, one call | `tools`, one call |
|
||||
//!
|
||||
//! Every path may end with the message close, and a stop token is legal only
|
||||
//! after the last channel closes.
|
||||
|
||||
mod arguments;
|
||||
|
||||
use xgrammar_structural_tag::NormalizedToolChoice;
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
use xgrammar_structural_tag::tool::BuilderToolChoice;
|
||||
|
||||
use self::arguments::call_tag;
|
||||
use super::{
|
||||
CLOSE, END_OF_MSG, KimiK3Mode, MESSAGE_CLOSE, OPEN, RESPONSE_CLOSE, RESPONSE_OPEN, THINK_CLOSE,
|
||||
THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN,
|
||||
};
|
||||
use crate::output_grammar::{
|
||||
self, BuiltOutputGrammar, OutputGrammarContext, normalize_tool_choice,
|
||||
};
|
||||
|
||||
/// Build the grammar of everything generated after a prompt that left the
|
||||
/// parser in `mode`, or `None` when the request asks for no tool grammar.
|
||||
pub(super) fn build_output_grammar(
|
||||
mode: &KimiK3Mode,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
let Some(NormalizedToolChoice {
|
||||
function_tools,
|
||||
choice,
|
||||
..
|
||||
}) = normalize_tool_choice(ctx)?
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let single_call = choice == BuilderToolChoice::Forced || !ctx.parallel_tool_calls;
|
||||
let tools = Format::tag(
|
||||
TOOLS_OPEN,
|
||||
Format::tags_with_separator(
|
||||
function_tools.iter().map(call_tag).collect(),
|
||||
"",
|
||||
true,
|
||||
single_call,
|
||||
),
|
||||
TOOLS_CLOSE,
|
||||
);
|
||||
|
||||
let mut elements = match mode {
|
||||
KimiK3Mode::Reasoning => vec![think_body(), after_reasoning(choice, tools)],
|
||||
KimiK3Mode::Response => vec![response_body(), after_response(choice, tools)],
|
||||
KimiK3Mode::Idle => vec![
|
||||
Format::optional(Format::tag(THINK_OPEN, think_text(), THINK_CLOSE)),
|
||||
after_reasoning(choice, tools),
|
||||
],
|
||||
KimiK3Mode::Epilogue | KimiK3Mode::Tools | KimiK3Mode::Call { .. } | KimiK3Mode::Done => {
|
||||
unreachable!("initialization leaves the parser idle or inside `think` or `response`")
|
||||
}
|
||||
};
|
||||
elements.push(Format::optional(Format::const_string(MESSAGE_CLOSE)));
|
||||
|
||||
Ok(Some(BuiltOutputGrammar::from_token_zero(Format::sequence(
|
||||
elements,
|
||||
))))
|
||||
}
|
||||
|
||||
/// Channels that may follow the closed `think` channel.
|
||||
fn after_reasoning(tool_choice: BuilderToolChoice, tools: Format) -> Format {
|
||||
let response = Format::tag(RESPONSE_OPEN, response_text(), RESPONSE_CLOSE);
|
||||
match tool_choice {
|
||||
BuilderToolChoice::Auto => Format::or(vec![
|
||||
Format::sequence(vec![response, Format::optional(tools.clone())]),
|
||||
tools,
|
||||
]),
|
||||
BuilderToolChoice::Required | BuilderToolChoice::Forced => {
|
||||
Format::sequence(vec![Format::optional(response), tools])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Channels that may follow the closed `response` channel.
|
||||
fn after_response(tool_choice: BuilderToolChoice, tools: Format) -> Format {
|
||||
match tool_choice {
|
||||
BuilderToolChoice::Auto => Format::optional(tools),
|
||||
BuilderToolChoice::Required | BuilderToolChoice::Forced => tools,
|
||||
}
|
||||
}
|
||||
|
||||
/// The rest of the `think` channel the prompt opened.
|
||||
fn think_body() -> Format {
|
||||
Format::tag("", think_text(), THINK_CLOSE)
|
||||
}
|
||||
|
||||
/// The rest of the `response` channel the prompt opened.
|
||||
fn response_body() -> Format {
|
||||
Format::tag("", response_text(), RESPONSE_CLOSE)
|
||||
}
|
||||
|
||||
fn think_text() -> Format {
|
||||
Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG])
|
||||
}
|
||||
|
||||
fn response_text() -> Format {
|
||||
// Keep marker-looking prefixes out of response text so the grammar must
|
||||
// resolve them through a valid channel boundary.
|
||||
Format::any_text_excluding(&[OPEN, CLOSE, END_OF_MSG])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use expect_test::{Expect, expect};
|
||||
use serde_json::json;
|
||||
use xgrammar_structural_tag::ToolChoice;
|
||||
use xgrammar_structural_tag::format::{EndBoundary, TagFormat};
|
||||
|
||||
use super::super::CALL_CLOSE;
|
||||
use super::*;
|
||||
use crate::output_grammar::test_utils::outline;
|
||||
use crate::output_grammar::{GrammarCoverage, ToolStrictLevel};
|
||||
use crate::tool::Tool;
|
||||
|
||||
fn tools() -> Vec<Tool> {
|
||||
["get_weather", "add"]
|
||||
.map(|name| Tool {
|
||||
name: name.to_string(),
|
||||
description: None,
|
||||
parameters: json!({ "type": "object", "properties": {} }),
|
||||
strict: None,
|
||||
defer_loading: None,
|
||||
})
|
||||
.into()
|
||||
}
|
||||
|
||||
fn check(
|
||||
mode: KimiK3Mode,
|
||||
tool_choice: ToolChoice,
|
||||
parallel_tool_calls: bool,
|
||||
expected: Expect,
|
||||
) {
|
||||
let tools = tools();
|
||||
let grammar = build_output_grammar(
|
||||
&mode,
|
||||
&OutputGrammarContext {
|
||||
tools: &tools,
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: ToolStrictLevel::Function,
|
||||
parallel_tool_calls,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(grammar.coverage, GrammarCoverage::FromTokenZero);
|
||||
let mut format = grammar.format;
|
||||
elide_call_bodies(&mut format);
|
||||
expected.assert_eq(&outline(&format));
|
||||
}
|
||||
|
||||
/// Replace each call body with `..`, so that outlines show the channel
|
||||
/// structure. The `arguments` tests cover call bodies.
|
||||
fn elide_call_bodies(format: &mut Format) {
|
||||
match format {
|
||||
Format::Sequence(format) => format.elements.iter_mut().for_each(elide_call_bodies),
|
||||
Format::Or(format) => format.elements.iter_mut().for_each(elide_call_bodies),
|
||||
Format::Optional(format) => elide_call_bodies(&mut format.content),
|
||||
Format::Tag(tag) => elide_tag(tag),
|
||||
Format::TagsWithSeparator(format) => format.tags.iter_mut().for_each(elide_tag),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn elide_tag(tag: &mut TagFormat) {
|
||||
if matches!(&tag.end, EndBoundary::Text(end) if end == CALL_CLOSE) {
|
||||
*tag.content = Format::const_string("..");
|
||||
} else {
|
||||
elide_call_bodies(&mut tag.content);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_opened_think_channel() {
|
||||
check(
|
||||
KimiK3Mode::Reasoning,
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
|
||||
or
|
||||
sequence
|
||||
tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
|
||||
optional
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
check(
|
||||
KimiK3Mode::Reasoning,
|
||||
ToolChoice::required(),
|
||||
true,
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
|
||||
sequence
|
||||
optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
check(
|
||||
KimiK3Mode::Reasoning,
|
||||
ToolChoice::function("add"),
|
||||
true,
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
|
||||
sequence
|
||||
optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one stop_after_first
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompt_opened_response_channel() {
|
||||
check(
|
||||
KimiK3Mode::Response,
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
|
||||
optional
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
check(
|
||||
KimiK3Mode::Response,
|
||||
ToolChoice::required(),
|
||||
true,
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
check(
|
||||
KimiK3Mode::Response,
|
||||
ToolChoice::function("add"),
|
||||
true,
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one stop_after_first
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prompt_opened_channel() {
|
||||
check(
|
||||
KimiK3Mode::Idle,
|
||||
ToolChoice::auto(),
|
||||
true,
|
||||
expect![[r#"
|
||||
sequence
|
||||
optional tag `<|open|>think<|sep|>` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
|
||||
or
|
||||
sequence
|
||||
tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
|
||||
optional
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serial_tool_calls_stop_after_one_call() {
|
||||
check(
|
||||
KimiK3Mode::Reasoning,
|
||||
ToolChoice::required(),
|
||||
false,
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
|
||||
sequence
|
||||
optional tag `<|open|>response<|sep|>` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one stop_after_first
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_strict_auto_builds_no_grammar() {
|
||||
let tools = tools();
|
||||
let grammar = build_output_grammar(
|
||||
&KimiK3Mode::Reasoning,
|
||||
&OutputGrammarContext {
|
||||
tools: &tools,
|
||||
tool_choice: &ToolChoice::auto(),
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: true,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(grammar, None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Kimi K3 XTML tool-call grammar: call tags and their typed arguments.
|
||||
|
||||
use serde_json::Value;
|
||||
use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, TagFormat};
|
||||
use xgrammar_structural_tag::tool::{FunctionToolParam, function_parameters};
|
||||
|
||||
use super::super::{ARG_CLOSE, ARG_OPEN, CALL_CLOSE, JSON_CLOSE, JSON_OPEN, OPEN, SEP};
|
||||
use crate::output_grammar::arguments::{
|
||||
self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, ValueOption, group_by, one_of,
|
||||
};
|
||||
|
||||
/// XTML arguments:
|
||||
/// `<|open|>argument key="KEY" type="TYPE"<|sep|>VALUE<|close|>argument<|sep|>`,
|
||||
/// adjacent to each other.
|
||||
///
|
||||
/// The parser decodes each value by its `type` attribute, so every value option
|
||||
/// sits under the attribute of its own JSON type. String values are raw text
|
||||
/// and every other value is JSON.
|
||||
struct XtmlArguments;
|
||||
|
||||
impl ArgumentSyntax for XtmlArguments {
|
||||
fn separator(&self) -> Option<Format> {
|
||||
None
|
||||
}
|
||||
|
||||
fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option<Format> {
|
||||
let escaped;
|
||||
let key = match key {
|
||||
ParameterKey::Declared(key) => {
|
||||
escaped = escape_attr_value(key);
|
||||
ParameterKey::Declared(&escaped)
|
||||
}
|
||||
ParameterKey::Free => ParameterKey::Free,
|
||||
};
|
||||
let arguments = group_by(options, |option| xtml_type(option.ty))
|
||||
.into_iter()
|
||||
.filter_map(|(type_name, options)| {
|
||||
let values = options
|
||||
.into_iter()
|
||||
.filter_map(|option| match option.ty {
|
||||
// The parser ends the call at its close marker, so a
|
||||
// string value may contain neither close marker.
|
||||
JsonType::String => option.raw_string(&[ARG_CLOSE, CALL_CLOSE]),
|
||||
_ => Some(option.json()),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
(!values.is_empty()).then(|| {
|
||||
let suffix = format!("\" type=\"{type_name}\"{SEP}");
|
||||
key.tag(
|
||||
&format!("{ARG_OPEN} key=\""),
|
||||
&suffix,
|
||||
one_of(values),
|
||||
ARG_CLOSE,
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
(!arguments.is_empty()).then(|| one_of(arguments))
|
||||
}
|
||||
}
|
||||
|
||||
/// The `type` attribute K3 renders for a value of type `ty`, as
|
||||
/// `_xtml_type` in the checkpoint's `encoding_k3.py`: integers and floats are
|
||||
/// both `number`.
|
||||
fn xtml_type(ty: JsonType) -> &'static str {
|
||||
match ty {
|
||||
JsonType::String => "string",
|
||||
JsonType::Integer | JsonType::Number => "number",
|
||||
JsonType::Boolean => "boolean",
|
||||
JsonType::Null => "null",
|
||||
JsonType::Object => "object",
|
||||
JsonType::Array => "array",
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the tag for one call to `tool`.
|
||||
pub(super) fn call_tag(tool: &FunctionToolParam) -> TagFormat {
|
||||
let parameters = function_parameters(&tool.function);
|
||||
let typed_arguments =
|
||||
arguments::arguments(¶meters, &XtmlArguments, &ArgumentOptions::default());
|
||||
let call_body = Format::or(vec![typed_arguments, raw_json_arguments(¶meters)]);
|
||||
|
||||
TagFormat::new(
|
||||
format!(
|
||||
"{OPEN}call tool=\"{}\" index=\"",
|
||||
escape_attr_value(&tool.function.name)
|
||||
),
|
||||
Format::sequence(vec![
|
||||
Format::regex("[1-9][0-9]*"),
|
||||
Format::const_string(format!("\"{SEP}")),
|
||||
call_body,
|
||||
]),
|
||||
CALL_CLOSE,
|
||||
)
|
||||
}
|
||||
|
||||
fn raw_json_arguments(parameters: &Value) -> Format {
|
||||
Format::tag(
|
||||
format!("{JSON_OPEN} type=\"object\"{SEP}"),
|
||||
Format::JsonSchema(JsonSchemaFormat::new(parameters.clone())),
|
||||
JSON_CLOSE,
|
||||
)
|
||||
}
|
||||
|
||||
fn escape_attr_value(value: &str) -> String {
|
||||
value.replace('&', "&").replace('"', """)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use expect_test::{Expect, expect};
|
||||
use serde_json::json;
|
||||
use xgrammar_structural_tag::FunctionDefinition;
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
use xgrammar_structural_tag::tool::FunctionToolParam;
|
||||
|
||||
use crate::output_grammar::test_utils::outline;
|
||||
|
||||
fn check(parameters: serde_json::Value, expected: Expect) {
|
||||
let tool = FunctionToolParam::new(
|
||||
FunctionDefinition::new("get_weather").with_parameters(parameters),
|
||||
);
|
||||
expected.assert_eq(&outline(&Format::Tag(super::call_tag(&tool))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn call_tag_matches_xtml_arguments() {
|
||||
check(
|
||||
json!({
|
||||
"$defs": {
|
||||
"place": { "type": "object", "properties": { "city": { "type": "string" } } }
|
||||
},
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"unit": { "type": "string", "enum": ["celsius", "fahrenheit"] },
|
||||
"place": { "$ref": "#/$defs/place", "type": "object" }
|
||||
},
|
||||
"required": ["place"]
|
||||
}),
|
||||
expect![[r#"
|
||||
tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>`
|
||||
sequence
|
||||
/[1-9][0-9]*/
|
||||
`"<|sep|>`
|
||||
or
|
||||
sequence
|
||||
optional
|
||||
tag `<|open|>argument key="unit" type="string"<|sep|>` .. `<|close|>argument<|sep|>`
|
||||
or
|
||||
`celsius`
|
||||
`fahrenheit`
|
||||
tag `<|open|>argument key="place" type="object"<|sep|>` json({ city?: string } where place = { city?: string }) `<|close|>argument<|sep|>`
|
||||
tag `<|open|>json type="object"<|sep|>` json({ unit?: "celsius" | "fahrenheit", place: place & object } where place = { city?: string }) `<|close|>json<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn union_argument_content_matches_its_xtml_type() {
|
||||
check(
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"count": { "type": ["integer", "null"] }
|
||||
}
|
||||
}),
|
||||
expect![[r#"
|
||||
tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>`
|
||||
sequence
|
||||
/[1-9][0-9]*/
|
||||
`"<|sep|>`
|
||||
or
|
||||
optional
|
||||
or
|
||||
tag `<|open|>argument key="count" type="number"<|sep|>` json(integer) `<|close|>argument<|sep|>`
|
||||
tag `<|open|>argument key="count" type="null"<|sep|>` json(null) `<|close|>argument<|sep|>`
|
||||
tag `<|open|>json type="object"<|sep|>` json({ count?: integer | null }) `<|close|>json<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,343 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Structural-tag grammar for Kimi K3 XTML tool calls.
|
||||
|
||||
use serde_json::Value;
|
||||
use xgrammar_structural_tag::Result;
|
||||
use xgrammar_structural_tag::builders::{
|
||||
ReasoningMode, StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
|
||||
};
|
||||
use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat};
|
||||
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters};
|
||||
|
||||
use crate::output_grammar::arguments::{
|
||||
self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, ValueOption, group_by, one_of,
|
||||
};
|
||||
|
||||
use super::{
|
||||
ARG_CLOSE, ARG_OPEN, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN,
|
||||
RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN,
|
||||
};
|
||||
|
||||
pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder =
|
||||
KimiK3StructuralTagBuilder;
|
||||
|
||||
/// XTML arguments:
|
||||
/// `<|open|>argument key="KEY" type="TYPE"<|sep|>VALUE<|close|>argument<|sep|>`,
|
||||
/// adjacent to each other.
|
||||
///
|
||||
/// The parser decodes each value by its `type` attribute, so every value option
|
||||
/// sits under the attribute of its own JSON type. String values are raw text
|
||||
/// and every other value is JSON.
|
||||
struct XtmlArguments;
|
||||
|
||||
impl ArgumentSyntax for XtmlArguments {
|
||||
fn separator(&self) -> Option<Format> {
|
||||
None
|
||||
}
|
||||
|
||||
fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option<Format> {
|
||||
let escaped;
|
||||
let key = match key {
|
||||
ParameterKey::Declared(key) => {
|
||||
escaped = escape_attr_value(key);
|
||||
ParameterKey::Declared(&escaped)
|
||||
}
|
||||
ParameterKey::Free => ParameterKey::Free,
|
||||
};
|
||||
let arguments = group_by(options, |option| xtml_type(option.ty))
|
||||
.into_iter()
|
||||
.filter_map(|(type_name, options)| {
|
||||
let values = options
|
||||
.into_iter()
|
||||
.filter_map(|option| match option.ty {
|
||||
// The parser ends the call at its close marker, so a
|
||||
// string value may contain neither close marker.
|
||||
JsonType::String => option.raw_string(&[ARG_CLOSE, CALL_CLOSE]),
|
||||
_ => Some(option.json()),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
(!values.is_empty()).then(|| {
|
||||
let suffix = format!("\" type=\"{type_name}\"{SEP}");
|
||||
key.tag(
|
||||
&format!("{ARG_OPEN} key=\""),
|
||||
&suffix,
|
||||
one_of(values),
|
||||
ARG_CLOSE,
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
(!arguments.is_empty()).then(|| one_of(arguments))
|
||||
}
|
||||
}
|
||||
|
||||
/// The `type` attribute K3 renders for a value of type `ty`, as
|
||||
/// `_xtml_type` in the checkpoint's `encoding_k3.py`: integers and floats are
|
||||
/// both `number`.
|
||||
fn xtml_type(ty: JsonType) -> &'static str {
|
||||
match ty {
|
||||
JsonType::String => "string",
|
||||
JsonType::Integer | JsonType::Number => "number",
|
||||
JsonType::Boolean => "boolean",
|
||||
JsonType::Null => "null",
|
||||
JsonType::Object => "object",
|
||||
JsonType::Array => "array",
|
||||
}
|
||||
}
|
||||
|
||||
/// Kimi K3 XTML structural-tag builder.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct KimiK3StructuralTagBuilder;
|
||||
|
||||
impl StructuralTagBuilder for KimiK3StructuralTagBuilder {
|
||||
fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
|
||||
let mut elements = response_prefix(ctx.options.reasoning);
|
||||
|
||||
let tools = match ctx.tool_choice {
|
||||
// Serving lowering filters empty tools before calling the builder,
|
||||
// while direct `build_structural_tag(..., auto, ...)` calls do not.
|
||||
BuilderToolChoice::Auto if ctx.function_tools.is_empty() => None,
|
||||
BuilderToolChoice::Auto => Some(Format::optional(tools_channel(
|
||||
ctx.function_tools,
|
||||
ctx.options,
|
||||
))),
|
||||
BuilderToolChoice::Forced | BuilderToolChoice::Required => {
|
||||
Some(tools_channel(ctx.function_tools, ctx.options))
|
||||
}
|
||||
};
|
||||
if let Some(tools) = tools {
|
||||
elements.push(tools);
|
||||
}
|
||||
elements.push(Format::optional(Format::const_string(MESSAGE_CLOSE)));
|
||||
|
||||
Ok(StructuralTag::new(Format::sequence(elements)))
|
||||
}
|
||||
}
|
||||
|
||||
fn response_prefix(reasoning: ReasoningMode) -> Vec<Format> {
|
||||
let mut elements = Vec::new();
|
||||
if reasoning != ReasoningMode::Disabled {
|
||||
let prefix = Format::tag(
|
||||
if reasoning == ReasoningMode::Auto {
|
||||
THINK_OPEN
|
||||
} else {
|
||||
""
|
||||
},
|
||||
Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]),
|
||||
THINK_CLOSE,
|
||||
);
|
||||
elements.push(if reasoning == ReasoningMode::Auto {
|
||||
Format::optional(prefix)
|
||||
} else {
|
||||
prefix
|
||||
});
|
||||
elements.push(Format::const_string(RESPONSE_OPEN));
|
||||
} else {
|
||||
elements.push(Format::optional(Format::const_string(RESPONSE_OPEN)));
|
||||
}
|
||||
elements.push(Format::tag(
|
||||
"",
|
||||
// Keep marker-looking prefixes out of response text so the grammar
|
||||
// must resolve them through a valid channel boundary.
|
||||
Format::any_text_excluding(&[OPEN, CLOSE, END_OF_MSG]),
|
||||
RESPONSE_CLOSE,
|
||||
));
|
||||
elements
|
||||
}
|
||||
|
||||
fn tools_channel(tools: &[FunctionToolParam], options: StructuralTagOptions) -> Format {
|
||||
let calls = tools.iter().map(|tool| call_tag(tool, options)).collect();
|
||||
Format::tag(
|
||||
TOOLS_OPEN,
|
||||
Format::tags_with_separator(calls, "", true, false),
|
||||
TOOLS_CLOSE,
|
||||
)
|
||||
}
|
||||
|
||||
fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat {
|
||||
let parameters = function_parameters(&tool.function);
|
||||
let typed_arguments = arguments::arguments(
|
||||
¶meters,
|
||||
&XtmlArguments,
|
||||
&ArgumentOptions {
|
||||
any_order: options.any_order,
|
||||
max_whitespace_cnt: options.max_whitespace_cnt,
|
||||
excludes: vec![],
|
||||
},
|
||||
);
|
||||
let call_body = Format::or(vec![
|
||||
typed_arguments,
|
||||
raw_json_arguments(¶meters, options),
|
||||
]);
|
||||
|
||||
TagFormat::new(
|
||||
format!(
|
||||
"{OPEN}call tool=\"{}\" index=\"",
|
||||
escape_attr_value(&tool.function.name)
|
||||
),
|
||||
Format::sequence(vec![
|
||||
Format::regex("[1-9][0-9]*"),
|
||||
Format::const_string(format!("\"{SEP}")),
|
||||
call_body,
|
||||
]),
|
||||
CALL_CLOSE,
|
||||
)
|
||||
}
|
||||
|
||||
fn raw_json_arguments(parameters: &Value, options: StructuralTagOptions) -> Format {
|
||||
Format::tag(
|
||||
format!("{JSON_OPEN} type=\"object\"{SEP}"),
|
||||
json_schema(parameters.clone(), options),
|
||||
JSON_CLOSE,
|
||||
)
|
||||
}
|
||||
|
||||
fn json_schema(schema: Value, options: StructuralTagOptions) -> Format {
|
||||
Format::JsonSchema(
|
||||
JsonSchemaFormat::new(schema)
|
||||
.with_any_order(options.any_order)
|
||||
.with_max_whitespace_cnt(options.max_whitespace_cnt),
|
||||
)
|
||||
}
|
||||
|
||||
fn escape_attr_value(value: &str) -> String {
|
||||
value.replace('&', "&").replace('"', """)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use expect_test::expect;
|
||||
use serde_json::json;
|
||||
use xgrammar_structural_tag::builders::StructuralTagOptions;
|
||||
use xgrammar_structural_tag::{
|
||||
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
|
||||
};
|
||||
|
||||
use super::KimiK3StructuralTagBuilder;
|
||||
use crate::output_grammar::test_utils::outline;
|
||||
|
||||
fn tool(name: &str, parameters: serde_json::Value) -> ToolParam {
|
||||
ToolParam::Function(FunctionToolParam::new(
|
||||
FunctionDefinition::new(name).with_parameters(parameters),
|
||||
))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_structural_tag_matches_xtml_channels() {
|
||||
let tools = vec![tool(
|
||||
"get_weather",
|
||||
json!({
|
||||
"$defs": {
|
||||
"place": { "type": "object", "properties": { "city": { "type": "string" } } }
|
||||
},
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"unit": { "type": "string", "enum": ["celsius", "fahrenheit"] },
|
||||
"place": { "$ref": "#/$defs/place", "type": "object" }
|
||||
},
|
||||
"required": ["place"]
|
||||
}),
|
||||
)];
|
||||
let tag = build_structural_tag(
|
||||
KimiK3StructuralTagBuilder,
|
||||
&tools,
|
||||
ToolChoice::required(),
|
||||
StructuralTagOptions::default().with_reasoning(false),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
expect![[r#"
|
||||
sequence
|
||||
optional `<|open|>response<|sep|>`
|
||||
tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>`
|
||||
sequence
|
||||
/[1-9][0-9]*/
|
||||
`"<|sep|>`
|
||||
or
|
||||
sequence
|
||||
optional
|
||||
tag `<|open|>argument key="unit" type="string"<|sep|>` .. `<|close|>argument<|sep|>`
|
||||
or
|
||||
`celsius`
|
||||
`fahrenheit`
|
||||
tag `<|open|>argument key="place" type="object"<|sep|>` json({ city?: string } where place = { city?: string }) `<|close|>argument<|sep|>`
|
||||
tag `<|open|>json type="object"<|sep|>` json({ unit?: "celsius" | "fahrenheit", place: place & object } where place = { city?: string }) `<|close|>json<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]].assert_eq(&outline(&tag.format));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_grammar_starts_inside_prefilled_think_channel() {
|
||||
let tools = vec![tool("ping", json!({ "type": "object", "properties": {} }))];
|
||||
let tag = build_structural_tag(
|
||||
KimiK3StructuralTagBuilder,
|
||||
&tools,
|
||||
ToolChoice::auto(),
|
||||
StructuralTagOptions::default().with_reasoning(true),
|
||||
)
|
||||
.unwrap();
|
||||
let value = serde_json::to_value(tag).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
value["format"]["elements"][0]["end"],
|
||||
"<|close|>think<|sep|>"
|
||||
);
|
||||
assert_eq!(
|
||||
value["format"]["elements"][1]["value"],
|
||||
"<|open|>response<|sep|>"
|
||||
);
|
||||
assert_eq!(value["format"]["elements"][3]["type"], "optional");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forced_choice_keeps_only_the_named_tool() {
|
||||
let tools = vec![
|
||||
tool("search", json!({ "type": "object" })),
|
||||
tool("lookup", json!({ "type": "object" })),
|
||||
];
|
||||
let tag = build_structural_tag(
|
||||
KimiK3StructuralTagBuilder,
|
||||
&tools,
|
||||
ToolChoice::function("lookup"),
|
||||
StructuralTagOptions::default().with_reasoning(false),
|
||||
)
|
||||
.unwrap()
|
||||
.to_json_string()
|
||||
.unwrap();
|
||||
|
||||
assert!(tag.contains("lookup"));
|
||||
assert!(!tag.contains("search"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn union_argument_content_matches_its_xtml_type() {
|
||||
let tools = vec![tool(
|
||||
"set_count",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"count": { "type": ["integer", "null"] }
|
||||
}
|
||||
}),
|
||||
)];
|
||||
let tag = build_structural_tag(
|
||||
KimiK3StructuralTagBuilder,
|
||||
&tools,
|
||||
ToolChoice::required(),
|
||||
StructuralTagOptions::default(),
|
||||
)
|
||||
.unwrap()
|
||||
.to_json_string()
|
||||
.unwrap();
|
||||
|
||||
assert!(tag.contains(r#"type=\"number\""#));
|
||||
assert!(tag.contains(r#""json_schema":{"type":"integer"}"#), "{tag}");
|
||||
assert!(tag.contains(r#"type=\"null\""#));
|
||||
assert!(tag.contains(r#""json_schema":{"type":"null"}"#), "{tag}");
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,7 @@ pub use gemma4::Gemma4UnifiedParser;
|
||||
pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate};
|
||||
pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser};
|
||||
pub use inkling::InklingUnifiedParser;
|
||||
pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser};
|
||||
pub use kimi_k3::KimiK3UnifiedParser;
|
||||
use thiserror::Error;
|
||||
use thiserror_ext::Macro;
|
||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||
|
||||
Reference in New Issue
Block a user