[Rust Frontend] Compose answer constraints into parser-owned output grammars (#60781)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
0466bd36ea
commit
5a09062632
@@ -132,6 +132,10 @@ pub struct OutputGrammarContext<'a> {
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
/// Whether the request permits more than one tool call.
|
||||
pub parallel_tool_calls: bool,
|
||||
/// The user's constraint on the answer text, such as a `response_format`
|
||||
/// JSON schema, normalized to a format. A builder that composes it inserts
|
||||
/// it where the model writes its answer; `None` leaves the answer free.
|
||||
pub answer: Option<&'a Format>,
|
||||
}
|
||||
|
||||
/// Errors produced while building an output grammar.
|
||||
@@ -243,9 +247,12 @@ fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
|
||||
|
||||
match ctx.tool_choice {
|
||||
ToolChoice::Value(ToolChoiceValue::None) => false,
|
||||
// With an answer constraint, the grammar must hold the calls `auto`
|
||||
// allows, or the answer would exclude them.
|
||||
ToolChoice::Value(ToolChoiceValue::Auto) => {
|
||||
ctx.tool_strict_level >= ToolStrictLevel::Function
|
||||
|| ctx.tools.iter().any(|tool| tool.strict == Some(true))
|
||||
|| ctx.answer.is_some()
|
||||
}
|
||||
ToolChoice::Value(ToolChoiceValue::Required)
|
||||
| ToolChoice::NamedFunction(_)
|
||||
@@ -284,6 +291,7 @@ mod tests {
|
||||
tool_choice,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
@@ -310,6 +318,7 @@ mod tests {
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: level,
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
|
||||
@@ -122,7 +122,7 @@ impl ToolParser for MinimaxM2ToolParser {
|
||||
Some(xgrammar_structural_tag::Model::Minimax.builder())
|
||||
}
|
||||
|
||||
fn build_visible_format(
|
||||
fn build_call_format(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<Format>> {
|
||||
|
||||
@@ -20,6 +20,7 @@ mod seed_oss;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub mod test_utils;
|
||||
use std::collections::{BTreeMap, btree_map};
|
||||
use std::sync::Once;
|
||||
|
||||
pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser, DeepSeekV41ToolParser};
|
||||
pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser};
|
||||
@@ -39,8 +40,11 @@ pub use qwen_coder::Qwen3CoderToolParser;
|
||||
pub use seed_oss::SeedOssToolParser;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use tracing::warn;
|
||||
use xgrammar_structural_tag::ToolChoice;
|
||||
pub use xgrammar_structural_tag::builders::StructuralTagBuilder;
|
||||
use xgrammar_structural_tag::format::Format;
|
||||
use xgrammar_structural_tag::tool::{AllowedToolsMode, ToolChoiceValue};
|
||||
|
||||
use crate::output_grammar::{self, OutputGrammarContext};
|
||||
use crate::utils;
|
||||
@@ -196,21 +200,54 @@ pub trait ToolParser: Send {
|
||||
}
|
||||
|
||||
/// Build the language of everything the model may emit after reasoning
|
||||
/// ends: visible text plus tool calls. `None` means no tool grammar applies.
|
||||
/// ends for the context's tool choice: visible text plus tool calls. `None`
|
||||
/// means no tool grammar applies.
|
||||
///
|
||||
/// The default forwards to [`Self::structural_tag_builder`] with reasoning
|
||||
/// disabled, which is exactly today's strict-tool-calling grammar.
|
||||
fn build_visible_format(
|
||||
fn build_call_format(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<Format>> {
|
||||
output_grammar::visible_format_from_builder(self.structural_tag_builder(), ctx)
|
||||
}
|
||||
|
||||
/// Build the visible language with the request's answer constraint
|
||||
/// composed in, as the Python frontend does: an optional call becomes
|
||||
/// `Or[calls built as required, answer]`, and a tool choice that obliges a
|
||||
/// call ignores the answer. Parsers override [`Self::build_call_format`]
|
||||
/// instead.
|
||||
fn build_visible_format(
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<Format>> {
|
||||
let Some(answer) = ctx.answer else {
|
||||
return self.build_call_format(ctx);
|
||||
};
|
||||
match call_required(ctx.tool_choice) {
|
||||
// Calls built from the optional choice would also admit free text
|
||||
// in place of the answer.
|
||||
Some(tool_choice) => {
|
||||
let calls = self.build_call_format(&OutputGrammarContext {
|
||||
tool_choice: &tool_choice,
|
||||
..*ctx
|
||||
})?;
|
||||
Ok(calls.map(|calls| Format::or(vec![calls, answer.clone()])))
|
||||
}
|
||||
None => {
|
||||
let calls = self.build_call_format(ctx)?;
|
||||
if calls.is_some() {
|
||||
warn_answer_ignored_once();
|
||||
}
|
||||
Ok(calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the xgrammar structural-tag builder used for strict tool calling.
|
||||
///
|
||||
/// To be deprecated: this only exists to back the default
|
||||
/// [`Self::build_visible_format`]. New parsers should override that method
|
||||
/// [`Self::build_call_format`]. New parsers should override that method
|
||||
/// directly instead of exposing a builder.
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
None
|
||||
@@ -243,6 +280,32 @@ pub trait ToolParser: Send {
|
||||
fn reset(&mut self) -> String;
|
||||
}
|
||||
|
||||
/// The tool choice with an optional call made obligatory, or `None` when the
|
||||
/// choice forbids or already obliges a call.
|
||||
fn call_required(tool_choice: &ToolChoice) -> Option<ToolChoice> {
|
||||
let mut required = tool_choice.clone();
|
||||
match &mut required {
|
||||
ToolChoice::Value(value @ ToolChoiceValue::Auto) => *value = ToolChoiceValue::Required,
|
||||
ToolChoice::AllowedTools(choice) if choice.allowed_tools.mode == AllowedToolsMode::Auto => {
|
||||
choice.allowed_tools.mode = AllowedToolsMode::Required;
|
||||
}
|
||||
ToolChoice::FlatAllowedTools(choice) if choice.mode == AllowedToolsMode::Auto => {
|
||||
choice.mode = AllowedToolsMode::Required;
|
||||
}
|
||||
_ => return None,
|
||||
}
|
||||
Some(required)
|
||||
}
|
||||
|
||||
/// Warn that a request's answer constraint is ignored because its tool
|
||||
/// choice obliges a call, as the Python frontend does.
|
||||
fn warn_answer_ignored_once() {
|
||||
static ONCE: Once = Once::new();
|
||||
ONCE.call_once(|| {
|
||||
warn!("structured outputs are ignored because tool_choice forces a tool call");
|
||||
});
|
||||
}
|
||||
|
||||
/// Extension methods for easily testing `ToolParser` implementations.
|
||||
///
|
||||
/// These helpers do not handle partial parsing or error recovery, so they are
|
||||
|
||||
@@ -87,10 +87,19 @@ impl UnifiedParser for CombinedParser {
|
||||
&self,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
let Some(tool) = self.tool.as_ref() else {
|
||||
// Without an active reasoning or tool parser, the request's own
|
||||
// constraint applies unchanged.
|
||||
if self.reasoning.is_none() && self.tool.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// The tool parser composes the answer constraint into its calls where
|
||||
// it can; without a tool grammar, the answer is the visible language.
|
||||
let calls = match self.tool.as_ref() {
|
||||
Some(tool) => tool.build_visible_format(ctx)?,
|
||||
None => None,
|
||||
};
|
||||
let Some(visible) = tool.build_visible_format(ctx)? else {
|
||||
let Some(visible) = calls.or_else(|| ctx.answer.cloned()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
// The reasoning parser may wrap that language with the reasoning phase
|
||||
@@ -165,7 +174,8 @@ mod tests {
|
||||
|
||||
use super::CombinedParser;
|
||||
use crate::output_grammar::{
|
||||
GrammarCoverage, OutputGrammarContext, full_format_from_builder_for_test,
|
||||
BuiltOutputGrammar, GrammarCoverage, OutputGrammarContext,
|
||||
full_format_from_builder_for_test,
|
||||
};
|
||||
use crate::reasoning::{
|
||||
DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser, Glm45ReasoningParser,
|
||||
@@ -235,6 +245,7 @@ mod tests {
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
};
|
||||
let tool = T::create(&tools).unwrap();
|
||||
let expected = full_format_from_builder_for_test(
|
||||
@@ -524,6 +535,7 @@ mod tests {
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
};
|
||||
let reasoning = Qwen3ReasoningParser::create(Arc::new(tokenizer())).unwrap();
|
||||
let tool = Qwen3XmlToolParser::create(&tools).unwrap();
|
||||
@@ -553,6 +565,7 @@ mod tests {
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
};
|
||||
let visible = Format::const_string("answer");
|
||||
|
||||
@@ -586,6 +599,7 @@ mod tests {
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
};
|
||||
let visible = Format::const_string("visible");
|
||||
|
||||
@@ -622,6 +636,7 @@ mod tests {
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
};
|
||||
let tokenizer = Arc::new(tokenizer());
|
||||
let reasoning = Qwen3ReasoningParser::create(tokenizer.clone()).unwrap();
|
||||
@@ -650,4 +665,97 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn grammar_ctx<'a>(
|
||||
tools: &'a [Tool],
|
||||
tool_choice: &'a ToolChoice,
|
||||
answer: Option<&'a Format>,
|
||||
) -> OutputGrammarContext<'a> {
|
||||
OutputGrammarContext {
|
||||
tools,
|
||||
tool_choice,
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
answer,
|
||||
}
|
||||
}
|
||||
|
||||
fn qwen3_parser(tools: &[Tool], with_tool: bool) -> CombinedParser {
|
||||
let reasoning = Qwen3ReasoningParser::create(Arc::new(tokenizer())).unwrap();
|
||||
let tool = with_tool.then(|| Qwen3XmlToolParser::create(tools).unwrap());
|
||||
let mut parser = CombinedParser::new(Some(reasoning), tool);
|
||||
parser.initialize(&[]).unwrap();
|
||||
parser
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_constraint_composes_with_the_tool_choice() {
|
||||
let tools = test_tools();
|
||||
let answer = Format::json_schema(serde_json::json!({ "type": "object" }));
|
||||
let reasoning = |visible| {
|
||||
Format::sequence(vec![
|
||||
Format::optional(Format::sequence(vec![
|
||||
Format::tag("<think>", Format::any_text(), "</think>"),
|
||||
Format::const_string("\n\n"),
|
||||
])),
|
||||
visible,
|
||||
])
|
||||
};
|
||||
let required = ToolChoice::required();
|
||||
let required_calls = Qwen3XmlToolParser::create(&tools)
|
||||
.unwrap()
|
||||
.build_visible_format(&grammar_ctx(&tools, &required, None))
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
// No callable tool: the answer alone, after the reasoning phase.
|
||||
for (tool_choice, with_tool) in [(ToolChoice::none(), true), (ToolChoice::auto(), false)] {
|
||||
let parser = qwen3_parser(&tools, with_tool);
|
||||
let ctx = grammar_ctx(&tools, &tool_choice, Some(&answer));
|
||||
let actual = parser.build_output_grammar(&ctx).unwrap().unwrap();
|
||||
assert_eq!(actual.coverage, GrammarCoverage::FromTokenZero);
|
||||
assert_eq!(actual.format, reasoning(answer.clone()), "{tool_choice:?}");
|
||||
}
|
||||
|
||||
// `auto`, strict or not, keeps tool calls next to the answer.
|
||||
let parser = qwen3_parser(&tools, true);
|
||||
let auto = ToolChoice::auto();
|
||||
let actual = parser.build_output_grammar(&grammar_ctx(&tools, &auto, Some(&answer)));
|
||||
let expected = Format::or(vec![required_calls, answer.clone()]);
|
||||
assert_eq!(actual.unwrap().unwrap().format, reasoning(expected));
|
||||
|
||||
// A required call ignores the answer constraint.
|
||||
for tool_choice in [ToolChoice::required(), ToolChoice::function("get_weather")] {
|
||||
let with_answer = grammar_ctx(&tools, &tool_choice, Some(&answer));
|
||||
let without = grammar_ctx(&tools, &tool_choice, None);
|
||||
assert_eq!(
|
||||
parser.build_output_grammar(&with_answer).unwrap(),
|
||||
parser.build_output_grammar(&without).unwrap(),
|
||||
"{tool_choice:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_without_a_reasoning_phase_covers_the_final_output() {
|
||||
let tools = test_tools();
|
||||
let answer = Format::regex("[a-z]+");
|
||||
let none = ToolChoice::none();
|
||||
let auto = ToolChoice::auto();
|
||||
|
||||
// Without an active parser, the request keeps its own constraint.
|
||||
let parser = CombinedParser::new(None, None);
|
||||
let ctx = grammar_ctx(&tools, &auto, Some(&answer));
|
||||
assert!(parser.build_output_grammar(&ctx).unwrap().is_none());
|
||||
|
||||
let parser = CombinedParser::new(None, Some(Qwen3XmlToolParser::create(&tools).unwrap()));
|
||||
let ctx = grammar_ctx(&tools, &none, Some(&answer));
|
||||
let alone = parser.build_output_grammar(&ctx).unwrap().unwrap();
|
||||
assert_eq!(alone, BuiltOutputGrammar::final_output_only(answer.clone()));
|
||||
|
||||
let ctx = grammar_ctx(&tools, &auto, Some(&answer));
|
||||
let composed = parser.build_output_grammar(&ctx).unwrap().unwrap();
|
||||
assert_eq!(composed.coverage, GrammarCoverage::FinalOutputOnly);
|
||||
assert!(matches!(composed.format, Format::Or(_)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,6 +164,7 @@ mod tests {
|
||||
tools: &tools,
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
tool_choice: &ToolChoice::required(),
|
||||
})
|
||||
.unwrap()
|
||||
@@ -194,6 +195,7 @@ mod tests {
|
||||
tool_choice: &ToolChoice::required(),
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
})
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
@@ -178,6 +178,7 @@ mod tests {
|
||||
tools: &tools,
|
||||
tool_strict_level: Default::default(),
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
tool_choice: &ToolChoice::required(),
|
||||
})
|
||||
.unwrap()
|
||||
|
||||
@@ -16,6 +16,10 @@
|
||||
//!
|
||||
//! Every path may end with the message close, and a stop token is legal only
|
||||
//! after the last channel closes.
|
||||
//!
|
||||
//! The user's answer constraint, if any, replaces the free text of the
|
||||
//! `response` channel. Without a callable tool, the `response` channel then
|
||||
//! follows `think` alone.
|
||||
|
||||
mod arguments;
|
||||
|
||||
@@ -33,38 +37,44 @@ use crate::output_grammar::{
|
||||
};
|
||||
|
||||
/// Build the grammar of everything generated after a prompt that left the
|
||||
/// parser in `mode`, or `None` when the request asks for no tool grammar.
|
||||
/// parser in `mode`, or `None` when the request asks for no tool grammar and
|
||||
/// has no answer constraint.
|
||||
pub(super) fn build_output_grammar(
|
||||
mode: &KimiK3Mode,
|
||||
ctx: &OutputGrammarContext<'_>,
|
||||
) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
|
||||
let Some(NormalizedToolChoice {
|
||||
function_tools,
|
||||
choice,
|
||||
..
|
||||
}) = normalize_tool_choice(ctx)?
|
||||
else {
|
||||
let normalized = normalize_tool_choice(ctx)?;
|
||||
if normalized.is_none() && ctx.answer.is_none() {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let single_call = choice == BuilderToolChoice::Forced || !ctx.parallel_tool_calls;
|
||||
let tools = Format::tag(
|
||||
TOOLS_OPEN,
|
||||
Format::tags_with_separator(
|
||||
function_tools.iter().map(call_tag).collect(),
|
||||
"",
|
||||
true,
|
||||
single_call,
|
||||
),
|
||||
TOOLS_CLOSE,
|
||||
}
|
||||
let calls = normalized.map(
|
||||
|NormalizedToolChoice {
|
||||
function_tools,
|
||||
choice,
|
||||
..
|
||||
}| {
|
||||
let single_call = choice == BuilderToolChoice::Forced || !ctx.parallel_tool_calls;
|
||||
let tools = Format::tag(
|
||||
TOOLS_OPEN,
|
||||
Format::tags_with_separator(
|
||||
function_tools.iter().map(call_tag).collect(),
|
||||
"",
|
||||
true,
|
||||
single_call,
|
||||
),
|
||||
TOOLS_CLOSE,
|
||||
);
|
||||
(choice, tools)
|
||||
},
|
||||
);
|
||||
let answer = ctx.answer;
|
||||
|
||||
let mut elements = match mode {
|
||||
KimiK3Mode::Reasoning => vec![think_body(), after_reasoning(choice, tools)],
|
||||
KimiK3Mode::Response => vec![response_body(), after_response(choice, tools)],
|
||||
KimiK3Mode::Reasoning => vec![think_body(), after_reasoning(calls, answer)],
|
||||
KimiK3Mode::Response => rest_of_response(calls, answer),
|
||||
KimiK3Mode::Idle => vec![
|
||||
Format::optional(Format::tag(THINK_OPEN, think_text(), THINK_CLOSE)),
|
||||
after_reasoning(choice, tools),
|
||||
after_reasoning(calls, answer),
|
||||
],
|
||||
KimiK3Mode::Epilogue | KimiK3Mode::Tools | KimiK3Mode::Call { .. } | KimiK3Mode::Done => {
|
||||
unreachable!("initialization leaves the parser idle or inside `think` or `response`")
|
||||
@@ -77,38 +87,73 @@ pub(super) fn build_output_grammar(
|
||||
))))
|
||||
}
|
||||
|
||||
/// Channels that may follow the closed `think` channel.
|
||||
fn after_reasoning(tool_choice: BuilderToolChoice, tools: Format) -> Format {
|
||||
let response = Format::tag(RESPONSE_OPEN, response_text(), RESPONSE_CLOSE);
|
||||
match tool_choice {
|
||||
BuilderToolChoice::Auto => Format::or(vec![
|
||||
/// Channels that may follow the closed `think` channel, given the tool
|
||||
/// choice and `tools` channel when a tool is callable, and the user's answer
|
||||
/// constraint for the `response` channel.
|
||||
fn after_reasoning(calls: Option<(BuilderToolChoice, Format)>, answer: Option<&Format>) -> Format {
|
||||
let response = Format::tag(RESPONSE_OPEN, response_content(answer), RESPONSE_CLOSE);
|
||||
let Some((choice, tools)) = calls else {
|
||||
return response;
|
||||
};
|
||||
let empty = || Format::const_string(format!("{RESPONSE_OPEN}{RESPONSE_CLOSE}"));
|
||||
match (choice, answer) {
|
||||
(BuilderToolChoice::Auto, None) => Format::or(vec![
|
||||
Format::sequence(vec![response, Format::optional(tools.clone())]),
|
||||
tools,
|
||||
]),
|
||||
BuilderToolChoice::Required | BuilderToolChoice::Forced => {
|
||||
(BuilderToolChoice::Auto, Some(_)) => Format::or(vec![
|
||||
Format::sequence(vec![response, Format::optional(tools.clone())]),
|
||||
Format::sequence(vec![Format::optional(empty()), tools]),
|
||||
]),
|
||||
(BuilderToolChoice::Required | BuilderToolChoice::Forced, None) => {
|
||||
Format::sequence(vec![Format::optional(response), tools])
|
||||
}
|
||||
(BuilderToolChoice::Required | BuilderToolChoice::Forced, Some(_)) => {
|
||||
Format::sequence(vec![
|
||||
Format::optional(Format::or(vec![response, empty()])),
|
||||
tools,
|
||||
])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Channels that may follow the closed `response` channel.
|
||||
fn after_response(tool_choice: BuilderToolChoice, tools: Format) -> Format {
|
||||
match tool_choice {
|
||||
BuilderToolChoice::Auto => Format::optional(tools),
|
||||
BuilderToolChoice::Required | BuilderToolChoice::Forced => tools,
|
||||
/// The rest of the `response` channel the prompt opened, then the channels
|
||||
/// that may follow it.
|
||||
fn rest_of_response(
|
||||
calls: Option<(BuilderToolChoice, Format)>,
|
||||
answer: Option<&Format>,
|
||||
) -> Vec<Format> {
|
||||
let response = Format::tag("", response_content(answer), RESPONSE_CLOSE);
|
||||
let Some((choice, tools)) = calls else {
|
||||
return vec![response];
|
||||
};
|
||||
let empty = || Format::const_string(RESPONSE_CLOSE);
|
||||
match (choice, answer) {
|
||||
(BuilderToolChoice::Auto, None) => vec![response, Format::optional(tools)],
|
||||
(BuilderToolChoice::Auto, Some(_)) => vec![Format::or(vec![
|
||||
Format::sequence(vec![response, Format::optional(tools.clone())]),
|
||||
Format::sequence(vec![empty(), tools]),
|
||||
])],
|
||||
(BuilderToolChoice::Required | BuilderToolChoice::Forced, None) => vec![response, tools],
|
||||
(BuilderToolChoice::Required | BuilderToolChoice::Forced, Some(_)) => {
|
||||
vec![Format::or(vec![response, empty()]), tools]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The text of the `response` channel: the user's answer constraint, or free
|
||||
/// text. A constrained answer rules out the empty `response` channel the chat
|
||||
/// template writes before tool calls, so callers allow that channel where
|
||||
/// calls follow.
|
||||
fn response_content(answer: Option<&Format>) -> Format {
|
||||
answer.cloned().unwrap_or_else(response_text)
|
||||
}
|
||||
|
||||
/// The rest of the `think` channel the prompt opened.
|
||||
fn think_body() -> Format {
|
||||
Format::tag("", think_text(), THINK_CLOSE)
|
||||
}
|
||||
|
||||
/// The rest of the `response` channel the prompt opened.
|
||||
fn response_body() -> Format {
|
||||
Format::tag("", response_text(), RESPONSE_CLOSE)
|
||||
}
|
||||
|
||||
fn think_text() -> Format {
|
||||
Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG])
|
||||
}
|
||||
@@ -158,6 +203,28 @@ mod tests {
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: ToolStrictLevel::Function,
|
||||
parallel_tool_calls,
|
||||
answer: None,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(grammar.coverage, GrammarCoverage::FromTokenZero);
|
||||
let mut format = grammar.format;
|
||||
elide_call_bodies(&mut format);
|
||||
expected.assert_eq(&outline(&format));
|
||||
}
|
||||
|
||||
/// Like [`check`], for non-strict tools and a JSON answer constraint.
|
||||
fn check_answer(mode: KimiK3Mode, tools: &[Tool], tool_choice: ToolChoice, expected: Expect) {
|
||||
let answer = Format::json_schema(json!({ "type": "object" }));
|
||||
let grammar = build_output_grammar(
|
||||
&mode,
|
||||
&OutputGrammarContext {
|
||||
tools,
|
||||
tool_choice: &tool_choice,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: true,
|
||||
answer: Some(&answer),
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
@@ -348,9 +415,120 @@ mod tests {
|
||||
tool_choice: &ToolChoice::auto(),
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
parallel_tool_calls: true,
|
||||
answer: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(grammar, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_constraint_fills_the_response_channel() {
|
||||
// Non-strict `auto` keeps the calls next to the constrained answer.
|
||||
// Calls may follow the empty `response` channel the chat template
|
||||
// writes before them.
|
||||
check_answer(
|
||||
KimiK3Mode::Reasoning,
|
||||
&tools(),
|
||||
ToolChoice::auto(),
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
|
||||
or
|
||||
sequence
|
||||
tag `<|open|>response<|sep|>` json(object) `<|close|>response<|sep|>`
|
||||
optional
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
sequence
|
||||
optional `<|open|>response<|sep|><|close|>response<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
// A required call may follow a constrained answer.
|
||||
check_answer(
|
||||
KimiK3Mode::Reasoning,
|
||||
&tools(),
|
||||
ToolChoice::required(),
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
|
||||
sequence
|
||||
optional
|
||||
or
|
||||
tag `<|open|>response<|sep|>` json(object) `<|close|>response<|sep|>`
|
||||
`<|open|>response<|sep|><|close|>response<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
check_answer(
|
||||
KimiK3Mode::Response,
|
||||
&tools(),
|
||||
ToolChoice::auto(),
|
||||
expect![[r#"
|
||||
sequence
|
||||
or
|
||||
sequence
|
||||
tag `` json(object) `<|close|>response<|sep|>`
|
||||
optional
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
sequence
|
||||
`<|close|>response<|sep|>`
|
||||
tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>`
|
||||
tags_with_separator `` at_least_one
|
||||
tag `<|open|>call tool="get_weather" index="` `..` `<|close|>call<|sep|>`
|
||||
tag `<|open|>call tool="add" index="` `..` `<|close|>call<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn answer_constraint_without_a_callable_tool_requires_the_response() {
|
||||
check_answer(
|
||||
KimiK3Mode::Idle,
|
||||
&tools(),
|
||||
ToolChoice::none(),
|
||||
expect![[r#"
|
||||
sequence
|
||||
optional tag `<|open|>think<|sep|>` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
|
||||
tag `<|open|>response<|sep|>` json(object) `<|close|>response<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
check_answer(
|
||||
KimiK3Mode::Reasoning,
|
||||
&[],
|
||||
ToolChoice::auto(),
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` text excluding [`<|close|>think<|sep|>`, `<|end_of_msg|>`] `<|close|>think<|sep|>`
|
||||
tag `<|open|>response<|sep|>` json(object) `<|close|>response<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
check_answer(
|
||||
KimiK3Mode::Response,
|
||||
&[],
|
||||
ToolChoice::auto(),
|
||||
expect![[r#"
|
||||
sequence
|
||||
tag `` json(object) `<|close|>response<|sep|>`
|
||||
optional `<|close|>message<|sep|>`
|
||||
"#]],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user