[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: lazymio <mio@lazym.io>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
lazymio
2026-09-18 04:32:39 +00:00
committed by GitHub
co-authored by Claude Opus 5 Bugen Zhao
parent d12c276853
commit dee37d8911
35 changed files with 742 additions and 61 deletions
+2
View File
@@ -134,6 +134,7 @@ impl ChatBackend for HfChatBackend {
self.tokenizer.clone(),
options.tool_call_parser,
options.reasoning_parser,
options.tool_strict_level,
)?))
}
}
@@ -343,6 +344,7 @@ mod tests {
let error = match backend.new_chat_output_processor(
&mut request,
NewChatOutputProcessorOptions {
tool_strict_level: crate::ToolStrictLevel::Auto,
tool_call_parser: &ParserSelection::Explicit("json".to_string()),
reasoning_parser: &ParserSelection::Auto,
},
+2 -1
View File
@@ -13,7 +13,7 @@ use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};
use crate::output::DynChatOutputProcessor;
use crate::renderer::DynChatRenderer;
use crate::request::ChatRequest;
use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection, ToolStrictLevel};
pub mod hf;
@@ -21,6 +21,7 @@ pub mod hf;
pub struct NewChatOutputProcessorOptions<'a> {
pub tool_call_parser: &'a ParserSelection,
pub reasoning_parser: &'a ParserSelection,
pub tool_strict_level: ToolStrictLevel,
}
/// Minimal prompt-processing backend needed by `vllm-chat`.
+18 -1
View File
@@ -30,7 +30,7 @@ pub use parser::reasoning::{
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory,
};
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
pub use parser::{ParserSelection, validate_parser_overrides};
pub use parser::{ParserSelection, ToolStrictLevel, validate_parser_overrides};
pub use reasoning::EffortValue;
pub use renderer::hf::ChatTemplateContentFormatOption;
pub use renderer::{
@@ -74,6 +74,8 @@ pub struct ChatRequestProcessor {
tool_call_parser: ParserSelection,
/// Reasoning parser selection used when preparing generation requests.
reasoning_parser: ParserSelection,
/// Server-side floor for tool-call structural tags.
tool_strict_level: ToolStrictLevel,
}
impl ChatRequestProcessor {
@@ -85,6 +87,7 @@ impl ChatRequestProcessor {
model_dtype: Some(model_dtype),
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
tool_strict_level: ToolStrictLevel::Auto,
}
}
@@ -95,6 +98,7 @@ impl ChatRequestProcessor {
model_dtype: None,
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
tool_strict_level: ToolStrictLevel::Auto,
}
}
@@ -109,6 +113,12 @@ impl ChatRequestProcessor {
self
}
/// Configure the server-side floor for tool-call structural tags.
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
self.tool_strict_level = tool_strict_level;
self
}
async fn finalize_rendered_prompt(
&self,
request: &ChatRequest,
@@ -207,6 +217,7 @@ impl ChatRequestProcessor {
NewChatOutputProcessorOptions {
tool_call_parser: &self.tool_call_parser,
reasoning_parser: &self.reasoning_parser,
tool_strict_level: self.tool_strict_level,
},
)?;
let text_request = self.prepare_text_request(request).await?;
@@ -255,6 +266,12 @@ impl ChatLlm {
self
}
/// Set the server-side floor for tool-call structural tags.
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
self.processor.tool_strict_level = tool_strict_level;
self
}
/// Override the effective model dtype used for multimodal tensor encoding.
pub fn with_model_dtype(mut self, model_dtype: ModelDtype) -> Self {
self.processor.model_dtype = Some(model_dtype);
+12 -3
View File
@@ -18,10 +18,10 @@ use self::unified::unified_event_stream;
use super::structured::structured_chat_event_stream;
use crate::error::Result;
use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream};
use crate::parser::ParserSelection;
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
use crate::parser::tool::{ToolParser, ToolParserFactory};
use crate::parser::unified::UnifiedParserFactory;
use crate::parser::{ParserSelection, ToolStrictLevel};
use crate::request::{ChatRequest, ChatTool};
use crate::{Error, Result as ChatResult};
@@ -49,6 +49,7 @@ impl DefaultChatOutputProcessor {
tokenizer: DynTokenizer,
tool_call_parser: &ParserSelection,
reasoning_parser: &ParserSelection,
tool_strict_level: ToolStrictLevel,
) -> ChatResult<Self> {
let parser = if let Some(parser) = Self::resolve_optional_unified_parser(
request.tools(),
@@ -73,7 +74,11 @@ impl DefaultChatOutputProcessor {
Box::new(CombinedParser::new(reasoning_parser, tool_parser)) as Box<dyn UnifiedParser>
};
apply_structural_tag_constraint(request, parser.structural_tag_builder())?;
apply_structural_tag_constraint(
request,
parser.structural_tag_builder(),
tool_strict_level,
)?;
if parser.preserve_special_tokens() {
request.decode_options.skip_special_tokens = false;
@@ -191,7 +196,7 @@ mod tests {
use vllm_tokenizer::test_utils::TestTokenizer;
use super::DefaultChatOutputProcessor;
use crate::parser::ParserSelection;
use crate::parser::{ParserSelection, ToolStrictLevel};
use crate::request::ChatRequest;
fn tokenizer() -> Arc<TestTokenizer> {
@@ -213,6 +218,7 @@ mod tests {
tokenizer(),
&selection,
&selection,
ToolStrictLevel::Auto,
)
.unwrap();
}
@@ -227,6 +233,7 @@ mod tests {
tokenizer(),
&ParserSelection::Auto,
&ParserSelection::Auto,
ToolStrictLevel::Auto,
)
.unwrap();
}
@@ -244,6 +251,7 @@ mod tests {
tokenizer(),
tool,
reasoning,
ToolStrictLevel::Auto,
)
.unwrap();
}
@@ -258,6 +266,7 @@ mod tests {
tokenizer(),
&ParserSelection::Auto,
&ParserSelection::Explicit("gemma4".to_string()),
ToolStrictLevel::Auto,
) {
Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"),
Err(error) => error,
@@ -14,19 +14,21 @@ use xgrammar_structural_tag::{
build_structural_tag,
};
use crate::parser::ToolStrictLevel;
use crate::request::{ChatRequest, ChatToolChoice};
use crate::{Error, Result as ChatResult};
/// Apply structural tag constraints to the request based on the tool parser's structural tag
/// support and the request's tool choice.
/// support, the request's tool choice, and the server-side strictness floor.
pub(super) fn apply_structural_tag_constraint(
request: &mut ChatRequest,
builder: Option<&dyn StructuralTagBuilder>,
strict_level: ToolStrictLevel,
) -> ChatResult<()> {
let Some(builder) = builder else {
return Ok(());
};
let Some(tool_choice) = structural_tag_tool_choice(request) else {
let Some(tool_choice) = structural_tag_tool_choice(request, strict_level) else {
return Ok(());
};
@@ -34,11 +36,14 @@ pub(super) fn apply_structural_tag_constraint(
.tools()
.iter()
.map(|tool| {
// A tool without an explicit `strict` is non-strict: its call envelope is
// constrained, but its arguments stay free unless the server level is `parameter`.
let strict = strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true);
ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
name: tool.name.clone(),
description: tool.description.clone(),
parameters: Some(tool.parameters.clone()),
strict: tool.strict,
strict: Some(strict),
}))
})
.collect::<Vec<_>>();
@@ -66,17 +71,21 @@ pub(super) fn apply_structural_tag_constraint(
/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request.
///
/// Returns `None` if no structural tag constraints should be applied.
fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagToolChoice> {
fn structural_tag_tool_choice(
request: &ChatRequest,
strict_level: ToolStrictLevel,
) -> Option<StructuralTagToolChoice> {
if request.tools().is_empty() {
return None;
}
match request.tool_choice() {
// For `Auto`, only apply the structural tag if there's at least one strict tool.
ChatToolChoice::Auto if request.tools().iter().any(|tool| tool.strict == Some(true)) => {
Some(StructuralTagToolChoice::auto())
}
ChatToolChoice::Auto | ChatToolChoice::None => None,
ChatToolChoice::None => None,
// For `Auto`, apply the structural tag only when the server raises the floor or at
// least one tool opts in with `strict: true`.
ChatToolChoice::Auto => (strict_level >= ToolStrictLevel::Function
|| request.tools().iter().any(|tool| tool.strict == Some(true)))
.then(StructuralTagToolChoice::auto),
ChatToolChoice::Required => Some(StructuralTagToolChoice::required()),
ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())),
@@ -148,8 +157,12 @@ mod tests {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
@@ -172,8 +185,12 @@ mod tests {
};
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert!(tag.to_string().contains("lookup"));
@@ -199,8 +216,12 @@ mod tests {
};
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request).to_string();
assert!(tag.contains("search"));
@@ -212,8 +233,12 @@ mod tests {
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag decision should succeed");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag decision should succeed");
assert!(request.sampling_params.structured_outputs.is_none());
}
@@ -227,8 +252,12 @@ mod tests {
});
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let params = structured_outputs(&request);
assert!(params.constraint.is_structural_tag());
@@ -242,8 +271,12 @@ mod tests {
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
@@ -259,8 +292,12 @@ mod tests {
});
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let params = structured_outputs(&request);
assert!(params.constraint.is_structural_tag());
@@ -279,8 +316,12 @@ mod tests {
);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag should build");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag should build");
let tag = structural_tag_value(&request).to_string();
assert!(tag.contains("lookup"));
@@ -292,8 +333,12 @@ mod tests {
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag decision should succeed");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag decision should succeed");
assert!(request.sampling_params.structured_outputs.is_none());
}
@@ -307,10 +352,127 @@ mod tests {
});
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
.expect("structural tag decision should succeed");
apply_structural_tag_constraint(
&mut request,
parser.structural_tag_builder(),
ToolStrictLevel::Auto,
)
.expect("structural tag decision should succeed");
let params = structured_outputs(&request);
assert!(params.constraint.is_json_object());
}
fn build(tool_choice: ChatToolChoice, tools: Vec<Tool>, level: ToolStrictLevel) -> ChatRequest {
let mut request = request(tool_choice, tools);
let parser = qwen3_coder_parser(request.tools());
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder(), level)
.expect("structural tag decision should succeed");
request
}
fn tag_string(request: &ChatRequest) -> String {
structural_tag_value(request).to_string()
}
#[test]
fn function_level_constrains_auto_without_strict_tools() {
let request = build(
ChatToolChoice::Auto,
vec![chat_tool("search", None)],
ToolStrictLevel::Function,
);
let tag = structural_tag_value(&request);
assert_eq!(tag["type"], "structural_tag");
assert!(tag.to_string().contains("search"));
}
#[test]
fn absent_strict_is_non_strict() {
let unset = build(
ChatToolChoice::Required,
vec![chat_tool("search", None)],
ToolStrictLevel::Auto,
);
let non_strict = build(
ChatToolChoice::Required,
vec![chat_tool("search", Some(false))],
ToolStrictLevel::Auto,
);
let strict = build(
ChatToolChoice::Required,
vec![chat_tool("search", Some(true))],
ToolStrictLevel::Auto,
);
assert_eq!(tag_string(&unset), tag_string(&non_strict));
assert_ne!(tag_string(&unset), tag_string(&strict));
}
#[test]
fn strict_tool_does_not_pin_its_neighbour() {
let mixed = build(
ChatToolChoice::Auto,
vec![chat_tool("weather", Some(true)), chat_tool("search", None)],
ToolStrictLevel::Auto,
);
let explicit = build(
ChatToolChoice::Auto,
vec![
chat_tool("weather", Some(true)),
chat_tool("search", Some(false)),
],
ToolStrictLevel::Auto,
);
assert_eq!(tag_string(&mixed), tag_string(&explicit));
}
#[test]
fn parameter_level_pins_argument_schemas() {
let pinned = build(
ChatToolChoice::Auto,
vec![chat_tool("search", None)],
ToolStrictLevel::Parameter,
);
let strict = build(
ChatToolChoice::Auto,
vec![chat_tool("search", Some(true))],
ToolStrictLevel::Auto,
);
let envelope_only = build(
ChatToolChoice::Auto,
vec![chat_tool("search", None)],
ToolStrictLevel::Function,
);
assert_eq!(tag_string(&pinned), tag_string(&strict));
assert_ne!(tag_string(&pinned), tag_string(&envelope_only));
}
#[test]
fn tool_strict_level_parses_case_insensitively() {
assert_eq!("AUTO".parse::<ToolStrictLevel>(), Ok(ToolStrictLevel::Auto));
assert_eq!(
"PARAMETER".parse::<ToolStrictLevel>(),
Ok(ToolStrictLevel::Parameter)
);
assert_eq!(
"function".parse::<ToolStrictLevel>(),
Ok(ToolStrictLevel::Function)
);
assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto);
assert_eq!(
serde_json::to_value(ToolStrictLevel::Auto).unwrap(),
json!("auto")
);
assert_eq!(
serde_json::from_value::<ToolStrictLevel>(json!("auto")).unwrap(),
ToolStrictLevel::Auto
);
assert!("off".parse::<ToolStrictLevel>().is_err());
assert!("strict".parse::<ToolStrictLevel>().is_err());
}
}
+60
View File
@@ -57,6 +57,66 @@ impl ParserSelection {
}
}
/// Server-side floor for tool-call structural tags (`--tool-strict-level`).
#[derive(
Debug,
Clone,
Copy,
PartialEq,
Eq,
PartialOrd,
Ord,
Default,
DeserializeFromStr,
SerializeDisplay,
)]
pub enum ToolStrictLevel {
/// Derive tool constraints from the request's tool choice and per-tool strictness.
#[default]
Auto,
/// Constrain the tool-call envelope for every request with tools.
Function,
/// Additionally pin argument schemas, as if every tool were `strict: true`.
Parameter,
}
impl ToolStrictLevel {
pub const AUTO_LITERAL: &str = "auto";
pub const FUNCTION_LITERAL: &str = "function";
pub const PARAMETER_LITERAL: &str = "parameter";
}
impl FromStr for ToolStrictLevel {
type Err = String;
fn from_str(value: &str) -> Result<Self, Self::Err> {
if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
Ok(Self::Auto)
} else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) {
Ok(Self::Function)
} else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) {
Ok(Self::Parameter)
} else {
Err(format!(
"unknown tool strict level {value:?}; expected one of {}, {}, {}",
Self::AUTO_LITERAL,
Self::FUNCTION_LITERAL,
Self::PARAMETER_LITERAL
))
}
}
}
impl fmt::Display for ToolStrictLevel {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Auto => Self::AUTO_LITERAL,
Self::Function => Self::FUNCTION_LITERAL,
Self::Parameter => Self::PARAMETER_LITERAL,
})
}
}
/// Validate explicit parser override names without starting request processing.
pub fn validate_parser_overrides(
tool_call_parser: &ParserSelection,
+1
View File
@@ -227,6 +227,7 @@ impl ChatBackend for FakeChatBackend {
self.tokenizer(),
options.tool_call_parser,
options.reasoning_parser,
options.tool_strict_level,
)?))
}
}
+1
View File
@@ -717,6 +717,7 @@ async fn parse_completion(
let processor = backends.chat_backend.new_chat_output_processor(
&mut request,
NewChatOutputProcessorOptions {
tool_strict_level: vllm_chat::ToolStrictLevel::Auto,
tool_call_parser: &case.tool_call_parser,
reasoning_parser: &case.reasoning_parser,
},