[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: lazymio <mio@lazym.io> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5
Bugen Zhao
parent
d12c276853
commit
dee37d8911
@@ -134,6 +134,7 @@ impl ChatBackend for HfChatBackend {
|
||||
self.tokenizer.clone(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
@@ -343,6 +344,7 @@ mod tests {
|
||||
let error = match backend.new_chat_output_processor(
|
||||
&mut request,
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_strict_level: crate::ToolStrictLevel::Auto,
|
||||
tool_call_parser: &ParserSelection::Explicit("json".to_string()),
|
||||
reasoning_parser: &ParserSelection::Auto,
|
||||
},
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};
|
||||
use crate::output::DynChatOutputProcessor;
|
||||
use crate::renderer::DynChatRenderer;
|
||||
use crate::request::ChatRequest;
|
||||
use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
|
||||
use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection, ToolStrictLevel};
|
||||
|
||||
pub mod hf;
|
||||
|
||||
@@ -21,6 +21,7 @@ pub mod hf;
|
||||
pub struct NewChatOutputProcessorOptions<'a> {
|
||||
pub tool_call_parser: &'a ParserSelection,
|
||||
pub reasoning_parser: &'a ParserSelection,
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
}
|
||||
|
||||
/// Minimal prompt-processing backend needed by `vllm-chat`.
|
||||
|
||||
@@ -30,7 +30,7 @@ pub use parser::reasoning::{
|
||||
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory,
|
||||
};
|
||||
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
|
||||
pub use parser::{ParserSelection, validate_parser_overrides};
|
||||
pub use parser::{ParserSelection, ToolStrictLevel, validate_parser_overrides};
|
||||
pub use reasoning::EffortValue;
|
||||
pub use renderer::hf::ChatTemplateContentFormatOption;
|
||||
pub use renderer::{
|
||||
@@ -74,6 +74,8 @@ pub struct ChatRequestProcessor {
|
||||
tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection used when preparing generation requests.
|
||||
reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for tool-call structural tags.
|
||||
tool_strict_level: ToolStrictLevel,
|
||||
}
|
||||
|
||||
impl ChatRequestProcessor {
|
||||
@@ -85,6 +87,7 @@ impl ChatRequestProcessor {
|
||||
model_dtype: Some(model_dtype),
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,6 +98,7 @@ impl ChatRequestProcessor {
|
||||
model_dtype: None,
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,6 +113,12 @@ impl ChatRequestProcessor {
|
||||
self
|
||||
}
|
||||
|
||||
/// Configure the server-side floor for tool-call structural tags.
|
||||
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
|
||||
self.tool_strict_level = tool_strict_level;
|
||||
self
|
||||
}
|
||||
|
||||
async fn finalize_rendered_prompt(
|
||||
&self,
|
||||
request: &ChatRequest,
|
||||
@@ -207,6 +217,7 @@ impl ChatRequestProcessor {
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_call_parser: &self.tool_call_parser,
|
||||
reasoning_parser: &self.reasoning_parser,
|
||||
tool_strict_level: self.tool_strict_level,
|
||||
},
|
||||
)?;
|
||||
let text_request = self.prepare_text_request(request).await?;
|
||||
@@ -255,6 +266,12 @@ impl ChatLlm {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the server-side floor for tool-call structural tags.
|
||||
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
|
||||
self.processor.tool_strict_level = tool_strict_level;
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the effective model dtype used for multimodal tensor encoding.
|
||||
pub fn with_model_dtype(mut self, model_dtype: ModelDtype) -> Self {
|
||||
self.processor.model_dtype = Some(model_dtype);
|
||||
|
||||
@@ -18,10 +18,10 @@ use self::unified::unified_event_stream;
|
||||
use super::structured::structured_chat_event_stream;
|
||||
use crate::error::Result;
|
||||
use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream};
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
|
||||
use crate::parser::tool::{ToolParser, ToolParserFactory};
|
||||
use crate::parser::unified::UnifiedParserFactory;
|
||||
use crate::parser::{ParserSelection, ToolStrictLevel};
|
||||
use crate::request::{ChatRequest, ChatTool};
|
||||
use crate::{Error, Result as ChatResult};
|
||||
|
||||
@@ -49,6 +49,7 @@ impl DefaultChatOutputProcessor {
|
||||
tokenizer: DynTokenizer,
|
||||
tool_call_parser: &ParserSelection,
|
||||
reasoning_parser: &ParserSelection,
|
||||
tool_strict_level: ToolStrictLevel,
|
||||
) -> ChatResult<Self> {
|
||||
let parser = if let Some(parser) = Self::resolve_optional_unified_parser(
|
||||
request.tools(),
|
||||
@@ -73,7 +74,11 @@ impl DefaultChatOutputProcessor {
|
||||
Box::new(CombinedParser::new(reasoning_parser, tool_parser)) as Box<dyn UnifiedParser>
|
||||
};
|
||||
|
||||
apply_structural_tag_constraint(request, parser.structural_tag_builder())?;
|
||||
apply_structural_tag_constraint(
|
||||
request,
|
||||
parser.structural_tag_builder(),
|
||||
tool_strict_level,
|
||||
)?;
|
||||
|
||||
if parser.preserve_special_tokens() {
|
||||
request.decode_options.skip_special_tokens = false;
|
||||
@@ -191,7 +196,7 @@ mod tests {
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::DefaultChatOutputProcessor;
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::parser::{ParserSelection, ToolStrictLevel};
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
fn tokenizer() -> Arc<TestTokenizer> {
|
||||
@@ -213,6 +218,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&selection,
|
||||
&selection,
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -227,6 +233,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&ParserSelection::Auto,
|
||||
&ParserSelection::Auto,
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -244,6 +251,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
tool,
|
||||
reasoning,
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -258,6 +266,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&ParserSelection::Auto,
|
||||
&ParserSelection::Explicit("gemma4".to_string()),
|
||||
ToolStrictLevel::Auto,
|
||||
) {
|
||||
Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"),
|
||||
Err(error) => error,
|
||||
|
||||
@@ -14,19 +14,21 @@ use xgrammar_structural_tag::{
|
||||
build_structural_tag,
|
||||
};
|
||||
|
||||
use crate::parser::ToolStrictLevel;
|
||||
use crate::request::{ChatRequest, ChatToolChoice};
|
||||
use crate::{Error, Result as ChatResult};
|
||||
|
||||
/// Apply structural tag constraints to the request based on the tool parser's structural tag
|
||||
/// support and the request's tool choice.
|
||||
/// support, the request's tool choice, and the server-side strictness floor.
|
||||
pub(super) fn apply_structural_tag_constraint(
|
||||
request: &mut ChatRequest,
|
||||
builder: Option<&dyn StructuralTagBuilder>,
|
||||
strict_level: ToolStrictLevel,
|
||||
) -> ChatResult<()> {
|
||||
let Some(builder) = builder else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(tool_choice) = structural_tag_tool_choice(request) else {
|
||||
let Some(tool_choice) = structural_tag_tool_choice(request, strict_level) else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
@@ -34,11 +36,14 @@ pub(super) fn apply_structural_tag_constraint(
|
||||
.tools()
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
// A tool without an explicit `strict` is non-strict: its call envelope is
|
||||
// constrained, but its arguments stay free unless the server level is `parameter`.
|
||||
let strict = strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true);
|
||||
ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
|
||||
name: tool.name.clone(),
|
||||
description: tool.description.clone(),
|
||||
parameters: Some(tool.parameters.clone()),
|
||||
strict: tool.strict,
|
||||
strict: Some(strict),
|
||||
}))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
@@ -66,17 +71,21 @@ pub(super) fn apply_structural_tag_constraint(
|
||||
/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request.
|
||||
///
|
||||
/// Returns `None` if no structural tag constraints should be applied.
|
||||
fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagToolChoice> {
|
||||
fn structural_tag_tool_choice(
|
||||
request: &ChatRequest,
|
||||
strict_level: ToolStrictLevel,
|
||||
) -> Option<StructuralTagToolChoice> {
|
||||
if request.tools().is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
match request.tool_choice() {
|
||||
// For `Auto`, only apply the structural tag if there's at least one strict tool.
|
||||
ChatToolChoice::Auto if request.tools().iter().any(|tool| tool.strict == Some(true)) => {
|
||||
Some(StructuralTagToolChoice::auto())
|
||||
}
|
||||
ChatToolChoice::Auto | ChatToolChoice::None => None,
|
||||
ChatToolChoice::None => None,
|
||||
// For `Auto`, apply the structural tag only when the server raises the floor or at
|
||||
// least one tool opts in with `strict: true`.
|
||||
ChatToolChoice::Auto => (strict_level >= ToolStrictLevel::Function
|
||||
|| request.tools().iter().any(|tool| tool.strict == Some(true)))
|
||||
.then(StructuralTagToolChoice::auto),
|
||||
|
||||
ChatToolChoice::Required => Some(StructuralTagToolChoice::required()),
|
||||
ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())),
|
||||
@@ -148,8 +157,12 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
assert_eq!(tag["type"], "structural_tag");
|
||||
@@ -172,8 +185,12 @@ mod tests {
|
||||
};
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
assert!(tag.to_string().contains("lookup"));
|
||||
@@ -199,8 +216,12 @@ mod tests {
|
||||
};
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request).to_string();
|
||||
assert!(tag.contains("search"));
|
||||
@@ -212,8 +233,12 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag decision should succeed");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
assert!(request.sampling_params.structured_outputs.is_none());
|
||||
}
|
||||
@@ -227,8 +252,12 @@ mod tests {
|
||||
});
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
assert!(params.constraint.is_structural_tag());
|
||||
@@ -242,8 +271,12 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
assert_eq!(tag["type"], "structural_tag");
|
||||
@@ -259,8 +292,12 @@ mod tests {
|
||||
});
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
assert!(params.constraint.is_structural_tag());
|
||||
@@ -279,8 +316,12 @@ mod tests {
|
||||
);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request).to_string();
|
||||
assert!(tag.contains("lookup"));
|
||||
@@ -292,8 +333,12 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag decision should succeed");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
assert!(request.sampling_params.structured_outputs.is_none());
|
||||
}
|
||||
@@ -307,10 +352,127 @@ mod tests {
|
||||
});
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag decision should succeed");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
assert!(params.constraint.is_json_object());
|
||||
}
|
||||
|
||||
fn build(tool_choice: ChatToolChoice, tools: Vec<Tool>, level: ToolStrictLevel) -> ChatRequest {
|
||||
let mut request = request(tool_choice, tools);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder(), level)
|
||||
.expect("structural tag decision should succeed");
|
||||
request
|
||||
}
|
||||
|
||||
fn tag_string(request: &ChatRequest) -> String {
|
||||
structural_tag_value(request).to_string()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_level_constrains_auto_without_strict_tools() {
|
||||
let request = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![chat_tool("search", None)],
|
||||
ToolStrictLevel::Function,
|
||||
);
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
assert_eq!(tag["type"], "structural_tag");
|
||||
assert!(tag.to_string().contains("search"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_strict_is_non_strict() {
|
||||
let unset = build(
|
||||
ChatToolChoice::Required,
|
||||
vec![chat_tool("search", None)],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let non_strict = build(
|
||||
ChatToolChoice::Required,
|
||||
vec![chat_tool("search", Some(false))],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let strict = build(
|
||||
ChatToolChoice::Required,
|
||||
vec![chat_tool("search", Some(true))],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
|
||||
assert_eq!(tag_string(&unset), tag_string(&non_strict));
|
||||
assert_ne!(tag_string(&unset), tag_string(&strict));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_tool_does_not_pin_its_neighbour() {
|
||||
let mixed = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![chat_tool("weather", Some(true)), chat_tool("search", None)],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let explicit = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![
|
||||
chat_tool("weather", Some(true)),
|
||||
chat_tool("search", Some(false)),
|
||||
],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
|
||||
assert_eq!(tag_string(&mixed), tag_string(&explicit));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parameter_level_pins_argument_schemas() {
|
||||
let pinned = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![chat_tool("search", None)],
|
||||
ToolStrictLevel::Parameter,
|
||||
);
|
||||
let strict = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![chat_tool("search", Some(true))],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let envelope_only = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![chat_tool("search", None)],
|
||||
ToolStrictLevel::Function,
|
||||
);
|
||||
|
||||
assert_eq!(tag_string(&pinned), tag_string(&strict));
|
||||
assert_ne!(tag_string(&pinned), tag_string(&envelope_only));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_strict_level_parses_case_insensitively() {
|
||||
assert_eq!("AUTO".parse::<ToolStrictLevel>(), Ok(ToolStrictLevel::Auto));
|
||||
assert_eq!(
|
||||
"PARAMETER".parse::<ToolStrictLevel>(),
|
||||
Ok(ToolStrictLevel::Parameter)
|
||||
);
|
||||
assert_eq!(
|
||||
"function".parse::<ToolStrictLevel>(),
|
||||
Ok(ToolStrictLevel::Function)
|
||||
);
|
||||
assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto);
|
||||
assert_eq!(
|
||||
serde_json::to_value(ToolStrictLevel::Auto).unwrap(),
|
||||
json!("auto")
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ToolStrictLevel>(json!("auto")).unwrap(),
|
||||
ToolStrictLevel::Auto
|
||||
);
|
||||
assert!("off".parse::<ToolStrictLevel>().is_err());
|
||||
assert!("strict".parse::<ToolStrictLevel>().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,66 @@ impl ParserSelection {
|
||||
}
|
||||
}
|
||||
|
||||
/// Server-side floor for tool-call structural tags (`--tool-strict-level`).
|
||||
#[derive(
|
||||
Debug,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
Eq,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
Default,
|
||||
DeserializeFromStr,
|
||||
SerializeDisplay,
|
||||
)]
|
||||
pub enum ToolStrictLevel {
|
||||
/// Derive tool constraints from the request's tool choice and per-tool strictness.
|
||||
#[default]
|
||||
Auto,
|
||||
/// Constrain the tool-call envelope for every request with tools.
|
||||
Function,
|
||||
/// Additionally pin argument schemas, as if every tool were `strict: true`.
|
||||
Parameter,
|
||||
}
|
||||
|
||||
impl ToolStrictLevel {
|
||||
pub const AUTO_LITERAL: &str = "auto";
|
||||
pub const FUNCTION_LITERAL: &str = "function";
|
||||
pub const PARAMETER_LITERAL: &str = "parameter";
|
||||
}
|
||||
|
||||
impl FromStr for ToolStrictLevel {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
|
||||
Ok(Self::Auto)
|
||||
} else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) {
|
||||
Ok(Self::Function)
|
||||
} else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) {
|
||||
Ok(Self::Parameter)
|
||||
} else {
|
||||
Err(format!(
|
||||
"unknown tool strict level {value:?}; expected one of {}, {}, {}",
|
||||
Self::AUTO_LITERAL,
|
||||
Self::FUNCTION_LITERAL,
|
||||
Self::PARAMETER_LITERAL
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ToolStrictLevel {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::Auto => Self::AUTO_LITERAL,
|
||||
Self::Function => Self::FUNCTION_LITERAL,
|
||||
Self::Parameter => Self::PARAMETER_LITERAL,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate explicit parser override names without starting request processing.
|
||||
pub fn validate_parser_overrides(
|
||||
tool_call_parser: &ParserSelection,
|
||||
|
||||
@@ -227,6 +227,7 @@ impl ChatBackend for FakeChatBackend {
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -717,6 +717,7 @@ async fn parse_completion(
|
||||
let processor = backends.chat_backend.new_chat_output_processor(
|
||||
&mut request,
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_strict_level: vllm_chat::ToolStrictLevel::Auto,
|
||||
tool_call_parser: &case.tool_call_parser,
|
||||
reasoning_parser: &case.reasoning_parser,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user