[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: lazymio <mio@lazym.io> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5
Bugen Zhao
parent
d12c276853
commit
dee37d8911
@@ -134,6 +134,7 @@ impl ChatBackend for HfChatBackend {
|
||||
self.tokenizer.clone(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
@@ -343,6 +344,7 @@ mod tests {
|
||||
let error = match backend.new_chat_output_processor(
|
||||
&mut request,
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_strict_level: crate::ToolStrictLevel::Auto,
|
||||
tool_call_parser: &ParserSelection::Explicit("json".to_string()),
|
||||
reasoning_parser: &ParserSelection::Auto,
|
||||
},
|
||||
|
||||
@@ -13,7 +13,7 @@ use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo};
|
||||
use crate::output::DynChatOutputProcessor;
|
||||
use crate::renderer::DynChatRenderer;
|
||||
use crate::request::ChatRequest;
|
||||
use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
|
||||
use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection, ToolStrictLevel};
|
||||
|
||||
pub mod hf;
|
||||
|
||||
@@ -21,6 +21,7 @@ pub mod hf;
|
||||
pub struct NewChatOutputProcessorOptions<'a> {
|
||||
pub tool_call_parser: &'a ParserSelection,
|
||||
pub reasoning_parser: &'a ParserSelection,
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
}
|
||||
|
||||
/// Minimal prompt-processing backend needed by `vllm-chat`.
|
||||
|
||||
@@ -30,7 +30,7 @@ pub use parser::reasoning::{
|
||||
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory,
|
||||
};
|
||||
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
|
||||
pub use parser::{ParserSelection, validate_parser_overrides};
|
||||
pub use parser::{ParserSelection, ToolStrictLevel, validate_parser_overrides};
|
||||
pub use reasoning::EffortValue;
|
||||
pub use renderer::hf::ChatTemplateContentFormatOption;
|
||||
pub use renderer::{
|
||||
@@ -74,6 +74,8 @@ pub struct ChatRequestProcessor {
|
||||
tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection used when preparing generation requests.
|
||||
reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for tool-call structural tags.
|
||||
tool_strict_level: ToolStrictLevel,
|
||||
}
|
||||
|
||||
impl ChatRequestProcessor {
|
||||
@@ -85,6 +87,7 @@ impl ChatRequestProcessor {
|
||||
model_dtype: Some(model_dtype),
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,6 +98,7 @@ impl ChatRequestProcessor {
|
||||
model_dtype: None,
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,6 +113,12 @@ impl ChatRequestProcessor {
|
||||
self
|
||||
}
|
||||
|
||||
/// Configure the server-side floor for tool-call structural tags.
|
||||
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
|
||||
self.tool_strict_level = tool_strict_level;
|
||||
self
|
||||
}
|
||||
|
||||
async fn finalize_rendered_prompt(
|
||||
&self,
|
||||
request: &ChatRequest,
|
||||
@@ -207,6 +217,7 @@ impl ChatRequestProcessor {
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_call_parser: &self.tool_call_parser,
|
||||
reasoning_parser: &self.reasoning_parser,
|
||||
tool_strict_level: self.tool_strict_level,
|
||||
},
|
||||
)?;
|
||||
let text_request = self.prepare_text_request(request).await?;
|
||||
@@ -255,6 +266,12 @@ impl ChatLlm {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the server-side floor for tool-call structural tags.
|
||||
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
|
||||
self.processor.tool_strict_level = tool_strict_level;
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the effective model dtype used for multimodal tensor encoding.
|
||||
pub fn with_model_dtype(mut self, model_dtype: ModelDtype) -> Self {
|
||||
self.processor.model_dtype = Some(model_dtype);
|
||||
|
||||
@@ -18,10 +18,10 @@ use self::unified::unified_event_stream;
|
||||
use super::structured::structured_chat_event_stream;
|
||||
use crate::error::Result;
|
||||
use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream};
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
|
||||
use crate::parser::tool::{ToolParser, ToolParserFactory};
|
||||
use crate::parser::unified::UnifiedParserFactory;
|
||||
use crate::parser::{ParserSelection, ToolStrictLevel};
|
||||
use crate::request::{ChatRequest, ChatTool};
|
||||
use crate::{Error, Result as ChatResult};
|
||||
|
||||
@@ -49,6 +49,7 @@ impl DefaultChatOutputProcessor {
|
||||
tokenizer: DynTokenizer,
|
||||
tool_call_parser: &ParserSelection,
|
||||
reasoning_parser: &ParserSelection,
|
||||
tool_strict_level: ToolStrictLevel,
|
||||
) -> ChatResult<Self> {
|
||||
let parser = if let Some(parser) = Self::resolve_optional_unified_parser(
|
||||
request.tools(),
|
||||
@@ -73,7 +74,11 @@ impl DefaultChatOutputProcessor {
|
||||
Box::new(CombinedParser::new(reasoning_parser, tool_parser)) as Box<dyn UnifiedParser>
|
||||
};
|
||||
|
||||
apply_structural_tag_constraint(request, parser.structural_tag_builder())?;
|
||||
apply_structural_tag_constraint(
|
||||
request,
|
||||
parser.structural_tag_builder(),
|
||||
tool_strict_level,
|
||||
)?;
|
||||
|
||||
if parser.preserve_special_tokens() {
|
||||
request.decode_options.skip_special_tokens = false;
|
||||
@@ -191,7 +196,7 @@ mod tests {
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::DefaultChatOutputProcessor;
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::parser::{ParserSelection, ToolStrictLevel};
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
fn tokenizer() -> Arc<TestTokenizer> {
|
||||
@@ -213,6 +218,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&selection,
|
||||
&selection,
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -227,6 +233,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&ParserSelection::Auto,
|
||||
&ParserSelection::Auto,
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -244,6 +251,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
tool,
|
||||
reasoning,
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
@@ -258,6 +266,7 @@ mod tests {
|
||||
tokenizer(),
|
||||
&ParserSelection::Auto,
|
||||
&ParserSelection::Explicit("gemma4".to_string()),
|
||||
ToolStrictLevel::Auto,
|
||||
) {
|
||||
Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"),
|
||||
Err(error) => error,
|
||||
|
||||
@@ -14,19 +14,21 @@ use xgrammar_structural_tag::{
|
||||
build_structural_tag,
|
||||
};
|
||||
|
||||
use crate::parser::ToolStrictLevel;
|
||||
use crate::request::{ChatRequest, ChatToolChoice};
|
||||
use crate::{Error, Result as ChatResult};
|
||||
|
||||
/// Apply structural tag constraints to the request based on the tool parser's structural tag
|
||||
/// support and the request's tool choice.
|
||||
/// support, the request's tool choice, and the server-side strictness floor.
|
||||
pub(super) fn apply_structural_tag_constraint(
|
||||
request: &mut ChatRequest,
|
||||
builder: Option<&dyn StructuralTagBuilder>,
|
||||
strict_level: ToolStrictLevel,
|
||||
) -> ChatResult<()> {
|
||||
let Some(builder) = builder else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(tool_choice) = structural_tag_tool_choice(request) else {
|
||||
let Some(tool_choice) = structural_tag_tool_choice(request, strict_level) else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
@@ -34,11 +36,14 @@ pub(super) fn apply_structural_tag_constraint(
|
||||
.tools()
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
// A tool without an explicit `strict` is non-strict: its call envelope is
|
||||
// constrained, but its arguments stay free unless the server level is `parameter`.
|
||||
let strict = strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true);
|
||||
ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
|
||||
name: tool.name.clone(),
|
||||
description: tool.description.clone(),
|
||||
parameters: Some(tool.parameters.clone()),
|
||||
strict: tool.strict,
|
||||
strict: Some(strict),
|
||||
}))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
@@ -66,17 +71,21 @@ pub(super) fn apply_structural_tag_constraint(
|
||||
/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request.
|
||||
///
|
||||
/// Returns `None` if no structural tag constraints should be applied.
|
||||
fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagToolChoice> {
|
||||
fn structural_tag_tool_choice(
|
||||
request: &ChatRequest,
|
||||
strict_level: ToolStrictLevel,
|
||||
) -> Option<StructuralTagToolChoice> {
|
||||
if request.tools().is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
match request.tool_choice() {
|
||||
// For `Auto`, only apply the structural tag if there's at least one strict tool.
|
||||
ChatToolChoice::Auto if request.tools().iter().any(|tool| tool.strict == Some(true)) => {
|
||||
Some(StructuralTagToolChoice::auto())
|
||||
}
|
||||
ChatToolChoice::Auto | ChatToolChoice::None => None,
|
||||
ChatToolChoice::None => None,
|
||||
// For `Auto`, apply the structural tag only when the server raises the floor or at
|
||||
// least one tool opts in with `strict: true`.
|
||||
ChatToolChoice::Auto => (strict_level >= ToolStrictLevel::Function
|
||||
|| request.tools().iter().any(|tool| tool.strict == Some(true)))
|
||||
.then(StructuralTagToolChoice::auto),
|
||||
|
||||
ChatToolChoice::Required => Some(StructuralTagToolChoice::required()),
|
||||
ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())),
|
||||
@@ -148,8 +157,12 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
assert_eq!(tag["type"], "structural_tag");
|
||||
@@ -172,8 +185,12 @@ mod tests {
|
||||
};
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
assert!(tag.to_string().contains("lookup"));
|
||||
@@ -199,8 +216,12 @@ mod tests {
|
||||
};
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request).to_string();
|
||||
assert!(tag.contains("search"));
|
||||
@@ -212,8 +233,12 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag decision should succeed");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
assert!(request.sampling_params.structured_outputs.is_none());
|
||||
}
|
||||
@@ -227,8 +252,12 @@ mod tests {
|
||||
});
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
assert!(params.constraint.is_structural_tag());
|
||||
@@ -242,8 +271,12 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
assert_eq!(tag["type"], "structural_tag");
|
||||
@@ -259,8 +292,12 @@ mod tests {
|
||||
});
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
assert!(params.constraint.is_structural_tag());
|
||||
@@ -279,8 +316,12 @@ mod tests {
|
||||
);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag should build");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request).to_string();
|
||||
assert!(tag.contains("lookup"));
|
||||
@@ -292,8 +333,12 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag decision should succeed");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
assert!(request.sampling_params.structured_outputs.is_none());
|
||||
}
|
||||
@@ -307,10 +352,127 @@ mod tests {
|
||||
});
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
|
||||
.expect("structural tag decision should succeed");
|
||||
apply_structural_tag_constraint(
|
||||
&mut request,
|
||||
parser.structural_tag_builder(),
|
||||
ToolStrictLevel::Auto,
|
||||
)
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
assert!(params.constraint.is_json_object());
|
||||
}
|
||||
|
||||
fn build(tool_choice: ChatToolChoice, tools: Vec<Tool>, level: ToolStrictLevel) -> ChatRequest {
|
||||
let mut request = request(tool_choice, tools);
|
||||
let parser = qwen3_coder_parser(request.tools());
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_builder(), level)
|
||||
.expect("structural tag decision should succeed");
|
||||
request
|
||||
}
|
||||
|
||||
fn tag_string(request: &ChatRequest) -> String {
|
||||
structural_tag_value(request).to_string()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn function_level_constrains_auto_without_strict_tools() {
|
||||
let request = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![chat_tool("search", None)],
|
||||
ToolStrictLevel::Function,
|
||||
);
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
assert_eq!(tag["type"], "structural_tag");
|
||||
assert!(tag.to_string().contains("search"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absent_strict_is_non_strict() {
|
||||
let unset = build(
|
||||
ChatToolChoice::Required,
|
||||
vec![chat_tool("search", None)],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let non_strict = build(
|
||||
ChatToolChoice::Required,
|
||||
vec![chat_tool("search", Some(false))],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let strict = build(
|
||||
ChatToolChoice::Required,
|
||||
vec![chat_tool("search", Some(true))],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
|
||||
assert_eq!(tag_string(&unset), tag_string(&non_strict));
|
||||
assert_ne!(tag_string(&unset), tag_string(&strict));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strict_tool_does_not_pin_its_neighbour() {
|
||||
let mixed = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![chat_tool("weather", Some(true)), chat_tool("search", None)],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let explicit = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![
|
||||
chat_tool("weather", Some(true)),
|
||||
chat_tool("search", Some(false)),
|
||||
],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
|
||||
assert_eq!(tag_string(&mixed), tag_string(&explicit));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parameter_level_pins_argument_schemas() {
|
||||
let pinned = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![chat_tool("search", None)],
|
||||
ToolStrictLevel::Parameter,
|
||||
);
|
||||
let strict = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![chat_tool("search", Some(true))],
|
||||
ToolStrictLevel::Auto,
|
||||
);
|
||||
let envelope_only = build(
|
||||
ChatToolChoice::Auto,
|
||||
vec![chat_tool("search", None)],
|
||||
ToolStrictLevel::Function,
|
||||
);
|
||||
|
||||
assert_eq!(tag_string(&pinned), tag_string(&strict));
|
||||
assert_ne!(tag_string(&pinned), tag_string(&envelope_only));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_strict_level_parses_case_insensitively() {
|
||||
assert_eq!("AUTO".parse::<ToolStrictLevel>(), Ok(ToolStrictLevel::Auto));
|
||||
assert_eq!(
|
||||
"PARAMETER".parse::<ToolStrictLevel>(),
|
||||
Ok(ToolStrictLevel::Parameter)
|
||||
);
|
||||
assert_eq!(
|
||||
"function".parse::<ToolStrictLevel>(),
|
||||
Ok(ToolStrictLevel::Function)
|
||||
);
|
||||
assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto);
|
||||
assert_eq!(
|
||||
serde_json::to_value(ToolStrictLevel::Auto).unwrap(),
|
||||
json!("auto")
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ToolStrictLevel>(json!("auto")).unwrap(),
|
||||
ToolStrictLevel::Auto
|
||||
);
|
||||
assert!("off".parse::<ToolStrictLevel>().is_err());
|
||||
assert!("strict".parse::<ToolStrictLevel>().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,66 @@ impl ParserSelection {
|
||||
}
|
||||
}
|
||||
|
||||
/// Server-side floor for tool-call structural tags (`--tool-strict-level`).
|
||||
#[derive(
|
||||
Debug,
|
||||
Clone,
|
||||
Copy,
|
||||
PartialEq,
|
||||
Eq,
|
||||
PartialOrd,
|
||||
Ord,
|
||||
Default,
|
||||
DeserializeFromStr,
|
||||
SerializeDisplay,
|
||||
)]
|
||||
pub enum ToolStrictLevel {
|
||||
/// Derive tool constraints from the request's tool choice and per-tool strictness.
|
||||
#[default]
|
||||
Auto,
|
||||
/// Constrain the tool-call envelope for every request with tools.
|
||||
Function,
|
||||
/// Additionally pin argument schemas, as if every tool were `strict: true`.
|
||||
Parameter,
|
||||
}
|
||||
|
||||
impl ToolStrictLevel {
|
||||
pub const AUTO_LITERAL: &str = "auto";
|
||||
pub const FUNCTION_LITERAL: &str = "function";
|
||||
pub const PARAMETER_LITERAL: &str = "parameter";
|
||||
}
|
||||
|
||||
impl FromStr for ToolStrictLevel {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
|
||||
Ok(Self::Auto)
|
||||
} else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) {
|
||||
Ok(Self::Function)
|
||||
} else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) {
|
||||
Ok(Self::Parameter)
|
||||
} else {
|
||||
Err(format!(
|
||||
"unknown tool strict level {value:?}; expected one of {}, {}, {}",
|
||||
Self::AUTO_LITERAL,
|
||||
Self::FUNCTION_LITERAL,
|
||||
Self::PARAMETER_LITERAL
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ToolStrictLevel {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::Auto => Self::AUTO_LITERAL,
|
||||
Self::Function => Self::FUNCTION_LITERAL,
|
||||
Self::Parameter => Self::PARAMETER_LITERAL,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate explicit parser override names without starting request processing.
|
||||
pub fn validate_parser_overrides(
|
||||
tool_call_parser: &ParserSelection,
|
||||
|
||||
@@ -227,6 +227,7 @@ impl ChatBackend for FakeChatBackend {
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -717,6 +717,7 @@ async fn parse_completion(
|
||||
let processor = backends.chat_backend.new_chat_output_processor(
|
||||
&mut request,
|
||||
NewChatOutputProcessorOptions {
|
||||
tool_strict_level: vllm_chat::ToolStrictLevel::Auto,
|
||||
tool_call_parser: &case.tool_call_parser,
|
||||
reasoning_parser: &case.reasoning_parser,
|
||||
},
|
||||
|
||||
@@ -24,6 +24,7 @@ use serde_with::{DefaultOnNull, OneOrMany, serde_as};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use uuid::Uuid;
|
||||
use vllm_chat::GenerationConfigMode;
|
||||
use vllm_chat::ToolStrictLevel;
|
||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_managed_engine::ManagedEngineConfig;
|
||||
@@ -134,6 +135,10 @@ pub struct RenderArgs {
|
||||
/// `none` to disable parsing.
|
||||
#[arg(long, default_value_t)]
|
||||
reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for structural-tag based tool calling: `auto`,
|
||||
/// `function`, or `parameter`.
|
||||
#[arg(long, default_value_t)]
|
||||
tool_strict_level: ToolStrictLevel,
|
||||
/// Select the native chat renderer implementation.
|
||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||
renderer: RendererSelection,
|
||||
@@ -171,6 +176,7 @@ impl RenderArgs {
|
||||
port: self.port,
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
tool_strict_level: self.tool_strict_level,
|
||||
renderer: self.renderer,
|
||||
chat_template: self.chat_template,
|
||||
default_chat_template_kwargs: self.default_chat_template_kwargs.unwrap_or_default(),
|
||||
@@ -241,6 +247,14 @@ pub struct SharedRuntimeArgs {
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default = "default_py_bootstrap_parser_selection")]
|
||||
pub reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for structural-tag based tool calling, applied on
|
||||
/// top of the per-tool `strict` field: `auto` follows the request's
|
||||
/// tool choice and per-tool strictness,
|
||||
/// `function` constrains the tool-call envelope for every request with
|
||||
/// tools, `parameter` additionally pins argument schemas.
|
||||
#[arg(long, default_value_t)]
|
||||
#[serde(default)]
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
/// Select the chat renderer implementation.
|
||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||
#[serde(default, rename = "tokenizer_mode")]
|
||||
@@ -517,6 +531,7 @@ impl SharedRuntimeArgs {
|
||||
listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
tool_strict_level: self.tool_strict_level,
|
||||
renderer: self.renderer,
|
||||
language_model_only: self.language_model_only,
|
||||
chat_template: self.chat_template,
|
||||
@@ -574,6 +589,7 @@ impl SharedRuntimeArgs {
|
||||
listener_mode,
|
||||
tool_call_parser: self.tool_call_parser,
|
||||
reasoning_parser: self.reasoning_parser,
|
||||
tool_strict_level: self.tool_strict_level,
|
||||
renderer: self.renderer,
|
||||
language_model_only: self.language_model_only,
|
||||
chat_template: self.chat_template,
|
||||
|
||||
@@ -184,6 +184,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
@@ -1023,6 +1024,7 @@ fn frontend_args_accept_json() {
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: None,
|
||||
reasoning_parser: None,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
@@ -1693,6 +1695,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
engine_ready_timeout_secs: 600,
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
@@ -1850,6 +1853,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
@@ -1942,6 +1946,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
},
|
||||
tool_call_parser: Auto,
|
||||
reasoning_parser: Auto,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
@@ -2055,6 +2060,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
},
|
||||
tool_call_parser: None,
|
||||
reasoning_parser: None,
|
||||
tool_strict_level: Auto,
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
|
||||
@@ -16,6 +16,7 @@ use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_chat::ToolStrictLevel;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_server::{
|
||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
||||
@@ -70,6 +71,7 @@ async fn main() -> Result<()> {
|
||||
},
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
renderer: RendererSelection::Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
|
||||
@@ -14,6 +14,7 @@ use serde_json::Value;
|
||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||
use vllm_chat::{
|
||||
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
|
||||
ToolStrictLevel,
|
||||
};
|
||||
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
@@ -229,6 +230,8 @@ pub struct Config {
|
||||
pub tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection.
|
||||
pub reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for tool-call structural tags.
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
/// Chat renderer selection.
|
||||
pub renderer: RendererSelection,
|
||||
/// Disable frontend-side multimodal preprocessing and render the model as
|
||||
|
||||
@@ -244,6 +244,7 @@ impl ChatBackend for FakeTextBackend {
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
@@ -292,6 +293,7 @@ impl ChatBackend for FakeMultimodalBackend {
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,7 +139,8 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
|
||||
let chat = ChatLlm::new(text, chat_backend)
|
||||
.with_tool_call_parser(config.tool_call_parser.clone())
|
||||
.with_reasoning_parser(config.reasoning_parser.clone());
|
||||
.with_reasoning_parser(config.reasoning_parser.clone())
|
||||
.with_tool_strict_level(config.tool_strict_level);
|
||||
|
||||
let state = Arc::new(
|
||||
AppState::new(served_model_names, chat)
|
||||
|
||||
@@ -10,7 +10,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::info;
|
||||
use vllm_chat::{
|
||||
ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions,
|
||||
ParserSelection, RendererSelection, load_model_backends,
|
||||
ParserSelection, RendererSelection, ToolStrictLevel, load_model_backends,
|
||||
};
|
||||
use vllm_text::TextRequestProcessor;
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
@@ -32,6 +32,7 @@ pub struct RenderConfig {
|
||||
pub port: u16,
|
||||
pub tool_call_parser: ParserSelection,
|
||||
pub reasoning_parser: ParserSelection,
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
pub renderer: RendererSelection,
|
||||
pub chat_template: Option<String>,
|
||||
pub default_chat_template_kwargs: HashMap<String, Value>,
|
||||
@@ -88,10 +89,12 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
|
||||
let max_model_len = config.max_model_len.unwrap_or(u32::MAX);
|
||||
let text = TextRequestProcessor::new(loaded.text_backend, max_model_len)
|
||||
.with_max_logprobs(config.max_logprobs);
|
||||
let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections(
|
||||
config.tool_call_parser.clone(),
|
||||
config.reasoning_parser.clone(),
|
||||
);
|
||||
let chat = ChatRequestProcessor::render_only(loaded.chat_backend)
|
||||
.with_parser_selections(
|
||||
config.tool_call_parser.clone(),
|
||||
config.reasoning_parser.clone(),
|
||||
)
|
||||
.with_tool_strict_level(config.tool_strict_level);
|
||||
Ok(Arc::new(RenderState {
|
||||
model: config.model.clone(),
|
||||
served_model_names,
|
||||
@@ -154,6 +157,7 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
let error = serve_render(
|
||||
RenderConfig {
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
model: "test-model".to_string(),
|
||||
revision: None,
|
||||
hf_overrides: Default::default(),
|
||||
|
||||
@@ -168,6 +168,7 @@ impl ChatBackend for FakeChatBackend {
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -517,6 +517,7 @@ impl ChatBackend for FakeChatBackend {
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user