[Frontend] Let an unset --tool-strict-level fall back to the tool parser's default (#61328)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
91ed2859c7
commit
8fa5ae844a
@@ -146,7 +146,7 @@ Most OpenAI-compatible clients and agent frameworks never set `strict` on their
|
|||||||
|
|
||||||
| Value | Behavior |
|
| Value | Behavior |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `auto` (default) | Follow the request's tool choice and per-tool strictness. Required/named choices activate structural tags; `tool_choice="auto"` activates them when at least one tool sets `strict: true`. |
|
| `auto` | Follow the request's tool choice and per-tool strictness. Required/named choices activate structural tags; `tool_choice="auto"` activates them when at least one tool sets `strict: true`. |
|
||||||
| `function` | Constrain the tool-call envelope (markup and the function name) for every request with tools, leaving argument contents free unless the client marked the tool `strict: true`. |
|
| `function` | Constrain the tool-call envelope (markup and the function name) for every request with tools, leaving argument contents free unless the client marked the tool `strict: true`. |
|
||||||
| `parameter` | Additionally pin argument schemas for every tool, as if every tool had `strict: true`. |
|
| `parameter` | Additionally pin argument schemas for every tool, as if every tool had `strict: true`. |
|
||||||
|
|
||||||
@@ -154,6 +154,8 @@ Most OpenAI-compatible clients and agent frameworks never set `strict` on their
|
|||||||
vllm serve ... --tool-strict-level function
|
vllm serve ... --tool-strict-level function
|
||||||
```
|
```
|
||||||
|
|
||||||
|
When the option is unset, the tool parser's default level applies: `function` for `glm47`, `auto` for the other parsers. Passing `--tool-strict-level auto` overrides a parser default.
|
||||||
|
|
||||||
The floor never relaxes a constraint the request would already receive: tools the client marked `strict: true` keep their schemas at every level. With `tool_choice="auto"`, the grammar does not force a tool call; a plain text response stays valid. `VLLM_ENFORCE_STRICT_TOOL_CALLING=false` disables structural tags entirely and takes precedence over this option.
|
The floor never relaxes a constraint the request would already receive: tools the client marked `strict: true` keep their schemas at every level. With `tool_choice="auto"`, the grammar does not force a tool call; a plain text response stays valid. `VLLM_ENFORCE_STRICT_TOOL_CALLING=false` disables structural tags entirely and takes precedence over this option.
|
||||||
|
|
||||||
## Automatic Function Calling
|
## Automatic Function Calling
|
||||||
|
|||||||
@@ -380,7 +380,7 @@ mod tests {
|
|||||||
let error = match backend.new_chat_output_processor(
|
let error = match backend.new_chat_output_processor(
|
||||||
&mut request,
|
&mut request,
|
||||||
NewChatOutputProcessorOptions {
|
NewChatOutputProcessorOptions {
|
||||||
tool_strict_level: crate::ToolStrictLevel::Auto,
|
tool_strict_level: None,
|
||||||
tool_call_parser: &ParserSelection::Explicit("json".to_string()),
|
tool_call_parser: &ParserSelection::Explicit("json".to_string()),
|
||||||
reasoning_parser: &ParserSelection::Auto,
|
reasoning_parser: &ParserSelection::Auto,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -21,7 +21,9 @@ pub mod hf;
|
|||||||
pub struct NewChatOutputProcessorOptions<'a> {
|
pub struct NewChatOutputProcessorOptions<'a> {
|
||||||
pub tool_call_parser: &'a ParserSelection,
|
pub tool_call_parser: &'a ParserSelection,
|
||||||
pub reasoning_parser: &'a ParserSelection,
|
pub reasoning_parser: &'a ParserSelection,
|
||||||
pub tool_strict_level: ToolStrictLevel,
|
/// Server-side floor for tool-call structural tags, or `None` for the
|
||||||
|
/// tool parser's default level.
|
||||||
|
pub tool_strict_level: Option<ToolStrictLevel>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Minimal prompt-processing backend needed by `vllm-chat`.
|
/// Minimal prompt-processing backend needed by `vllm-chat`.
|
||||||
|
|||||||
@@ -79,8 +79,9 @@ pub struct ChatRequestProcessor {
|
|||||||
tool_call_parser: ParserSelection,
|
tool_call_parser: ParserSelection,
|
||||||
/// Reasoning parser selection used when preparing generation requests.
|
/// Reasoning parser selection used when preparing generation requests.
|
||||||
reasoning_parser: ParserSelection,
|
reasoning_parser: ParserSelection,
|
||||||
/// Server-side floor for tool-call structural tags.
|
/// Server-side floor for tool-call structural tags, or `None` for the
|
||||||
tool_strict_level: ToolStrictLevel,
|
/// tool parser's default level.
|
||||||
|
tool_strict_level: Option<ToolStrictLevel>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ChatRequestProcessor {
|
impl ChatRequestProcessor {
|
||||||
@@ -93,7 +94,7 @@ impl ChatRequestProcessor {
|
|||||||
max_model_len: Some(max_model_len as usize),
|
max_model_len: Some(max_model_len as usize),
|
||||||
tool_call_parser: ParserSelection::Auto,
|
tool_call_parser: ParserSelection::Auto,
|
||||||
reasoning_parser: ParserSelection::Auto,
|
reasoning_parser: ParserSelection::Auto,
|
||||||
tool_strict_level: ToolStrictLevel::Auto,
|
tool_strict_level: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -105,7 +106,7 @@ impl ChatRequestProcessor {
|
|||||||
max_model_len: None,
|
max_model_len: None,
|
||||||
tool_call_parser: ParserSelection::Auto,
|
tool_call_parser: ParserSelection::Auto,
|
||||||
reasoning_parser: ParserSelection::Auto,
|
reasoning_parser: ParserSelection::Auto,
|
||||||
tool_strict_level: ToolStrictLevel::Auto,
|
tool_strict_level: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,8 +121,9 @@ impl ChatRequestProcessor {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Configure the server-side floor for tool-call structural tags.
|
/// Configure the server-side floor for tool-call structural tags, or
|
||||||
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
|
/// `None` for the tool parser's default level.
|
||||||
|
pub fn with_tool_strict_level(mut self, tool_strict_level: Option<ToolStrictLevel>) -> Self {
|
||||||
self.tool_strict_level = tool_strict_level;
|
self.tool_strict_level = tool_strict_level;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
@@ -295,8 +297,9 @@ impl ChatLlm {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Set the server-side floor for tool-call structural tags.
|
/// Set the server-side floor for tool-call structural tags, or `None` for
|
||||||
pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self {
|
/// the tool parser's default level.
|
||||||
|
pub fn with_tool_strict_level(mut self, tool_strict_level: Option<ToolStrictLevel>) -> Self {
|
||||||
self.processor.tool_strict_level = tool_strict_level;
|
self.processor.tool_strict_level = tool_strict_level;
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ impl DefaultChatOutputProcessor {
|
|||||||
tokenizer: DynTokenizer,
|
tokenizer: DynTokenizer,
|
||||||
tool_call_parser: &ParserSelection,
|
tool_call_parser: &ParserSelection,
|
||||||
reasoning_parser: &ParserSelection,
|
reasoning_parser: &ParserSelection,
|
||||||
tool_strict_level: ToolStrictLevel,
|
tool_strict_level: Option<ToolStrictLevel>,
|
||||||
) -> ChatResult<Self> {
|
) -> ChatResult<Self> {
|
||||||
Self::with_response_template(
|
Self::with_response_template(
|
||||||
request,
|
request,
|
||||||
@@ -92,7 +92,7 @@ impl DefaultChatOutputProcessor {
|
|||||||
response_template: &std::result::Result<Arc<ResponseTemplate>, HfTemplateError>,
|
response_template: &std::result::Result<Arc<ResponseTemplate>, HfTemplateError>,
|
||||||
tool_call_parser: &ParserSelection,
|
tool_call_parser: &ParserSelection,
|
||||||
reasoning_parser: &ParserSelection,
|
reasoning_parser: &ParserSelection,
|
||||||
tool_strict_level: ToolStrictLevel,
|
tool_strict_level: Option<ToolStrictLevel>,
|
||||||
) -> ChatResult<Self> {
|
) -> ChatResult<Self> {
|
||||||
let tool_name = tool_call_parser.resolve_tool_name(model_id);
|
let tool_name = tool_call_parser.resolve_tool_name(model_id);
|
||||||
let reasoning_name = reasoning_parser.resolve_reasoning_name(model_id);
|
let reasoning_name = reasoning_parser.resolve_reasoning_name(model_id);
|
||||||
@@ -135,6 +135,8 @@ impl DefaultChatOutputProcessor {
|
|||||||
if parser.preserve_special_tokens() {
|
if parser.preserve_special_tokens() {
|
||||||
request.decode_options.skip_special_tokens = false;
|
request.decode_options.skip_special_tokens = false;
|
||||||
}
|
}
|
||||||
|
let tool_strict_level =
|
||||||
|
tool_strict_level.unwrap_or_else(|| parser.default_tool_strict_level());
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
parser,
|
parser,
|
||||||
@@ -292,6 +294,7 @@ mod tests {
|
|||||||
|
|
||||||
use thiserror_ext::AsReport as _;
|
use thiserror_ext::AsReport as _;
|
||||||
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
||||||
|
use vllm_parser::output_grammar::BuiltOutputGrammar;
|
||||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||||
use xgrammar_structural_tag::format::Format;
|
use xgrammar_structural_tag::format::Format;
|
||||||
|
|
||||||
@@ -309,36 +312,45 @@ mod tests {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The output grammar for one `search` tool with `strict` under `parser`.
|
||||||
|
fn search_grammar(
|
||||||
|
parser: &str,
|
||||||
|
level: Option<ToolStrictLevel>,
|
||||||
|
strict: Option<bool>,
|
||||||
|
choice: ChatToolChoice,
|
||||||
|
) -> Option<BuiltOutputGrammar> {
|
||||||
|
let tools = vec![ChatTool {
|
||||||
|
name: "search".to_string(),
|
||||||
|
description: None,
|
||||||
|
parameters: serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": { "query": { "type": "string" } },
|
||||||
|
"required": ["query"]
|
||||||
|
}),
|
||||||
|
strict,
|
||||||
|
defer_loading: None,
|
||||||
|
}];
|
||||||
|
let mut request = ChatRequest {
|
||||||
|
tool_context: ResolvedToolContext::new(&[], tools, Some(choice), true).unwrap(),
|
||||||
|
..ChatRequest::for_test()
|
||||||
|
};
|
||||||
|
let mut processor = DefaultChatOutputProcessor::new(
|
||||||
|
&mut request,
|
||||||
|
"other-model",
|
||||||
|
tokenizer(),
|
||||||
|
&ParserSelection::Explicit(parser.to_string()),
|
||||||
|
&ParserSelection::None,
|
||||||
|
level,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
processor.initialize(&[]).unwrap();
|
||||||
|
processor.build_output_grammar().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn output_grammar_preserves_tool_strict_level() {
|
fn output_grammar_preserves_tool_strict_level() {
|
||||||
let build = |level, strict, choice| {
|
let build =
|
||||||
let tools = vec![ChatTool {
|
|level, strict, choice| search_grammar("qwen3_coder", Some(level), strict, choice);
|
||||||
name: "search".to_string(),
|
|
||||||
description: None,
|
|
||||||
parameters: serde_json::json!({
|
|
||||||
"type": "object",
|
|
||||||
"properties": { "query": { "type": "string" } },
|
|
||||||
"required": ["query"]
|
|
||||||
}),
|
|
||||||
strict,
|
|
||||||
defer_loading: None,
|
|
||||||
}];
|
|
||||||
let mut request = ChatRequest {
|
|
||||||
tool_context: ResolvedToolContext::new(&[], tools, Some(choice), true).unwrap(),
|
|
||||||
..ChatRequest::for_test()
|
|
||||||
};
|
|
||||||
let mut processor = DefaultChatOutputProcessor::new(
|
|
||||||
&mut request,
|
|
||||||
"other-model",
|
|
||||||
tokenizer(),
|
|
||||||
&ParserSelection::Explicit("qwen3_coder".to_string()),
|
|
||||||
&ParserSelection::None,
|
|
||||||
level,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
processor.initialize(&[]).unwrap();
|
|
||||||
processor.build_output_grammar().unwrap()
|
|
||||||
};
|
|
||||||
|
|
||||||
assert!(build(ToolStrictLevel::Auto, None, ChatToolChoice::Auto).is_none());
|
assert!(build(ToolStrictLevel::Auto, None, ChatToolChoice::Auto).is_none());
|
||||||
let envelope = build(ToolStrictLevel::Function, None, ChatToolChoice::Auto).unwrap();
|
let envelope = build(ToolStrictLevel::Function, None, ChatToolChoice::Auto).unwrap();
|
||||||
@@ -349,6 +361,19 @@ mod tests {
|
|||||||
assert!(build(ToolStrictLevel::Parameter, None, ChatToolChoice::None).is_none());
|
assert!(build(ToolStrictLevel::Parameter, None, ChatToolChoice::None).is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unset_tool_strict_level_falls_back_to_the_parser_default() {
|
||||||
|
let build = |parser, level| search_grammar(parser, level, None, ChatToolChoice::Auto);
|
||||||
|
|
||||||
|
// GLM-4.7 constrains auto tool calls unless the level is set explicitly.
|
||||||
|
let function = build("glm47", Some(ToolStrictLevel::Function));
|
||||||
|
assert_eq!(build("glm47", None), function);
|
||||||
|
assert!(function.is_some());
|
||||||
|
assert!(build("glm47", Some(ToolStrictLevel::Auto)).is_none());
|
||||||
|
// Other parsers default to `auto`.
|
||||||
|
assert!(build("qwen3_coder", None).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn answer_constraint_keeps_optional_tool_calls() {
|
fn answer_constraint_keeps_optional_tool_calls() {
|
||||||
let tools = vec![ChatTool {
|
let tools = vec![ChatTool {
|
||||||
@@ -377,7 +402,7 @@ mod tests {
|
|||||||
tokenizer(),
|
tokenizer(),
|
||||||
&ParserSelection::Explicit("qwen3_coder".to_string()),
|
&ParserSelection::Explicit("qwen3_coder".to_string()),
|
||||||
&ParserSelection::None,
|
&ParserSelection::None,
|
||||||
ToolStrictLevel::Auto,
|
None,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
processor.initialize(&[]).unwrap();
|
processor.initialize(&[]).unwrap();
|
||||||
@@ -420,7 +445,7 @@ mod tests {
|
|||||||
tokenizer(),
|
tokenizer(),
|
||||||
&ParserSelection::Explicit("qwen3_coder".to_string()),
|
&ParserSelection::Explicit("qwen3_coder".to_string()),
|
||||||
&ParserSelection::None,
|
&ParserSelection::None,
|
||||||
ToolStrictLevel::Auto,
|
None,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
processor.initialize(&[]).unwrap();
|
processor.initialize(&[]).unwrap();
|
||||||
@@ -444,7 +469,7 @@ mod tests {
|
|||||||
tokenizer(),
|
tokenizer(),
|
||||||
&selection,
|
&selection,
|
||||||
&selection,
|
&selection,
|
||||||
ToolStrictLevel::Auto,
|
None,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
@@ -459,7 +484,7 @@ mod tests {
|
|||||||
tokenizer(),
|
tokenizer(),
|
||||||
&ParserSelection::Auto,
|
&ParserSelection::Auto,
|
||||||
&ParserSelection::Auto,
|
&ParserSelection::Auto,
|
||||||
ToolStrictLevel::Auto,
|
None,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
@@ -477,7 +502,7 @@ mod tests {
|
|||||||
tokenizer(),
|
tokenizer(),
|
||||||
tool,
|
tool,
|
||||||
reasoning,
|
reasoning,
|
||||||
ToolStrictLevel::Auto,
|
None,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
@@ -492,7 +517,7 @@ mod tests {
|
|||||||
tokenizer(),
|
tokenizer(),
|
||||||
&ParserSelection::Auto,
|
&ParserSelection::Auto,
|
||||||
&ParserSelection::Explicit("gemma4".to_string()),
|
&ParserSelection::Explicit("gemma4".to_string()),
|
||||||
ToolStrictLevel::Auto,
|
None,
|
||||||
) {
|
) {
|
||||||
Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"),
|
Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"),
|
||||||
Err(error) => error,
|
Err(error) => error,
|
||||||
@@ -539,7 +564,7 @@ mod tests {
|
|||||||
template,
|
template,
|
||||||
tool,
|
tool,
|
||||||
reasoning,
|
reasoning,
|
||||||
ToolStrictLevel::Auto,
|
None,
|
||||||
)?;
|
)?;
|
||||||
Ok(!request.decode_options.skip_special_tokens)
|
Ok(!request.decode_options.skip_special_tokens)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -958,7 +958,7 @@ async fn parse_completion(
|
|||||||
NewChatOutputProcessorOptions {
|
NewChatOutputProcessorOptions {
|
||||||
// Pin every tool schema so that builder-backed parsers build an
|
// Pin every tool schema so that builder-backed parsers build an
|
||||||
// output grammar for replay. The strict level does not affect parsing.
|
// output grammar for replay. The strict level does not affect parsing.
|
||||||
tool_strict_level: ToolStrictLevel::Parameter,
|
tool_strict_level: Some(ToolStrictLevel::Parameter),
|
||||||
tool_call_parser: &case.tool_call_parser,
|
tool_call_parser: &case.tool_call_parser,
|
||||||
reasoning_parser: &case.reasoning_parser,
|
reasoning_parser: &case.reasoning_parser,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -137,9 +137,10 @@ pub struct RenderArgs {
|
|||||||
#[arg(long, default_value_t)]
|
#[arg(long, default_value_t)]
|
||||||
reasoning_parser: ParserSelection,
|
reasoning_parser: ParserSelection,
|
||||||
/// Server-side floor for structural-tag based tool calling: `auto`,
|
/// Server-side floor for structural-tag based tool calling: `auto`,
|
||||||
/// `function`, or `parameter`.
|
/// `function`, or `parameter`. When unset, the tool parser's default
|
||||||
#[arg(long, default_value_t)]
|
/// level applies, else `auto`.
|
||||||
tool_strict_level: ToolStrictLevel,
|
#[arg(long)]
|
||||||
|
tool_strict_level: Option<ToolStrictLevel>,
|
||||||
/// Select the native chat renderer implementation.
|
/// Select the native chat renderer implementation.
|
||||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||||
renderer: RendererSelection,
|
renderer: RendererSelection,
|
||||||
@@ -252,10 +253,11 @@ pub struct SharedRuntimeArgs {
|
|||||||
/// top of the per-tool `strict` field: `auto` follows the request's
|
/// top of the per-tool `strict` field: `auto` follows the request's
|
||||||
/// tool choice and per-tool strictness,
|
/// tool choice and per-tool strictness,
|
||||||
/// `function` constrains the tool-call envelope for every request with
|
/// `function` constrains the tool-call envelope for every request with
|
||||||
/// tools, `parameter` additionally pins argument schemas.
|
/// tools, `parameter` additionally pins argument schemas. When unset, the
|
||||||
#[arg(long, default_value_t)]
|
/// tool parser's default level applies, else `auto`.
|
||||||
|
#[arg(long)]
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub tool_strict_level: ToolStrictLevel,
|
pub tool_strict_level: Option<ToolStrictLevel>,
|
||||||
/// Select the chat renderer implementation.
|
/// Select the chat renderer implementation.
|
||||||
#[arg(long = "tokenizer-mode", default_value_t)]
|
#[arg(long = "tokenizer-mode", default_value_t)]
|
||||||
#[serde(default, rename = "tokenizer_mode")]
|
#[serde(default, rename = "tokenizer_mode")]
|
||||||
|
|||||||
@@ -185,7 +185,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
|||||||
engine_ready_timeout_secs: 600,
|
engine_ready_timeout_secs: 600,
|
||||||
tool_call_parser: Auto,
|
tool_call_parser: Auto,
|
||||||
reasoning_parser: Auto,
|
reasoning_parser: Auto,
|
||||||
tool_strict_level: Auto,
|
tool_strict_level: None,
|
||||||
renderer: Auto,
|
renderer: Auto,
|
||||||
language_model_only: false,
|
language_model_only: false,
|
||||||
max_logprobs: None,
|
max_logprobs: None,
|
||||||
@@ -1088,7 +1088,7 @@ fn frontend_args_accept_json() {
|
|||||||
engine_ready_timeout_secs: 600,
|
engine_ready_timeout_secs: 600,
|
||||||
tool_call_parser: None,
|
tool_call_parser: None,
|
||||||
reasoning_parser: None,
|
reasoning_parser: None,
|
||||||
tool_strict_level: Auto,
|
tool_strict_level: None,
|
||||||
renderer: Auto,
|
renderer: Auto,
|
||||||
language_model_only: false,
|
language_model_only: false,
|
||||||
max_logprobs: None,
|
max_logprobs: None,
|
||||||
@@ -1760,7 +1760,7 @@ fn serve_args_accept_handshake_aliases() {
|
|||||||
engine_ready_timeout_secs: 600,
|
engine_ready_timeout_secs: 600,
|
||||||
tool_call_parser: Auto,
|
tool_call_parser: Auto,
|
||||||
reasoning_parser: Auto,
|
reasoning_parser: Auto,
|
||||||
tool_strict_level: Auto,
|
tool_strict_level: None,
|
||||||
renderer: Auto,
|
renderer: Auto,
|
||||||
language_model_only: false,
|
language_model_only: false,
|
||||||
max_logprobs: None,
|
max_logprobs: None,
|
||||||
@@ -1919,7 +1919,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
|||||||
grpc_listener_mode: None,
|
grpc_listener_mode: None,
|
||||||
tool_call_parser: Auto,
|
tool_call_parser: Auto,
|
||||||
reasoning_parser: Auto,
|
reasoning_parser: Auto,
|
||||||
tool_strict_level: Auto,
|
tool_strict_level: None,
|
||||||
renderer: Auto,
|
renderer: Auto,
|
||||||
language_model_only: false,
|
language_model_only: false,
|
||||||
chat_template: None,
|
chat_template: None,
|
||||||
@@ -2014,7 +2014,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
|||||||
grpc_listener_mode: None,
|
grpc_listener_mode: None,
|
||||||
tool_call_parser: Auto,
|
tool_call_parser: Auto,
|
||||||
reasoning_parser: Auto,
|
reasoning_parser: Auto,
|
||||||
tool_strict_level: Auto,
|
tool_strict_level: None,
|
||||||
renderer: Auto,
|
renderer: Auto,
|
||||||
language_model_only: false,
|
language_model_only: false,
|
||||||
chat_template: None,
|
chat_template: None,
|
||||||
@@ -2153,7 +2153,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
|||||||
grpc_listener_mode: None,
|
grpc_listener_mode: None,
|
||||||
tool_call_parser: None,
|
tool_call_parser: None,
|
||||||
reasoning_parser: None,
|
reasoning_parser: None,
|
||||||
tool_strict_level: Auto,
|
tool_strict_level: None,
|
||||||
renderer: Auto,
|
renderer: Auto,
|
||||||
language_model_only: false,
|
language_model_only: false,
|
||||||
chat_template: None,
|
chat_template: None,
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
use super::{GlmXmlToolParser, Separator};
|
use super::{GlmXmlToolParser, Separator};
|
||||||
|
use crate::output_grammar::ToolStrictLevel;
|
||||||
use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
/// Tool parser for GLM-4.7 MoE XML-style tool calls.
|
/// Tool parser for GLM-4.7 MoE XML-style tool calls.
|
||||||
@@ -25,6 +26,11 @@ impl ToolParser for Glm47MoeToolParser {
|
|||||||
Ok(Box::new(Self::new(tools)))
|
Ok(Box::new(Self::new(tools)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Constrain auto tool calls by default, like the Python `glm47` parser.
|
||||||
|
fn default_tool_strict_level(&self) -> ToolStrictLevel {
|
||||||
|
ToolStrictLevel::Function
|
||||||
|
}
|
||||||
|
|
||||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||||
Some(xgrammar_structural_tag::Model::Glm47.builder())
|
Some(xgrammar_structural_tag::Model::Glm47.builder())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ pub use xgrammar_structural_tag::builders::StructuralTagBuilder;
|
|||||||
use xgrammar_structural_tag::format::Format;
|
use xgrammar_structural_tag::format::Format;
|
||||||
use xgrammar_structural_tag::tool::{AllowedToolsMode, ToolChoiceValue};
|
use xgrammar_structural_tag::tool::{AllowedToolsMode, ToolChoiceValue};
|
||||||
|
|
||||||
use crate::output_grammar::{self, OutputGrammarContext};
|
use crate::output_grammar::{self, OutputGrammarContext, ToolStrictLevel};
|
||||||
use crate::utils;
|
use crate::utils;
|
||||||
|
|
||||||
/// One function-style tool made available to the model.
|
/// One function-style tool made available to the model.
|
||||||
@@ -199,6 +199,12 @@ pub trait ToolParser: Send {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Return the tool strict level applied when the operator leaves
|
||||||
|
/// `--tool-strict-level` unset.
|
||||||
|
fn default_tool_strict_level(&self) -> ToolStrictLevel {
|
||||||
|
ToolStrictLevel::Auto
|
||||||
|
}
|
||||||
|
|
||||||
/// Build the language of everything the model may emit after reasoning
|
/// Build the language of everything the model may emit after reasoning
|
||||||
/// ends for the context's tool choice: visible text plus tool calls. `None`
|
/// ends for the context's tool choice: visible text plus tool calls. `None`
|
||||||
/// means no tool grammar applies.
|
/// means no tool grammar applies.
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||||
|
|
||||||
use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
|
use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
|
||||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
|
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext, ToolStrictLevel};
|
||||||
use crate::reasoning::ReasoningParser;
|
use crate::reasoning::ReasoningParser;
|
||||||
use crate::tool::{Tool, ToolParser, ToolParserOutput};
|
use crate::tool::{Tool, ToolParser, ToolParserOutput};
|
||||||
|
|
||||||
@@ -83,6 +83,12 @@ impl UnifiedParser for CombinedParser {
|
|||||||
|| self.tool.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
|
|| self.tool.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_tool_strict_level(&self) -> ToolStrictLevel {
|
||||||
|
self.tool.as_ref().map_or(ToolStrictLevel::Auto, |parser| {
|
||||||
|
parser.default_tool_strict_level()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn build_output_grammar(
|
fn build_output_grammar(
|
||||||
&self,
|
&self,
|
||||||
ctx: &OutputGrammarContext<'_>,
|
ctx: &OutputGrammarContext<'_>,
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use thiserror::Error;
|
|||||||
use thiserror_ext::Macro;
|
use thiserror_ext::Macro;
|
||||||
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
use vllm_tokenizer::{DecodedText, DynTokenizer};
|
||||||
|
|
||||||
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
|
use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext, ToolStrictLevel};
|
||||||
use crate::reasoning::ReasoningError;
|
use crate::reasoning::ReasoningError;
|
||||||
use crate::tool::{Tool, ToolCallDelta, ToolParserError, ToolParserEvent, ToolParserOutput};
|
use crate::tool::{Tool, ToolCallDelta, ToolParserError, ToolParserEvent, ToolParserOutput};
|
||||||
use crate::utils::SpecialToken;
|
use crate::utils::SpecialToken;
|
||||||
@@ -242,6 +242,12 @@ pub trait UnifiedParser: Send {
|
|||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Return the tool strict level applied when the operator leaves
|
||||||
|
/// `--tool-strict-level` unset.
|
||||||
|
fn default_tool_strict_level(&self) -> ToolStrictLevel {
|
||||||
|
ToolStrictLevel::Auto
|
||||||
|
}
|
||||||
|
|
||||||
/// Build the request output grammar after prompt-based initialization.
|
/// Build the request output grammar after prompt-based initialization.
|
||||||
fn build_output_grammar(
|
fn build_output_grammar(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ use clap::Parser;
|
|||||||
use futures::StreamExt as _;
|
use futures::StreamExt as _;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use tracing_subscriber::EnvFilter;
|
use tracing_subscriber::EnvFilter;
|
||||||
use vllm_chat::ToolStrictLevel;
|
|
||||||
use vllm_engine_core_client::TransportMode;
|
use vllm_engine_core_client::TransportMode;
|
||||||
use vllm_server::{
|
use vllm_server::{
|
||||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
||||||
@@ -73,7 +72,7 @@ async fn main() -> Result<()> {
|
|||||||
grpc_listener_mode: None,
|
grpc_listener_mode: None,
|
||||||
tool_call_parser: ParserSelection::Auto,
|
tool_call_parser: ParserSelection::Auto,
|
||||||
reasoning_parser: ParserSelection::Auto,
|
reasoning_parser: ParserSelection::Auto,
|
||||||
tool_strict_level: ToolStrictLevel::Auto,
|
tool_strict_level: None,
|
||||||
renderer: RendererSelection::Auto,
|
renderer: RendererSelection::Auto,
|
||||||
language_model_only: false,
|
language_model_only: false,
|
||||||
chat_template: None,
|
chat_template: None,
|
||||||
|
|||||||
@@ -238,8 +238,9 @@ pub struct Config {
|
|||||||
pub tool_call_parser: ParserSelection,
|
pub tool_call_parser: ParserSelection,
|
||||||
/// Reasoning parser selection.
|
/// Reasoning parser selection.
|
||||||
pub reasoning_parser: ParserSelection,
|
pub reasoning_parser: ParserSelection,
|
||||||
/// Server-side floor for tool-call structural tags.
|
/// Server-side floor for tool-call structural tags, or `None` for the
|
||||||
pub tool_strict_level: ToolStrictLevel,
|
/// tool parser's default level.
|
||||||
|
pub tool_strict_level: Option<ToolStrictLevel>,
|
||||||
/// Chat renderer selection.
|
/// Chat renderer selection.
|
||||||
pub renderer: RendererSelection,
|
pub renderer: RendererSelection,
|
||||||
/// Disable frontend-side multimodal preprocessing and render the model as
|
/// Disable frontend-side multimodal preprocessing and render the model as
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ pub struct RenderConfig {
|
|||||||
pub port: u16,
|
pub port: u16,
|
||||||
pub tool_call_parser: ParserSelection,
|
pub tool_call_parser: ParserSelection,
|
||||||
pub reasoning_parser: ParserSelection,
|
pub reasoning_parser: ParserSelection,
|
||||||
pub tool_strict_level: ToolStrictLevel,
|
pub tool_strict_level: Option<ToolStrictLevel>,
|
||||||
pub renderer: RendererSelection,
|
pub renderer: RendererSelection,
|
||||||
pub chat_template: Option<String>,
|
pub chat_template: Option<String>,
|
||||||
pub default_chat_template_kwargs: HashMap<String, Value>,
|
pub default_chat_template_kwargs: HashMap<String, Value>,
|
||||||
@@ -157,7 +157,7 @@ mod tests {
|
|||||||
shutdown.cancel();
|
shutdown.cancel();
|
||||||
let error = serve_render(
|
let error = serve_render(
|
||||||
RenderConfig {
|
RenderConfig {
|
||||||
tool_strict_level: ToolStrictLevel::Auto,
|
tool_strict_level: None,
|
||||||
model: "test-model".to_string(),
|
model: "test-model".to_string(),
|
||||||
revision: None,
|
revision: None,
|
||||||
hf_overrides: Default::default(),
|
hf_overrides: Default::default(),
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
|||||||
from vllm.entrypoints.openai.responses.utils import construct_tool_dicts
|
from vllm.entrypoints.openai.responses.utils import construct_tool_dicts
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.parser.abstract_parser import DelegatingParser
|
from vllm.parser.abstract_parser import DelegatingParser
|
||||||
|
from vllm.parser.parser_manager import ParserManager
|
||||||
from vllm.parser.plamo3 import (
|
from vllm.parser.plamo3 import (
|
||||||
BEGIN_TOOL_ARGUMENTS,
|
BEGIN_TOOL_ARGUMENTS,
|
||||||
BEGIN_TOOL_NAME,
|
BEGIN_TOOL_NAME,
|
||||||
@@ -1991,6 +1992,34 @@ def test_glm47_operator_level_overrides_parser_default(
|
|||||||
assert content["json_schema"] == sample_tools[0].function.parameters
|
assert content["json_schema"] == sample_tools[0].function.parameters
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("tool_strict_level", "constrained"), [(None, True), ("auto", False)]
|
||||||
|
)
|
||||||
|
def test_glm47_default_level_applies_only_when_the_level_is_unset(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
sample_tools: list[ChatCompletionToolsParam],
|
||||||
|
tool_strict_level: str | None,
|
||||||
|
constrained: bool,
|
||||||
|
):
|
||||||
|
monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", True)
|
||||||
|
parser_cls = ParserManager.get_parser(
|
||||||
|
tool_parser_name="glm47",
|
||||||
|
enable_auto_tools=True,
|
||||||
|
tool_strict_level=tool_strict_level,
|
||||||
|
)
|
||||||
|
assert parser_cls is not None
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
messages=[],
|
||||||
|
model="m",
|
||||||
|
tools=[tool.model_dump(exclude_none=True) for tool in sample_tools],
|
||||||
|
tool_choice="auto",
|
||||||
|
)
|
||||||
|
|
||||||
|
out = parser_cls(MagicMock(), tools=sample_tools).adjust_request(request)
|
||||||
|
|
||||||
|
assert (out.structured_outputs is not None) is constrained
|
||||||
|
|
||||||
|
|
||||||
def test_glm47_required_with_non_strict_tools_attaches_shallow_tag(
|
def test_glm47_required_with_non_strict_tools_attaches_shallow_tag(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
sample_tools: list[ChatCompletionToolsParam],
|
sample_tools: list[ChatCompletionToolsParam],
|
||||||
|
|||||||
@@ -160,7 +160,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
|||||||
reasoning_parser: str = "",
|
reasoning_parser: str = "",
|
||||||
enable_auto_tools: bool = False,
|
enable_auto_tools: bool = False,
|
||||||
tool_parser: str | None = None,
|
tool_parser: str | None = None,
|
||||||
tool_strict_level: str = "auto",
|
tool_strict_level: str | None = None,
|
||||||
enable_prompt_tokens_details: bool = False,
|
enable_prompt_tokens_details: bool = False,
|
||||||
enable_force_include_usage: bool = False,
|
enable_force_include_usage: bool = False,
|
||||||
enable_log_outputs: bool = False,
|
enable_log_outputs: bool = False,
|
||||||
|
|||||||
@@ -251,7 +251,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
|||||||
reasoning_parser: str = "",
|
reasoning_parser: str = "",
|
||||||
enable_auto_tools: bool = False,
|
enable_auto_tools: bool = False,
|
||||||
tool_parser: str | None = None,
|
tool_parser: str | None = None,
|
||||||
tool_strict_level: str = "auto",
|
tool_strict_level: str | None = None,
|
||||||
enable_prompt_tokens_details: bool = False,
|
enable_prompt_tokens_details: bool = False,
|
||||||
enable_force_include_usage: bool = False,
|
enable_force_include_usage: bool = False,
|
||||||
enable_log_outputs: bool = False,
|
enable_log_outputs: bool = False,
|
||||||
|
|||||||
@@ -128,13 +128,14 @@ class BaseFrontendArgs:
|
|||||||
"""Special the tool parser plugin write to parse the model-generated tool
|
"""Special the tool parser plugin write to parse the model-generated tool
|
||||||
into OpenAI API format, the name register in this plugin can be used in
|
into OpenAI API format, the name register in this plugin can be used in
|
||||||
`--tool-call-parser`."""
|
`--tool-call-parser`."""
|
||||||
tool_strict_level: ToolStrictLevelName = "auto"
|
tool_strict_level: ToolStrictLevelName | None = None
|
||||||
"""Server-side floor for structural-tag based tool calling, applied on top
|
"""Server-side floor for structural-tag based tool calling, applied on top
|
||||||
of the per-tool `strict` field. `auto` follows the request's tool choice
|
of the per-tool `strict` field. `auto` follows the request's tool choice
|
||||||
and per-tool strictness; `function` constrains the
|
and per-tool strictness; `function` constrains the
|
||||||
tool-call envelope (markup and function name) for every request with
|
tool-call envelope (markup and function name) for every request with
|
||||||
tools; `parameter` additionally pins argument schemas, as if every tool
|
tools; `parameter` additionally pins argument schemas, as if every tool
|
||||||
were `strict: true`."""
|
were `strict: true`. When unset, the tool parser's default level applies,
|
||||||
|
which is `auto` for most parsers."""
|
||||||
tool_server: str | None = None
|
tool_server: str | None = None
|
||||||
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
|
"""Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
|
||||||
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
|
Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
|
||||||
|
|||||||
@@ -132,7 +132,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
|||||||
enable_auto_tools: bool = False,
|
enable_auto_tools: bool = False,
|
||||||
exclude_tools_when_tool_choice_none: bool = False,
|
exclude_tools_when_tool_choice_none: bool = False,
|
||||||
tool_parser: str | None = None,
|
tool_parser: str | None = None,
|
||||||
tool_strict_level: str = "auto",
|
tool_strict_level: str | None = None,
|
||||||
enable_prompt_tokens_details: bool = False,
|
enable_prompt_tokens_details: bool = False,
|
||||||
enable_force_include_usage: bool = False,
|
enable_force_include_usage: bool = False,
|
||||||
enable_log_outputs: bool = False,
|
enable_log_outputs: bool = False,
|
||||||
|
|||||||
@@ -113,7 +113,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
reasoning_parser: str = "",
|
reasoning_parser: str = "",
|
||||||
enable_auto_tools: bool = False,
|
enable_auto_tools: bool = False,
|
||||||
tool_parser: str | None = None,
|
tool_parser: str | None = None,
|
||||||
tool_strict_level: str = "auto",
|
tool_strict_level: str | None = None,
|
||||||
tool_server: ToolServer | None = None,
|
tool_server: ToolServer | None = None,
|
||||||
enable_prompt_tokens_details: bool = False,
|
enable_prompt_tokens_details: bool = False,
|
||||||
enable_force_include_usage: bool = False,
|
enable_force_include_usage: bool = False,
|
||||||
|
|||||||
@@ -118,8 +118,9 @@ class Parser:
|
|||||||
# Subclasses should override these if they use specific parser classes
|
# Subclasses should override these if they use specific parser classes
|
||||||
reasoning_parser_cls: type[ReasoningParser] | None = None
|
reasoning_parser_cls: type[ReasoningParser] | None = None
|
||||||
tool_parser_cls: type[ToolParser] | None = None
|
tool_parser_cls: type[ToolParser] | None = None
|
||||||
# Server-side floor for tool-call structural tags (--tool-strict-level).
|
# Server-side floor for tool-call structural tags (--tool-strict-level), or
|
||||||
tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO
|
# None for the tool parser's default level.
|
||||||
|
tool_strict_level: ToolStrictLevel | None = None
|
||||||
always_adjust_request: bool = False
|
always_adjust_request: bool = False
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -519,10 +520,7 @@ class DelegatingParser(Parser):
|
|||||||
is_auto = request.tool_choice == "auto"
|
is_auto = request.tool_choice == "auto"
|
||||||
single_call = request.parallel_tool_calls is False
|
single_call = request.parallel_tool_calls is False
|
||||||
strict_level = self.tool_strict_level
|
strict_level = self.tool_strict_level
|
||||||
if (
|
if strict_level is None:
|
||||||
strict_level == ToolStrictLevel.AUTO
|
|
||||||
and tool_parser.default_tool_strict_level is not None
|
|
||||||
):
|
|
||||||
strict_level = tool_parser.default_tool_strict_level
|
strict_level = tool_parser.default_tool_strict_level
|
||||||
if single_call and not (is_auto and structured_outputs is not None):
|
if single_call and not (is_auto and structured_outputs is not None):
|
||||||
# Limiting the call count needs the call envelope in the grammar.
|
# Limiting the call count needs the call envelope in the grammar.
|
||||||
|
|||||||
@@ -104,7 +104,7 @@ class ParserManager:
|
|||||||
enable_auto_tools: bool = False,
|
enable_auto_tools: bool = False,
|
||||||
model_name: str | None = None,
|
model_name: str | None = None,
|
||||||
is_harmony: bool = False,
|
is_harmony: bool = False,
|
||||||
tool_strict_level: str = "auto",
|
tool_strict_level: str | None = None,
|
||||||
tokenizer: TokenizerLike | None = None,
|
tokenizer: TokenizerLike | None = None,
|
||||||
) -> type[Parser] | None:
|
) -> type[Parser] | None:
|
||||||
"""Get a Parser that handles both reasoning and tool parsing.
|
"""Get a Parser that handles both reasoning and tool parsing.
|
||||||
@@ -119,7 +119,8 @@ class ParserManager:
|
|||||||
is_harmony: Whether the selected model uses the Harmony format.
|
is_harmony: Whether the selected model uses the Harmony format.
|
||||||
If True, HarmonyParser is always returned.
|
If True, HarmonyParser is always returned.
|
||||||
tool_strict_level: Server-side floor for tool-call structural
|
tool_strict_level: Server-side floor for tool-call structural
|
||||||
tags (``--tool-strict-level``).
|
tags (``--tool-strict-level``), or ``None`` for the tool
|
||||||
|
parser's default level.
|
||||||
tokenizer: Tokenizer the composed parser is validated against
|
tokenizer: Tokenizer the composed parser is validated against
|
||||||
at startup (see :meth:`validate_parser_tokenizer`). ``None``
|
at startup (see :meth:`validate_parser_tokenizer`). ``None``
|
||||||
skips the check (e.g. ``skip_tokenizer_init``).
|
skips the check (e.g. ``skip_tokenizer_init``).
|
||||||
@@ -155,7 +156,7 @@ class ParserManager:
|
|||||||
enable_auto_tools: bool,
|
enable_auto_tools: bool,
|
||||||
model_name: str | None,
|
model_name: str | None,
|
||||||
is_harmony: bool,
|
is_harmony: bool,
|
||||||
tool_strict_level: str,
|
tool_strict_level: str | None,
|
||||||
tokenizer: TokenizerLike | None,
|
tokenizer: TokenizerLike | None,
|
||||||
) -> type[Parser] | None:
|
) -> type[Parser] | None:
|
||||||
if not tool_parser_name and not reasoning_parser_name:
|
if not tool_parser_name and not reasoning_parser_name:
|
||||||
@@ -169,7 +170,11 @@ class ParserManager:
|
|||||||
if reasoning_parser_cls is None and tool_parser_cls is None:
|
if reasoning_parser_cls is None and tool_parser_cls is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
strict_level = ToolStrictLevel.from_name(tool_strict_level)
|
strict_level = (
|
||||||
|
None
|
||||||
|
if tool_strict_level is None
|
||||||
|
else ToolStrictLevel.from_name(tool_strict_level)
|
||||||
|
)
|
||||||
|
|
||||||
if is_harmony:
|
if is_harmony:
|
||||||
from vllm.parser.harmony import HarmonyParser
|
from vllm.parser.harmony import HarmonyParser
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ class OnlineDerenderer:
|
|||||||
exclude_tools_when_tool_choice_none: bool = False,
|
exclude_tools_when_tool_choice_none: bool = False,
|
||||||
tool_parser: str | None = None,
|
tool_parser: str | None = None,
|
||||||
reasoning_parser: str | None = None,
|
reasoning_parser: str | None = None,
|
||||||
tool_strict_level: str = "auto",
|
tool_strict_level: str | None = None,
|
||||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||||
log_error_stack: bool = False,
|
log_error_stack: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -165,7 +165,7 @@ class OnlineRenderer:
|
|||||||
exclude_tools_when_tool_choice_none: bool = False,
|
exclude_tools_when_tool_choice_none: bool = False,
|
||||||
tool_parser: str | None = None,
|
tool_parser: str | None = None,
|
||||||
reasoning_parser: str | None = None,
|
reasoning_parser: str | None = None,
|
||||||
tool_strict_level: str = "auto",
|
tool_strict_level: str | None = None,
|
||||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||||
log_error_stack: bool = False,
|
log_error_stack: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -60,9 +60,9 @@ class ToolParser:
|
|||||||
# their parsed tool-call syntax matches a builtin xgrammar format.
|
# their parsed tool-call syntax matches a builtin xgrammar format.
|
||||||
structural_tag_model: str | None = None
|
structural_tag_model: str | None = None
|
||||||
# Strict level applied when the operator leaves ``--tool-strict-level``
|
# Strict level applied when the operator leaves ``--tool-strict-level``
|
||||||
# at auto. Operator flags override it, and
|
# unset. Operator flags override it, and
|
||||||
# ``VLLM_ENFORCE_STRICT_TOOL_CALLING=0`` still disables structural tags.
|
# ``VLLM_ENFORCE_STRICT_TOOL_CALLING=0`` still disables structural tags.
|
||||||
default_tool_strict_level: ToolStrictLevel | None = None
|
default_tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO
|
||||||
engine_based_streaming: bool = False
|
engine_based_streaming: bool = False
|
||||||
|
|
||||||
def __init_subclass__(cls, **kwargs: Any) -> None:
|
def __init_subclass__(cls, **kwargs: Any) -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user