[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: lazymio <mio@lazym.io>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
lazymio
2026-09-18 04:32:39 +00:00
committed by GitHub
co-authored by Claude Opus 5 Bugen Zhao
parent d12c276853
commit dee37d8911
35 changed files with 742 additions and 61 deletions
@@ -16,6 +16,7 @@ use clap::Parser;
use futures::StreamExt as _;
use tokio_util::sync::CancellationToken;
use tracing_subscriber::EnvFilter;
use vllm_chat::ToolStrictLevel;
use vllm_engine_core_client::TransportMode;
use vllm_server::{
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
@@ -70,6 +71,7 @@ async fn main() -> Result<()> {
},
tool_call_parser: ParserSelection::Auto,
reasoning_parser: ParserSelection::Auto,
tool_strict_level: ToolStrictLevel::Auto,
renderer: RendererSelection::Auto,
language_model_only: false,
chat_template: None,
+3
View File
@@ -14,6 +14,7 @@ use serde_json::Value;
use vllm_chat::multimodal::MmLimitPerPrompt;
use vllm_chat::{
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
ToolStrictLevel,
};
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
use vllm_text::backend::hf::HfOverrides;
@@ -229,6 +230,8 @@ pub struct Config {
pub tool_call_parser: ParserSelection,
/// Reasoning parser selection.
pub reasoning_parser: ParserSelection,
/// Server-side floor for tool-call structural tags.
pub tool_strict_level: ToolStrictLevel,
/// Chat renderer selection.
pub renderer: RendererSelection,
/// Disable frontend-side multimodal preprocessing and render the model as
+2
View File
@@ -244,6 +244,7 @@ impl ChatBackend for FakeTextBackend {
self.tokenizer(),
options.tool_call_parser,
options.reasoning_parser,
options.tool_strict_level,
)?))
}
}
@@ -292,6 +293,7 @@ impl ChatBackend for FakeMultimodalBackend {
self.tokenizer(),
options.tool_call_parser,
options.reasoning_parser,
options.tool_strict_level,
)?))
}
}
+2 -1
View File
@@ -139,7 +139,8 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
let chat = ChatLlm::new(text, chat_backend)
.with_tool_call_parser(config.tool_call_parser.clone())
.with_reasoning_parser(config.reasoning_parser.clone());
.with_reasoning_parser(config.reasoning_parser.clone())
.with_tool_strict_level(config.tool_strict_level);
let state = Arc::new(
AppState::new(served_model_names, chat)
+9 -5
View File
@@ -10,7 +10,7 @@ use tokio_util::sync::CancellationToken;
use tracing::info;
use vllm_chat::{
ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions,
ParserSelection, RendererSelection, load_model_backends,
ParserSelection, RendererSelection, ToolStrictLevel, load_model_backends,
};
use vllm_text::TextRequestProcessor;
use vllm_text::backend::hf::HfOverrides;
@@ -32,6 +32,7 @@ pub struct RenderConfig {
pub port: u16,
pub tool_call_parser: ParserSelection,
pub reasoning_parser: ParserSelection,
pub tool_strict_level: ToolStrictLevel,
pub renderer: RendererSelection,
pub chat_template: Option<String>,
pub default_chat_template_kwargs: HashMap<String, Value>,
@@ -88,10 +89,12 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
let max_model_len = config.max_model_len.unwrap_or(u32::MAX);
let text = TextRequestProcessor::new(loaded.text_backend, max_model_len)
.with_max_logprobs(config.max_logprobs);
let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections(
config.tool_call_parser.clone(),
config.reasoning_parser.clone(),
);
let chat = ChatRequestProcessor::render_only(loaded.chat_backend)
.with_parser_selections(
config.tool_call_parser.clone(),
config.reasoning_parser.clone(),
)
.with_tool_strict_level(config.tool_strict_level);
Ok(Arc::new(RenderState {
model: config.model.clone(),
served_model_names,
@@ -154,6 +157,7 @@ mod tests {
shutdown.cancel();
let error = serve_render(
RenderConfig {
tool_strict_level: ToolStrictLevel::Auto,
model: "test-model".to_string(),
revision: None,
hf_overrides: Default::default(),
@@ -168,6 +168,7 @@ impl ChatBackend for FakeChatBackend {
self.tokenizer(),
options.tool_call_parser,
options.reasoning_parser,
options.tool_strict_level,
)?))
}
}
+1
View File
@@ -517,6 +517,7 @@ impl ChatBackend for FakeChatBackend {
self.tokenizer(),
options.tool_call_parser,
options.reasoning_parser,
options.tool_strict_level,
)?))
}
}