[Frontend] Add --tool-strict-level for server-side control for structural tag activation (#56268)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: lazymio <mio@lazym.io> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5
Bugen Zhao
parent
d12c276853
commit
dee37d8911
@@ -16,6 +16,7 @@ use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_chat::ToolStrictLevel;
|
||||
use vllm_engine_core_client::TransportMode;
|
||||
use vllm_server::{
|
||||
ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
|
||||
@@ -70,6 +71,7 @@ async fn main() -> Result<()> {
|
||||
},
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
renderer: RendererSelection::Auto,
|
||||
language_model_only: false,
|
||||
chat_template: None,
|
||||
|
||||
@@ -14,6 +14,7 @@ use serde_json::Value;
|
||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||
use vllm_chat::{
|
||||
ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection,
|
||||
ToolStrictLevel,
|
||||
};
|
||||
use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
@@ -229,6 +230,8 @@ pub struct Config {
|
||||
pub tool_call_parser: ParserSelection,
|
||||
/// Reasoning parser selection.
|
||||
pub reasoning_parser: ParserSelection,
|
||||
/// Server-side floor for tool-call structural tags.
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
/// Chat renderer selection.
|
||||
pub renderer: RendererSelection,
|
||||
/// Disable frontend-side multimodal preprocessing and render the model as
|
||||
|
||||
@@ -244,6 +244,7 @@ impl ChatBackend for FakeTextBackend {
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
@@ -292,6 +293,7 @@ impl ChatBackend for FakeMultimodalBackend {
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,7 +139,8 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
|
||||
let chat = ChatLlm::new(text, chat_backend)
|
||||
.with_tool_call_parser(config.tool_call_parser.clone())
|
||||
.with_reasoning_parser(config.reasoning_parser.clone());
|
||||
.with_reasoning_parser(config.reasoning_parser.clone())
|
||||
.with_tool_strict_level(config.tool_strict_level);
|
||||
|
||||
let state = Arc::new(
|
||||
AppState::new(served_model_names, chat)
|
||||
|
||||
@@ -10,7 +10,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::info;
|
||||
use vllm_chat::{
|
||||
ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions,
|
||||
ParserSelection, RendererSelection, load_model_backends,
|
||||
ParserSelection, RendererSelection, ToolStrictLevel, load_model_backends,
|
||||
};
|
||||
use vllm_text::TextRequestProcessor;
|
||||
use vllm_text::backend::hf::HfOverrides;
|
||||
@@ -32,6 +32,7 @@ pub struct RenderConfig {
|
||||
pub port: u16,
|
||||
pub tool_call_parser: ParserSelection,
|
||||
pub reasoning_parser: ParserSelection,
|
||||
pub tool_strict_level: ToolStrictLevel,
|
||||
pub renderer: RendererSelection,
|
||||
pub chat_template: Option<String>,
|
||||
pub default_chat_template_kwargs: HashMap<String, Value>,
|
||||
@@ -88,10 +89,12 @@ async fn build_state(config: &RenderConfig) -> Result<Arc<RenderState>> {
|
||||
let max_model_len = config.max_model_len.unwrap_or(u32::MAX);
|
||||
let text = TextRequestProcessor::new(loaded.text_backend, max_model_len)
|
||||
.with_max_logprobs(config.max_logprobs);
|
||||
let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections(
|
||||
config.tool_call_parser.clone(),
|
||||
config.reasoning_parser.clone(),
|
||||
);
|
||||
let chat = ChatRequestProcessor::render_only(loaded.chat_backend)
|
||||
.with_parser_selections(
|
||||
config.tool_call_parser.clone(),
|
||||
config.reasoning_parser.clone(),
|
||||
)
|
||||
.with_tool_strict_level(config.tool_strict_level);
|
||||
Ok(Arc::new(RenderState {
|
||||
model: config.model.clone(),
|
||||
served_model_names,
|
||||
@@ -154,6 +157,7 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
let error = serve_render(
|
||||
RenderConfig {
|
||||
tool_strict_level: ToolStrictLevel::Auto,
|
||||
model: "test-model".to_string(),
|
||||
revision: None,
|
||||
hf_overrides: Default::default(),
|
||||
|
||||
@@ -168,6 +168,7 @@ impl ChatBackend for FakeChatBackend {
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -517,6 +517,7 @@ impl ChatBackend for FakeChatBackend {
|
||||
self.tokenizer(),
|
||||
options.tool_call_parser,
|
||||
options.reasoning_parser,
|
||||
options.tool_strict_level,
|
||||
)?))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user