[Rust Frontend] Resolve unified and split parser selections consistently (#56018)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -34,6 +34,10 @@ pub enum Error {
|
|||||||
},
|
},
|
||||||
#[error("{kind} parsing is disabled by frontend configuration")]
|
#[error("{kind} parsing is disabled by frontend configuration")]
|
||||||
ParserDisabled { kind: &'static str },
|
ParserDisabled { kind: &'static str },
|
||||||
|
#[error(
|
||||||
|
"unified parsing requires the tool and reasoning selections to resolve to the same parser; resolved tool={tool}, reasoning={reasoning}"
|
||||||
|
)]
|
||||||
|
IncompatibleParserSelections { tool: String, reasoning: String },
|
||||||
#[error(
|
#[error(
|
||||||
"{kind} parser `{name}` is not registered{}",
|
"{kind} parser `{name}` is not registered{}",
|
||||||
available_parser_hint(.available_names)
|
available_parser_hint(.available_names)
|
||||||
|
|||||||
@@ -26,11 +26,11 @@ pub use output::{
|
|||||||
ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
|
ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
|
||||||
HarmonyChatOutputProcessor,
|
HarmonyChatOutputProcessor,
|
||||||
};
|
};
|
||||||
pub use parser::ParserSelection;
|
|
||||||
pub use parser::reasoning::{
|
pub use parser::reasoning::{
|
||||||
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory,
|
ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory,
|
||||||
};
|
};
|
||||||
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
|
pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
|
||||||
|
pub use parser::{ParserSelection, validate_parser_overrides};
|
||||||
pub use renderer::hf::ChatTemplateContentFormatOption;
|
pub use renderer::hf::ChatTemplateContentFormatOption;
|
||||||
pub use renderer::{
|
pub use renderer::{
|
||||||
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
|
ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
|
||||||
@@ -62,36 +62,6 @@ use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
|||||||
use vllm_llm::Llm;
|
use vllm_llm::Llm;
|
||||||
use vllm_text::{Prompt, TextLlm, TextRequest};
|
use vllm_text::{Prompt, TextLlm, TextRequest};
|
||||||
|
|
||||||
/// Validate explicit parser override names without starting request processing.
|
|
||||||
pub fn validate_parser_overrides(
|
|
||||||
tool_call_parser: &ParserSelection,
|
|
||||||
reasoning_parser: &ParserSelection,
|
|
||||||
) -> Result<()> {
|
|
||||||
let tool_parser_factory = ToolParserFactory::global();
|
|
||||||
if let ParserSelection::Explicit(name) = tool_call_parser
|
|
||||||
&& !tool_parser_factory.contains(name)
|
|
||||||
{
|
|
||||||
return Err(Error::ParserUnavailableByName {
|
|
||||||
kind: "tool",
|
|
||||||
name: name.clone(),
|
|
||||||
available_names: tool_parser_factory.list(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let reasoning_parser_factory = ReasoningParserFactory::global();
|
|
||||||
if let ParserSelection::Explicit(name) = reasoning_parser
|
|
||||||
&& !reasoning_parser_factory.contains(name)
|
|
||||||
{
|
|
||||||
return Err(Error::ParserUnavailableByName {
|
|
||||||
kind: "reasoning",
|
|
||||||
name: name.clone(),
|
|
||||||
available_names: reasoning_parser_factory.list(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Chat request preparation shared by inference and render-only frontends.
|
/// Chat request preparation shared by inference and render-only frontends.
|
||||||
pub struct ChatRequestProcessor {
|
pub struct ChatRequestProcessor {
|
||||||
backend: DynChatBackend,
|
backend: DynChatBackend,
|
||||||
@@ -326,24 +296,12 @@ impl ChatLlm {
|
|||||||
|
|
||||||
/// Effective tool-call parser name for this model, if parsing is enabled.
|
/// Effective tool-call parser name for this model, if parsing is enabled.
|
||||||
pub fn tool_call_parser_name(&self) -> Option<&str> {
|
pub fn tool_call_parser_name(&self) -> Option<&str> {
|
||||||
match &self.processor.tool_call_parser {
|
self.processor.tool_call_parser.resolve_tool_name(self.model_id())
|
||||||
ParserSelection::Auto => {
|
|
||||||
ToolParserFactory::global().resolve_name_for_model(self.model_id())
|
|
||||||
}
|
|
||||||
ParserSelection::None => None,
|
|
||||||
ParserSelection::Explicit(name) => Some(name),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Effective reasoning parser name for this model, if parsing is enabled.
|
/// Effective reasoning parser name for this model, if parsing is enabled.
|
||||||
pub fn reasoning_parser_name(&self) -> Option<&str> {
|
pub fn reasoning_parser_name(&self) -> Option<&str> {
|
||||||
match &self.processor.reasoning_parser {
|
self.processor.reasoning_parser.resolve_reasoning_name(self.model_id())
|
||||||
ParserSelection::Auto => {
|
|
||||||
ReasoningParserFactory::global().resolve_name_for_model(self.model_id())
|
|
||||||
}
|
|
||||||
ParserSelection::None => None,
|
|
||||||
ParserSelection::Explicit(name) => Some(name),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Render, tokenize, and submit one chat request.
|
/// Render, tokenize, and submit one chat request.
|
||||||
|
|||||||
@@ -50,13 +50,12 @@ impl DefaultChatOutputProcessor {
|
|||||||
tool_call_parser: &ParserSelection,
|
tool_call_parser: &ParserSelection,
|
||||||
reasoning_parser: &ParserSelection,
|
reasoning_parser: &ParserSelection,
|
||||||
) -> ChatResult<Self> {
|
) -> ChatResult<Self> {
|
||||||
let parser = if tool_call_parser == reasoning_parser
|
let parser = if let Some(parser) = Self::resolve_optional_unified_parser(
|
||||||
&& let Some(parser) = Self::resolve_optional_unified_parser(
|
request.tools(),
|
||||||
request.tools(),
|
tokenizer.clone(),
|
||||||
model_id,
|
tool_call_parser.resolve_tool_name(model_id),
|
||||||
tokenizer.clone(),
|
reasoning_parser.resolve_reasoning_name(model_id),
|
||||||
tool_call_parser,
|
)? {
|
||||||
)? {
|
|
||||||
parser
|
parser
|
||||||
} else {
|
} else {
|
||||||
let tool_parsing_enabled = request.tool_parsing_enabled();
|
let tool_parsing_enabled = request.tool_parsing_enabled();
|
||||||
@@ -105,7 +104,7 @@ impl DefaultChatOutputProcessor {
|
|||||||
) -> ChatResult<Box<dyn ToolParser>> {
|
) -> ChatResult<Box<dyn ToolParser>> {
|
||||||
let factory = ToolParserFactory::global();
|
let factory = ToolParserFactory::global();
|
||||||
let parser_name = match selection {
|
let parser_name = match selection {
|
||||||
ParserSelection::Auto => factory.resolve_name_for_model(model_id).ok_or_else(|| {
|
ParserSelection::Auto => selection.resolve_tool_name(model_id).ok_or_else(|| {
|
||||||
Error::ParserUnavailableForModel {
|
Error::ParserUnavailableForModel {
|
||||||
kind: "tool",
|
kind: "tool",
|
||||||
model_id: model_id.to_string(),
|
model_id: model_id.to_string(),
|
||||||
@@ -123,21 +122,22 @@ impl DefaultChatOutputProcessor {
|
|||||||
|
|
||||||
fn resolve_optional_unified_parser(
|
fn resolve_optional_unified_parser(
|
||||||
tools: &[ChatTool],
|
tools: &[ChatTool],
|
||||||
model_id: &str,
|
|
||||||
tokenizer: DynTokenizer,
|
tokenizer: DynTokenizer,
|
||||||
selection: &ParserSelection,
|
tool_name: Option<&str>,
|
||||||
|
reasoning_name: Option<&str>,
|
||||||
) -> ChatResult<Option<Box<dyn UnifiedParser>>> {
|
) -> ChatResult<Option<Box<dyn UnifiedParser>>> {
|
||||||
let factory = UnifiedParserFactory::global();
|
let factory = UnifiedParserFactory::global();
|
||||||
let parser_name = match selection {
|
let Some(parser_name) =
|
||||||
ParserSelection::Auto => factory.resolve_name_for_model(model_id),
|
tool_name.into_iter().chain(reasoning_name).find(|name| factory.contains(name))
|
||||||
ParserSelection::None => None,
|
else {
|
||||||
ParserSelection::Explicit(name) if factory.contains(name) => Some(name.as_str()),
|
|
||||||
ParserSelection::Explicit(_) => None,
|
|
||||||
};
|
|
||||||
|
|
||||||
let Some(parser_name) = parser_name else {
|
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
|
if tool_name != reasoning_name {
|
||||||
|
return Err(Error::IncompatibleParserSelections {
|
||||||
|
tool: tool_name.unwrap_or("none").to_owned(),
|
||||||
|
reasoning: reasoning_name.unwrap_or("none").to_owned(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
let parser = factory.create(parser_name, tools, tokenizer)?;
|
let parser = factory.create(parser_name, tools, tokenizer)?;
|
||||||
|
|
||||||
@@ -151,11 +151,7 @@ impl DefaultChatOutputProcessor {
|
|||||||
selection: &ParserSelection,
|
selection: &ParserSelection,
|
||||||
) -> ChatResult<Option<Box<dyn ReasoningParser>>> {
|
) -> ChatResult<Option<Box<dyn ReasoningParser>>> {
|
||||||
let factory = ReasoningParserFactory::global();
|
let factory = ReasoningParserFactory::global();
|
||||||
let parser_name = match selection {
|
let parser_name = selection.resolve_reasoning_name(model_id);
|
||||||
ParserSelection::Auto => factory.resolve_name_for_model(model_id),
|
|
||||||
ParserSelection::None => None,
|
|
||||||
ParserSelection::Explicit(name) => Some(name.as_str()),
|
|
||||||
};
|
|
||||||
|
|
||||||
let Some(parser_name) = parser_name else {
|
let Some(parser_name) = parser_name else {
|
||||||
REASONING_PARSER_LOG_ONCE.call_once(|| info!("reasoning parsing disabled"));
|
REASONING_PARSER_LOG_ONCE.call_once(|| info!("reasoning parsing disabled"));
|
||||||
@@ -195,7 +191,6 @@ mod tests {
|
|||||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||||
|
|
||||||
use super::DefaultChatOutputProcessor;
|
use super::DefaultChatOutputProcessor;
|
||||||
use crate::Error;
|
|
||||||
use crate::parser::ParserSelection;
|
use crate::parser::ParserSelection;
|
||||||
use crate::request::ChatRequest;
|
use crate::request::ChatRequest;
|
||||||
|
|
||||||
@@ -237,11 +232,29 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mixed_gemma4_selection_uses_split_dummy_error() {
|
fn auto_and_explicit_gemma4_selections_use_unified_parser() {
|
||||||
|
let explicit = ParserSelection::Explicit("gemma4".to_string());
|
||||||
|
for (tool, reasoning) in [
|
||||||
|
(&ParserSelection::Auto, &explicit),
|
||||||
|
(&explicit, &ParserSelection::Auto),
|
||||||
|
] {
|
||||||
|
DefaultChatOutputProcessor::new(
|
||||||
|
&mut ChatRequest::for_test(),
|
||||||
|
"google/gemma-4-27b-it",
|
||||||
|
tokenizer(),
|
||||||
|
tool,
|
||||||
|
reasoning,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn conflicting_unified_parser_selections_report_resolved_names() {
|
||||||
let mut request = ChatRequest::for_test();
|
let mut request = ChatRequest::for_test();
|
||||||
let error = match DefaultChatOutputProcessor::new(
|
let error = match DefaultChatOutputProcessor::new(
|
||||||
&mut request,
|
&mut request,
|
||||||
"other-model",
|
"Qwen/Qwen3-8B",
|
||||||
tokenizer(),
|
tokenizer(),
|
||||||
&ParserSelection::Auto,
|
&ParserSelection::Auto,
|
||||||
&ParserSelection::Explicit("gemma4".to_string()),
|
&ParserSelection::Explicit("gemma4".to_string()),
|
||||||
@@ -250,12 +263,7 @@ mod tests {
|
|||||||
Err(error) => error,
|
Err(error) => error,
|
||||||
};
|
};
|
||||||
|
|
||||||
let Error::ParserInitialization { error, .. } = error else {
|
expect_test::expect!["unified parsing requires the tool and reasoning selections to resolve to the same parser; resolved tool=qwen3_xml, reasoning=gemma4"]
|
||||||
panic!("expected parser initialization error");
|
.assert_eq(&format!("{error}"));
|
||||||
};
|
|
||||||
assert_eq!(
|
|
||||||
error.to_string(),
|
|
||||||
"`gemma4` only provides a unified parser; the same reasoning parser and tool parser should be specified together"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,6 +12,10 @@ use std::str::FromStr;
|
|||||||
|
|
||||||
use serde_with::{DeserializeFromStr, SerializeDisplay};
|
use serde_with::{DeserializeFromStr, SerializeDisplay};
|
||||||
|
|
||||||
|
use self::reasoning::ReasoningParserFactory;
|
||||||
|
use self::tool::ToolParserFactory;
|
||||||
|
use self::unified::UnifiedParserFactory;
|
||||||
|
|
||||||
/// Specify which reasoning or tool-call parser implementation to use.
|
/// Specify which reasoning or tool-call parser implementation to use.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
|
#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)]
|
||||||
pub enum ParserSelection {
|
pub enum ParserSelection {
|
||||||
@@ -27,6 +31,66 @@ pub enum ParserSelection {
|
|||||||
impl ParserSelection {
|
impl ParserSelection {
|
||||||
pub const AUTO_LITERAL: &str = "auto";
|
pub const AUTO_LITERAL: &str = "auto";
|
||||||
pub const NONE_LITERAL: &str = "none";
|
pub const NONE_LITERAL: &str = "none";
|
||||||
|
|
||||||
|
/// Resolve a reasoning parser name, preferring a matching unified parser.
|
||||||
|
pub fn resolve_reasoning_name(&self, model_id: &str) -> Option<&str> {
|
||||||
|
self.resolve_name(model_id, ReasoningParserFactory::global())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve a tool parser name, preferring a matching unified parser.
|
||||||
|
pub fn resolve_tool_name(&self, model_id: &str) -> Option<&str> {
|
||||||
|
self.resolve_name(model_id, ToolParserFactory::global())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_name<'a, C>(
|
||||||
|
&'a self,
|
||||||
|
model_id: &str,
|
||||||
|
factory: &'a ParserFactory<C>,
|
||||||
|
) -> Option<&'a str> {
|
||||||
|
match self {
|
||||||
|
Self::Auto => UnifiedParserFactory::global()
|
||||||
|
.resolve_name_for_model(model_id)
|
||||||
|
.or_else(|| factory.resolve_name_for_model(model_id)),
|
||||||
|
Self::None => None,
|
||||||
|
Self::Explicit(name) => Some(name),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validate explicit parser override names without starting request processing.
|
||||||
|
pub fn validate_parser_overrides(
|
||||||
|
tool_call_parser: &ParserSelection,
|
||||||
|
reasoning_parser: &ParserSelection,
|
||||||
|
) -> crate::Result<()> {
|
||||||
|
validate_selection(tool_call_parser, "tool", ToolParserFactory::global())?;
|
||||||
|
validate_selection(
|
||||||
|
reasoning_parser,
|
||||||
|
"reasoning",
|
||||||
|
ReasoningParserFactory::global(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_selection<C>(
|
||||||
|
selection: &ParserSelection,
|
||||||
|
kind: &'static str,
|
||||||
|
factory: &ParserFactory<C>,
|
||||||
|
) -> crate::Result<()> {
|
||||||
|
let unified = UnifiedParserFactory::global();
|
||||||
|
if let ParserSelection::Explicit(name) = selection
|
||||||
|
&& !factory.contains(name)
|
||||||
|
&& !unified.contains(name)
|
||||||
|
{
|
||||||
|
let mut available_names = factory.list();
|
||||||
|
available_names.extend(unified.list());
|
||||||
|
available_names.sort_unstable();
|
||||||
|
available_names.dedup();
|
||||||
|
return Err(crate::Error::ParserUnavailableByName {
|
||||||
|
kind,
|
||||||
|
name: name.clone(),
|
||||||
|
available_names,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FromStr for ParserSelection {
|
impl FromStr for ParserSelection {
|
||||||
|
|||||||
@@ -22,15 +22,10 @@ pub mod names {
|
|||||||
pub const DEEPSEEK_R1: &str = "deepseek_r1";
|
pub const DEEPSEEK_R1: &str = "deepseek_r1";
|
||||||
pub const DEEPSEEK_V3: &str = "deepseek_v3";
|
pub const DEEPSEEK_V3: &str = "deepseek_v3";
|
||||||
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
||||||
pub const GEMMA4: &str = "gemma4";
|
|
||||||
pub const INKLING: &str = "inkling";
|
|
||||||
pub const GLM45: &str = "glm45";
|
pub const GLM45: &str = "glm45";
|
||||||
pub const GLM47: &str = "glm47";
|
pub const GLM47: &str = "glm47";
|
||||||
pub const HY_V3: &str = "hy_v3";
|
|
||||||
pub const HY_V4: &str = "hy_v4";
|
|
||||||
pub const KIMI: &str = "kimi";
|
pub const KIMI: &str = "kimi";
|
||||||
pub const KIMI_K2: &str = "kimi_k2";
|
pub const KIMI_K2: &str = "kimi_k2";
|
||||||
pub const KIMI_K3: &str = "kimi_k3";
|
|
||||||
pub const MINIMAX_M2: &str = "minimax_m2";
|
pub const MINIMAX_M2: &str = "minimax_m2";
|
||||||
pub const MINIMAX_M3: &str = "minimax_m3";
|
pub const MINIMAX_M3: &str = "minimax_m3";
|
||||||
pub const NEMOTRON_V3: &str = "nemotron_v3";
|
pub const NEMOTRON_V3: &str = "nemotron_v3";
|
||||||
@@ -67,15 +62,10 @@ impl ReasoningParserFactory {
|
|||||||
.register_parser::<DeepSeekR1ReasoningParser>(names::DEEPSEEK_R1)
|
.register_parser::<DeepSeekR1ReasoningParser>(names::DEEPSEEK_R1)
|
||||||
.register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
|
.register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
|
||||||
.register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
|
.register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
|
||||||
.register_unified_dummy(names::GEMMA4)
|
|
||||||
.register_unified_dummy(names::INKLING)
|
|
||||||
.register_parser::<Glm45ReasoningParser>(names::GLM45)
|
.register_parser::<Glm45ReasoningParser>(names::GLM45)
|
||||||
.register_parser::<Glm47ReasoningParser>(names::GLM47)
|
.register_parser::<Glm47ReasoningParser>(names::GLM47)
|
||||||
.register_unified_dummy(names::HY_V3)
|
|
||||||
.register_unified_dummy(names::HY_V4)
|
|
||||||
.register_parser::<KimiReasoningParser>(names::KIMI)
|
.register_parser::<KimiReasoningParser>(names::KIMI)
|
||||||
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
|
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
|
||||||
.register_unified_dummy(names::KIMI_K3)
|
|
||||||
.register_parser::<MiniMaxM2ReasoningParser>(names::MINIMAX_M2)
|
.register_parser::<MiniMaxM2ReasoningParser>(names::MINIMAX_M2)
|
||||||
.register_parser::<MiniMaxM3ReasoningParser>(names::MINIMAX_M3)
|
.register_parser::<MiniMaxM3ReasoningParser>(names::MINIMAX_M3)
|
||||||
.register_parser::<NemotronV3ReasoningParser>(names::NEMOTRON_V3)
|
.register_parser::<NemotronV3ReasoningParser>(names::NEMOTRON_V3)
|
||||||
@@ -89,16 +79,12 @@ impl ReasoningParserFactory {
|
|||||||
.register_pattern("deepseek-v4", names::DEEPSEEK_V4)
|
.register_pattern("deepseek-v4", names::DEEPSEEK_V4)
|
||||||
.register_pattern("deepseek_v4", names::DEEPSEEK_V4)
|
.register_pattern("deepseek_v4", names::DEEPSEEK_V4)
|
||||||
.register_pattern("deepseek-v3", names::DEEPSEEK_V3)
|
.register_pattern("deepseek-v3", names::DEEPSEEK_V3)
|
||||||
.register_pattern("gemma-4", names::GEMMA4)
|
|
||||||
.register_pattern("gemma4", names::GEMMA4)
|
|
||||||
.register_pattern("qwq", names::DEEPSEEK_R1)
|
.register_pattern("qwq", names::DEEPSEEK_R1)
|
||||||
.register_pattern("qwen3", names::QWEN3)
|
.register_pattern("qwen3", names::QWEN3)
|
||||||
.register_pattern("glm-5", names::GLM47)
|
.register_pattern("glm-5", names::GLM47)
|
||||||
.register_pattern("glm-4.7", names::GLM47)
|
.register_pattern("glm-4.7", names::GLM47)
|
||||||
.register_pattern("glm-4.6", names::GLM45)
|
.register_pattern("glm-4.6", names::GLM45)
|
||||||
.register_pattern("glm-4.5", names::GLM45)
|
.register_pattern("glm-4.5", names::GLM45)
|
||||||
.register_pattern("hy3", names::HY_V3)
|
|
||||||
.register_pattern("hy4", names::HY_V4)
|
|
||||||
.register_pattern("kimi-k2", names::KIMI_K2)
|
.register_pattern("kimi-k2", names::KIMI_K2)
|
||||||
.register_pattern("kimi", names::KIMI)
|
.register_pattern("kimi", names::KIMI)
|
||||||
// step3p5 patterns must precede `step3`: substring matching would
|
// step3p5 patterns must precede `step3`: substring matching would
|
||||||
@@ -129,16 +115,6 @@ impl ReasoningParserFactory {
|
|||||||
self.register_creator(name, Arc::new(T::create))
|
self.register_creator(name, Arc::new(T::create))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Register one unified-only parser name in the split reasoning registry.
|
|
||||||
pub fn register_unified_dummy(&mut self, name: &str) -> &mut Self {
|
|
||||||
let name = name.to_string();
|
|
||||||
let registered_name = name.clone();
|
|
||||||
self.register_creator(
|
|
||||||
®istered_name,
|
|
||||||
Arc::new(move |_| Err(ReasoningError::DummyUnifiedParser { name: name.clone() })),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Construct a parser from an exact name.
|
/// Construct a parser from an exact name.
|
||||||
pub fn create(
|
pub fn create(
|
||||||
&self,
|
&self,
|
||||||
|
|||||||
@@ -15,13 +15,11 @@ fn factory_contains_and_lists_registered_parsers() {
|
|||||||
assert!(factory.contains(names::SEED_OSS));
|
assert!(factory.contains(names::SEED_OSS));
|
||||||
assert!(factory.contains(names::STEP3P5));
|
assert!(factory.contains(names::STEP3P5));
|
||||||
assert!(factory.contains(names::MINIMAX_M3));
|
assert!(factory.contains(names::MINIMAX_M3));
|
||||||
assert!(factory.contains(names::GEMMA4));
|
|
||||||
assert!(factory.list().contains(&names::QWEN3.to_string()));
|
assert!(factory.list().contains(&names::QWEN3.to_string()));
|
||||||
assert!(factory.list().contains(&names::DEEPSEEK_V4.to_string()));
|
assert!(factory.list().contains(&names::DEEPSEEK_V4.to_string()));
|
||||||
assert!(factory.list().contains(&names::SEED_OSS.to_string()));
|
assert!(factory.list().contains(&names::SEED_OSS.to_string()));
|
||||||
assert!(factory.list().contains(&names::STEP3P5.to_string()));
|
assert!(factory.list().contains(&names::STEP3P5.to_string()));
|
||||||
assert!(factory.list().contains(&names::MINIMAX_M3.to_string()));
|
assert!(factory.list().contains(&names::MINIMAX_M3.to_string()));
|
||||||
assert!(factory.list().contains(&names::GEMMA4.to_string()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -24,17 +24,12 @@ pub mod names {
|
|||||||
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
pub const DEEPSEEK_V4: &str = "deepseek_v4";
|
||||||
pub const GLM45: &str = "glm45";
|
pub const GLM45: &str = "glm45";
|
||||||
pub const GLM47: &str = "glm47";
|
pub const GLM47: &str = "glm47";
|
||||||
pub const GEMMA4: &str = "gemma4";
|
|
||||||
pub const INKLING: &str = "inkling";
|
|
||||||
pub const GRANITE4: &str = "granite4";
|
pub const GRANITE4: &str = "granite4";
|
||||||
pub const HERMES: &str = "hermes";
|
pub const HERMES: &str = "hermes";
|
||||||
pub const HY_V3: &str = "hy_v3";
|
|
||||||
pub const HY_V4: &str = "hy_v4";
|
|
||||||
// Matches the Python CLI name `--tool-call-parser internlm`, which Python
|
// Matches the Python CLI name `--tool-call-parser internlm`, which Python
|
||||||
// also routes to `Internlm2ToolParser` despite the version-agnostic name.
|
// also routes to `Internlm2ToolParser` despite the version-agnostic name.
|
||||||
pub const INTERNLM: &str = "internlm";
|
pub const INTERNLM: &str = "internlm";
|
||||||
pub const KIMI_K2: &str = "kimi_k2";
|
pub const KIMI_K2: &str = "kimi_k2";
|
||||||
pub const KIMI_K3: &str = "kimi_k3";
|
|
||||||
pub const LLAMA3_JSON: &str = "llama3_json";
|
pub const LLAMA3_JSON: &str = "llama3_json";
|
||||||
pub const LLAMA4_JSON: &str = "llama4_json";
|
pub const LLAMA4_JSON: &str = "llama4_json";
|
||||||
pub const MINIMAX_M2: &str = "minimax_m2";
|
pub const MINIMAX_M2: &str = "minimax_m2";
|
||||||
@@ -73,15 +68,10 @@ impl ToolParserFactory {
|
|||||||
.register_parser::<DeepSeekV4ToolParser>(names::DEEPSEEK_V4)
|
.register_parser::<DeepSeekV4ToolParser>(names::DEEPSEEK_V4)
|
||||||
.register_parser::<Glm45MoeToolParser>(names::GLM45)
|
.register_parser::<Glm45MoeToolParser>(names::GLM45)
|
||||||
.register_parser::<Glm47MoeToolParser>(names::GLM47)
|
.register_parser::<Glm47MoeToolParser>(names::GLM47)
|
||||||
.register_unified_dummy(names::GEMMA4)
|
|
||||||
.register_unified_dummy(names::INKLING)
|
|
||||||
.register_parser::<Granite4ToolParser>(names::GRANITE4)
|
.register_parser::<Granite4ToolParser>(names::GRANITE4)
|
||||||
.register_parser::<HermesToolParser>(names::HERMES)
|
.register_parser::<HermesToolParser>(names::HERMES)
|
||||||
.register_unified_dummy(names::HY_V3)
|
|
||||||
.register_unified_dummy(names::HY_V4)
|
|
||||||
.register_parser::<Internlm2ToolParser>(names::INTERNLM)
|
.register_parser::<Internlm2ToolParser>(names::INTERNLM)
|
||||||
.register_parser::<KimiK2ToolParser>(names::KIMI_K2)
|
.register_parser::<KimiK2ToolParser>(names::KIMI_K2)
|
||||||
.register_unified_dummy(names::KIMI_K3)
|
|
||||||
.register_parser::<Llama3JsonToolParser>(names::LLAMA3_JSON)
|
.register_parser::<Llama3JsonToolParser>(names::LLAMA3_JSON)
|
||||||
.register_parser::<Llama3JsonToolParser>(names::LLAMA4_JSON)
|
.register_parser::<Llama3JsonToolParser>(names::LLAMA4_JSON)
|
||||||
.register_parser::<MinimaxM2ToolParser>(names::MINIMAX_M2)
|
.register_parser::<MinimaxM2ToolParser>(names::MINIMAX_M2)
|
||||||
@@ -101,8 +91,6 @@ impl ToolParserFactory {
|
|||||||
.register_pattern("qwen2.5", names::HERMES)
|
.register_pattern("qwen2.5", names::HERMES)
|
||||||
.register_pattern("qwen3", names::QWEN3_XML)
|
.register_pattern("qwen3", names::QWEN3_XML)
|
||||||
.register_pattern("hermes", names::HERMES)
|
.register_pattern("hermes", names::HERMES)
|
||||||
.register_pattern("hy3", names::HY_V3)
|
|
||||||
.register_pattern("hy4", names::HY_V4)
|
|
||||||
// Narrow to `internlm2` substring so it matches `internlm2-chat-7b`
|
// Narrow to `internlm2` substring so it matches `internlm2-chat-7b`
|
||||||
// and `internlm2_5-7b-chat` but NOT `internlm-chat-7b` (InternLM v1,
|
// and `internlm2_5-7b-chat` but NOT `internlm-chat-7b` (InternLM v1,
|
||||||
// routes to Llama), `internlm3-*` (also Llama-architecture per
|
// routes to Llama), `internlm3-*` (also Llama-architecture per
|
||||||
@@ -122,8 +110,6 @@ impl ToolParserFactory {
|
|||||||
.register_pattern("glm-4.7", names::GLM47)
|
.register_pattern("glm-4.7", names::GLM47)
|
||||||
.register_pattern("glm-4.6", names::GLM45)
|
.register_pattern("glm-4.6", names::GLM45)
|
||||||
.register_pattern("glm-4.5", names::GLM45)
|
.register_pattern("glm-4.5", names::GLM45)
|
||||||
.register_pattern("gemma4", names::GEMMA4)
|
|
||||||
.register_pattern("gemma-4", names::GEMMA4)
|
|
||||||
.register_pattern("granite-4", names::GRANITE4)
|
.register_pattern("granite-4", names::GRANITE4)
|
||||||
.register_pattern("kimi-k2", names::KIMI_K2)
|
.register_pattern("kimi-k2", names::KIMI_K2)
|
||||||
.register_pattern("minimax-m3", names::MINIMAX_M3)
|
.register_pattern("minimax-m3", names::MINIMAX_M3)
|
||||||
@@ -144,16 +130,6 @@ impl ToolParserFactory {
|
|||||||
self.register_creator(name, Arc::new(T::create))
|
self.register_creator(name, Arc::new(T::create))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Register one unified-only parser name in the split tool registry.
|
|
||||||
pub fn register_unified_dummy(&mut self, name: &str) -> &mut Self {
|
|
||||||
let name = name.to_string();
|
|
||||||
let registered_name = name.clone();
|
|
||||||
self.register_creator(
|
|
||||||
®istered_name,
|
|
||||||
Arc::new(move |_| Err(ToolParserError::DummyUnifiedParser { name: name.clone() })),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Construct a parser from an exact name.
|
/// Construct a parser from an exact name.
|
||||||
pub fn create(&self, name: &str, tools: &[ChatTool]) -> crate::Result<Box<dyn ToolParser>> {
|
pub fn create(&self, name: &str, tools: &[ChatTool]) -> crate::Result<Box<dyn ToolParser>> {
|
||||||
let creator = self.creator(name).ok_or_else(|| crate::Error::ParserUnavailableByName {
|
let creator = self.creator(name).ok_or_else(|| crate::Error::ParserUnavailableByName {
|
||||||
|
|||||||
@@ -167,10 +167,6 @@ fn factory_new_resolves_default_patterns() {
|
|||||||
factory.resolve_name_for_model("glm-4.7"),
|
factory.resolve_name_for_model("glm-4.7"),
|
||||||
Some(names::GLM47)
|
Some(names::GLM47)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
|
||||||
factory.resolve_name_for_model("google/gemma-4-27b-it"),
|
|
||||||
Some(names::GEMMA4)
|
|
||||||
);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
factory.resolve_name_for_model("ibm-granite/granite-4.0-h-tiny"),
|
factory.resolve_name_for_model("ibm-granite/granite-4.0-h-tiny"),
|
||||||
Some(names::GRANITE4)
|
Some(names::GRANITE4)
|
||||||
@@ -179,14 +175,6 @@ fn factory_new_resolves_default_patterns() {
|
|||||||
factory.resolve_name_for_model("NousResearch/Hermes-3-Llama-3.1-8B"),
|
factory.resolve_name_for_model("NousResearch/Hermes-3-Llama-3.1-8B"),
|
||||||
Some(names::HERMES)
|
Some(names::HERMES)
|
||||||
);
|
);
|
||||||
assert_eq!(
|
|
||||||
factory.resolve_name_for_model("tencent/Hy3-preview"),
|
|
||||||
Some(names::HY_V3)
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
factory.resolve_name_for_model("tencent/Hy4-preview"),
|
|
||||||
Some(names::HY_V4)
|
|
||||||
);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
factory.resolve_name_for_model("MiniMax/MiniMax-M3-Text"),
|
factory.resolve_name_for_model("MiniMax/MiniMax-M3-Text"),
|
||||||
Some(names::MINIMAX_M3)
|
Some(names::MINIMAX_M3)
|
||||||
|
|||||||
@@ -23,8 +23,8 @@ use serde_json::Value;
|
|||||||
use serde_with::{DefaultOnNull, OneOrMany, serde_as};
|
use serde_with::{DefaultOnNull, OneOrMany, serde_as};
|
||||||
use thiserror_ext::AsReport as _;
|
use thiserror_ext::AsReport as _;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
use vllm_chat::GenerationConfigMode;
|
||||||
use vllm_chat::multimodal::MmLimitPerPrompt;
|
use vllm_chat::multimodal::MmLimitPerPrompt;
|
||||||
use vllm_chat::{GenerationConfigMode, ReasoningParserFactory};
|
|
||||||
use vllm_engine_core_client::TransportMode;
|
use vllm_engine_core_client::TransportMode;
|
||||||
use vllm_managed_engine::ManagedEngineConfig;
|
use vllm_managed_engine::ManagedEngineConfig;
|
||||||
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
|
||||||
@@ -756,13 +756,7 @@ impl ServeArgs {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn effective_engine_reasoning_parser(selection: &ParserSelection, model: &str) -> Option<String> {
|
fn effective_engine_reasoning_parser(selection: &ParserSelection, model: &str) -> Option<String> {
|
||||||
match selection {
|
selection.resolve_reasoning_name(model).map(str::to_owned)
|
||||||
ParserSelection::Auto => ReasoningParserFactory::global()
|
|
||||||
.resolve_name_for_model(model)
|
|
||||||
.map(str::to_string),
|
|
||||||
ParserSelection::None => None,
|
|
||||||
ParserSelection::Explicit(name) => Some(name.clone()),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Allocate fresh IPC endpoints for one managed frontend instance.
|
/// Allocate fresh IPC endpoints for one managed frontend instance.
|
||||||
|
|||||||
@@ -413,6 +413,21 @@ fn serve_args_resolve_auto_reasoning_parser_for_managed_engine() {
|
|||||||
.assert_debug_eq(&config.python_args);
|
.assert_debug_eq(&config.python_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn serve_args_resolve_unified_reasoning_parser_for_managed_engine() {
|
||||||
|
let cli = Cli::try_parse_from(["vllm-rs", "serve", "moonshotai/Kimi-K3"]).unwrap();
|
||||||
|
let Command::Serve(args) = cli.command else {
|
||||||
|
panic!("expected serve args");
|
||||||
|
};
|
||||||
|
expect![[r#"
|
||||||
|
[
|
||||||
|
"--reasoning-parser",
|
||||||
|
"kimi_k3",
|
||||||
|
]
|
||||||
|
"#]]
|
||||||
|
.assert_debug_eq(&args.to_managed_engine_config(5555).python_args);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn serve_args_forward_explicit_reasoning_parser_to_managed_engine() {
|
fn serve_args_forward_explicit_reasoning_parser_to_managed_engine() {
|
||||||
let cli = Cli::try_parse_from([
|
let cli = Cli::try_parse_from([
|
||||||
|
|||||||
Reference in New Issue
Block a user