[Rust Frontend] Migrate gemma4 to unified parser (#46602)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -18,7 +18,8 @@ use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEvent
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
|
||||
use crate::parser::tool::{ToolParser, ToolParserFactory};
|
||||
use crate::request::ChatRequest;
|
||||
use crate::parser::unified::UnifiedParserFactory;
|
||||
use crate::request::{ChatRequest, ChatTool};
|
||||
use crate::{Error, Result as ChatResult};
|
||||
|
||||
/// Default request-scoped output processor used by Hugging Face style chat
|
||||
@@ -46,20 +47,31 @@ impl DefaultChatOutputProcessor {
|
||||
tool_call_parser: &ParserSelection,
|
||||
reasoning_parser: &ParserSelection,
|
||||
) -> ChatResult<Self> {
|
||||
let tool_parsing_enabled = request.tool_parsing_enabled();
|
||||
let tool_parser = if tool_parsing_enabled {
|
||||
Some(Self::resolve_tool_parser(
|
||||
request,
|
||||
let parser = if tool_call_parser == reasoning_parser
|
||||
&& let Some(parser) = Self::resolve_optional_unified_parser(
|
||||
&request.tools,
|
||||
model_id,
|
||||
tokenizer.clone(),
|
||||
tool_call_parser,
|
||||
)?)
|
||||
)? {
|
||||
parser
|
||||
} else {
|
||||
None
|
||||
let tool_parsing_enabled = request.tool_parsing_enabled();
|
||||
let tool_parser = if tool_parsing_enabled {
|
||||
Some(Self::resolve_tool_parser(
|
||||
&request.tools,
|
||||
model_id,
|
||||
tool_call_parser,
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let reasoning_parser =
|
||||
Self::resolve_optional_reasoning_parser(model_id, tokenizer, reasoning_parser)?;
|
||||
Box::new(CombinedParser::new(reasoning_parser, tool_parser)) as Box<dyn UnifiedParser>
|
||||
};
|
||||
let reasoning_parser =
|
||||
Self::resolve_optional_reasoning_parser(model_id, tokenizer, reasoning_parser)?;
|
||||
let parser: Box<dyn UnifiedParser> =
|
||||
Box::new(CombinedParser::new(reasoning_parser, tool_parser));
|
||||
|
||||
apply_structural_tag_constraint(request, parser.structural_tag_model())?;
|
||||
|
||||
if parser.preserve_special_tokens() {
|
||||
request.decode_options.skip_special_tokens = false;
|
||||
@@ -84,7 +96,7 @@ impl DefaultChatOutputProcessor {
|
||||
}
|
||||
|
||||
fn resolve_tool_parser(
|
||||
request: &mut ChatRequest,
|
||||
tools: &[ChatTool],
|
||||
model_id: &str,
|
||||
selection: &ParserSelection,
|
||||
) -> ChatResult<Box<dyn ToolParser>> {
|
||||
@@ -100,14 +112,36 @@ impl DefaultChatOutputProcessor {
|
||||
ParserSelection::Explicit(name) => name.as_str(),
|
||||
};
|
||||
|
||||
let parser = factory.create(parser_name, &request.tools)?;
|
||||
|
||||
apply_structural_tag_constraint(request, parser.as_ref())?;
|
||||
let parser = factory.create(parser_name, tools)?;
|
||||
|
||||
TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
|
||||
Ok(parser)
|
||||
}
|
||||
|
||||
fn resolve_optional_unified_parser(
|
||||
tools: &[ChatTool],
|
||||
model_id: &str,
|
||||
tokenizer: DynTokenizer,
|
||||
selection: &ParserSelection,
|
||||
) -> ChatResult<Option<Box<dyn UnifiedParser>>> {
|
||||
let factory = UnifiedParserFactory::global();
|
||||
let parser_name = match selection {
|
||||
ParserSelection::Auto => factory.resolve_name_for_model(model_id),
|
||||
ParserSelection::None => None,
|
||||
ParserSelection::Explicit(name) if factory.contains(name) => Some(name.as_str()),
|
||||
ParserSelection::Explicit(_) => None,
|
||||
};
|
||||
|
||||
let Some(parser_name) = parser_name else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let parser = factory.create(parser_name, tools, tokenizer)?;
|
||||
|
||||
UNIFIED_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using unified parser"));
|
||||
Ok(Some(parser))
|
||||
}
|
||||
|
||||
fn resolve_optional_reasoning_parser(
|
||||
model_id: &str,
|
||||
tokenizer: DynTokenizer,
|
||||
@@ -134,6 +168,7 @@ impl DefaultChatOutputProcessor {
|
||||
|
||||
static TOOL_PARSER_LOG_ONCE: Once = Once::new();
|
||||
static REASONING_PARSER_LOG_ONCE: Once = Once::new();
|
||||
static UNIFIED_PARSER_LOG_ONCE: Once = Once::new();
|
||||
|
||||
impl ChatOutputProcessor for DefaultChatOutputProcessor {
|
||||
/// Transforms a raw generate-output token stream into structured chat
|
||||
@@ -149,3 +184,102 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {
|
||||
Ok(structured.boxed())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
use super::DefaultChatOutputProcessor;
|
||||
use crate::Error;
|
||||
use crate::parser::ParserSelection;
|
||||
use crate::request::ChatRequest;
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<|channel>" => Some(1),
|
||||
"<channel|>" => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn tokenizer() -> Arc<FakeTokenizer> {
|
||||
Arc::new(FakeTokenizer)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn equal_explicit_gemma4_uses_unified_parser() {
|
||||
let mut request = ChatRequest::for_test();
|
||||
let selection = ParserSelection::Explicit("gemma4".to_string());
|
||||
|
||||
DefaultChatOutputProcessor::new(
|
||||
&mut request,
|
||||
"other-model",
|
||||
tokenizer(),
|
||||
&selection,
|
||||
&selection,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_auto_gemma4_model_uses_unified_parser() {
|
||||
let mut request = ChatRequest::for_test();
|
||||
|
||||
DefaultChatOutputProcessor::new(
|
||||
&mut request,
|
||||
"google/gemma-4-27b-it",
|
||||
tokenizer(),
|
||||
&ParserSelection::Auto,
|
||||
&ParserSelection::Auto,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_gemma4_selection_uses_split_dummy_error() {
|
||||
let mut request = ChatRequest::for_test();
|
||||
let error = match DefaultChatOutputProcessor::new(
|
||||
&mut request,
|
||||
"other-model",
|
||||
tokenizer(),
|
||||
&ParserSelection::Auto,
|
||||
&ParserSelection::Explicit("gemma4".to_string()),
|
||||
) {
|
||||
Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"),
|
||||
Err(error) => error,
|
||||
};
|
||||
|
||||
let Error::ParserInitialization { error, .. } = error else {
|
||||
panic!("expected parser initialization error");
|
||||
};
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"`gemma4` only provides a unified parser; the same reasoning parser and tool parser should be specified together"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
|
||||
use thiserror_ext::AsReport;
|
||||
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
|
||||
use vllm_parser::tool::StructuralTagModel;
|
||||
use xgrammar_structural_tag::{
|
||||
FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam,
|
||||
build_structural_tag,
|
||||
};
|
||||
|
||||
use crate::parser::tool::ToolParser;
|
||||
use crate::request::{ChatRequest, ChatToolChoice};
|
||||
use crate::{Error, Result as ChatResult};
|
||||
|
||||
@@ -15,9 +15,9 @@ use crate::{Error, Result as ChatResult};
|
||||
/// support and the request's tool choice.
|
||||
pub(super) fn apply_structural_tag_constraint(
|
||||
request: &mut ChatRequest,
|
||||
parser: &dyn ToolParser,
|
||||
model: Option<StructuralTagModel>,
|
||||
) -> ChatResult<()> {
|
||||
let Some(model) = parser.structural_tag_model() else {
|
||||
let Some(model) = model else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(tool_choice) = structural_tag_tool_choice(request) else {
|
||||
@@ -77,7 +77,7 @@ fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagTool
|
||||
mod tests {
|
||||
use serde_json::{Value, json};
|
||||
use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
|
||||
use vllm_parser::tool::{Qwen3CoderToolParser, Tool};
|
||||
use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -134,7 +134,7 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_model())
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
@@ -147,7 +147,7 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_model())
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
assert!(request.sampling_params.structured_outputs.is_none());
|
||||
@@ -163,7 +163,7 @@ mod tests {
|
||||
});
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_model())
|
||||
.expect("structural tag should build");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
@@ -179,7 +179,7 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_model())
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request);
|
||||
@@ -197,7 +197,7 @@ mod tests {
|
||||
});
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_model())
|
||||
.expect("structural tag should build");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
@@ -218,7 +218,7 @@ mod tests {
|
||||
);
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_model())
|
||||
.expect("structural tag should build");
|
||||
|
||||
let tag = structural_tag_value(&request).to_string();
|
||||
@@ -231,7 +231,7 @@ mod tests {
|
||||
let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_model())
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
assert!(request.sampling_params.structured_outputs.is_none());
|
||||
@@ -247,7 +247,7 @@ mod tests {
|
||||
});
|
||||
let parser = qwen3_coder_parser(&request.tools);
|
||||
|
||||
apply_structural_tag_constraint(&mut request, parser.as_ref())
|
||||
apply_structural_tag_constraint(&mut request, parser.structural_tag_model())
|
||||
.expect("structural tag decision should succeed");
|
||||
|
||||
let params = structured_outputs(&request);
|
||||
|
||||
@@ -317,7 +317,7 @@ mod tests {
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
unreachable!("ScriptedParser is constructed directly in tests")
|
||||
Ok(Box::new(Self::new([])))
|
||||
}
|
||||
|
||||
fn parse_into(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
pub mod reasoning;
|
||||
pub mod tool;
|
||||
pub mod unified;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::convert::Infallible;
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
//! Reasoning parser registration and selection boundary for `vllm-chat`.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
pub use vllm_parser::reasoning::{
|
||||
CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
|
||||
DeepSeekV4ReasoningParser, Gemma4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser,
|
||||
KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
|
||||
NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, ReasoningError,
|
||||
ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, Step3p5ReasoningParser,
|
||||
DeepSeekV4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser, KimiReasoningParser,
|
||||
MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser, NemotronV3ReasoningParser,
|
||||
Qwen3ReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, SeedOssReasoningParser,
|
||||
Step3ReasoningParser, Step3p5ReasoningParser,
|
||||
};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
@@ -33,8 +33,9 @@ pub mod names {
|
||||
}
|
||||
|
||||
/// Constructor signature for one registered reasoning parser implementation.
|
||||
type ReasoningParserCreator =
|
||||
fn(DynTokenizer) -> vllm_parser::reasoning::Result<Box<dyn ReasoningParser>>;
|
||||
type ReasoningParserCreator = Arc<
|
||||
dyn Fn(DynTokenizer) -> vllm_parser::reasoning::Result<Box<dyn ReasoningParser>> + Send + Sync,
|
||||
>;
|
||||
|
||||
/// Registry and model matcher for reasoning parsers.
|
||||
pub type ReasoningParserFactory = ParserFactory<ReasoningParserCreator>;
|
||||
@@ -58,7 +59,7 @@ impl ReasoningParserFactory {
|
||||
.register_parser::<DeepSeekR1ReasoningParser>(names::DEEPSEEK_R1)
|
||||
.register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
|
||||
.register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
|
||||
.register_parser::<Gemma4ReasoningParser>(names::GEMMA4)
|
||||
.register_unified_dummy(names::GEMMA4)
|
||||
.register_parser::<Glm45ReasoningParser>(names::GLM45)
|
||||
.register_parser::<KimiReasoningParser>(names::KIMI)
|
||||
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
|
||||
@@ -109,7 +110,17 @@ impl ReasoningParserFactory {
|
||||
where
|
||||
T: ReasoningParser + 'static,
|
||||
{
|
||||
self.register_creator(name, T::create)
|
||||
self.register_creator(name, Arc::new(T::create))
|
||||
}
|
||||
|
||||
/// Register one unified-only parser name in the split reasoning registry.
|
||||
pub fn register_unified_dummy(&mut self, name: &str) -> &mut Self {
|
||||
let name = name.to_string();
|
||||
let registered_name = name.clone();
|
||||
self.register_creator(
|
||||
®istered_name,
|
||||
Arc::new(move |_| Err(ReasoningError::DummyUnifiedParser { name: name.clone() })),
|
||||
)
|
||||
}
|
||||
|
||||
/// Construct a parser from an exact name.
|
||||
@@ -124,7 +135,7 @@ impl ReasoningParserFactory {
|
||||
available_names: self.list(),
|
||||
})?;
|
||||
|
||||
creator(tokenizer).map_err(|error| crate::Error::ParserInitialization {
|
||||
creator.as_ref()(tokenizer).map_err(|error| crate::Error::ParserInitialization {
|
||||
kind: "reasoning",
|
||||
name: name.to_string(),
|
||||
error: error.into(),
|
||||
|
||||
@@ -35,11 +35,13 @@ fn factory_contains_and_lists_registered_parsers() {
|
||||
assert!(factory.contains(names::SEED_OSS));
|
||||
assert!(factory.contains(names::STEP3P5));
|
||||
assert!(factory.contains(names::MINIMAX_M3));
|
||||
assert!(factory.contains(names::GEMMA4));
|
||||
assert!(factory.list().contains(&names::QWEN3.to_string()));
|
||||
assert!(factory.list().contains(&names::DEEPSEEK_V4.to_string()));
|
||||
assert!(factory.list().contains(&names::SEED_OSS.to_string()));
|
||||
assert!(factory.list().contains(&names::STEP3P5.to_string()));
|
||||
assert!(factory.list().contains(&names::MINIMAX_M3.to_string()));
|
||||
assert!(factory.list().contains(&names::GEMMA4.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
//! Tool parser registration and selection boundary for `vllm-chat`.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
pub use vllm_parser::tool::{
|
||||
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
|
||||
Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
|
||||
HyV3ToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
|
||||
MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
|
||||
Qwen3CoderToolParser, Qwen3XmlToolParser, ToolParser, ToolParserError,
|
||||
Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser, HyV3ToolParser,
|
||||
Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser,
|
||||
MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser,
|
||||
Qwen3XmlToolParser, ToolParser, ToolParserError,
|
||||
};
|
||||
|
||||
use crate::parser::ParserFactory;
|
||||
@@ -40,7 +40,8 @@ pub mod names {
|
||||
}
|
||||
|
||||
/// Constructor signature for one registered tool parser implementation.
|
||||
type ToolParserCreator = fn(&[ChatTool]) -> vllm_parser::tool::Result<Box<dyn ToolParser>>;
|
||||
type ToolParserCreator =
|
||||
Arc<dyn Fn(&[ChatTool]) -> vllm_parser::tool::Result<Box<dyn ToolParser>> + Send + Sync>;
|
||||
|
||||
/// Registry and model matcher for tool parsers.
|
||||
pub type ToolParserFactory = ParserFactory<ToolParserCreator>;
|
||||
@@ -65,7 +66,7 @@ impl ToolParserFactory {
|
||||
.register_parser::<DeepSeekV4ToolParser>(names::DEEPSEEK_V4)
|
||||
.register_parser::<Glm45MoeToolParser>(names::GLM45)
|
||||
.register_parser::<Glm47MoeToolParser>(names::GLM47)
|
||||
.register_parser::<Gemma4ToolParser>(names::GEMMA4)
|
||||
.register_unified_dummy(names::GEMMA4)
|
||||
.register_parser::<Granite4ToolParser>(names::GRANITE4)
|
||||
.register_parser::<HermesToolParser>(names::HERMES)
|
||||
.register_parser::<HyV3ToolParser>(names::HY_V3)
|
||||
@@ -126,7 +127,17 @@ impl ToolParserFactory {
|
||||
where
|
||||
T: ToolParser + 'static,
|
||||
{
|
||||
self.register_creator(name, T::create)
|
||||
self.register_creator(name, Arc::new(T::create))
|
||||
}
|
||||
|
||||
/// Register one unified-only parser name in the split tool registry.
|
||||
pub fn register_unified_dummy(&mut self, name: &str) -> &mut Self {
|
||||
let name = name.to_string();
|
||||
let registered_name = name.clone();
|
||||
self.register_creator(
|
||||
®istered_name,
|
||||
Arc::new(move |_| Err(ToolParserError::DummyUnifiedParser { name: name.clone() })),
|
||||
)
|
||||
}
|
||||
|
||||
/// Construct a parser from an exact name.
|
||||
@@ -137,7 +148,7 @@ impl ToolParserFactory {
|
||||
available_names: self.list(),
|
||||
})?;
|
||||
|
||||
creator(tools).map_err(|error| crate::Error::ParserInitialization {
|
||||
creator.as_ref()(tools).map_err(|error| crate::Error::ParserInitialization {
|
||||
kind: "tool",
|
||||
name: name.to_string(),
|
||||
error: error.into(),
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
//! Unified parser registration and selection boundary for `vllm-chat`.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub use vllm_parser::unified::{Gemma4UnifiedParser, UnifiedParser};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use crate::parser::ParserFactory;
|
||||
use crate::request::ChatTool;
|
||||
|
||||
/// Canonical public names for registered unified parsers.
|
||||
pub mod names {
|
||||
pub const GEMMA4: &str = "gemma4";
|
||||
}
|
||||
|
||||
/// Constructor signature for one registered unified parser implementation.
|
||||
type UnifiedParserCreator =
|
||||
fn(&[ChatTool], DynTokenizer) -> vllm_parser::unified::Result<Box<dyn UnifiedParser>>;
|
||||
|
||||
/// Registry and model matcher for unified parsers.
|
||||
pub type UnifiedParserFactory = ParserFactory<UnifiedParserCreator>;
|
||||
|
||||
impl UnifiedParserFactory {
|
||||
/// Get the global unified parser factory with built-in registrations and
|
||||
/// model mappings.
|
||||
pub fn global() -> &'static Self {
|
||||
static INSTANCE: LazyLock<UnifiedParserFactory> = LazyLock::new(UnifiedParserFactory::new);
|
||||
&INSTANCE
|
||||
}
|
||||
|
||||
/// Create the default registry with built-in parser names and model
|
||||
/// mappings.
|
||||
pub fn new() -> Self {
|
||||
let mut factory = Self::default();
|
||||
|
||||
factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
|
||||
|
||||
factory
|
||||
.register_pattern("gemma-4", names::GEMMA4)
|
||||
.register_pattern("gemma4", names::GEMMA4);
|
||||
|
||||
factory
|
||||
}
|
||||
|
||||
/// Register one parser type that exposes a static `create()` constructor.
|
||||
pub fn register_parser<T>(&mut self, name: &str) -> &mut Self
|
||||
where
|
||||
T: UnifiedParser + 'static,
|
||||
{
|
||||
self.register_creator(name, T::create)
|
||||
}
|
||||
|
||||
/// Construct a parser from an exact name.
|
||||
pub fn create(
|
||||
&self,
|
||||
name: &str,
|
||||
tools: &[ChatTool],
|
||||
tokenizer: DynTokenizer,
|
||||
) -> crate::Result<Box<dyn UnifiedParser>> {
|
||||
let creator = self.creator(name).ok_or_else(|| crate::Error::ParserUnavailableByName {
|
||||
kind: "unified",
|
||||
name: name.to_string(),
|
||||
available_names: self.list(),
|
||||
})?;
|
||||
|
||||
creator(tools, tokenizer).map_err(|error| crate::Error::ParserInitialization {
|
||||
kind: "unified",
|
||||
name: name.to_string(),
|
||||
error: error.into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
use super::{UnifiedParserFactory, names};
|
||||
|
||||
struct FakeTokenizer;
|
||||
|
||||
impl Tokenizer for FakeTokenizer {
|
||||
fn encode(
|
||||
&self,
|
||||
text: &str,
|
||||
_add_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<Vec<u32>> {
|
||||
Ok(text.chars().map(u32::from).collect())
|
||||
}
|
||||
|
||||
fn decode(
|
||||
&self,
|
||||
token_ids: &[u32],
|
||||
_skip_special_tokens: bool,
|
||||
) -> vllm_tokenizer::Result<String> {
|
||||
Ok(token_ids
|
||||
.iter()
|
||||
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
match token {
|
||||
"<|channel>" => Some(1),
|
||||
"<channel|>" => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_registers_gemma4() {
|
||||
let factory = UnifiedParserFactory::new();
|
||||
|
||||
assert!(factory.contains(names::GEMMA4));
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("google/gemma-4-27b-it"),
|
||||
Some(names::GEMMA4)
|
||||
);
|
||||
factory.create(names::GEMMA4, &[], Arc::new(FakeTokenizer)).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,8 @@ struct RoundtripCase {
|
||||
/// JSON formatting expected after this model's template has materialized
|
||||
/// tool-call arguments.
|
||||
json_fmt: JsonFmt,
|
||||
/// Whether the template renders tool-call argument object keys in sorted order.
|
||||
sort_json_keys: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -81,6 +83,7 @@ impl RoundtripCase {
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Toggleable { default: true },
|
||||
json_fmt: spaced_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,6 +96,7 @@ impl RoundtripCase {
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Toggleable { default: true },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,6 +109,7 @@ impl RoundtripCase {
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Always { value: true },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,6 +122,7 @@ impl RoundtripCase {
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Toggleable { default: false },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,6 +135,20 @@ impl RoundtripCase {
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Toggleable { default: true },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Gemma4 channel reasoning with custom function-call arguments.
|
||||
fn gemma4() -> Self {
|
||||
Self {
|
||||
model_id: "google/gemma-4-E4B-it",
|
||||
assistant_stop_suffix: "<|tool_response>",
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Always { value: true },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,6 +162,7 @@ impl RoundtripCase {
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Toggleable { default: true },
|
||||
json_fmt: spaced_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,6 +175,7 @@ impl RoundtripCase {
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Always { value: true },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,6 +188,7 @@ impl RoundtripCase {
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::Always { value: true },
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -195,8 +218,8 @@ roundtrip_tests! {
|
||||
seed_oss => [reasoning_and_content],
|
||||
step3p5 => [reasoning_and_content],
|
||||
|
||||
// Note: Kimi K2.5 strips the reasoning content in history.
|
||||
kimi_k25 => [tool_call_mix],
|
||||
gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call
|
||||
kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
|
||||
}
|
||||
|
||||
/// Run the fixed reasoning+content fixture for one model/parser case.
|
||||
@@ -347,14 +370,38 @@ fn spaced_json_fmt() -> JsonFmt {
|
||||
/// Pass in a raw JSON string instead of a structured value to ensure the exact precision and
|
||||
/// formatting of numbers are preserved.
|
||||
fn expected_arguments(case: &RoundtripCase, raw_json: &str) -> Result<String> {
|
||||
let value: serde_json::Value =
|
||||
let mut value: serde_json::Value =
|
||||
serde_json::from_str(raw_json).context("invalid expected tool-call arguments")?;
|
||||
if case.sort_json_keys {
|
||||
sort_json_value(&mut value);
|
||||
}
|
||||
|
||||
case.json_fmt
|
||||
.format_to_string(&value)
|
||||
.context("failed to format expected tool-call arguments")
|
||||
}
|
||||
|
||||
/// Sort JSON object keys recursively to match templates that render mappings with `dictsort`.
|
||||
fn sort_json_value(value: &mut serde_json::Value) {
|
||||
match value {
|
||||
serde_json::Value::Object(map) => {
|
||||
for value in map.values_mut() {
|
||||
sort_json_value(value);
|
||||
}
|
||||
|
||||
let mut entries = std::mem::take(map).into_iter().collect::<Vec<_>>();
|
||||
entries.sort_by(|(left, _), (right, _)| left.cmp(right));
|
||||
map.extend(entries);
|
||||
}
|
||||
serde_json::Value::Array(values) => {
|
||||
for value in values {
|
||||
sort_json_value(value);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Load the real model chat/text backend for one roundtrip case.
|
||||
async fn load_roundtrip_backends(case: &RoundtripCase) -> Result<vllm_chat::LoadedModelBackends> {
|
||||
load_model_backends(
|
||||
|
||||
Reference in New Issue
Block a user