[Rust Frontend] Add HY3 unified parser and local XGrammar structural-tag builder (#53054)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -415,6 +415,6 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ pub mod names {
|
||||
pub const GEMMA4: &str = "gemma4";
|
||||
pub const INKLING: &str = "inkling";
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const HY_V3: &str = "hy_v3";
|
||||
pub const KIMI: &str = "kimi";
|
||||
pub const KIMI_K2: &str = "kimi_k2";
|
||||
pub const KIMI_K3: &str = "kimi_k3";
|
||||
@@ -67,6 +68,7 @@ impl ReasoningParserFactory {
|
||||
.register_unified_dummy(names::GEMMA4)
|
||||
.register_unified_dummy(names::INKLING)
|
||||
.register_parser::<Glm45ReasoningParser>(names::GLM45)
|
||||
.register_unified_dummy(names::HY_V3)
|
||||
.register_parser::<KimiReasoningParser>(names::KIMI)
|
||||
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
|
||||
.register_unified_dummy(names::KIMI_K3)
|
||||
@@ -91,6 +93,7 @@ impl ReasoningParserFactory {
|
||||
.register_pattern("glm-4.7", names::GLM45)
|
||||
.register_pattern("glm-4.6", names::GLM45)
|
||||
.register_pattern("glm-4.5", names::GLM45)
|
||||
.register_pattern("hy3", names::HY_V3)
|
||||
.register_pattern("kimi-k2", names::KIMI_K2)
|
||||
.register_pattern("kimi", names::KIMI)
|
||||
// step3p5 patterns must precede `step3`: substring matching would
|
||||
|
||||
@@ -7,7 +7,7 @@ use std::sync::{Arc, LazyLock};
|
||||
|
||||
pub use vllm_parser::tool::{
|
||||
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
|
||||
Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser, HyV3ToolParser,
|
||||
Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
|
||||
Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser,
|
||||
MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser,
|
||||
Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
|
||||
@@ -76,7 +76,7 @@ impl ToolParserFactory {
|
||||
.register_unified_dummy(names::INKLING)
|
||||
.register_parser::<Granite4ToolParser>(names::GRANITE4)
|
||||
.register_parser::<HermesToolParser>(names::HERMES)
|
||||
.register_parser::<HyV3ToolParser>(names::HY_V3)
|
||||
.register_unified_dummy(names::HY_V3)
|
||||
.register_parser::<Internlm2ToolParser>(names::INTERNLM)
|
||||
.register_parser::<KimiK2ToolParser>(names::KIMI_K2)
|
||||
.register_unified_dummy(names::KIMI_K3)
|
||||
@@ -100,7 +100,6 @@ impl ToolParserFactory {
|
||||
.register_pattern("qwen3", names::QWEN3_XML)
|
||||
.register_pattern("hermes", names::HERMES)
|
||||
.register_pattern("hy3", names::HY_V3)
|
||||
.register_pattern("hy_v3", names::HY_V3)
|
||||
// Narrow to `internlm2` substring so it matches `internlm2-chat-7b`
|
||||
// and `internlm2_5-7b-chat` but NOT `internlm-chat-7b` (InternLM v1,
|
||||
// routes to Llama), `internlm3-*` (also Llama-architecture per
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub use vllm_parser::unified::{
|
||||
Gemma4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, UnifiedParser,
|
||||
Gemma4UnifiedParser, HyV3UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser,
|
||||
UnifiedParser,
|
||||
};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
@@ -16,6 +17,7 @@ use crate::request::ChatTool;
|
||||
/// Canonical public names for registered unified parsers.
|
||||
pub mod names {
|
||||
pub const GEMMA4: &str = "gemma4";
|
||||
pub const HY_V3: &str = "hy_v3";
|
||||
pub const INKLING: &str = "inkling";
|
||||
pub const KIMI_K3: &str = "kimi_k3";
|
||||
}
|
||||
@@ -41,12 +43,14 @@ impl UnifiedParserFactory {
|
||||
let mut factory = Self::default();
|
||||
|
||||
factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
|
||||
factory.register_parser::<HyV3UnifiedParser>(names::HY_V3);
|
||||
factory.register_parser::<InklingUnifiedParser>(names::INKLING);
|
||||
factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
|
||||
|
||||
factory
|
||||
.register_pattern("gemma-4", names::GEMMA4)
|
||||
.register_pattern("gemma4", names::GEMMA4)
|
||||
.register_pattern("hy3", names::HY_V3)
|
||||
.register_pattern("inkling", names::INKLING)
|
||||
.register_pattern("kimi-k3", names::KIMI_K3)
|
||||
.register_pattern("kimi_k3", names::KIMI_K3);
|
||||
|
||||
@@ -55,6 +55,12 @@ enum ThinkingBehavior {
|
||||
Toggleable { default: bool },
|
||||
/// The chat template always behaves as `value` for this fixture.
|
||||
Always { value: bool },
|
||||
/// The chat template selects thinking mode through `reasoning_effort`.
|
||||
ReasoningEffort {
|
||||
default: bool,
|
||||
enabled: &'static str,
|
||||
disabled: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
impl ThinkingBehavior {
|
||||
@@ -62,6 +68,7 @@ impl ThinkingBehavior {
|
||||
match self {
|
||||
Self::Toggleable { default } => default,
|
||||
Self::Always { value } => value,
|
||||
Self::ReasoningEffort { default, .. } => default,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,6 +83,33 @@ impl ThinkingBehavior {
|
||||
Some(value), // explicitly request the supported thinking behavior
|
||||
None, // use default template behavior
|
||||
],
|
||||
Self::ReasoningEffort { .. } => vec![
|
||||
Some(true), // explicitly enable thinking
|
||||
Some(false), // explicitly disable thinking
|
||||
None, // use default template behavior
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn apply(self, request: &mut ChatRequest, thinking: Option<bool>) {
|
||||
let Some(thinking) = thinking else {
|
||||
return;
|
||||
};
|
||||
|
||||
match self {
|
||||
Self::Toggleable { .. } | Self::Always { .. } => {
|
||||
for key in ["thinking", "enable_thinking"] {
|
||||
request.chat_options.template_kwargs.insert(key.to_string(), thinking.into());
|
||||
}
|
||||
}
|
||||
Self::ReasoningEffort {
|
||||
enabled, disabled, ..
|
||||
} => {
|
||||
request.chat_options.template_kwargs.insert(
|
||||
"reasoning_effort".to_string(),
|
||||
if thinking { enabled } else { disabled }.into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -242,6 +276,23 @@ impl RoundtripCase {
|
||||
}
|
||||
}
|
||||
|
||||
/// HY3 suffixed reasoning/tool markers discovered from tokenizer added vocab.
|
||||
fn hy_v3() -> Self {
|
||||
Self {
|
||||
model_id: "tencent/Hy3",
|
||||
assistant_stop_suffix: "<|hy_eos:opensource|>",
|
||||
tool_call_parser: ParserSelection::Auto,
|
||||
reasoning_parser: ParserSelection::Auto,
|
||||
thinking_behavior: ThinkingBehavior::ReasoningEffort {
|
||||
default: false,
|
||||
enabled: "high",
|
||||
disabled: "no_think",
|
||||
},
|
||||
json_fmt: compact_json_fmt(),
|
||||
sort_json_keys: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// SeedOSS with `<seed:think>` / `</seed:think>` reasoning tags.
|
||||
fn seed_oss() -> Self {
|
||||
Self {
|
||||
@@ -345,6 +396,9 @@ roundtrip_tests! {
|
||||
kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
|
||||
// K3 drops plain-assistant reasoning in history; tool-call turns keep it.
|
||||
kimi_k3 => [tool_call_mix],
|
||||
// HY3's final plain-assistant history omits EOS; tool-call history keeps it
|
||||
// and exercises both stages of the tokenizer-derived unified parser.
|
||||
hy_v3 => [tool_call_mix],
|
||||
gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
|
||||
inkling => [reasoning_and_content, tool_call_mix],
|
||||
}
|
||||
@@ -370,6 +424,7 @@ async fn run_roundtrip_reasoning_and_content_inner(
|
||||
vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")],
|
||||
Vec::new(),
|
||||
thinking,
|
||||
case.thinking_behavior,
|
||||
);
|
||||
let expected_reasoning = "Need compute 2 + 2 directly.";
|
||||
let expected_text = "The answer is 4.";
|
||||
@@ -417,6 +472,7 @@ async fn run_roundtrip_tool_call_mix(
|
||||
)],
|
||||
test_tools(),
|
||||
Some(true), // always enable thinking in this fixture
|
||||
case.thinking_behavior,
|
||||
);
|
||||
let expected_reasoning = "Need call the weather and add tools.";
|
||||
let expected_text = "I will call the tools.";
|
||||
@@ -838,6 +894,7 @@ fn roundtrip_request(
|
||||
messages: Vec<ChatMessage>,
|
||||
tools: Vec<ChatTool>,
|
||||
thinking: Option<bool>,
|
||||
thinking_behavior: ThinkingBehavior,
|
||||
) -> ChatRequest {
|
||||
let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true)
|
||||
.expect("tool context should resolve");
|
||||
@@ -848,13 +905,9 @@ fn roundtrip_request(
|
||||
..ChatRequest::for_test()
|
||||
};
|
||||
|
||||
// Explicitly enable or disable thinking so that rendering and parsing the reasoning block is
|
||||
// exercised or skipped in the roundtrip. If unspecified, use the default template behavior.
|
||||
if let Some(thinking) = thinking {
|
||||
for key in ["thinking", "enable_thinking"] {
|
||||
request.chat_options.template_kwargs.insert(key.to_string(), thinking.into());
|
||||
}
|
||||
}
|
||||
// Explicitly enable or disable thinking using the controls understood by
|
||||
// this model's template. If unspecified, use the template default.
|
||||
thinking_behavior.apply(&mut request, thinking);
|
||||
|
||||
request
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user