From b2f685834a6456197e7033966fdef52a23f1abcd Mon Sep 17 00:00:00 2001 From: Cheng Jiang Date: Sat, 29 Aug 2026 12:07:28 +0800 Subject: [PATCH] [Hy4] support Hy4-preview model (#54160) Signed-off-by: chengvjiang Signed-off-by: Jee Jee Li Signed-off-by: Bugen Zhao Signed-off-by: Jee Jee Li Co-authored-by: chengvjiang Co-authored-by: Jee Jee Li Co-authored-by: Bugen Zhao Co-authored-by: russellfeng Co-authored-by: Jee Jee Li --- docs/models/supported_models.md | 3 +- rust/src/chat/src/lib.rs | 4 +- rust/src/chat/src/parser/reasoning/mod.rs | 3 + rust/src/chat/src/parser/tool/mod.rs | 3 + rust/src/chat/src/parser/tool/tests.rs | 4 + rust/src/chat/src/parser/unified.rs | 36 +- .../parser/src/reasoning/{hy_v3.rs => hy.rs} | 14 +- rust/src/parser/src/reasoning/mod.rs | 4 +- rust/src/parser/src/tool/{hy_v3.rs => hy.rs} | 368 ++++-- .../src/tool/{hy_v3 => hy}/structural_tag.rs | 109 +- rust/src/parser/src/tool/mod.rs | 4 +- rust/src/parser/src/unified/{ => hy}/hy_v3.rs | 48 +- rust/src/parser/src/unified/hy/hy_v4.rs | 194 +++ rust/src/parser/src/unified/hy/mod.rs | 39 + rust/src/parser/src/unified/mod.rs | 4 +- tests/models/registry.py | 7 + tests/models/test_registry.py | 5 + tests/models/utils.py | 5 + vllm/config/speculative.py | 8 + vllm/config/vllm.py | 2 + vllm/envs.py | 10 + .../fused_moe/experts/trtllm_fp8_moe.py | 25 +- vllm/model_executor/layers/hpc/__init__.py | 14 + vllm/model_executor/layers/hpc/gated_mla.py | 113 ++ vllm/model_executor/layers/hpc/hpc_ihc.py | 261 ++++ .../model_executor/layers/logits_processor.py | 8 +- vllm/model_executor/models/deepseek_v2.py | 12 +- vllm/model_executor/models/registry.py | 2 + vllm/models/hy_v4/__init__.py | 36 + vllm/models/hy_v4/nvidia/__init__.py | 2 + vllm/models/hy_v4/nvidia/attention.py | 765 +++++++++++ vllm/models/hy_v4/nvidia/flashmla_sparse.py | 283 ++++ vllm/models/hy_v4/nvidia/hc.py | 370 ++++++ vllm/models/hy_v4/nvidia/model.py | 716 ++++++++++ vllm/models/hy_v4/nvidia/moe.py | 190 +++ vllm/models/hy_v4/nvidia/mtp.py | 937 +++++++++++++ vllm/parser/harmony.py | 3 +- vllm/reasoning/__init__.py | 4 + vllm/reasoning/hy_v4_reasoning_parser.py | 324 +++++ vllm/tool_parsers/__init__.py | 4 + vllm/tool_parsers/hy_v4_tool_parser.py | 1166 +++++++++++++++++ vllm/tool_parsers/structural_tag_registry.py | 163 ++- vllm/transformers_utils/config.py | 1 + vllm/transformers_utils/configs/__init__.py | 2 + vllm/transformers_utils/configs/hy_v4.py | 172 +++ .../model_arch_config_convertor.py | 2 + vllm/v1/worker/gpu/spec_decode/speculator.py | 28 +- 47 files changed, 6260 insertions(+), 217 deletions(-) rename rust/src/parser/src/reasoning/{hy_v3.rs => hy.rs} (76%) rename rust/src/parser/src/tool/{hy_v3.rs => hy.rs} (61%) rename rust/src/parser/src/tool/{hy_v3 => hy}/structural_tag.rs (74%) rename rust/src/parser/src/unified/{ => hy}/hy_v3.rs (79%) create mode 100644 rust/src/parser/src/unified/hy/hy_v4.rs create mode 100644 rust/src/parser/src/unified/hy/mod.rs create mode 100644 vllm/model_executor/layers/hpc/gated_mla.py create mode 100644 vllm/model_executor/layers/hpc/hpc_ihc.py create mode 100644 vllm/models/hy_v4/__init__.py create mode 100644 vllm/models/hy_v4/nvidia/__init__.py create mode 100644 vllm/models/hy_v4/nvidia/attention.py create mode 100644 vllm/models/hy_v4/nvidia/flashmla_sparse.py create mode 100644 vllm/models/hy_v4/nvidia/hc.py create mode 100644 vllm/models/hy_v4/nvidia/model.py create mode 100644 vllm/models/hy_v4/nvidia/moe.py create mode 100644 vllm/models/hy_v4/nvidia/mtp.py create mode 100644 vllm/reasoning/hy_v4_reasoning_parser.py create mode 100644 vllm/tool_parsers/hy_v4_tool_parser.py create mode 100644 vllm/transformers_utils/configs/hy_v4.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index a696b02ede..609ca54564 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -405,7 +405,8 @@ th { | `GraniteMoeSWAForCausalLM` | Granite MoE SWA | `ibm-granite/granite-swash-3b-a600m` | ✅︎ | ✅︎ | | `GraniteSWAForCausalLM` | Granite SWA | `ibm-granite/granite-swash-2b` | ✅︎ | ✅︎ | | `HrmTextForCausalLM` | HRM-Text | `sapientinc/HRM-Text-1B`, etc. | | | -| `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview` | ✅︎ | ✅︎ | +| `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview`, `tencent/Hy3`, `tencent/Hy3-FP8` | ✅︎ | ✅︎ | +| `HYV4ForCausalLM` | HY4 | `tencent/Hy4-preview`, `tencent/Hy4-preview-FP8` | ✅︎ | ✅︎ | | `HyperCLOVAXForCausalLM` | HyperCLOVAX-SEED-Think-14B | `naver-hyperclovax/HyperCLOVAX-SEED-Think-14B` | ✅︎ | ✅︎ | | `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ | | `InternLM3ForCausalLM` | InternLM3 | `internlm/internlm3-8b-instruct`, etc. | ✅︎ | ✅︎ | diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 4c68819d88..8a7cac7b2c 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -405,7 +405,7 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string()); + expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string()); } #[test] @@ -416,6 +416,6 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); } } diff --git a/rust/src/chat/src/parser/reasoning/mod.rs b/rust/src/chat/src/parser/reasoning/mod.rs index c8e1c2df39..6942d05371 100644 --- a/rust/src/chat/src/parser/reasoning/mod.rs +++ b/rust/src/chat/src/parser/reasoning/mod.rs @@ -26,6 +26,7 @@ pub mod names { pub const INKLING: &str = "inkling"; pub const GLM45: &str = "glm45"; pub const HY_V3: &str = "hy_v3"; + pub const HY_V4: &str = "hy_v4"; pub const KIMI: &str = "kimi"; pub const KIMI_K2: &str = "kimi_k2"; pub const KIMI_K3: &str = "kimi_k3"; @@ -69,6 +70,7 @@ impl ReasoningParserFactory { .register_unified_dummy(names::INKLING) .register_parser::(names::GLM45) .register_unified_dummy(names::HY_V3) + .register_unified_dummy(names::HY_V4) .register_parser::(names::KIMI) .register_parser::(names::KIMI_K2) .register_unified_dummy(names::KIMI_K3) @@ -94,6 +96,7 @@ impl ReasoningParserFactory { .register_pattern("glm-4.6", names::GLM45) .register_pattern("glm-4.5", names::GLM45) .register_pattern("hy3", names::HY_V3) + .register_pattern("hy4", names::HY_V4) .register_pattern("kimi-k2", names::KIMI_K2) .register_pattern("kimi", names::KIMI) // step3p5 patterns must precede `step3`: substring matching would diff --git a/rust/src/chat/src/parser/tool/mod.rs b/rust/src/chat/src/parser/tool/mod.rs index 580ede0019..d64a49b2ec 100644 --- a/rust/src/chat/src/parser/tool/mod.rs +++ b/rust/src/chat/src/parser/tool/mod.rs @@ -29,6 +29,7 @@ pub mod names { pub const GRANITE4: &str = "granite4"; pub const HERMES: &str = "hermes"; pub const HY_V3: &str = "hy_v3"; + pub const HY_V4: &str = "hy_v4"; // Matches the Python CLI name `--tool-call-parser internlm`, which Python // also routes to `Internlm2ToolParser` despite the version-agnostic name. pub const INTERNLM: &str = "internlm"; @@ -77,6 +78,7 @@ impl ToolParserFactory { .register_parser::(names::GRANITE4) .register_parser::(names::HERMES) .register_unified_dummy(names::HY_V3) + .register_unified_dummy(names::HY_V4) .register_parser::(names::INTERNLM) .register_parser::(names::KIMI_K2) .register_unified_dummy(names::KIMI_K3) @@ -100,6 +102,7 @@ impl ToolParserFactory { .register_pattern("qwen3", names::QWEN3_XML) .register_pattern("hermes", names::HERMES) .register_pattern("hy3", names::HY_V3) + .register_pattern("hy4", names::HY_V4) // Narrow to `internlm2` substring so it matches `internlm2-chat-7b` // and `internlm2_5-7b-chat` but NOT `internlm-chat-7b` (InternLM v1, // routes to Llama), `internlm3-*` (also Llama-architecture per diff --git a/rust/src/chat/src/parser/tool/tests.rs b/rust/src/chat/src/parser/tool/tests.rs index d398d84e35..5c7ee3d85e 100644 --- a/rust/src/chat/src/parser/tool/tests.rs +++ b/rust/src/chat/src/parser/tool/tests.rs @@ -183,6 +183,10 @@ fn factory_new_resolves_default_patterns() { factory.resolve_name_for_model("tencent/Hy3-preview"), Some(names::HY_V3) ); + assert_eq!( + factory.resolve_name_for_model("tencent/Hy4-preview"), + Some(names::HY_V4) + ); assert_eq!( factory.resolve_name_for_model("MiniMax/MiniMax-M3-Text"), Some(names::MINIMAX_M3) diff --git a/rust/src/chat/src/parser/unified.rs b/rust/src/chat/src/parser/unified.rs index ccd0fff637..5ab830150a 100644 --- a/rust/src/chat/src/parser/unified.rs +++ b/rust/src/chat/src/parser/unified.rs @@ -6,8 +6,8 @@ use std::sync::LazyLock; pub use vllm_parser::unified::{ - Gemma4UnifiedParser, HyV3UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, - UnifiedParser, + Gemma4UnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser, InklingUnifiedParser, + KimiK3UnifiedParser, UnifiedParser, }; use vllm_tokenizer::DynTokenizer; @@ -18,6 +18,7 @@ use crate::request::ChatTool; pub mod names { pub const GEMMA4: &str = "gemma4"; pub const HY_V3: &str = "hy_v3"; + pub const HY_V4: &str = "hy_v4"; pub const INKLING: &str = "inkling"; pub const KIMI_K3: &str = "kimi_k3"; } @@ -44,6 +45,7 @@ impl UnifiedParserFactory { factory.register_parser::(names::GEMMA4); factory.register_parser::(names::HY_V3); + factory.register_parser::(names::HY_V4); factory.register_parser::(names::INKLING); factory.register_parser::(names::KIMI_K3); @@ -51,6 +53,7 @@ impl UnifiedParserFactory { .register_pattern("gemma-4", names::GEMMA4) .register_pattern("gemma4", names::GEMMA4) .register_pattern("hy3", names::HY_V3) + .register_pattern("hy4", names::HY_V4) .register_pattern("inkling", names::INKLING) .register_pattern("kimi-k3", names::KIMI_K3) .register_pattern("kimi_k3", names::KIMI_K3); @@ -147,4 +150,33 @@ mod tests { ); factory.create(names::KIMI_K3, &[], Arc::new(tokenizer)).unwrap(); } + + #[test] + fn factory_registers_hy_v4() { + let factory = UnifiedParserFactory::new(); + let tokenizer = [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + ] + .into_iter() + .enumerate() + .fold(TestTokenizer::new(), |tokenizer, (index, token)| { + tokenizer.with_regular_token(token, 1000 + index as u32) + }); + + assert!(factory.contains(names::HY_V4)); + assert_eq!( + factory.resolve_name_for_model("tencent/Hy4-preview"), + Some(names::HY_V4) + ); + factory.create(names::HY_V4, &[], Arc::new(tokenizer)).unwrap(); + } } diff --git a/rust/src/parser/src/reasoning/hy_v3.rs b/rust/src/parser/src/reasoning/hy.rs similarity index 76% rename from rust/src/parser/src/reasoning/hy_v3.rs rename to rust/src/parser/src/reasoning/hy.rs index f25f4a22e0..b61d987e82 100644 --- a/rust/src/parser/src/reasoning/hy_v3.rs +++ b/rust/src/parser/src/reasoning/hy.rs @@ -5,13 +5,13 @@ use vllm_tokenizer::DynTokenizer; use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, Result}; -/// Internal HY3 reasoning stage used by the unified HY3 parser. -pub(crate) struct HyV3ReasoningParser { +/// Internal HY reasoning stage used by the unified HY parsers. +pub(crate) struct HyReasoningParser { inner: DelimitedReasoningParser, } -impl HyV3ReasoningParser { - /// Create a HY3 reasoning parser for the tokenizer-specific marker suffix. +impl HyReasoningParser { + /// Create a HY reasoning parser for the tokenizer-specific marker suffix. pub(crate) fn new(tokenizer: DynTokenizer, suffix: &str) -> Result { Ok(Self { inner: DelimitedReasoningParser::new( @@ -24,15 +24,15 @@ impl HyV3ReasoningParser { } } -impl ReasoningParser for HyV3ReasoningParser { - // Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and +impl ReasoningParser for HyReasoningParser { + // Suffix discovery belongs to the unified HY parser so its reasoning and // tool delimiters always use the same tokenizer-derived value. fn create(_tokenizer: DynTokenizer) -> Result> where Self: Sized + 'static, { Err(ReasoningError::DummyUnifiedParser { - name: "hy_v3".to_string(), + name: "hy".to_string(), }) } diff --git a/rust/src/parser/src/reasoning/mod.rs b/rust/src/parser/src/reasoning/mod.rs index 7588fe8f58..ed355f1caf 100644 --- a/rust/src/parser/src/reasoning/mod.rs +++ b/rust/src/parser/src/reasoning/mod.rs @@ -20,7 +20,7 @@ mod cohere_cmd; mod deepseek_r1; mod delimited; -mod hy_v3; +mod hy; mod kimi; mod minimax_m3; mod qwen3; @@ -33,7 +33,7 @@ use vllm_tokenizer::DynTokenizer; pub use self::cohere_cmd::CohereCmdReasoningParser; pub use self::deepseek_r1::DeepSeekR1ReasoningParser; pub(crate) use self::delimited::{DelimitedReasoningParser, last_reasoning_boundary}; -pub(crate) use self::hy_v3::HyV3ReasoningParser; +pub(crate) use self::hy::HyReasoningParser; pub use self::kimi::KimiReasoningParser; pub use self::minimax_m3::MiniMaxM3ReasoningParser; pub use self::qwen3::Qwen3ReasoningParser; diff --git a/rust/src/parser/src/tool/hy_v3.rs b/rust/src/parser/src/tool/hy.rs similarity index 61% rename from rust/src/parser/src/tool/hy_v3.rs rename to rust/src/parser/src/tool/hy.rs index e289ab66ff..03ef1f3c41 100644 --- a/rust/src/parser/src/tool/hy_v3.rs +++ b/rust/src/parser/src/tool/hy.rs @@ -11,33 +11,56 @@ use winnow::prelude::*; use winnow::stream::Partial; use winnow::token::{literal, rest, take_until}; -use self::structural_tag::HyV3StructuralTagBuilder; +use self::structural_tag::HyStructuralTagBuilder; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, ToolCallDelta, ToolParser, ToolParserError, ToolParserOutput}; use crate::tool::{StructuralTagBuilder, Tool}; +/// Wire-level HY tool-call dialect. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HyDialect { + V3, + V4, +} + +impl HyDialect { + fn separator(self) -> &'static str { + match self { + Self::V3 => "\n", + Self::V4 => "", + } + } + + fn label(self) -> &'static str { + match self { + Self::V3 => "HY3", + Self::V4 => "HY4", + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct HyV3ToolMarkers { +pub(crate) struct HyToolMarkers { tool_calls_start: String, tool_calls_end: String, tool_call_start: String, tool_call_end: String, - tool_sep: String, + tool_sep: Option, arg_key_start: String, arg_key_end: String, arg_value_start: String, arg_value_end: String, } -impl HyV3ToolMarkers { - pub(crate) fn new(suffix: &str) -> Self { +impl HyToolMarkers { + pub(crate) fn new(suffix: &str, dialect: HyDialect) -> Self { Self { tool_calls_start: format!(""), tool_calls_end: format!(""), tool_call_start: format!(""), tool_call_end: format!(""), - tool_sep: format!(""), + tool_sep: (dialect == HyDialect::V3).then(|| format!("")), arg_key_start: format!(""), arg_key_end: format!(""), arg_value_start: format!(""), @@ -47,31 +70,32 @@ impl HyV3ToolMarkers { pub(crate) fn iter(&self) -> impl Iterator { [ - self.tool_calls_start.as_str(), - self.tool_calls_end.as_str(), - self.tool_call_start.as_str(), - self.tool_call_end.as_str(), - self.tool_sep.as_str(), - self.arg_key_start.as_str(), - self.arg_key_end.as_str(), - self.arg_value_start.as_str(), - self.arg_value_end.as_str(), + Some(self.tool_calls_start.as_str()), + Some(self.tool_calls_end.as_str()), + Some(self.tool_call_start.as_str()), + Some(self.tool_call_end.as_str()), + self.tool_sep.as_deref(), + Some(self.arg_key_start.as_str()), + Some(self.arg_key_end.as_str()), + Some(self.arg_value_start.as_str()), + Some(self.arg_value_end.as_str()), ] .into_iter() + .flatten() } } -type HyV3Input<'i> = Partial<&'i str>; +type HyInput<'i> = Partial<&'i str>; #[derive(Debug, Clone, PartialEq, Eq)] -enum HyV3Mode { +enum HyMode { Text, ToolBlock { tool_call_end_scan: MarkerScanState }, Done, } #[derive(Debug, Clone, PartialEq, Eq)] -enum HyV3Event { +enum HyEvent { Text { len: usize, }, @@ -84,7 +108,7 @@ enum HyV3Event { IgnoredRest, } -/// Tool parser for HY3 XML-style tool calls. +/// Tool parser for HY XML-style tool calls. /// /// Example tool call content: /// @@ -97,43 +121,45 @@ enum HyV3Event { /// ``` /// /// Arguments are emitted only after a full `` block is parsed. -/// HY3 marker tokens are added-vocabulary tokens rather than tokenizer special +/// HY marker tokens are added-vocabulary tokens rather than tokenizer special /// tokens, so the default `preserve_special_tokens() == false` is sufficient. -pub(crate) struct HyV3ToolParser { +pub(crate) struct HyToolParser { buffer: String, - mode: HyV3Mode, + mode: HyMode, emitted_tool_count: usize, tool_parameters: ToolSchemas, - markers: Arc, - structural_tag_builder: HyV3StructuralTagBuilder, + dialect: HyDialect, + markers: Arc, + structural_tag_builder: HyStructuralTagBuilder, } -impl HyV3ToolParser { - /// Create a HY3 tool parser. - pub(crate) fn new(tools: &[Tool], suffix: &str) -> Self { - let markers = Arc::new(HyV3ToolMarkers::new(suffix)); +impl HyToolParser { + /// Create a HY tool parser for one wire dialect. + pub(crate) fn new(tools: &[Tool], suffix: &str, dialect: HyDialect) -> Self { + let markers = Arc::new(HyToolMarkers::new(suffix, dialect)); Self { buffer: String::new(), - mode: HyV3Mode::Text, + mode: HyMode::Text, emitted_tool_count: 0, tool_parameters: ToolSchemas::from_tools(tools), + dialect, markers: Arc::clone(&markers), - structural_tag_builder: HyV3StructuralTagBuilder::new(markers), + structural_tag_builder: HyStructuralTagBuilder::new(markers, dialect), } } - /// Apply one parsed HY3 event to parser state and output. - fn apply_event(&mut self, event: HyV3Event, output: &mut ToolParserOutput) -> Result<()> { + /// Apply one parsed HY event to parser state and output. + fn apply_event(&mut self, event: HyEvent, output: &mut ToolParserOutput) -> Result<()> { match event { - HyV3Event::Text { len: consumed_len } => { + HyEvent::Text { len: consumed_len } => { output.push_text(&self.buffer[..consumed_len]); } - HyV3Event::ToolBlockStart => { - self.mode = HyV3Mode::ToolBlock { + HyEvent::ToolBlockStart => { + self.mode = HyMode::ToolBlock { tool_call_end_scan: MarkerScanState::default(), }; } - HyV3Event::ToolCall { name, raw_params } => { + HyEvent::ToolCall { name, raw_params } => { let arguments = self.tool_parameters.convert_params_with_schema(&name, raw_params); let arguments = serde_json::to_string(&arguments) .map_err(|error| parsing_failed!("failed to serialize arguments: {}", error))?; @@ -145,22 +171,22 @@ impl HyV3ToolParser { }); self.emitted_tool_count += 1; } - HyV3Event::ToolBlockEnd => self.mode = HyV3Mode::Done, - HyV3Event::IgnoredRest => {} + HyEvent::ToolBlockEnd => self.mode = HyMode::Done, + HyEvent::IgnoredRest => {} } Ok(()) } } -impl ToolParser for HyV3ToolParser { - // Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and +impl ToolParser for HyToolParser { + // Suffix discovery belongs to the unified HY parser so its reasoning and // tool delimiters always use the same tokenizer-derived value. fn create(_tools: &[Tool]) -> Result> where Self: Sized + 'static, { Err(ToolParserError::DummyUnifiedParser { - name: "hy_v3".to_string(), + name: "hy".to_string(), }) } @@ -172,7 +198,7 @@ impl ToolParser for HyV3ToolParser { self.buffer.push_str(chunk); while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| { - parse_next_hy_v3_event(input, &mut self.mode, &self.markers) + parse_next_hy_event(input, &mut self.mode, &self.markers, self.dialect) })? { self.apply_event(event, output)?; self.buffer.drain(..consumed_len); @@ -184,113 +210,145 @@ impl ToolParser for HyV3ToolParser { fn finish(&mut self) -> Result { let mut output = ToolParserOutput::default(); match self.mode { - HyV3Mode::Text => output.push_text(&self.buffer), - HyV3Mode::ToolBlock { .. } => return Err(parsing_failed!("incomplete HY3 tool call")), - HyV3Mode::Done => {} + HyMode::Text => output.push_text(&self.buffer), + HyMode::ToolBlock { .. } => { + return Err(parsing_failed!( + "incomplete {} tool call", + self.dialect.label() + )); + } + HyMode::Done => {} } let _ = self.reset(); Ok(output) } fn reset(&mut self) -> String { - self.mode = HyV3Mode::Text; + self.mode = HyMode::Text; self.emitted_tool_count = 0; std::mem::take(&mut self.buffer) } } -/// Parse a HY3 event for the current parser mode. -fn parse_next_hy_v3_event( - input: &mut HyV3Input<'_>, - mode: &mut HyV3Mode, - markers: &HyV3ToolMarkers, -) -> ModalResult { +/// Parse a HY event for the current parser mode. +fn parse_next_hy_event( + input: &mut HyInput<'_>, + mode: &mut HyMode, + markers: &HyToolMarkers, + dialect: HyDialect, +) -> ModalResult { match mode { - HyV3Mode::Text => parse_text_event(input, markers), - HyV3Mode::ToolBlock { tool_call_end_scan } => { - parse_tool_block_event(input, tool_call_end_scan, markers) + HyMode::Text => parse_text_event(input, markers), + HyMode::ToolBlock { tool_call_end_scan } => { + parse_tool_block_event(input, tool_call_end_scan, markers, dialect) } - HyV3Mode::Done => ignored_rest_event(input), + HyMode::Done => ignored_rest_event(input), } } -/// Parse a text-mode HY3 event. -fn parse_text_event( - input: &mut HyV3Input<'_>, - markers: &HyV3ToolMarkers, -) -> ModalResult { +/// Parse a text-mode HY event. +fn parse_text_event(input: &mut HyInput<'_>, markers: &HyToolMarkers) -> ModalResult { alt(( - |input: &mut HyV3Input<'_>| tool_block_start_event(input, markers), - |input: &mut HyV3Input<'_>| safe_text_event(input, markers), + |input: &mut HyInput<'_>| tool_block_start_event(input, markers), + |input: &mut HyInput<'_>| safe_text_event(input, markers), )) .parse_next(input) } -/// Parse a HY3 tool-block start marker. +/// Parse a HY tool-block start marker. fn tool_block_start_event( - input: &mut HyV3Input<'_>, - markers: &HyV3ToolMarkers, -) -> ModalResult { + input: &mut HyInput<'_>, + markers: &HyToolMarkers, +) -> ModalResult { literal(markers.tool_calls_start.as_str()) - .value(HyV3Event::ToolBlockStart) + .value(HyEvent::ToolBlockStart) .parse_next(input) } -/// Parse a safe text run before the next HY3 marker. -fn safe_text_event(input: &mut HyV3Input<'_>, markers: &HyV3ToolMarkers) -> ModalResult { - safe_text_len(input, &markers.tool_calls_start).map(|len| HyV3Event::Text { len }) +/// Parse a safe text run before the next HY marker. +fn safe_text_event(input: &mut HyInput<'_>, markers: &HyToolMarkers) -> ModalResult { + safe_text_len(input, &markers.tool_calls_start).map(|len| HyEvent::Text { len }) } -/// Parse one event inside a HY3 tool block. +/// Parse one event inside a HY tool block. fn parse_tool_block_event( - input: &mut HyV3Input<'_>, + input: &mut HyInput<'_>, tool_call_end_scan: &mut MarkerScanState, - markers: &HyV3ToolMarkers, -) -> ModalResult { + markers: &HyToolMarkers, + dialect: HyDialect, +) -> ModalResult { alt(( - |input: &mut HyV3Input<'_>| tool_block_end_event(input, markers), - |input: &mut HyV3Input<'_>| tool_call_event(input, tool_call_end_scan, markers), + |input: &mut HyInput<'_>| tool_block_end_event(input, markers), + |input: &mut HyInput<'_>| tool_call_event(input, tool_call_end_scan, markers, dialect), )) .parse_next(input) } -/// Parse a HY3 tool-block end marker. -fn tool_block_end_event( - input: &mut HyV3Input<'_>, - markers: &HyV3ToolMarkers, -) -> ModalResult { +/// Parse a HY tool-block end marker. +fn tool_block_end_event(input: &mut HyInput<'_>, markers: &HyToolMarkers) -> ModalResult { (ws0, literal(markers.tool_calls_end.as_str())) - .value(HyV3Event::ToolBlockEnd) + .value(HyEvent::ToolBlockEnd) .parse_next(input) } -/// Parse a complete HY3 tool-call block. +/// Parse a complete HY tool-call block. fn tool_call_event( - input: &mut HyV3Input<'_>, + input: &mut HyInput<'_>, tool_call_end_scan: &mut MarkerScanState, - markers: &HyV3ToolMarkers, -) -> ModalResult { - let (name, body) = seq!( + markers: &HyToolMarkers, + dialect: HyDialect, +) -> ModalResult { + let (body,) = seq!( _: ws0, _: literal(markers.tool_call_start.as_str()), - take_until(0.., markers.tool_sep.as_str()), - _: literal(markers.tool_sep.as_str()), take_until_marker(markers.tool_call_end.as_str(), tool_call_end_scan), _: literal(markers.tool_call_end.as_str()), ) .parse_next(input)?; - let raw_params = parse_tool_call_params(body, markers)?; + let mut body_input = body; + let (name, params) = parse_tool_call_body(&mut body_input, markers, dialect)?; + let raw_params = parse_tool_call_params(params, markers)?; - Ok(HyV3Event::ToolCall { + Ok(HyEvent::ToolCall { name: name.trim().to_string(), raw_params, }) } -/// Parse all parameter blocks inside a complete HY3 tool call. +/// Parse a complete HY tool-call body according to its dialect. +fn parse_tool_call_body<'i>( + input: &mut &'i str, + markers: &HyToolMarkers, + dialect: HyDialect, +) -> ModalResult<(&'i str, &'i str)> { + match dialect { + HyDialect::V3 => { + let tool_sep = markers.tool_sep.as_deref().expect("HY3 has a tool separator"); + terminated( + seq!( + take_until(0.., tool_sep), + _: literal(tool_sep), + rest, + ), + eof, + ) + .parse_next(input) + } + HyDialect::V4 => terminated( + alt(( + seq!(take_until(0.., markers.arg_key_start.as_str()), rest,), + rest.map(|name| (name, "")), + )), + eof, + ) + .parse_next(input), + } +} + +/// Parse all parameter blocks inside a complete HY tool call. fn parse_tool_call_params( tool_call_body: &str, - markers: &HyV3ToolMarkers, + markers: &HyToolMarkers, ) -> ModalResult> { let mut input = tool_call_body; delimited( @@ -304,8 +362,8 @@ fn parse_tool_call_params( .parse_next(&mut input) } -/// Parse a HY3 argument key/value block. -fn parameter(input: &mut &str, markers: &HyV3ToolMarkers) -> ModalResult<(String, String)> { +/// Parse a HY argument key/value block. +fn parameter(input: &mut &str, markers: &HyToolMarkers) -> ModalResult<(String, String)> { let (name, value) = seq!( _: literal(markers.arg_key_start.as_str()), take_until(0.., markers.arg_key_end.as_str()), @@ -320,9 +378,9 @@ fn parameter(input: &mut &str, markers: &HyV3ToolMarkers) -> ModalResult<(String Ok((name.trim().to_string(), value.to_string())) } -/// Parse ignored rest after the HY3 tool block ends. -fn ignored_rest_event(input: &mut HyV3Input<'_>) -> ModalResult { - rest.value(HyV3Event::IgnoredRest).parse_next(input) +/// Parse ignored rest after the HY tool block ends. +fn ignored_rest_event(input: &mut HyInput<'_>) -> ModalResult { + rest.value(HyEvent::IgnoredRest).parse_next(input) } #[cfg(test)] @@ -331,7 +389,7 @@ mod tests { use serde_json::{Value, json}; use thiserror_ext::AsReport; - use super::{HyV3ToolParser, ToolParser}; + use super::{HyDialect, HyToolParser, ToolParser}; use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; use crate::tool::{ToolParserOutput, ToolParserTestExt as _}; @@ -354,14 +412,14 @@ mod tests { #[test] fn hy_v3_does_not_preserve_special_tokens() { - let parser = HyV3ToolParser::new(&test_tools(), ""); + let parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); assert!(!parser.preserve_special_tokens()); } #[test] fn hy_v3_parse_complete_without_tool_call_keeps_text() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let output = parser.parse_complete("This is a plain response.").unwrap(); assert_eq!(output.normal_text(), "This is a plain response."); @@ -370,7 +428,7 @@ mod tests { #[test] fn hy_v3_parse_complete_extracts_zero_arg_inline_tool_call() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let output = parser .parse_complete( "get_current_date", @@ -385,7 +443,7 @@ mod tests { #[test] fn hy_v3_parse_complete_extracts_zero_arg_newline_tool_call() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let output = parser .parse_complete( "\nget_current_date\n\n", @@ -398,7 +456,7 @@ mod tests { #[test] fn hy_v3_parse_complete_extracts_arguments_on_same_line() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let output = parser .parse_complete( "get_weathercityBeijingdate2026-03-30", @@ -413,7 +471,7 @@ mod tests { #[test] fn hy_v3_parse_complete_extracts_arguments_with_newlines() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let output = parser .parse_complete(&build_tool_calls(&[build_tool_call( "get_weather", @@ -429,7 +487,7 @@ mod tests { #[test] fn hy_v3_parse_complete_preserves_prefix_and_ignores_trailing_text() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let output = parser .parse_complete(&format!( "Checking.{} trailing text", @@ -443,7 +501,7 @@ mod tests { #[test] fn hy_v3_parse_complete_extracts_multiple_tool_calls_in_one_block() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let output = parser .parse_complete(&build_tool_calls(&[ build_tool_call( @@ -486,7 +544,7 @@ mod tests { #[test] fn hy_v3_parse_complete_converts_schema_types() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let output = parser .parse_complete(&build_tool_calls(&[build_tool_call( "convert", @@ -512,7 +570,7 @@ mod tests { #[test] fn hy_v3_streaming_without_tool_call_emits_text_incrementally() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let mut output = ToolParserOutput::default(); output.append(parser.parse_chunk("This is ").unwrap()); @@ -526,7 +584,7 @@ mod tests { #[test] fn hy_v3_streaming_extracts_zero_arg_tool_call() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let chunks = [ "", "\n", @@ -545,7 +603,7 @@ mod tests { #[test] fn hy_v3_streaming_extracts_arguments() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let chunks = [ "", "\n", @@ -571,7 +629,7 @@ mod tests { #[test] fn hy_v3_streaming_preserves_prefix_text() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let chunks = [ "Checking.", "", @@ -601,7 +659,7 @@ mod tests { ), ]); let chunks = split_by_chars(&input, 9); - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let output = collect_stream(&mut parser, &chunks); @@ -617,7 +675,7 @@ mod tests { build_tool_calls(&[build_tool_call("get_weather", &[("city", "Beijing")])]) ); let chunks = split_by_chars(&input, 5); - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let output = collect_stream(&mut parser, &chunks); @@ -628,7 +686,7 @@ mod tests { #[test] fn hy_v3_streaming_does_not_emit_incomplete_tool_call() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let mut output = ToolParserOutput::default(); parser @@ -644,7 +702,7 @@ mod tests { #[test] fn hy_v3_finish_fails_incomplete_tool_call() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); parser.parse_chunk("get_weather").unwrap(); let error = parser.finish().unwrap_err(); @@ -655,7 +713,7 @@ mod tests { #[test] fn hy_v3_malformed_tool_call_fails_fast() { - let mut parser = HyV3ToolParser::new(&test_tools(), ""); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3); let error = parser .parse_complete( "get_weathercityBeijing", @@ -664,4 +722,78 @@ mod tests { assert!(error.to_report_string().starts_with("tool parser parsing failed:")); } + + #[test] + fn hy_v4_parse_complete_extracts_compact_calls_with_and_without_arguments() { + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4); + let output = parser + .parse_complete( + "get_weathercityBeijingdate2026-03-30get_current_date", + ) + .unwrap(); + + expect![[r#" + ToolParserOutput { + events: [ + ToolCall( + ToolCallDelta { + tool_index: 0, + name: Some( + "get_weather", + ), + arguments: "{\"city\":\"Beijing\",\"date\":\"2026-03-30\"}", + }, + ), + ToolCall( + ToolCallDelta { + tool_index: 1, + name: Some( + "get_current_date", + ), + arguments: "{}", + }, + ), + ], + } + "#]] + .assert_debug_eq(&output); + } + + #[test] + fn hy_v4_streaming_handles_markers_split_across_chunks() { + let input = "prefixget_weathercity上海get_current_date"; + let chunks = split_by_chars(input, 7); + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4); + + let output = collect_stream(&mut parser, &chunks); + + assert_eq!(output.normal_text(), "prefix"); + assert_eq!(output.calls().len(), 2); + assert_eq!(parsed_arguments(&output, 0), json!({ "city": "上海" })); + assert_eq!(output.calls()[1].name.as_deref(), Some("get_current_date")); + assert_eq!(parsed_arguments(&output, 1), json!({})); + } + + #[test] + fn hy_v4_finish_reports_its_dialect_for_incomplete_calls() { + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4); + parser.parse_chunk("get_weather").unwrap(); + + let error = parser.finish().unwrap_err(); + + expect!["tool parser parsing failed: incomplete HY4 tool call"] + .assert_eq(&error.to_report_string()); + } + + #[test] + fn hy_v4_malformed_argument_pair_fails_fast() { + let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4); + let error = parser + .parse_complete( + "get_weathercityBeijing", + ) + .unwrap_err(); + + assert!(error.to_report_string().starts_with("tool parser parsing failed:")); + } } diff --git a/rust/src/parser/src/tool/hy_v3/structural_tag.rs b/rust/src/parser/src/tool/hy/structural_tag.rs similarity index 74% rename from rust/src/parser/src/tool/hy_v3/structural_tag.rs rename to rust/src/parser/src/tool/hy/structural_tag.rs index ef182da267..cce98d5b09 100644 --- a/rust/src/parser/src/tool/hy_v3/structural_tag.rs +++ b/rust/src/parser/src/tool/hy/structural_tag.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project -//! Structural-tag grammar for HY3 XML-style tool calls. +//! Structural-tag grammar for HY XML-style tool calls. use std::collections::HashSet; use std::sync::Arc; @@ -12,31 +12,40 @@ use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagConte use xgrammar_structural_tag::format::{Format, StructuralTag, TagFormat}; use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam}; -use super::HyV3ToolMarkers; +use super::{HyDialect, HyToolMarkers}; -/// HY3 structural-tag builder using tokenizer-specific structural markers. +/// HY structural-tag builder using tokenizer-specific structural markers. #[derive(Debug, Clone)] -pub(super) struct HyV3StructuralTagBuilder { - markers: Arc, +pub(super) struct HyStructuralTagBuilder { + markers: Arc, + dialect: HyDialect, } -impl HyV3StructuralTagBuilder { - pub(super) fn new(markers: Arc) -> Self { - Self { markers } +impl HyStructuralTagBuilder { + pub(super) fn new(markers: Arc, dialect: HyDialect) -> Self { + Self { markers, dialect } } fn argument_pair(&self, key: &str) -> Format { let excludes = self.markers.iter().collect::>(); - Format::sequence(vec![ + let separator = self.dialect.separator(); + let mut elements = vec![ Format::const_string(&self.markers.arg_key_start), Format::const_string(key), Format::const_string(&self.markers.arg_key_end), - Format::const_string("\n"), + ]; + if !separator.is_empty() { + elements.push(Format::const_string(separator)); + } + elements.extend([ Format::const_string(&self.markers.arg_value_start), Format::any_text_excluding(&excludes), Format::const_string(&self.markers.arg_value_end), - Format::const_string("\n"), - ]) + ]); + if !separator.is_empty() { + elements.push(Format::const_string(separator)); + } + Format::sequence(elements) } fn tool_call(&self, tool: &FunctionToolParam) -> TagFormat { @@ -56,14 +65,21 @@ impl HyV3StructuralTagBuilder { } let content = if elements.is_empty() { - Format::any_text() + match self.dialect { + HyDialect::V3 => Format::any_text(), + HyDialect::V4 => Format::const_string(""), + } } else { Format::sequence(elements) }; + let tool_sep = self.markers.tool_sep.as_deref().unwrap_or_default(); TagFormat::new( format!( - "{}{}{}\n", - self.markers.tool_call_start, tool.function.name, self.markers.tool_sep + "{}{}{}{}", + self.markers.tool_call_start, + tool.function.name, + tool_sep, + self.dialect.separator(), ), content, self.markers.tool_call_end.clone(), @@ -72,15 +88,16 @@ impl HyV3StructuralTagBuilder { fn tool_calls(&self, tools: &[FunctionToolParam], choice: BuilderToolChoice) -> Format { let mut calls = tools.iter().map(|tool| self.tool_call(tool)).collect::>(); - let begin = format!("{}\n", self.markers.tool_calls_start); - let end = format!("\n{}", self.markers.tool_calls_end); + let separator = self.dialect.separator(); + let begin = format!("{}{separator}", self.markers.tool_calls_start); + let end = format!("{separator}{}", self.markers.tool_calls_end); match choice { BuilderToolChoice::Auto if calls.is_empty() => Format::any_text(), BuilderToolChoice::Auto => { let outer = TagFormat::new( begin, - Format::tags_with_separator(calls, "\n", true, false), + Format::tags_with_separator(calls, separator, true, false), end, ); Format::triggered_tags(&[&self.markers.tool_calls_start], vec![outer]) @@ -92,14 +109,14 @@ impl HyV3StructuralTagBuilder { ]), BuilderToolChoice::Required => Format::sequence(vec![ Format::const_string(begin), - Format::tags_with_separator(calls, "\n", true, false), + Format::tags_with_separator(calls, separator, true, false), Format::const_string(end), ]), } } } -impl StructuralTagBuilder for HyV3StructuralTagBuilder { +impl StructuralTagBuilder for HyStructuralTagBuilder { fn build(&self, ctx: StructuralTagContext<'_>) -> Result { Ok(StructuralTag::new( self.tool_calls(ctx.function_tools, ctx.tool_choice), @@ -152,8 +169,8 @@ mod tests { FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag, }; - use super::HyV3StructuralTagBuilder; - use crate::tool::HyV3ToolMarkers; + use super::HyStructuralTagBuilder; + use crate::tool::{HyDialect, HyToolMarkers}; fn tool(name: &str, parameters: Value) -> ToolParam { ToolParam::Function(FunctionToolParam::new( @@ -165,9 +182,18 @@ mod tests { suffix: &str, tools: &[ToolParam], choice: ToolChoice, + ) -> xgrammar_structural_tag::format::StructuralTag { + build_for_dialect(HyDialect::V3, suffix, tools, choice) + } + + fn build_for_dialect( + dialect: HyDialect, + suffix: &str, + tools: &[ToolParam], + choice: ToolChoice, ) -> xgrammar_structural_tag::format::StructuralTag { build_structural_tag( - HyV3StructuralTagBuilder::new(Arc::new(HyV3ToolMarkers::new(suffix))), + HyStructuralTagBuilder::new(Arc::new(HyToolMarkers::new(suffix, dialect)), dialect), tools, choice, StructuralTagOptions::default().with_reasoning(false), @@ -196,6 +222,25 @@ mod tests { expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tags_with_separator","tags":[{"begin":"get_weather\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"days"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""]},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}}]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); } + #[test] + fn required_uses_suffixed_compact_hy4_skeleton_and_bounded_values() { + let tag = build_for_dialect( + HyDialect::V4, + ":opensource", + &[tool( + "get_weather", + json!({ + "type": "object", + "properties": { "city": { "type": "string" } }, + "required": ["city"] + }), + )], + ToolChoice::required(), + ); + + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"tags_with_separator","tags":[{"begin":"get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","",""]},{"type":"const_string","value":""}]}]},"end":""}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":""}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); + } + #[test] fn optional_only_schema_keeps_declared_key_alternatives() { let tag = build( @@ -235,6 +280,24 @@ mod tests { expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[]},"end":""},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&forced); } + #[test] + fn hy_v4_zero_argument_tool_has_empty_compact_body() { + let tag = build_for_dialect( + HyDialect::V4, + "", + &[tool( + "get_current_date", + json!({ "type": "object", "properties": {} }), + )], + ToolChoice::function("get_current_date"), + ); + let value = serde_json::to_value(tag).unwrap(); + + expect![[r#"{"type":"tag","begin":"get_current_date","content":{"type":"const_string","value":""},"end":""}"#]].assert_eq( + &serde_json::to_string(&value["format"]["elements"][1]).unwrap(), + ); + } + #[test] fn missing_required_property_is_still_emitted() { let tag = build( diff --git a/rust/src/parser/src/tool/mod.rs b/rust/src/parser/src/tool/mod.rs index 940072bdac..83510aeaf0 100644 --- a/rust/src/parser/src/tool/mod.rs +++ b/rust/src/parser/src/tool/mod.rs @@ -8,7 +8,7 @@ pub(crate) mod error; mod deepseek_dsml; pub(crate) mod deepseek_json; mod glm_xml; -mod hy_v3; +mod hy; pub(crate) mod json; mod kimi_k2; mod minimax_m2; @@ -24,7 +24,7 @@ pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser}; pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser}; pub use error::{Result, ToolParserError}; pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser}; -pub(crate) use hy_v3::{HyV3ToolMarkers, HyV3ToolParser}; +pub(crate) use hy::{HyDialect, HyToolMarkers, HyToolParser}; pub use json::{ Granite4ToolParser, HermesToolParser, Internlm2ToolParser, Llama3JsonToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3XmlToolParser, diff --git a/rust/src/parser/src/unified/hy_v3.rs b/rust/src/parser/src/unified/hy/hy_v3.rs similarity index 79% rename from rust/src/parser/src/unified/hy_v3.rs rename to rust/src/parser/src/unified/hy/hy_v3.rs index ede3b51dd2..069eb5de80 100644 --- a/rust/src/parser/src/unified/hy_v3.rs +++ b/rust/src/parser/src/unified/hy/hy_v3.rs @@ -1,20 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project -use vllm_tokenizer::{DynTokenizer, Tokenizer}; +use vllm_tokenizer::DynTokenizer; -use super::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id}; -use crate::reasoning::HyV3ReasoningParser; -use crate::tool::{HyV3ToolMarkers, HyV3ToolParser, StructuralTagBuilder, Tool}; - -const HY_V3_MARKER_STEMS: &[&str] = &[ - "think", - "tool_calls", - "tool_call", - "tool_sep", - "arg_key", - "arg_value", -]; +use super::detect_hy_token_suffix; +use crate::reasoning::HyReasoningParser; +use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, StructuralTagBuilder, Tool}; +use crate::unified::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id}; /// Unified reasoning and tool parser for HY3 output. pub struct HyV3UnifiedParser { @@ -24,14 +16,14 @@ pub struct HyV3UnifiedParser { impl HyV3UnifiedParser { /// Create a HY3 parser using the suffix encoded in tokenizer added tokens. pub fn new(tools: &[Tool], tokenizer: DynTokenizer) -> Result { - let suffix = detect_token_suffix(tokenizer.as_ref()); - let markers = HyV3ToolMarkers::new(&suffix); + let suffix = detect_hy_token_suffix(tokenizer.as_ref()); + let markers = HyToolMarkers::new(&suffix, HyDialect::V3); for marker in markers.iter() { token_id(tokenizer.as_ref(), marker)?; } - let reasoning = HyV3ReasoningParser::new(tokenizer, &suffix)?; - let tool = HyV3ToolParser::new(tools, &suffix); + let reasoning = HyReasoningParser::new(tokenizer, &suffix)?; + let tool = HyToolParser::new(tools, &suffix, HyDialect::V3); Ok(Self { inner: CombinedParser::new(Some(Box::new(reasoning)), Some(Box::new(tool))), }) @@ -75,28 +67,6 @@ impl UnifiedParser for HyV3UnifiedParser { } } -/// Detect the HY3 structural-token suffix from tokenizer added vocabulary. -fn detect_token_suffix(tokenizer: &dyn Tokenizer) -> String { - tokenizer - .added_vocab() - .iter() - .filter_map(|(token, id)| marker_suffix(token).map(|suffix| (*id, suffix))) - .min_by_key(|(id, _)| *id) - .map(|(_, suffix)| suffix.to_string()) - .unwrap_or_default() -} - -/// Extract the suffix from one opening or closing HY3 structural token. -fn marker_suffix(token: &str) -> Option<&str> { - let body = token.strip_prefix('<')?.strip_suffix('>')?; - let body = body.strip_prefix('/').unwrap_or(body); - - HY_V3_MARKER_STEMS.iter().find_map(|stem| { - let suffix = body.strip_prefix(stem)?; - (suffix.is_empty() || suffix.starts_with(':')).then_some(suffix) - }) -} - #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/rust/src/parser/src/unified/hy/hy_v4.rs b/rust/src/parser/src/unified/hy/hy_v4.rs new file mode 100644 index 0000000000..d3ad175bf9 --- /dev/null +++ b/rust/src/parser/src/unified/hy/hy_v4.rs @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use vllm_tokenizer::DynTokenizer; + +use super::detect_hy_token_suffix; +use crate::reasoning::HyReasoningParser; +use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, StructuralTagBuilder, Tool}; +use crate::unified::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id}; + +/// Unified reasoning and tool parser for HY4 output. +pub struct HyV4UnifiedParser { + inner: CombinedParser, +} + +impl HyV4UnifiedParser { + /// Create a HY4 parser using the suffix encoded in tokenizer added tokens. + pub fn new(tools: &[Tool], tokenizer: DynTokenizer) -> Result { + let suffix = detect_hy_token_suffix(tokenizer.as_ref()); + let markers = HyToolMarkers::new(&suffix, HyDialect::V4); + for marker in markers.iter() { + token_id(tokenizer.as_ref(), marker)?; + } + + let reasoning = HyReasoningParser::new(tokenizer, &suffix)?; + let tool = HyToolParser::new(tools, &suffix, HyDialect::V4); + Ok(Self { + inner: CombinedParser::new(Some(Box::new(reasoning)), Some(Box::new(tool))), + }) + } +} + +impl UnifiedParser for HyV4UnifiedParser { + fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result> + where + Self: Sized + 'static, + { + Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box) + } + + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + self.inner.initialize(prompt_token_ids) + } + + fn preserve_special_tokens(&self) -> bool { + self.inner.preserve_special_tokens() + } + + fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> { + self.inner.structural_tag_builder() + } + + fn tool_call_id(&self, tool_index: usize) -> Option<&str> { + self.inner.tool_call_id(tool_index) + } + + fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()> { + self.inner.parse_into(delta, output) + } + + fn finish(&mut self) -> Result { + self.inner.finish() + } + + fn reset(&mut self) -> String { + self.inner.reset() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use serde_json::json; + use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer}; + use xgrammar_structural_tag::builders::StructuralTagOptions; + use xgrammar_structural_tag::{ + FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag, + }; + + use super::{HyV4UnifiedParser, UnifiedParser}; + use crate::tool::Tool; + use crate::unified::{UnifiedParserEvent, UnifiedParserOutput}; + + fn tokenizer() -> TestTokenizer { + [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + ] + .into_iter() + .enumerate() + .fold(TestTokenizer::new(), |tokenizer, (index, token)| { + tokenizer.with_regular_token(token, 1000 + index as u32) + }) + } + + fn tools() -> Vec { + vec![ + Tool { + name: "get_weather".to_string(), + description: None, + parameters: json!({ + "type": "object", + "properties": { "city": { "type": "string" } }, + }), + strict: None, + }, + Tool { + name: "get_current_date".to_string(), + description: None, + parameters: json!({ "type": "object", "properties": {} }), + strict: None, + }, + ] + } + + #[test] + fn parses_suffixed_reasoning_and_compact_tool_calls_as_one_stream() { + let tokenizer = Arc::new(tokenizer()); + let think_start_id = tokenizer.token_to_id("").unwrap(); + let mut parser = HyV4UnifiedParser::new(&tools(), tokenizer).unwrap(); + parser.initialize(&[think_start_id]).unwrap(); + + let chunks = [ + "reasoninganswerget_weather", + "city", + "Beijing", + "get_current_date", + "", + ]; + let mut output = UnifiedParserOutput::default(); + for chunk in chunks { + parser.parse_into(chunk, &mut output).unwrap(); + } + output.append(parser.finish().unwrap()); + + assert_eq!( + output.events, + vec![ + UnifiedParserEvent::Reasoning("reasoning".to_string()), + UnifiedParserEvent::Text("answer".to_string()), + UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta { + tool_index: 0, + name: Some("get_weather".to_string()), + arguments: r#"{"city":"Beijing"}"#.to_string(), + }), + UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta { + tool_index: 1, + name: Some("get_current_date".to_string()), + arguments: "{}".to_string(), + }), + ] + ); + } + + #[test] + fn structural_tag_uses_compact_hy4_skeleton() { + let parser = HyV4UnifiedParser::new(&tools(), Arc::new(tokenizer())).unwrap(); + let structural_tools = [ToolParam::Function(FunctionToolParam::new( + FunctionDefinition::new("get_weather").with_parameters(json!({ + "type": "object", + "properties": { "city": { "type": "string" } }, + "required": ["city"] + })), + ))]; + + let tag = build_structural_tag( + parser.structural_tag_builder().unwrap(), + &structural_tools, + ToolChoice::required(), + StructuralTagOptions::default().with_reasoning(false), + ) + .unwrap() + .to_json_string() + .unwrap(); + + assert!(tag.contains("")); + assert!(tag.contains("get_weather")); + assert!(tag.contains("")); + assert!(tag.contains("")); + assert!(tag.contains("")); + assert!(!tag.contains("tool_sep")); + assert!(!tag.contains(r#"\n"#)); + } +} diff --git a/rust/src/parser/src/unified/hy/mod.rs b/rust/src/parser/src/unified/hy/mod.rs new file mode 100644 index 0000000000..dd707f05eb --- /dev/null +++ b/rust/src/parser/src/unified/hy/mod.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +mod hy_v3; +mod hy_v4; + +pub use hy_v3::HyV3UnifiedParser; +pub use hy_v4::HyV4UnifiedParser; + +const HY_MARKER_STEMS: &[&str] = &[ + "think", + "tool_calls", + "tool_call", + "tool_sep", + "arg_key", + "arg_value", +]; + +/// Detect the HY structural-token suffix from tokenizer added vocabulary. +fn detect_hy_token_suffix(tokenizer: &dyn vllm_tokenizer::Tokenizer) -> String { + tokenizer + .added_vocab() + .iter() + .filter_map(|(token, id)| hy_marker_suffix(token).map(|suffix| (*id, suffix))) + .min_by_key(|(id, _)| *id) + .map(|(_, suffix)| suffix.to_string()) + .unwrap_or_default() +} + +/// Extract the suffix from one opening or closing HY structural token. +fn hy_marker_suffix(token: &str) -> Option<&str> { + let body = token.strip_prefix('<')?.strip_suffix('>')?; + let body = body.strip_prefix('/').unwrap_or(body); + + HY_MARKER_STEMS.iter().find_map(|stem| { + let suffix = body.strip_prefix(stem)?; + (suffix.is_empty() || suffix.starts_with(':')).then_some(suffix) + }) +} diff --git a/rust/src/parser/src/unified/mod.rs b/rust/src/parser/src/unified/mod.rs index 9deac1853f..9f5547bcf2 100644 --- a/rust/src/parser/src/unified/mod.rs +++ b/rust/src/parser/src/unified/mod.rs @@ -5,13 +5,13 @@ mod combined; mod gemma4; -mod hy_v3; +mod hy; mod inkling; mod kimi_k3; pub use combined::CombinedParser; pub use gemma4::Gemma4UnifiedParser; -pub use hy_v3::HyV3UnifiedParser; +pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser}; pub use inkling::InklingUnifiedParser; pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser}; use thiserror::Error; diff --git a/tests/models/registry.py b/tests/models/registry.py index 819c7c4ff8..033ea454b6 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -331,6 +331,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"), "HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"), "HYV3ForCausalLM": _HfExamplesInfo("tencent/Hy3-preview", trust_remote_code=True), + # Internal checkpoint, not publicly available yet. + "HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True), "HyperCLOVAXForCausalLM": _HfExamplesInfo( "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B", min_transformers_version="5.9.0", @@ -1723,6 +1725,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { "tencent/Hy3-preview", speculative_model="tencent/Hy3-preview", ), + # Internal checkpoint, not publicly available yet. + "HYV4MTPModel": _HfExamplesInfo( + "tencent/Hy4-preview", + speculative_model="tencent/Hy4-preview", + ), "InklingMTPModel": _HfExamplesInfo( "thinkingmachines/Inkling-NVFP4", speculative_model="thinkingmachines/Inkling-NVFP4", diff --git a/tests/models/test_registry.py b/tests/models/test_registry.py index 81ccbf63a4..25eeb3f6a0 100644 --- a/tests/models/test_registry.py +++ b/tests/models/test_registry.py @@ -59,6 +59,11 @@ def test_registry_imports(model_arch): ): pytest.skip("Dots3 NOTE is only supported on CUDA") + if model_arch in ("HYV4ForCausalLM", "HYV4MTPModel") and not ( + current_platform.is_cuda() + ): + pytest.skip("HY V4 is only supported on CUDA") + # Ensure all model classes can be imported successfully model_cls = ModelRegistry._try_load_model_cls(model_arch) assert model_cls is not None diff --git a/tests/models/utils.py b/tests/models/utils.py index 6bd5431829..aff9fb0f5a 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -552,6 +552,11 @@ def dummy_hf_overrides( text_config.update(update_dict) + # Keep per-layer metadata consistent with the reduced layer count. + # HY V4 uses this to decide which sparse-attention indexers are shared. + if indexer_types := getattr(text_config, "indexer_types", None): + text_config.update({"indexer_types": indexer_types[:num_hidden_layers]}) + # Update n_layers and moe configs for Moondream3 model if model_arch in ("Moondream3ForCausalLM", "HfMoondream"): text_config.update( diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index d3b4af1cf0..564f13cddf 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -57,6 +57,7 @@ MTPModelTypes = Literal[ "pangu_ultra_moe_mtp", "step3p5_mtp", "hy_v3_mtp", + "hy_v4_mtp", "gemma4_mtp", "inkling_mtp", ] @@ -917,6 +918,13 @@ class SpeculativeConfig: {"n_predict": n_predict, "architectures": ["HYV3MTPModel"]} ) + if hf_config.model_type == "hy_v4": + hf_config.model_type = "hy_v4_mtp" + n_predict = getattr(hf_config, "num_nextn_predict_layers", None) + hf_config.update( + {"n_predict": n_predict, "architectures": ["HYV4MTPModel"]} + ) + if hf_config.model_type in ("inkling_mm_model", "inkling_model"): mtp_config = getattr(hf_config, "mtp_config", None) or {} hf_config = getattr(hf_config, "text_config", hf_config) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index a7fa4e1dca..0015d6693c 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -79,6 +79,8 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset( "DeepseekV4ForCausalLM", "DeepSeekV4MTPModel", "GlmMoeDsaForCausalLM", + "HYV4ForCausalLM", + "HYV4MTPModel", "InklingForCausalLM", "InklingForConditionalGeneration", "KimiK3ForConditionalGeneration", diff --git a/vllm/envs.py b/vllm/envs.py index 464d8d5479..6abcb4498a 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -321,6 +321,7 @@ if TYPE_CHECKING: VLLM_GPU_NIC_PCIE_MAPPING: str = "" VLLM_NIC_SELECTION_VARS: str = "" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: int | None = None + VLLM_ENABLE_HPC_OPS: bool = False def get_default_cache_root(): @@ -2155,6 +2156,15 @@ environment_variables: dict[str, Callable[[], Any]] = { # Each entry is VAR_NAME or VAR_NAME: (suffix appended to # RDMA device name). Must be set together with VLLM_GPU_NIC_PCIE_MAPPING. "VLLM_NIC_SELECTION_VARS": lambda: os.getenv("VLLM_NIC_SELECTION_VARS", ""), + # If set to 1, enable the HPC fused kernels (requires the hpc package + # (.so) and an sm100/sm103 device). Covers: + # * the HY V4 iHC ops -- each of the eager HYV4HCPreLayer / + # HYV4HCPostLayer / HYV4HCHeadLayer bodies becomes one kernel launch; + # * the gated-MLA output gating (attn_out * sigmoid(gate projection)), + # fused into gated_mla_gemm; elementwise gating only. + # Each op additionally checks its own shape / dtype constraints and falls + # back to the eager path when they do not hold. + "VLLM_ENABLE_HPC_OPS": lambda: bool(int(os.getenv("VLLM_ENABLE_HPC_OPS", "0"))), } diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py index 00ab06fe95..4cb6076c81 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py @@ -79,12 +79,27 @@ class TrtLlmFp8ExpertsBase: activation_key, activation_format, ) - if not supported or moe_config.num_experts <= 2048: + if not supported: return supported, reason - return False, ( - "FlashInfer TRTLLM routing supports at most 2048 experts, " - f"but got {moe_config.num_experts}" - ) + # FlashInfer accepts gemm1_alpha/beta/clamp_limit only for MXFP8; the + # DeepSeek-FP8 block-scale kernel rejects them and the per-tensor kernel + # has no clamp at all, so a model-requested SwiGLU clamp cannot be + # honored on those paths. + if ( + moe_config.swiglu_limit is not None + or moe_config.swiglu_alpha is not None + or moe_config.swiglu_beta is not None + ) and (weight_key, activation_key) != (kMxfp8Static, kMxfp8Dynamic): + return False, ( + "the TRTLLM FP8 kernels apply the SwiGLU alpha/beta/clamp " + "parameters only for MXFP8 weights" + ) + if moe_config.num_experts > 2048: + return False, ( + "FlashInfer TRTLLM routing supports at most 2048 experts, " + f"but got {moe_config.num_experts}" + ) + return True, None def __init__( self, diff --git a/vllm/model_executor/layers/hpc/__init__.py b/vllm/model_executor/layers/hpc/__init__.py index 47d39139d4..ac349578ad 100644 --- a/vllm/model_executor/layers/hpc/__init__.py +++ b/vllm/model_executor/layers/hpc/__init__.py @@ -1,11 +1,25 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm.model_executor.layers.hpc.gated_mla import ( + hpc_gated_mla_gemm, + hpc_gated_mla_supported, +) +from vllm.model_executor.layers.hpc.hpc_ihc import ( + HpcIHCHead, + HpcIHCPost, + HpcIHCPre, +) from vllm.model_executor.layers.hpc.hpc_module import HpcModule from vllm.model_executor.layers.hpc.rope_norm import HpcRopeNorm, QkNormPolicy __all__ = [ + "HpcIHCHead", + "HpcIHCPost", + "HpcIHCPre", "HpcModule", "HpcRopeNorm", "QkNormPolicy", + "hpc_gated_mla_gemm", + "hpc_gated_mla_supported", ] diff --git a/vllm/model_executor/layers/hpc/gated_mla.py b/vllm/model_executor/layers/hpc/gated_mla.py new file mode 100644 index 0000000000..f1e495a3a7 --- /dev/null +++ b/vllm/model_executor/layers/hpc/gated_mla.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Enablement gate for the HPC fused gated-MLA GEMM. + +``hpc.gated_mla_gemm`` fuses the MLA output gating -- the gate projection, the +sigmoid and the elementwise product -- into a single kernel launch, replacing +the eager ``attn_out * sigmoid(hidden_states @ weight.T)`` sequence. + +Kernel constraints: + - Requires VLLM_ENABLE_HPC_OPS=1 + - Requires the hpc package (.so) built for the current arch + - Only sm100 / sm103 (compute capability 100, 103) + - All three operands must be bfloat16 and contiguous + - The gate width must be a multiple of 256 + - Only elementwise gating (headwise has no fused form) +""" + +import torch + +from vllm import envs +from vllm.logger import init_logger +from vllm.utils.hpc import has_hpc + +logger = init_logger(__name__) + +# hpc.gated_mla_gemm fuses `attn_out * sigmoid(x @ weight.T)` into one launch. +# It is built for sm100/sm103 and needs all three operands bf16 + contiguous, +# with the gate width a multiple of 256. +_HPC_GATED_GEMM_N_ALIGN = 256 +_HPC_GATED_GEMM_CAPABILITIES: frozenset[int] = frozenset({100, 103}) + + +def hpc_gated_mla_supported( + gating_type: str | None, attn_output_gate: torch.nn.Module | None +) -> bool: + """Gate for running the MLA output gating through hpc.gated_mla_gemm.""" + if not envs.VLLM_ENABLE_HPC_OPS: + return False + + if attn_output_gate is None: + return False + + if not has_hpc(): + logger.warning_once( + "HPC gated MLA disabled: 'hpc' package is not installed. " + "Please install the HPC library (.so) to enable the fused kernel." + ) + return False + + from vllm.platforms import current_platform + + if not current_platform.is_cuda(): + logger.warning_once("HPC gated MLA disabled: only CUDA is supported.") + return False + + capability = current_platform.get_device_capability() + if capability is None or capability.to_int() not in _HPC_GATED_GEMM_CAPABILITIES: + logger.warning_once( + "HPC gated MLA disabled: compute capability %s not in %s.", + capability, + _HPC_GATED_GEMM_CAPABILITIES, + ) + return False + + # Headwise gating broadcasts one scalar per head over v_head_dim; the + # kernel only implements the elementwise product. + if gating_type != "elementwise": + logger.warning_once( + "HPC gated MLA disabled: gating_type '%s' has no fused form.", + gating_type, + ) + return False + + weight = getattr(attn_output_gate, "weight", None) + if weight is None or weight.dtype != torch.bfloat16: + logger.warning_once( + "HPC gated MLA disabled: gate weight dtype is %s, the kernel needs " + "bfloat16 (keep the layer out of quantization).", + None if weight is None else weight.dtype, + ) + return False + + if weight.shape[0] % _HPC_GATED_GEMM_N_ALIGN != 0: + logger.warning_once( + "HPC gated MLA disabled: gate width %d is not a multiple of %d.", + weight.shape[0], + _HPC_GATED_GEMM_N_ALIGN, + ) + return False + + logger.info_once("HPC gated MLA enabled by set VLLM_ENABLE_HPC_OPS.") + return True + + +def hpc_gated_mla_gemm( + x: torch.Tensor, + weight: torch.Tensor, + atten_output: torch.Tensor, +) -> torch.Tensor: + """Fused ``atten_output * sigmoid(x @ weight.T)``. + + Args: + x: ``[m, k]`` bfloat16, contiguous. + weight: ``[n, k]`` bfloat16, contiguous and row-major (the kernel + transposes it). ``n`` must be a multiple of 256. + atten_output: ``[m, n]`` bfloat16, contiguous. + + Returns: + ``[m, n]`` bfloat16. + """ + from hpc.gemm import gated_mla_gemm + + return gated_mla_gemm(x, weight, atten_output) diff --git a/vllm/model_executor/layers/hpc/hpc_ihc.py b/vllm/model_executor/layers/hpc/hpc_ihc.py new file mode 100644 index 0000000000..1a8681aa8e --- /dev/null +++ b/vllm/model_executor/layers/hpc/hpc_ihc.py @@ -0,0 +1,261 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""HPC fused iHC (independent Hyper-Connections) kernels for HY V4. + +Replaces the eager HYV4HCPreLayer / HYV4HCPostLayer / HYV4HCHeadLayer bodies +with single-kernel HPC implementations: + + pre : rms square sum + w projection + sigmoid gates + hc-dim weighted sum + post : y[n, i, :] = H_post[n, i] * x[n, :] + residual[n, i, :] + head : same as pre but hc_mult projection rows and no H_post output + +The eager path issues 20 / 5 / 15 kernels for pre / post / head; each HPC op is +a single kernel. + +Constraints: + - Requires VLLM_ENABLE_HPC_OPS=1 + - Requires the hpc package (.so) built for the current arch + - Only sm100 / sm103 (compute capability 100, 103) + - Only hc_mult == 4 and hidden_size in {4096, 6144} + - x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint + keeps them out of fp8 quantization via modules_to_not_convert) + +The fused modules own no parameters: they hold references to the weights of the +eager layer they replace, bound in ``process_weights_after_loading`` (invoked +generically for every ``HpcModule`` by the model loader). + +NOTE: The reference implementation also offers a cross-layer post+pre fusion +(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre. +It requires reworking the decoder-layer forward scheduling and is not ported +here. TODO: add it once the decoder dataflow is restructured for it. +""" + +import torch + +from vllm import envs +from vllm.logger import init_logger +from vllm.model_executor.layers.hpc.hpc_module import HpcModule +from vllm.utils.hpc import has_hpc + +logger = init_logger(__name__) + +# Shapes the HPC kernels are instantiated for. Other shapes launch nothing, +# so gate on this rather than relying on a silent no-op. +_SUPPORTED_HIDDEN_SIZES: frozenset[int] = frozenset({4096, 6144}) +_SUPPORTED_HC_MULTS: frozenset[int] = frozenset({4}) +_SUPPORTED_CAPABILITIES: frozenset[int] = frozenset({100, 103}) + + +def _ihc_supported(hc_mult: int, hidden_size: int) -> bool: + """Shared gate for all three iHC ops.""" + if not envs.VLLM_ENABLE_HPC_OPS: + return False + + if not has_hpc(): + logger.warning_once( + "HPC iHC disabled: 'hpc' package is not installed. " + "Please install the HPC library (.so) to enable fused kernels." + ) + return False + + from vllm.platforms import current_platform + + if not current_platform.is_cuda(): + logger.warning_once("HPC iHC disabled: only CUDA is supported.") + return False + + capability = current_platform.get_device_capability() + if capability is None or capability.to_int() not in _SUPPORTED_CAPABILITIES: + logger.warning_once( + "HPC iHC disabled: compute capability %s not in %s.", + capability, + _SUPPORTED_CAPABILITIES, + ) + return False + + if hc_mult not in _SUPPORTED_HC_MULTS: + logger.warning_once( + "HPC iHC disabled: hc_mult=%d not in %s.", hc_mult, _SUPPORTED_HC_MULTS + ) + return False + + if hidden_size not in _SUPPORTED_HIDDEN_SIZES: + logger.warning_once( + "HPC iHC disabled: hidden_size=%d not in %s.", + hidden_size, + _SUPPORTED_HIDDEN_SIZES, + ) + return False + + if envs.VLLM_BATCH_INVARIANT: + # Batch-invariant mode relies on torch's deterministic reductions; the + # HPC kernel reduces in a different order, so keep the two regimes + # apart. + logger.warning_once( + "HPC iHC disabled: not supported under VLLM_BATCH_INVARIANT." + ) + return False + + logger.info_once("HPC iHC enabled by set VLLM_ENABLE_HPC_OPS.") + return True + + +class HpcIHCPre(HpcModule): + """Fused iHC pre block. + + Computes, in one kernel: + x_flat = x.flatten(1) + r = rsqrt(x_flat.square().mean(-1) + rms_norm_eps) + mixes = (x_flat @ w.T) * r + H_pre = sigmoid(mixes[:, :hc] * hc_scale[0] + hc_base[:hc]) + hc_eps + H_post = magnitude * sigmoid( + mixes[:, hc:] * hc_scale[1] + hc_base[hc:]) + hc_eps + y = sum_i H_pre[:, i] * x[:, i, :] + + Args: + hc_mult: HC expand ratio. + hidden_size: Model hidden dimension. + magnitude: H_post multiplier (config.hc_magnitude). + hc_eps: Epsilon added to both gates (config.hc_eps). + norm_eps: Epsilon inside the rsqrt (config.rms_norm_eps). + fallback_op: The eager HYV4HCPreLayer to source weights from. + norm_owner: Optional RMSNorm that immediately follows this pre block. + When given, its weight/eps are folded into the kernel so a single + launch covers pre + RMSNorm (the caller then skips the separate + layernorm). None keeps the original pre-only behaviour. + """ + + def __init__( + self, + hc_mult: int, + hidden_size: int, + magnitude: float, + hc_eps: float, + norm_eps: float, + fallback_op: torch.nn.Module, + norm_owner: torch.nn.Module | None = None, + ) -> None: + super().__init__() + self.hc_mult = hc_mult + self.hidden_size = hidden_size + self.magnitude = magnitude + self.hc_eps = hc_eps + self.norm_eps = norm_eps + # Stash the owner outside nn.Module's attribute machinery: callers pass + # their own `self` here, and registering that as a submodule would make + # the module tree cyclic (state_dict() then hits RecursionError). + object.__setattr__(self, "_fallback_op", fallback_op) + # Same anti-cycle idiom for the fused RMSNorm owner (a sibling module). + object.__setattr__(self, "_norm_owner", norm_owner) + self.weight = None + self.hc_scale = None + self.hc_base = None + self.rms_weight = None + self.rms_eps = 0.0 + + @classmethod + def support(cls, hc_mult: int, hidden_size: int) -> bool: + return _ihc_supported(hc_mult, hidden_size) + + def process_weights_after_loading(self, model=None): + # hc_fn is a ReplicatedLinear whose weight is [2 * hc_mult, hc_mult * d]. + # It stays float32 -- the HPC kernel consumes fp32 weights directly, so + # there is no conversion (and no precision loss) here. + owner = self._fallback_op + self.weight = owner.hc_fn.weight + self.hc_scale = owner.hc_scale + self.hc_base = owner.hc_base + if self._norm_owner is not None: + self.rms_weight = self._norm_owner.weight + self.rms_eps = self._norm_owner.variance_epsilon + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + import hpc + + return hpc.fuse_ihc_pre( + x, + self.weight, + self.hc_scale, + self.hc_base, + self.norm_eps, + self.hc_eps, + self.magnitude, + self.rms_weight, + self.rms_eps, + ) + + +class HpcIHCPost(HpcModule): + """Fused iHC post block: H_post gating plus multi-channel residual add. + + Unlike mHC there is no comb matrix, so each output channel only needs its + own residual channel and the whole thing is one fused multiply-add per + element. + """ + + def __init__(self, hc_mult: int, hidden_size: int) -> None: + super().__init__() + self.hc_mult = hc_mult + self.hidden_size = hidden_size + + @classmethod + def support(cls, hc_mult: int, hidden_size: int) -> bool: + return _ihc_supported(hc_mult, hidden_size) + + def forward( + self, x: torch.Tensor, residual: torch.Tensor, H_post: torch.Tensor + ) -> torch.Tensor: + import hpc + + return hpc.fuse_ihc_post(x, residual, H_post) + + +class HpcIHCHead(HpcModule): + """Fused iHC head block: merge the hc channels back into one hidden state. + + Same structure as HpcIHCPre but the projection emits only hc_mult gate + logits and there is no H_post output. Called once per forward, after the + last decoder layer. + """ + + def __init__( + self, + hc_mult: int, + hidden_size: int, + hc_eps: float, + norm_eps: float, + fallback_op: torch.nn.Module, + ) -> None: + super().__init__() + self.hc_mult = hc_mult + self.hidden_size = hidden_size + self.hc_eps = hc_eps + self.norm_eps = norm_eps + # See HpcIHCPre: keep the owner out of the module tree. + object.__setattr__(self, "_fallback_op", fallback_op) + self.weight = None + self.hc_scale = None + self.hc_base = None + + @classmethod + def support(cls, hc_mult: int, hidden_size: int) -> bool: + return _ihc_supported(hc_mult, hidden_size) + + def process_weights_after_loading(self, model=None): + # hc_head_fn weight is [hc_mult, hc_mult * d], float32. + owner = self._fallback_op + self.weight = owner.hc_head_fn.weight + self.hc_scale = owner.hc_head_scale + self.hc_base = owner.hc_head_base + + def forward(self, x: torch.Tensor) -> torch.Tensor: + import hpc + + return hpc.fuse_ihc_head( + x, + self.weight, + self.hc_scale, + self.hc_base, + self.norm_eps, + self.hc_eps, + ) diff --git a/vllm/model_executor/layers/logits_processor.py b/vllm/model_executor/layers/logits_processor.py index c5a52cfd3a..694bb6b039 100644 --- a/vllm/model_executor/layers/logits_processor.py +++ b/vllm/model_executor/layers/logits_processor.py @@ -15,6 +15,7 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.custom_op import PluggableLayer +from vllm.model_executor.layers.linear import UnquantizedLinearMethod from vllm.model_executor.layers.vocab_parallel_embedding import ( UnquantizedEmbeddingMethod, VocabParallelEmbedding, @@ -144,7 +145,12 @@ class LogitsProcessor(PluggableLayer): lm_head, hidden_states, bias=embedding_bias ) - if not isinstance(lm_head.quant_method, UnquantizedEmbeddingMethod): + # A quant config that excludes lm_head hands out UnquantizedLinearMethod + # rather than UnquantizedEmbeddingMethod, so accept both: either way the + # weight is plain and `lm_head.weight` can be cast directly. + if not isinstance( + lm_head.quant_method, (UnquantizedEmbeddingMethod, UnquantizedLinearMethod) + ): raise ValueError( "A head_dtype different from the model dtype is only " "supported for an unquantized lm_head." diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index c45c907ada..bc99509c7f 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -874,12 +874,20 @@ def _try_load_fp8_indexer_wk( # We have both weight and scale: dequantize FP8 to BF16. weight_fp8, scale_inv = entry["weight"], entry["scale"] del buf[layer_prefix] + if scale_inv.dtype in (torch.uint8, torch.float8_e8m0fnu): + # MXFP8 checkpoints store power-of-two E8M0 scales, decode them to float. + scale_inv = scale_inv.view(torch.float8_e8m0fnu).to(torch.float32) if scale_inv.ndim == 1: # Per-channel scale: one scale per row of [out, in] group_shape = GroupShape(1, weight_fp8.shape[1]) else: - block_size = weight_fp8.shape[1] // scale_inv.shape[1] - group_shape = GroupShape(block_size, block_size) + # Derive the block size independently per dim so that per-channel + # ([out, 1]), MX ([out, in / 32]) and 2D block ([out / 128, in / 128]) + # scale layouts are all handled. + group_shape = GroupShape( + weight_fp8.shape[0] // scale_inv.shape[0], + weight_fp8.shape[1] // scale_inv.shape[1], + ) weight_bf16 = scaled_dequantize( weight_fp8, scale_inv, diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index cf1361bf40..05af1e04fe 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -128,6 +128,7 @@ _TEXT_GENERATION_MODELS = { "GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"), "HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"), "HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"), + "HYV4ForCausalLM": ("vllm.models.hy_v4", "HYV4ForCausalLM"), "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"), "HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"), "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"), @@ -663,6 +664,7 @@ _SPECULATIVE_DECODING_MODELS = { "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"), "InternS2MobiusMTP": ("interns2_mobius", "InternS2MobiusMTP"), "HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"), + "HYV4MTPModel": ("vllm.models.hy_v4", "HYV4MTP"), "KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"), # Temporarily disabled. # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1. diff --git a/vllm/models/hy_v4/__init__.py b/vllm/models/hy_v4/__init__.py new file mode 100644 index 0000000000..9fe80d568c --- /dev/null +++ b/vllm/models/hy_v4/__init__.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""HY V4 (``hy_v4``) model — hardware-isolated entry point. + +HY V4 combines three architectural pieces: + +- **iHC** (independent Hyper-Connections): the single residual stream is + replaced by ``hc_mult`` parallel residual channels, gated per sub-block. +- **MLA + lightning indexer**: multi-head latent attention with an optional + DSA-style sparse top-k selection, plus an output gate and a learnable sink. +- **MoE**: sigmoid-routed experts with a clamped SwiGLU and shared experts. + +The package is organized like `vllm.models.deepseek_v32`: this module is the +only public entry point and dispatches on the current platform, so registry +entries never reach into a platform subpackage. + +Only NVIDIA is supported for now. The port also drops the reference +implementation's HPC/TPCP fusion paths, which depend on infrastructure that +does not exist in this tree. +""" + +from vllm.platforms import current_platform + +if current_platform.is_rocm(): + raise NotImplementedError("hy_v4 does not yet support ROCm.") +elif current_platform.is_xpu(): + raise NotImplementedError("hy_v4 does not yet support XPU.") +else: + # Covers Blackwell (sm100) and all other CUDA devices. + from .nvidia.model import HYV4ForCausalLM + from .nvidia.mtp import HYV4MTP + +__all__ = [ + "HYV4ForCausalLM", + "HYV4MTP", +] diff --git a/vllm/models/hy_v4/nvidia/__init__.py b/vllm/models/hy_v4/nvidia/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/models/hy_v4/nvidia/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/models/hy_v4/nvidia/attention.py b/vllm/models/hy_v4/nvidia/attention.py new file mode 100644 index 0000000000..40a05f2c64 --- /dev/null +++ b/vllm/models/hy_v4/nvidia/attention.py @@ -0,0 +1,765 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MLA attention and lightning indexer for HY V4 (NVIDIA). + +NOTE: The reference implementation also offers a fused MLA preprocessing +wrapper, which is not available here, so this port keeps the eager MLA path +with an FP8 indexer cache. TODO: restore the fused preprocessing once the +kernel lands. The HPC gated-MLA GEMM *is* wired up: it replaces the eager +output gating when the hpc package is installed, ``VLLM_ENABLE_HPC_OPS=1`` +and the elementwise/dtype/alignment constraints hold. + +The per-head learnable sink is supported through `.flashmla_sparse`, which +subclasses the platform's sparse MLA backend to forward ``attn_sink``. +""" + +from typing import cast + +import regex as re +import torch +from torch import nn +from transformers import DeepseekV2Config, DeepseekV3Config, PretrainedConfig + +from vllm.compilation.breakable_cudagraph import eager_break_during_capture +from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config +from vllm.config.cache import CacheDType +from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.logger import init_logger +from vllm.model_executor.layers.attention import MLAAttention +from vllm.model_executor.layers.hpc import hpc_gated_mla_gemm, hpc_gated_mla_supported +from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + MergedColumnParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + per_token_group_quant_fp8, +) +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer +from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache +from vllm.platforms import current_platform +from vllm.v1.attention.backend import AttentionBackend, AttentionType +from vllm.v1.attention.selector import get_attn_backend + +logger = init_logger(__name__) + +_SPARSE_LAYER_TYPES = ("sparse_attention", "sparse", "deepseek_sparse_attention") +_WEIGHT_LAYER_INDEX_RE = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)") + + +def compute_skip_topk_layers(config: PretrainedConfig) -> set[int]: + """Return the backbone layers that reuse a previous layer's top-k indices. + + A "shared" indexer layer performs sparse attention with the indices computed + by the closest preceding "full" indexer layer, so it does not build its own + indexer and its checkpoint indexer weights must be skipped. + + Args: + config: The model config. + + Returns: + The set of layer indices that share another layer's top-k indices. + + Raises: + ValueError: If ``indexer_types`` has the wrong length or an unknown + entry, or if ``index_topk_freq`` is not a positive integer. + """ + if not hasattr(config, "index_topk"): + return set() + + num_hidden_layers = config.num_hidden_layers + indexer_types = getattr(config, "indexer_types", None) + if indexer_types is not None: + if len(indexer_types) != num_hidden_layers: + raise ValueError( + "indexer_types must contain one entry per hidden layer: " + f"expected {num_hidden_layers}, got {len(indexer_types)}." + ) + invalid_types = sorted(set(indexer_types) - {"full", "shared"}) + if invalid_types: + raise ValueError( + f"indexer_types only supports 'full' and 'shared', got {invalid_types}." + ) + return { + layer_idx + for layer_idx, indexer_type in enumerate(indexer_types) + if indexer_type == "shared" + } + + freq = getattr(config, "index_topk_freq", 1) + if not isinstance(freq, int) or freq <= 0: + raise ValueError(f"index_topk_freq must be a positive integer, got {freq!r}.") + pattern = getattr(config, "index_topk_pattern", None) + offset = getattr(config, "index_skip_topk_offset", 2) + skip_layers: set[int] = set() + for layer_idx in range(num_hidden_layers): + if pattern is None: + if max(layer_idx - offset + 1, 0) % freq != 0: + skip_layers.add(layer_idx) + elif 0 <= layer_idx < len(pattern) and pattern[layer_idx] == "S": + skip_layers.add(layer_idx) + return skip_layers + + +def is_skip_topk_indexer_weight(weight_name: str, skip_topk_layers: set[int]) -> bool: + """Return whether an indexer weight belongs to a top-k sharing layer. + + Args: + weight_name: Checkpoint weight name. + skip_topk_layers: Result of `compute_skip_topk_layers`. + + Returns: + True when the weight is an indexer weight of a layer that has no + indexer module and therefore must be dropped. + """ + if ".indexer." not in weight_name or not skip_topk_layers: + return False + match = _WEIGHT_LAYER_INDEX_RE.search(weight_name) + return match is not None and int(match.group(1)) in skip_topk_layers + + +class Indexer(nn.Module): + """Lightning indexer selecting the top-k tokens for sparse MLA.""" + + def __init__( + self, + vllm_config: VllmConfig, + config: DeepseekV2Config | DeepseekV3Config, + hidden_size: int, + q_lora_rank: int, + quant_config: QuantizationConfig | None, + cache_config: CacheConfig | None, + topk_indices_buffer: torch.Tensor | None, + prefix: str = "", + ): + super().__init__() + self.vllm_config = vllm_config + self.config = config + self.quant_config = quant_config + self.topk_tokens = config.index_topk + self.n_head = config.index_n_heads + self.head_dim = config.index_head_dim + self.rope_dim = config.qk_rope_head_dim + self.q_lora_rank = q_lora_rank + + # No tensor parallelism, just replicated. + self.wq_b = ReplicatedLinear( + self.q_lora_rank, + self.head_dim * self.n_head, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.wq_b", + ) + # Fused wk + weights_proj: single GEMM producing [head_dim + n_head]. + # FP8 wk weights are upcast to BF16 while loading to keep the fusion. + self.wk_weights_proj = MergedColumnParallelLinear( + hidden_size, + [self.head_dim, self.n_head], + bias=False, + quant_config=None, + disable_tp=True, + prefix=f"{prefix}.wk_weights_proj", + ) + self.k_norm = LayerNorm(self.head_dim, eps=1e-6) + self.softmax_scale = self.head_dim**-0.5 + + self.scale_fmt = "ue8m0" + self.quant_block_size = 128 + self.topk_indices_buffer = topk_indices_buffer + + # FP8 naive cache: values in fp8 plus one fp32 scale per + # ``quant_block_size`` elements. + assert cache_config is not None, "HYV4 indexer requires cache_config" + self.k_cache = DeepseekV32IndexerCache( + head_dim=self.head_dim + self.head_dim // self.quant_block_size * 4, + dtype=torch.uint8, + prefix=f"{prefix}.k_cache", + cache_config=cache_config, + ) + self.max_model_len = vllm_config.model_config.max_model_len + self.prefix = prefix + + from vllm.v1.attention.backends.mla.indexer import get_max_prefill_buffer_size + + self.max_total_seq_len = get_max_prefill_buffer_size(vllm_config) + self.indexer_op = SparseAttnIndexer( + self.k_cache, + self.quant_block_size, + self.scale_fmt, + self.topk_tokens, + self.head_dim, + self.max_model_len, + self.max_total_seq_len, + self.topk_indices_buffer, + ) + + def forward( + self, + hidden_states: torch.Tensor, + qr: torch.Tensor, + positions: torch.Tensor, + rotary_emb: nn.Module, + ) -> torch.Tensor: + hidden_states, q_quant, k, weights = self.prepare_inputs( + hidden_states, qr, positions, rotary_emb + ) + return self.indexer_op(hidden_states, q_quant, k, weights) + + def prepare_inputs( + self, + hidden_states: torch.Tensor, + qr: torch.Tensor, + positions: torch.Tensor, + rotary_emb: nn.Module, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Build the quantized query, key and per-head weights of the indexer.""" + q, _ = self.wq_b(qr) + q = q.view(-1, self.n_head, self.head_dim) + # Checkpoint (PTM) layout: pe occupies the LAST rope_dim dims. + q_nope, q_pe = torch.split( + q, [self.head_dim - self.rope_dim, self.rope_dim], dim=-1 + ) + + kw, _ = self.wk_weights_proj(hidden_states) + k = kw[:, : self.head_dim] + weights = kw[:, self.head_dim :] + + k = self.k_norm(k) + k_nope, k_pe = torch.split( + k, [self.head_dim - self.rope_dim, self.rope_dim], dim=-1 + ) + + q_pe, k_pe = rotary_emb(positions, q_pe, k_pe.unsqueeze(1)) + # RoPE (NeoX) can introduce extra leading dims, so flatten back to the + # token-major shapes. + q_pe = q_pe.reshape(-1, self.n_head, self.rope_dim) + k_pe = k_pe.reshape(-1, 1, self.rope_dim) + + # Reassemble with the original physical layout: no_pe first, pe last. + q = torch.cat([q_nope, q_pe], dim=-1) + # ``k_pe`` is [num_tokens, 1, rope_dim] (MQA). + k = torch.cat([k_nope, k_pe.squeeze(-2)], dim=-1) + + # Only q is quantized here; k quantization is fused with cache insertion. + q = q.view(-1, self.head_dim) + q_fp8, q_scale = per_token_group_quant_fp8( + q, + self.quant_block_size, + column_major_scales=False, + use_ue8m0=self.scale_fmt is not None, + ) + q_fp8 = q_fp8.view(-1, self.n_head, self.head_dim) + q_scale = q_scale.view(-1, self.n_head, 1) + + weights = ( + weights.unsqueeze(-1) * q_scale * self.softmax_scale * self.n_head**-0.5 + ) + weights = weights.squeeze(-1) + + return hidden_states, q_fp8, k, weights + + +class HYV4MLAAttention(nn.Module): + """Multi-head latent attention with optional sparse lightning indexer. + + Main reference: the DeepSeek-V2 paper and the FlashInfer implementation + (https://arxiv.org/abs/2405.04434). HY V4 additionally supports an output + gate (``gated_mla``) and a per-head learnable attention sink. + + The sink is applied by binding the sink-capable backend from + `.flashmla_sparse`; if no backend on this platform can consume sinks, the + weight is still loaded but the bias is disabled with a warning. + """ + + def __init__( + self, + vllm_config: VllmConfig, + config: PretrainedConfig, + hidden_size: int, + num_heads: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, + q_lora_rank: int | None, + kv_lora_rank: int, + max_position_embeddings: int = 8192, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + topk_indices_buffer: torch.Tensor | None = None, + layer_idx: int = 0, + ) -> None: + super().__init__() + self.config = config + self.hidden_size = hidden_size + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + self.v_head_dim = v_head_dim + self.layer_idx = layer_idx + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + + self.num_heads = num_heads + tp_size = get_tensor_model_parallel_world_size() + assert num_heads % tp_size == 0 + self.num_local_heads = num_heads // tp_size + + self.layer_id = int(prefix.split(".")[-2]) + layer_types = getattr(config, "layer_types", None) + requested_sparse = ( + hasattr(config, "index_topk") + and layer_types is not None + and self.layer_id < len(layer_types) + and layer_types[self.layer_id] in _SPARSE_LAYER_TYPES + ) + # Only actual sparse layers may share another layer's top-k indices. + self.skip_topk = requested_sparse and self.layer_id in compute_skip_topk_layers( + config + ) + # The skip pattern only governs backbone layers. MTP/nextn layers + # (layer_id >= num_hidden_layers) always build a full indexer: they + # compute indices at draft step 0 and toggle at runtime. + num_hidden_layers = getattr(config, "num_hidden_layers", None) + is_mtp_layer = ( + num_hidden_layers is not None and self.layer_id >= num_hidden_layers + ) + self.create_indexer = requested_sparse and (not self.skip_topk or is_mtp_layer) + self.is_sparse = requested_sparse + + # Do not silently degrade sparse layers into dense attention. Probe the + # sparse MLA backend directly and fail fast with the real error. + kv_cache_dtype = cache_config.cache_dtype if cache_config else "auto" + if self.is_sparse: + try: + get_attn_backend( + head_size=self.kv_lora_rank + self.qk_rope_head_dim, + dtype=torch.get_default_dtype(), + kv_cache_dtype=kv_cache_dtype, + use_mla=True, + has_sink=False, + use_sparse=True, + num_heads=self.num_local_heads, + ) + except Exception as exc: + raise RuntimeError( + "HYV4 sparse attention was requested, but no valid sparse MLA " + "backend is available for current runtime/config. " + "Refusing to fall back to dense attention." + ) from exc + + self.scaling = self.qk_head_dim**-0.5 + self.max_position_embeddings = max_position_embeddings + self.fused_qkv_a_proj = None + self.kv_a_proj_with_mqa = None + if self.q_lora_rank is not None: + # ``q_a_proj`` and ``kv_a_proj_with_mqa`` read the same + # ``hidden_states`` and are both TP-replicated, so they run as one + # GEMM. The checkpoint keeps them separate; `load_weights` merges + # them through ``stacked_params_mapping``. + self.fused_qkv_a_proj = MergedColumnParallelLinear( + self.hidden_size, + [self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim], + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.fused_qkv_a_proj", + disable_tp=True, + ) + else: + self.kv_a_proj_with_mqa = ReplicatedLinear( + self.hidden_size, + self.kv_lora_rank + self.qk_rope_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_a_proj_with_mqa", + ) + + self.q_a_layernorm = None + self.q_b_proj = None + self.q_proj = None + if self.q_lora_rank is not None: + self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps) + self.q_b_proj = ColumnParallelLinear( + self.q_lora_rank, + self.num_heads * self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_b_proj", + ) + else: + self.q_proj = ColumnParallelLinear( + self.hidden_size, + self.num_heads * self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_proj", + ) + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps) + self.kv_b_proj = ColumnParallelLinear( + self.kv_lora_rank, + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_b_proj", + ) + self.o_proj = RowParallelLinear( + self.num_heads * self.v_head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + self.rotary_emb = get_rope( + qk_rope_head_dim, + max_position=max_position_embeddings, + rope_parameters=config.rope_parameters, + is_neox_style=False, + ) + self.indexer_rope_emb: nn.Module | None + self.indexer: Indexer | None + if self.create_indexer: + # The checkpoint stores indexer q_pe/k_pe in interleaved + # (Megatron/PTM) layout, so the indexer must use interleaved RoPE + # (is_neox_style=False) like the main attention path. Using NeoX + # here loses the relative-position dependence and corrupts the DSA + # top-k selection. + self.indexer_rope_emb = get_rope( + qk_rope_head_dim, + max_position=max_position_embeddings, + rope_parameters=config.rope_parameters, + is_neox_style=False, + ) + # The indexer projects its queries from the MLA q_lora activations, + # so a sparse layer requires a query down-projection. + assert q_lora_rank is not None, ( + "HYV4 sparse attention requires q_lora_rank to be set" + ) + self.indexer = Indexer( + vllm_config, + config, + hidden_size, + q_lora_rank, + quant_config, + cache_config, + topk_indices_buffer, + f"{prefix}.indexer", + ) + else: + self.indexer_rope_emb = None + self.indexer = None + + self.gated_mla = bool(getattr(config, "gated_mla", False)) + self.linear_gate: ColumnParallelLinear | None + if self.gated_mla: + if config.gating_type == "headwise": + self.gate_projection_size_per_head = 1 + elif config.gating_type == "elementwise": + self.gate_projection_size_per_head = self.v_head_dim + else: + raise ValueError(f"Unknown gating type: {config.gating_type}") + self.linear_gate = ColumnParallelLinear( + self.hidden_size, + self.num_heads * self.gate_projection_size_per_head, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.linear_gate", + ) + self.use_hpc_gated_mla = hpc_gated_mla_supported( + config.gating_type, self.linear_gate + ) + else: + self.linear_gate = None + self.use_hpc_gated_mla = False + self.prefix = prefix + + # Per-head learnable attention sink. Created BEFORE ``MLAAttention`` so + # it can be forwarded as the ``sinks`` impl kwarg. The parameter always + # holds the local TP shard. + self.learnable_sink = bool(getattr(config, "learnable_sink", False)) + sinks = None + sink_backend: type[AttentionBackend] | None = None + if self.learnable_sink: + sink_backend = self._resolve_sink_backend(kv_cache_dtype) + enable_sink = sink_backend is not None + self.learnable_sink_param = nn.Parameter( + torch.empty( + self.num_local_heads, + # The kernels require fp32 sinks; the disabled path keeps + # the checkpoint dtype since the value is never consumed. + dtype=torch.float32 if enable_sink else torch.bfloat16, + ) + ) + if enable_sink: + sinks = self.learnable_sink_param + self._force_sparse_mqa() + + extra_impl_args = {} if sinks is None else {"sinks": sinks} + self.mla_attn = MLAAttention( + num_heads=self.num_local_heads, + scale=self.scaling, + qk_nope_head_dim=self.qk_nope_head_dim, + qk_rope_head_dim=self.qk_rope_head_dim, + v_head_dim=self.v_head_dim, + q_lora_rank=self.q_lora_rank, + kv_lora_rank=self.kv_lora_rank, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn", + kv_b_proj=self.kv_b_proj, + use_sparse=self.is_sparse, + indexer=self.indexer, + topk_indices_buffer=topk_indices_buffer, + attn_backend=sink_backend, + **extra_impl_args, + ) + + @property + def topk_indices_buffer(self) -> torch.Tensor | None: + """The sparse top-k index buffer this layer reads from. + + The real consumer is ``mla_attn.impl``, an ``AttentionImpl`` rather than + an ``nn.Module``, so a plain ``named_modules()`` walk cannot reach it. + The MTP proposer rebinds the draft layers onto the target model's buffer + through such a walk, so expose it here: without this the draft attention + would keep reading its own buffer while the indices it needs are written + to the target's. + """ + return getattr(self.mla_attn.impl, "topk_indices_buffer", None) + + @topk_indices_buffer.setter + def topk_indices_buffer(self, buffer: torch.Tensor) -> None: + self.mla_attn.impl.topk_indices_buffer = buffer # type: ignore[attr-defined] + + def _resolve_sink_backend( + self, kv_cache_dtype: str + ) -> type[AttentionBackend] | None: + """Return an MLA backend that can apply this layer's learnable sink. + + The sink is part of the architecture, so a backend that cannot apply it + changes the model's output. Resolution order: + + 1. If the backend the selector would pick already advertises + `supports_sink`, keep it — this also honours an explicit + ``--attention-backend`` choice. + 2. Otherwise fall back to the sink-capable ``FLASHMLA_SPARSE`` subclass + in `.flashmla_sparse`, whose kernels accept ``attn_sink``, provided + it validates against the current runtime configuration. + 3. Otherwise give up on the bias rather than failing the load. + + Args: + kv_cache_dtype: The layer's KV cache dtype string. + + Returns: + The backend class to bind, or None when no sink-capable backend is + available; the caller then loads the sink weight but disables the + bias. + """ + head_size = self.kv_lora_rank + self.qk_rope_head_dim + dtype = torch.get_default_dtype() + try: + selected_cls = get_attn_backend( + head_size=head_size, + dtype=dtype, + kv_cache_dtype=kv_cache_dtype, + use_mla=True, + use_sparse=self.is_sparse, + num_heads=self.num_local_heads, + ) + except Exception as exc: + # Stringify before logging: warning_once dedupes on the arguments, + # and a fresh exception object per layer would defeat it. + logger.warning_once( + "HYV4 failed to select an MLA backend for the learnable sink " + "(%s); the sink parameter is loaded but the sink bias is " + "disabled.", + str(exc), + ) + return None + + if selected_cls.supports_sink(): + return selected_cls + + from .flashmla_sparse import HYV4FlashMLASparseBackend + + # Mirror how the selector derives the configuration-dependent inputs so + # this check accepts exactly what the backend would accept at runtime. + capability = current_platform.get_device_capability() + if capability is None: + logger.warning_once( + "HYV4 learnable sink is unavailable: the device compute " + "capability is unknown. The sink parameter is loaded but the " + "sink bias is disabled." + ) + return None + cache_config = get_current_vllm_config().cache_config + block_size = ( + cache_config.block_size + if cache_config is not None and cache_config.user_specified_block_size + else None + ) + invalid_reasons = HYV4FlashMLASparseBackend.validate_configuration( + head_size=head_size, + dtype=dtype, + kv_cache_dtype=cast(CacheDType, kv_cache_dtype), + block_size=block_size, + use_mla=True, + has_sink=True, + use_sparse=self.is_sparse, + use_mm_prefix=False, + use_per_head_quant_scales=False, + device_capability=capability, + attn_type=AttentionType.DECODER, + ) + if invalid_reasons: + logger.warning_once( + "HYV4 learnable sink is unavailable: the selected backend %s " + "cannot apply sinks and the sink-capable FLASHMLA_SPARSE path " + "is invalid here (%s). The sink parameter is loaded but the " + "sink bias is disabled.", + selected_cls.get_name(), + ", ".join(invalid_reasons), + ) + return None + + logger.info_once( + "HYV4 learnable sink enabled: using the sink-capable " + "FLASHMLA_SPARSE impl instead of %s, which cannot apply sinks.", + selected_cls.get_name(), + ) + return HYV4FlashMLASparseBackend + + def _force_sparse_mqa(self) -> None: + """Keep every token on the sink-capable sparse MQA path. + + ``_resolve_sink_backend`` only binds the backend that serves decode. + ``MLAAttention`` additionally routes short prefills to a separate dense + MLA prefill backend, and none of those accept ``attn_sink``, so prefill + would silently drop the sink while decode applies it. Such a partially + applied sink is not the trained architecture and corrupts the output, so + opt out of the dense split instead. + + Prefills up to ``index_topk`` keep every token inside the sparse top-k, + making the sparse path numerically equivalent to the dense one apart from + also applying the sink. + """ + attention_config = get_current_vllm_config().attention_config + if attention_config.sparse_mla_force_mqa: + return + attention_config.sparse_mla_force_mqa = True + logger.info_once( + "HYV4 learnable sink enabled: forcing sparse MQA for prefill too, " + "as the dense MLA prefill backends cannot apply sinks." + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + llama_4_scaling: torch.Tensor | None = None, + ) -> torch.Tensor: + q_c = None + if self.q_lora_rank is not None: + assert self.fused_qkv_a_proj is not None + assert self.q_a_layernorm is not None + assert self.q_b_proj is not None + qkv_lora = self.fused_qkv_a_proj(hidden_states)[0] + q_c, kv_lora = qkv_lora.split( + [self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim], + dim=-1, + ) + q_c = self.q_a_layernorm(q_c) + q = self.q_b_proj(q_c)[0] + else: + assert self.q_proj is not None + assert self.kv_a_proj_with_mqa is not None + q = self.q_proj(hidden_states)[0] + kv_lora = self.kv_a_proj_with_mqa(hidden_states)[0] + + kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + kv_c_normed = self.kv_a_layernorm(kv_c) + + q = q.view(-1, self.num_local_heads, self.qk_head_dim) + # Add a head dim of 1 to k_pe. + k_pe = k_pe.unsqueeze(1) + q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb( + positions, q[..., self.qk_nope_head_dim :], k_pe + ) + + if llama_4_scaling is not None: + q *= llama_4_scaling + + output_shape = ( + hidden_states.shape[0], + self.num_local_heads * self.v_head_dim, + ) + # Single coarse eager break covering the indexer and MLA attention, as + # the breakable cudagraph contract requires: everything that reads + # per-batch metadata runs in one eager segment, so no tensor has to stay + # alive across a capture-segment boundary. + attn_out = torch.empty( + output_shape, dtype=hidden_states.dtype, device=hidden_states.device + ) + self._indexer_and_attn( + hidden_states, q_c, positions, q, kv_c_normed, k_pe, attn_out + ) + + if self.gated_mla and self.linear_gate is not None: + if self.use_hpc_gated_mla: + # Projection, sigmoid and the product in one launch. The gate + # is column-parallel and unbiased, so its local weight shard + # maps straight onto the local attn_out columns. + assert hidden_states.is_contiguous() and attn_out.is_contiguous() + attn_out = hpc_gated_mla_gemm( + hidden_states, + self.linear_gate.weight, + attn_out, + ) + else: + gate_score = self.linear_gate(hidden_states)[0] + if self.config.gating_type == "headwise": + gate_score = gate_score.unsqueeze(-1) + attn_out = attn_out.reshape( + *attn_out.shape[:-1], -1, self.v_head_dim + ) + attn_out = attn_out * torch.sigmoid(gate_score) + attn_out = attn_out.reshape(*attn_out.shape[:-2], -1) + else: + attn_out = attn_out * torch.sigmoid(gate_score) + + out, _ = self.o_proj(attn_out) + return out + + @eager_break_during_capture + def _indexer_and_attn( + self, + hidden_states: torch.Tensor, + q_c: torch.Tensor | None, + positions: torch.Tensor, + q: torch.Tensor, + kv_c_normed: torch.Tensor, + k_pe: torch.Tensor, + out: torch.Tensor, # [num_tokens, heads * v_head_dim], written in place + ) -> None: + """Run the lightning indexer and MLA attention in one eager segment. + + Both read per-batch attention metadata, so under the breakable cudagraph + they must not be captured. Keeping them in a single break (instead of one + break each) also means the attention inputs never have to survive a + capture-segment boundary. The nested ``sparse_attn_indexer`` and + ``unified_mla_attention_with_output`` breaks short-circuit here, since + the capture is no longer active inside an eager segment. + """ + if self.indexer is not None and self.is_sparse and not self.skip_topk: + self.indexer(hidden_states, q_c, positions, self.indexer_rope_emb) + out.copy_( + self.mla_attn( + q, + kv_c_normed, + k_pe, + output_shape=out.shape, + ) + ) diff --git a/vllm/models/hy_v4/nvidia/flashmla_sparse.py b/vllm/models/hy_v4/nvidia/flashmla_sparse.py new file mode 100644 index 0000000000..952dd8de5c --- /dev/null +++ b/vllm/models/hy_v4/nvidia/flashmla_sparse.py @@ -0,0 +1,283 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Sink-capable FlashMLA sparse backend for HY V4 (NVIDIA). + +HY V4 adds a per-head learnable attention sink on top of sparse MLA. The +vendored FlashMLA kernels already accept an ``attn_sink`` argument, but vLLM's +shared ``FLASHMLA_SPARSE`` backend neither advertises sink support nor forwards +the tensor, so the bias would be silently dropped. + +This module supplies the missing wiring inside the model package, mirroring how +`vllm.models.deepseek_v4.nvidia.flashinfer_sparse` hands ``sinks`` to the +FlashInfer sparse MLA kernels: subclass the platform backend, declare +`supports_sink`, and thread the sink into every kernel call. + +The subclass intentionally keeps the inherited ``get_name()`` +(``"FLASHMLA_SPARSE"``). Several shared code paths key off that exact string — +`_canonicalize_sparse_mla_kv_cache_dtype` promotes a quantized KV cache to +``fp8_ds_mla`` for it, and `FlashMLASparseImpl` asserts that layout — so a new +name would silently change KV cache behaviour. Only``supports_sink`` and the +two kernel wrappers differ from the parent. +""" + +from typing import TYPE_CHECKING + +import torch + +from vllm.logger import init_logger +from vllm.v1.attention.backends.mla.flashmla_sparse import ( + FlashMLASparseBackend, + FlashMLASparseImpl, + FlashMLASparseMetadata, +) +from vllm.v1.attention.ops.flashmla import ( + flash_mla_sparse_fwd, + flash_mla_with_kvcache, +) + +if TYPE_CHECKING: + from vllm.model_executor.models.deepseek_v2 import Indexer + +logger = init_logger(__name__) + + +class HYV4FlashMLASparseImpl(FlashMLASparseImpl): + """FlashMLA sparse impl that applies HY V4's per-head learnable sink. + + The sink enters as the ``sinks`` impl kwarg of + `vllm.model_executor.layers.attention.MLAAttention` and is consumed by the + FlashMLA kernels, which fold it into the softmax denominator: + ``out *= exp(lse) / (exp(lse) + exp(sink))``. + """ + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: list[float] | None, + sliding_window: int | None, + kv_cache_dtype: str, + logits_soft_cap: float | None, + attn_type: str, + kv_sharing_target_layer_name: str | None, + # MLA Specific Arguments + topk_indices_buffer: torch.Tensor | None = None, + indexer: "Indexer | None" = None, + **mla_args, + ) -> None: + # ``SparseMLACommonImpl`` takes explicit keyword arguments only, so the + # sink has to be removed before the base classes see``mla_args``. + sinks: torch.Tensor | None = mla_args.pop("sinks", None) + super().__init__( + num_heads, + head_size, + scale, + num_kv_heads, + alibi_slopes, + sliding_window, + kv_cache_dtype, + logits_soft_cap, + attn_type, + kv_sharing_target_layer_name, + topk_indices_buffer=topk_indices_buffer, + indexer=indexer, + **mla_args, + ) + self._validate_sinks(sinks, num_heads) + self.sinks = sinks + + @staticmethod + def _validate_sinks(sinks: torch.Tensor | None, num_heads: int) -> None: + """Reject sink tensors the FlashMLA kernels cannot consume. + + Args: + sinks: Candidate sink tensor, or None when the layer has no sink. + num_heads: Local (TP-sharded) query head count of this layer. + + Raises: + ValueError: If the dtype is not float32 or the shape is not + ``(num_heads,)``. + """ + if sinks is None: + return + if sinks.dtype != torch.float32: + raise ValueError( + "HYV4 FlashMLA sparse attention sinks must have dtype " + f"torch.float32, but got {sinks.dtype}." + ) + if sinks.ndim != 1 or sinks.shape[0] != num_heads: + raise ValueError( + "HYV4 FlashMLA sparse attention sinks must have shape " + f"({num_heads},), but got {tuple(sinks.shape)}." + ) + + def _sinks_for_query( + self, + q: torch.Tensor, + head_dim: int, + kernel_heads: int, + ) -> torch.Tensor | None: + """Return the sink laid out for the kernel's query head count. + + Args: + q: Query tensor, before any head padding. + head_dim: Axis of ``q`` holding the query heads. + kernel_heads: Head count the kernel is invoked with, which may + exceed the query head count because of padding. + + Returns: + The sink tensor padded to ``kernel_heads`` with ``-inf`` (a no-op + sink) for the padded lanes, or None when the layer has no sink. + + Raises: + ValueError: If the sink and query head layouts disagree, or if they + live on different devices. + """ + sinks = self.sinks + if sinks is None: + return None + + query_heads = q.shape[head_dim] + if sinks.shape[0] != query_heads: + raise ValueError( + "HYV4 FlashMLA sparse attention sink head count must match the " + f"runtime query layout: sinks={sinks.shape[0]}, " + f"query_heads={query_heads}. The sink must use the same " + "unpadded head layout as the query." + ) + if sinks.device != q.device: + raise ValueError( + "HYV4 FlashMLA sparse attention sinks and query must be on the " + f"same device, but got sinks={sinks.device}, query={q.device}." + ) + if kernel_heads < query_heads: + raise ValueError( + "HYV4 FlashMLA sparse kernel head count cannot be smaller than " + f"the runtime query layout: query_heads={query_heads}, " + f"kernel_heads={kernel_heads}." + ) + if kernel_heads == query_heads: + return sinks + + # Mirror the query padding the kernels require. Reading ``sinks`` here + # (rather than caching a padded copy at construction time) keeps the + # values correct for weights loaded after the module is built, and the + # allocation plus copy are captured in the CUDA graph. + padded_sinks = sinks.new_full((kernel_heads,), float("-inf")) + padded_sinks[:query_heads] = sinks + return padded_sinks + + def _fp8_flash_mla_kernel( + self, + q: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + topk_indices: torch.Tensor, + kernel_metadata: FlashMLASparseMetadata.FP8KernelMetadata, + ) -> tuple[torch.Tensor, torch.Tensor]: + # q shape: (batch, seq_len, num_heads, head_dim) + actual_num_heads = q.size(2) + padded_num_heads = self.fp8_decode_padded_heads + attn_sink = self._sinks_for_query(q, head_dim=2, kernel_heads=padded_num_heads) + + # Pad query if needed (kernel only supports h_q = 64 or 128) + if actual_num_heads < padded_num_heads: + logger.warning_once( + f"Padding num_heads from {actual_num_heads} to " + f"{padded_num_heads} for FP8 sparse decode kernel" + ) + q_padded = q.new_zeros((q.size(0), q.size(1), padded_num_heads, q.size(3))) + q_padded[:, :, :actual_num_heads, :] = q + q = q_padded + + out, lse = flash_mla_with_kvcache( + q=q, + k_cache=kv_c_and_k_pe_cache.view(torch.uint8).unsqueeze(-2), + block_table=kernel_metadata.dummy_block_table, + head_dim_v=512, + cache_seqlens=kernel_metadata.cache_lens, + tile_scheduler_metadata=kernel_metadata.scheduler_metadata, + is_fp8_kvcache=True, + indices=topk_indices, + softmax_scale=self.softmax_scale, + attn_sink=attn_sink, + ) + + # Slice output and lse back to actual head count if we padded + if actual_num_heads < padded_num_heads: + out = out[:, :, :actual_num_heads, :] + lse = lse[:, :actual_num_heads, :] + + return out, lse + + def _bf16_flash_mla_kernel( + self, + q: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + topk_indices: torch.Tensor, + topk_length: torch.Tensor | None = None, + actual_num_heads: int | None = None, + ) -> tuple[torch.Tensor, torch.Tensor]: + num_tokens = q.shape[0] + kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view( + -1, 1, kv_c_and_k_pe_cache.shape[-1] + ) + + # NOTE(Chen): kernel requires num_local_head to be a multiple of + # 64 on hopper and 128 on blackwell. Pad from q's head count, not + # self.num_heads: under DCP the heads are all-gathered before this. + if actual_num_heads is None: + actual_num_heads = q.shape[1] + padded_num_heads = ( + (actual_num_heads + self.prefill_padding - 1) + // self.prefill_padding + * self.prefill_padding + ) + # q_concat_buffer may already include the kernel-required padding. + # The sink remains defined only for the real query heads. + attn_sink = self._sinks_for_query( + q[:, :actual_num_heads], head_dim=1, kernel_heads=padded_num_heads + ) + + if q.shape[1] < padded_num_heads: + logger.warning_once( + f"Padding num_heads from {actual_num_heads} to " + f"{padded_num_heads} for BF16 sparse prefill kernel" + ) + # Zero (not new_empty) the padded lanes: topk_indices is shared by + # all heads, so the kernel reduces across the head group and NaNs + # from uninitialized memory would leak into the real heads. + q_padded = q.new_zeros((q.shape[0], padded_num_heads, q.shape[2])) + q_padded[:, :actual_num_heads, :] = q + q = q_padded + + topk_indices = topk_indices.view(num_tokens, 1, -1) + output, _, lse = flash_mla_sparse_fwd( + q, + kv_c_and_k_pe_cache, + topk_indices, + self.softmax_scale, + attn_sink=attn_sink, + topk_length=topk_length, + ) + + output = output[:, :actual_num_heads, :] + lse = lse[:, :actual_num_heads] + return output, lse + + +class HYV4FlashMLASparseBackend(FlashMLASparseBackend): + """``FLASHMLA_SPARSE`` with attention-sink support for HY V4. + + Keeps the parent's name, metadata and builder; only the impl class and the + sink capability differ. See the module docstring for why the name is reused. + """ + + @staticmethod + def get_impl_cls() -> type[HYV4FlashMLASparseImpl]: + return HYV4FlashMLASparseImpl + + @classmethod + def supports_sink(cls) -> bool: + return True diff --git a/vllm/models/hy_v4/nvidia/hc.py b/vllm/models/hy_v4/nvidia/hc.py new file mode 100644 index 0000000000..a89deed0f1 --- /dev/null +++ b/vllm/models/hy_v4/nvidia/hc.py @@ -0,0 +1,370 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""iHC (independent Hyper-Connections) layers for HY V4 (NVIDIA). + +iHC replaces the single residual stream of a standard transformer with +``hc_mult`` parallel residual channels. Each decoder sub-block reduces the +channels to one hidden state (``HYV4HCPreLayer``), runs the sub-block, then +scatters the result back over the channels (``HYV4HCPostLayer``). The final +``HYV4HCHeadLayer`` merges the channels before the model's output norm. + +NOTE: Each of the three steps has an optional single-kernel HPC replacement +(``HpcIHCPre`` / ``HpcIHCPost`` / ``HpcIHCHead``). They are only constructed +when the hpc package is installed, ``VLLM_ENABLE_HPC_OPS=1`` and the shape / +device constraints hold; otherwise the eager path below runs unchanged. +TODO: port the cross-layer post+pre fusion (``HpcIHCPostPre``) as well; it +requires restructuring the decoder-layer forward scheduling. +""" + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.model_executor.layers.hpc import HpcIHCHead, HpcIHCPost, HpcIHCPre +from vllm.model_executor.layers.linear import ReplicatedLinear + + +class HYV4HCPreLayer(nn.Module): + """iHC pre-processing layer (2D-activation adaptation). + + Steps: + 1. RMS-normalize the flattened ``[num_tokens, hc * d]`` input. + 2. Project to the pre/post gating logits. + 3. Turn the logits into sigmoid gates. + 4. Reduce over the channel dim with the pre gates. + """ + + def __init__( + self, + config: PretrainedConfig, + hidden_dim: int, + hc_mult: int = 4, + magnitude: float = 2.0, + init_std: float = 6e-3, + base_noise_std: float = 0.0, + hc_eps: float = 1e-6, + layernorm_epsilon: float = 1e-5, + prefix: str = "", + ): + super().__init__() + self.config = config + self.hidden_dim = hidden_dim + self.hc_mult = hc_mult + self.magnitude = magnitude + self.hc_eps = hc_eps + self.layernorm_epsilon = layernorm_epsilon + hc_in_dim = hc_mult * hidden_dim + mix_hc = 2 * hc_mult # pre + post only, no comb + + self.hc_fn = ReplicatedLinear( + input_size=hc_in_dim, + output_size=mix_hc, + params_dtype=torch.float32, + bias=False, + prefix=f"{prefix}.hc_fn", + ) + self.hc_scale = nn.Parameter(torch.empty(2, dtype=torch.float32)) + self.hc_base = nn.Parameter(torch.empty(mix_hc, dtype=torch.float32)) + + self.reset_parameters(init_std, base_noise_std) + + # Optional single-kernel HPC replacement for the whole forward below. + # ``norm_owner`` is left unset: the RMSNorm that follows the pre block + # lives in the decoder layer and is not folded in yet. + self.hpc_op: HpcIHCPre | None = None + if HpcIHCPre.support(hc_mult, hidden_dim): + self.hpc_op = HpcIHCPre( + hc_mult=hc_mult, + hidden_size=hidden_dim, + magnitude=magnitude, + hc_eps=hc_eps, + norm_eps=layernorm_epsilon, + fallback_op=self, + ) + + def reset_parameters(self, init_std: float, base_noise_std: float = 0.0) -> None: + """Initialize the gate scale and per-channel gate bias.""" + del init_std # hc_fn is initialized by ReplicatedLinear + nn.init.constant_(self.hc_scale, 0.01) + with torch.no_grad(): + self.hc_base[: self.hc_mult].fill_( + -torch.log(torch.tensor(self.hc_mult - 1.0, dtype=self.hc_base.dtype)) + ) + self.hc_base[self.hc_mult : 2 * self.hc_mult].fill_(0.0) + if base_noise_std > 0.0: + self.hc_base.add_(torch.randn_like(self.hc_base) * base_noise_std) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Reduce the iHC channels and emit the post gates. + + Args: + x: Input of shape ``[num_tokens, hc, d]``. + + Returns: + A tuple of the pre-gated reduction ``[num_tokens, d]`` and the post + gates ``[num_tokens, hc]`` consumed by `HYV4HCPostLayer`. + """ + if self.hpc_op is not None: + return self.hpc_op(x) + + shape = x.size() # [num_tokens, hc, d] + hc = self.hc_mult + hc_eps = self.hc_eps + + x_flat = x.flatten(1).float() # [num_tokens, hc*d] + + rsqrt = torch.rsqrt( + x_flat.square().mean(-1, keepdim=True) + self.layernorm_epsilon + ) + mixes = self.hc_fn(x_flat)[0] * rsqrt # [num_tokens, 2*hc] + + pre_raw = mixes[..., :hc] + post_raw = mixes[..., hc : 2 * hc] + + pre = ( + torch.sigmoid( + pre_raw * self.hc_scale[0].float() + self.hc_base[:hc].float() + ) + + hc_eps + ) + post = ( + self.magnitude + * torch.sigmoid( + post_raw * self.hc_scale[1].float() + self.hc_base[hc : 2 * hc].float() + ) + + hc_eps + ) + + y = torch.sum(pre.unsqueeze(-1) * x.reshape(shape), dim=1) # [num_tokens, d] + return y.to(x.dtype), post + + +class HYV4HCPostLayer(nn.Module): + """iHC post-processing layer (2D-activation adaptation). + + Applies post-gating to the sub-block output and adds the multi-channel + residual (no comb mixing):: + + y[n, i, d] = post[n, i] * x[n, d] + residual[n, i, d] + """ + + def __init__(self, config: PretrainedConfig): + super().__init__() + self.config = config + + # Optional single-kernel HPC replacement for the whole forward below. + # Only constructed under enable_ihc (see HYV4HCLayer), so hc_mult is + # present; getattr keeps this robust if that ever changes. + self.hpc_op: HpcIHCPost | None = None + hc_mult = getattr(config, "hc_mult", 0) + if HpcIHCPost.support(hc_mult, config.hidden_size): + self.hpc_op = HpcIHCPost(hc_mult=hc_mult, hidden_size=config.hidden_size) + + def forward( + self, x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor + ) -> torch.Tensor: + """Scatter the sub-block output back onto the iHC channels. + + Args: + x: Attention/MLP output of shape ``[num_tokens, d]``. + residual: Multi-channel residual ``[num_tokens, hc, d]``. + post: Post gates ``[num_tokens, hc]`` from `HYV4HCPreLayer`. + + Returns: + The updated residual channels ``[num_tokens, hc, d]``. + """ + if self.hpc_op is not None: + return self.hpc_op(x, residual, post) + + dtype = x.dtype + x = x.float() + residual = residual.float() + post = post.float() + + post_gated = post.unsqueeze(-1) * x.unsqueeze(-2) # [num_tokens, hc, d] + y = post_gated + residual + return y.to(dtype) + + +class HYV4HCHeadLayer(nn.Module): + """iHC head layer (2D-activation adaptation). + + Merges the iHC channels back into a single hidden state before the final + layer norm, using an RMS-normed projection plus sigmoid-gated reduction. + """ + + def __init__( + self, + config: PretrainedConfig, + hidden_size: int, + hc_mult: int = 4, + hc_eps: float = 1e-6, + init_std: float = 6e-3, + base_noise_std: float = 0.0, + prefix: str = "", + ): + super().__init__() + self.config = config + self.hidden_size = hidden_size + self.hc_mult = hc_mult + self.hc_eps = hc_eps + self.hc_head_fn = ReplicatedLinear( + input_size=hc_mult * hidden_size, + output_size=hc_mult, + params_dtype=torch.float32, + bias=False, + prefix=f"{prefix}.hc_head_fn", + ) + self.hc_head_base = nn.Parameter(torch.empty(hc_mult, dtype=torch.float32)) + self.hc_head_scale = nn.Parameter(torch.empty(1, dtype=torch.float32)) + + self.reset_parameters(init_std, base_noise_std) + + # Optional single-kernel HPC replacement for the whole forward below. + self.hpc_op: HpcIHCHead | None = None + if HpcIHCHead.support(hc_mult, hidden_size): + self.hpc_op = HpcIHCHead( + hc_mult=hc_mult, + hidden_size=hidden_size, + hc_eps=hc_eps, + norm_eps=config.rms_norm_eps, + fallback_op=self, + ) + + def reset_parameters( + self, init_std: float = 6e-3, base_noise_std: float = 0.0 + ) -> None: + """Initialize the head gate scale and per-channel gate bias.""" + del init_std # hc_head_fn is initialized by ReplicatedLinear + nn.init.constant_(self.hc_head_scale, 0.01) + with torch.no_grad(): + self.hc_head_base.fill_( + -torch.log( + torch.tensor(self.hc_mult - 1.0, dtype=self.hc_head_base.dtype) + ) + ) + if base_noise_std > 0.0: + self.hc_head_base.add_( + torch.randn_like(self.hc_head_base) * base_noise_std + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Merge the iHC channels into a single hidden state. + + Args: + x: Input of shape ``[num_tokens, hc, d]``. + + Returns: + The merged hidden state ``[num_tokens, d]``. + """ + if self.hpc_op is not None: + return self.hpc_op(x) + + shape, x_dtype = x.size(), x.dtype + + x = x.flatten(1).float() # [num_tokens, hc*d] + rsqrt = torch.rsqrt( + x.square().mean(-1, keepdim=True) + self.config.rms_norm_eps + ) + mixes = self.hc_head_fn(x)[0] * rsqrt # [num_tokens, hc] + pre = ( + torch.sigmoid( + mixes * self.hc_head_scale.float() + self.hc_head_base.float() + ) + + self.hc_eps + ) + y = torch.sum(pre.unsqueeze(-1) * x.reshape(shape), dim=1) # [num_tokens, d] + return y.to(x_dtype) + + +class HYV4HCLayer(nn.Module): + """Wrapper owning one iHC boundary (pre + post) of a decoder sub-block.""" + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + init_std: float = 6e-3, + base_noise_std: float = 0.0, + prefix: str = "", + ): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.enable_ihc = getattr(config, "enable_ihc", False) + if self.enable_ihc: + self.hc_pre = HYV4HCPreLayer( + config, + config.hidden_size, + config.hc_mult, + config.hc_magnitude, + init_std, + base_noise_std, + config.hc_eps, + config.rms_norm_eps, + prefix=f"{prefix}.hc_pre", + ) + self.hc_post = HYV4HCPostLayer(config) + + def prepare_input(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Normalize the sub-block input to 3D when iHC is enabled.""" + if not self.enable_ihc: + return hidden_states + return self._prepare_input_to_3d(hidden_states) + + def _prepare_input_to_3d(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Reshape the iHC input to ``[num_tokens, hc, h]``. + + Accepted inputs are ``[num_tokens, hc, h]`` (no-op), ``[num_tokens, h]`` + (broadcast over the channels) and ``[num_tokens, hc * h]`` (reshape). + """ + if hidden_states.dim() == 3: + return hidden_states + if hidden_states.dim() != 2: + raise RuntimeError( + f"HC expects a 2D/3D tensor, got shape={tuple(hidden_states.shape)}" + ) + + d0, d1 = hidden_states.shape + h = self.config.hidden_size + hc = self.config.hc_mult + + if d1 == h: + return hidden_states.unsqueeze(1).repeat(1, hc, 1) + + expected = hc * h + if d1 == expected: + return hidden_states.reshape(d0, hc, h) + + raise RuntimeError( + f"HC expects last dim to be hidden_size ({h})" + f"or hc_mult*hidden_size ({expected}), got {d1}. " + f"hc_mult={hc}, hidden_size={h}." + ) + + def pre( + self, hidden_states: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]: + """Reduce the iHC channels and produce the post gates. + + Returns: + A tuple of the reduced hidden states ``[num_tokens, d]``, the post + gates ``[num_tokens, hc]`` (``None`` when iHC is disabled) and the + residual (the untouched input). + """ + if not self.enable_ihc: + return hidden_states, None, hidden_states + reduced, post_gates = self.hc_pre(hidden_states) + return reduced, post_gates, hidden_states + + def post( + self, + output_with_bias: torch.Tensor, + residual: torch.Tensor, + post_gates: torch.Tensor | None, + ) -> torch.Tensor: + """Apply post-gating and add the residual.""" + if not self.enable_ihc: + return output_with_bias + residual + assert post_gates is not None + return self.hc_post(output_with_bias, residual, post_gates) diff --git a/vllm/models/hy_v4/nvidia/model.py b/vllm/models/hy_v4/nvidia/model.py new file mode 100644 index 0000000000..626d704f0e --- /dev/null +++ b/vllm/models/hy_v4/nvidia/model.py @@ -0,0 +1,716 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Adapted from +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only HY V4 model compatible with HuggingFace weights (NVIDIA).""" + +import typing +from collections.abc import Callable, Iterable, MutableSequence, Sequence +from itertools import islice + +import regex as re +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.config import CacheConfig, VllmConfig +from vllm.distributed import ( + get_pp_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.deepseek_v2 import _try_load_fp8_indexer_wk +from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP +from vllm.model_executor.models.utils import ( + AutoWeightsLoader, + PPMissingLayer, + WeightsMapper, + get_pp_missing_layer_names, + is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) +from vllm.platforms import current_platform +from vllm.sequence import IntermediateTensors + +from .attention import ( + HYV4MLAAttention, + compute_skip_topk_layers, + is_skip_topk_indexer_weight, +) +from .hc import HYV4HCHeadLayer, HYV4HCLayer +from .moe import HYV4FeedForward, HYV4MoEFused + +logger = init_logger(__name__) + + +def _normalize_hyv4_config(config: PretrainedConfig) -> PretrainedConfig: + """Populate the aliases consumed by the shared MoE implementation.""" + config.router_scaling_factor = config.routed_scaling_factor + config.num_experts = config.n_routed_experts + config.expert_hidden_dim = config.moe_intermediate_size + config.num_shared_experts = config.n_shared_experts + config.route_norm = config.norm_topk_prob + return config + + +class HYV4DecoderLayer(nn.Module): + """One HY V4 decoder layer: MLA attention plus a dense or MoE MLP. + + When``config.enable_ihc`` is set the layer runs on ``hc_mult`` residual + channels and each sub-block is wrapped by an `HYV4HCLayer` boundary; + otherwise it uses the standard single-stream residual. + """ + + def __init__( + self, + config: PretrainedConfig, + vllm_config: VllmConfig, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + topk_indices_buffer: torch.Tensor | None = None, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + self.config = config + self.enable_ihc = getattr(config, "enable_ihc", False) + layer_idx = int(prefix.split(".")[-1]) + self.layer_idx = layer_idx + + max_position_embeddings = getattr(config, "max_position_embeddings", 8192) + self.self_attn = HYV4MLAAttention( + vllm_config=vllm_config, + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + qk_nope_head_dim=config.qk_nope_head_dim, + qk_rope_head_dim=config.qk_rope_head_dim, + v_head_dim=config.v_head_dim, + q_lora_rank=config.q_lora_rank, + kv_lora_rank=config.kv_lora_rank, + max_position_embeddings=max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + layer_idx=layer_idx, + topk_indices_buffer=topk_indices_buffer, + ) + self.input_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps) + if config.mlp_layer_types[layer_idx] == "dense": + self.mlp = HYV4FeedForward( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + self.block_type = "feedforward" + else: + self.mlp = HYV4MoEFused( + config=config, quant_config=quant_config, prefix=f"{prefix}.mlp" + ) + self.block_type = "moe" + self.post_attention_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps) + self.hc_attn_layer = HYV4HCLayer( + config, layer_idx, prefix=f"{prefix}.hc_attn_layer" + ) + self.hc_mlp_layer = HYV4HCLayer( + config, layer_idx, prefix=f"{prefix}.hc_mlp_layer" + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + if self.enable_ihc: + return self._forward_ihc(positions, hidden_states) + return self._forward_normal(positions, hidden_states, residual) + + def _forward_normal( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Standard single-residual-stream forward (iHC disabled).""" + if residual is not None: + hidden_states = hidden_states + residual + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + ) + + hidden_states = hidden_states + residual + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + + hidden_states = self.mlp(hidden_states) + + return hidden_states, residual + + def _forward_ihc( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> tuple[torch.Tensor, None]: + """iHC forward: each sub-block reduces and re-scatters the channels.""" + hidden_states = self.hc_attn_layer.prepare_input(hidden_states) + hidden_states, post_gates, residual = self.hc_attn_layer.pre(hidden_states) + hidden_states = self.input_layernorm(hidden_states) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + ) + hidden_states = self.hc_attn_layer.post(hidden_states, residual, post_gates) + + hidden_states = self.hc_mlp_layer.prepare_input(hidden_states) + hidden_states, post_gates, residual = self.hc_mlp_layer.pre(hidden_states) + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = self.hc_mlp_layer.post(hidden_states, residual, post_gates) + + # Under iHC the residual is carried inside hidden_states. + return hidden_states, None + + +class HYV4Model(nn.Module): + """HY V4 backbone.""" + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + + config = _normalize_hyv4_config(vllm_config.model_config.hf_config) + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + parallel_config = vllm_config.parallel_config + eplb_config = parallel_config.eplb_config + self.num_redundant_experts = eplb_config.num_redundant_experts + self.device = current_platform.device_type + self.vocab_size = config.vocab_size + self.is_sparse = hasattr(config, "index_topk") + if self.is_sparse: + self.topk_indices_buffer = torch.empty( + vllm_config.scheduler_config.max_num_batched_tokens, + config.index_topk, + dtype=torch.int32, + device=self.device, + ) + else: + self.topk_indices_buffer = None + self.config = config + self.quant_config = quant_config + self.enable_ihc = getattr(config, "enable_ihc", False) + + if get_pp_group().is_first_rank or ( + self.config.tie_word_embeddings and get_pp_group().is_last_rank + ): + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + else: + self.embed_tokens = PPMissingLayer() + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: HYV4DecoderLayer( + config=config, + vllm_config=vllm_config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + topk_indices_buffer=self.topk_indices_buffer, + ), + prefix=f"{prefix}.layers", + ) + if get_pp_group().is_last_rank: + self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps) + else: + self.norm = PPMissingLayer() + if self.enable_ihc: + # iHC head layer: merge the residual channels into one stream. + if get_pp_group().is_last_rank: + self.hc_head = HYV4HCHeadLayer( + config, + hidden_size=config.hidden_size, + hc_mult=config.hc_mult, + hc_eps=config.hc_eps, + prefix=f"{prefix}.hc_head", + ) + else: + self.hc_head = PPMissingLayer() + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states"], config.hc_mult * config.hidden_size + ) + ) + else: + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size + ) + ) + + # MoE hyperparameters (consumed by EPLB). + self.expert_weights: MutableSequence[Sequence[torch.Tensor]] = [] + self.num_expert_groups = 1 + self.moe_layers: list[nn.Module] = [] + example_layer: HYV4MoEFused | None = None + for layer in self.layers: + if isinstance(layer, PPMissingLayer): + continue + + assert isinstance(layer, HYV4DecoderLayer) + if layer.block_type == "moe": + assert isinstance(layer.mlp, HYV4MoEFused) + example_layer = layer.mlp + self.moe_layers.append(layer.mlp.experts) + + if example_layer is None: + self.num_moe_layers = 0 + self.num_logical_experts = 0 + self.num_physical_experts = 0 + self.num_local_physical_experts = 0 + self.num_routed_experts = 0 + self.num_redundant_experts = 0 + return + + self.num_moe_layers = len(self.moe_layers) + self.num_logical_experts = example_layer.n_logical_experts + self.num_physical_experts = example_layer.n_physical_experts + self.num_local_physical_experts = example_layer.n_local_physical_experts + self.num_routed_experts = example_layer.n_routed_experts + self.num_redundant_experts = example_layer.n_redundant_experts + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def update_physical_experts_metadata( + self, + num_physical_experts: int, + num_local_physical_experts: int, + ) -> None: + assert self.num_local_physical_experts == num_local_physical_experts + self.num_physical_experts = num_physical_experts + self.num_local_physical_experts = num_local_physical_experts + self.num_redundant_experts = num_physical_experts - self.num_logical_experts + for layer in self.layers: + if isinstance(layer, PPMissingLayer): + continue + if isinstance(layer.mlp, HYV4MoEFused): + moe = layer.mlp + moe.n_local_physical_experts = num_local_physical_experts + moe.n_physical_experts = num_physical_experts + moe.n_redundant_experts = self.num_redundant_experts + moe.experts.update_expert_map() + + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + # (param_name, weight_name, expert_id, shard_id) for weights, fp8 + # weight scales and fp8 activation scales. + if not hasattr(self, "_cached_expert_params_mapping"): + self._cached_expert_params_mapping = fused_moe_make_expert_params_mapping( + self, + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.num_experts, + num_redundant_experts=self.num_redundant_experts, + ) + return self._cached_expert_params_mapping + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor | IntermediateTensors: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.embed_input_ids(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + # In iHC mode the flattened [num_tokens, hc*h] tensor from the + # previous PP stage is reshaped back to 3D by the first layer's + # prepare_input, and residual is unused. + residual = None if self.enable_ihc else intermediate_tensors["residual"] + + for layer in islice(self.layers, self.start_layer, self.end_layer): + hidden_states, residual = layer(positions, hidden_states, residual) + + if not get_pp_group().is_last_rank: + if self.enable_ihc: + # hidden_states is [num_tokens, hc, h]; flatten the channel dim + # for PP transfer (matches the 2D receive buffer). + return IntermediateTensors({"hidden_states": hidden_states.flatten(1)}) + return IntermediateTensors( + {"hidden_states": hidden_states, "residual": residual} + ) + if self.enable_ihc: + hidden_states = self.hc_head(hidden_states) + else: + hidden_states = hidden_states + residual + + return self.norm(hidden_states) + + def load_fused_expert_weights( + self, + name: str, + params_dict: dict, + loaded_weight: torch.Tensor, + shard_id: str, + num_experts: int, + ) -> bool: + param = params_dict[name] + weight_loader = typing.cast(Callable[..., bool], param.weight_loader) + loaded_local_expert = False + for expert_id in range(num_experts): + curr_expert_weight = loaded_weight[expert_id] + success = weight_loader( + param, + curr_expert_weight, + name, + shard_id, + expert_id, + return_success=True, + ) + if success: + loaded_local_expert = True + + return loaded_local_expert + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + # FP8 indexer wk dequant buffer (weight and scale arrive separately). + pending_wk_fp8: dict[str, dict[str, torch.Tensor]] = {} + pp_missing_layer_names = get_pp_missing_layer_names(self) + skip_topk_layers = compute_skip_topk_layers(self.config) + + # Must not be cached on `self`: `process_weights_after_loading` swaps in + # kernel-specific expert weights via `replace_parameter`, and a cache + # outliving this call would pin the pre-shuffle storage (OOM on large + # MoE) and make a later reload target orphaned tensors. + params_dict = dict(self.named_parameters()) + # Split per-expert mapping (V3 style): experts.0.gate_proj.weight + split_expert_params_mapping = self.get_expert_mapping() + loaded_params: set[str] = set() + + # Sink weights are sharded like the q/k/v linears. + sink_tp_size = get_tensor_model_parallel_world_size() + sink_tp_rank = get_tensor_model_parallel_rank() + n_local_head = self.config.num_attention_heads // sink_tp_size + head_rank_start = n_local_head * sink_tp_rank + head_rank_end = n_local_head * (sink_tp_rank + 1) + base_layer = ( + "base_layer." if any(".base_layer." in name for name in params_dict) else "" + ) + # Fused expert mapping: experts.gate_up_proj (all experts in one tensor). + # The packed weights are owned by the RoutedExperts submodule, so the + # targets are experts.routed_experts.[base_layer.]w{13,2}_weight. + fused_expert_prefix = f".experts.routed_experts.{base_layer}" + fused_expert_params_mapping = [ + (f"{fused_expert_prefix}w13_weight", ".experts.gate_up_proj", 0, "w1"), + (f"{fused_expert_prefix}w2_weight", ".experts.down_proj", 0, "w2"), + ] + num_experts = getattr(self.config, "num_experts", 0) + + def _should_skip_missing_param(param_name: str) -> bool: + # Sparse checkpoints may contain indexer weights for layers that + # fall back to dense MLA on unsupported GPUs. + return ".indexer." in param_name + + def _is_split_expert_weight(weight_name: str) -> bool: + """Whether this weight is in split (per-expert) format.""" + # Split format: mlp.experts..gate_proj.weight + return bool(re.search(r"\.experts\.\d+\.", weight_name)) + + def _is_fused_expert_weight(weight_name: str) -> bool: + """Whether this weight is in fused (all-experts-packed) format.""" + return ".experts.gate_up_proj" in weight_name or ( + ".experts.down_proj" in weight_name + and not _is_split_expert_weight(weight_name) + ) + + for name, loaded_weight in weights: + if is_skip_topk_indexer_weight(name, skip_topk_layers): + continue + if _try_load_fp8_indexer_wk( + name, + loaded_weight, + pending_wk_fp8, + params_dict, + loaded_params, + pp_missing_layer_names, + ): + continue + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + # Names and stacked shard ids are normalized upstream by + # AutoWeightsLoader, through quant_config.get_cache_scale_mapper() + # and hf_to_vllm_mapper. + stacked_shard_id = getattr(loaded_weight, "shard_id", None) + if ".indexer.wk." in name: + name = name.replace(".indexer.wk.", ".indexer.wk_weights_proj.") + stacked_shard_id = 0 + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + # Determine per-weight whether this is fused or split format. + is_fused_expert = _is_fused_expert_weight(name) + expert_params_mapping = ( + fused_expert_params_mapping + if is_fused_expert + else split_expert_params_mapping + ) + + is_expert_weight = False + loaded_expert_param_names: set[str] = set() + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + + # This is an expert weight and must not be attempted as any + # other kind of weight later on. + is_expert_weight = True + + # Do not modify `name`: the loop may continue past this point. + name_mapped = name.replace(weight_name, param_name) + if is_pp_missing_parameter(name_mapped, self): + continue + if is_fused_expert: + if "experts.gate_up_proj" in name: + chunks = loaded_weight.chunk(2, dim=-2) + success_w1 = self.load_fused_expert_weights( + name_mapped, params_dict, chunks[0], "w1", num_experts + ) + success_w3 = self.load_fused_expert_weights( + name_mapped, params_dict, chunks[1], "w3", num_experts + ) + success = success_w1 and success_w3 + else: + success = self.load_fused_expert_weights( + name_mapped, + params_dict, + loaded_weight, + shard_id, + num_experts, + ) + if success: + name = name_mapped + break + else: + # Split per-expert format (V3 style). + if name_mapped not in params_dict: + if _should_skip_missing_param(name_mapped): + continue + logger.warning_once( + "Skipping unknown checkpoint weight: %s", + name_mapped, + ) + continue + param = params_dict[name_mapped] + # Ask the weight loader whether it succeeded, otherwise we + # may skip experts that have other available replicas. + weight_loader: Callable[..., typing.Any] = param.weight_loader + success = weight_loader( + param, + loaded_weight, + name_mapped, + shard_id=shard_id, + expert_id=expert_id, + return_success=True, + ) + if success: + if not is_fused_expert: + loaded_expert_param_names.add(name_mapped) + continue + name = name_mapped + break + else: + if loaded_expert_param_names: + loaded_params.update(loaded_expert_param_names) + continue + if "learnable_sink_param" in name: + if is_pp_missing_parameter(name, self): + continue + narrow_weight = loaded_weight[head_rank_start:head_rank_end] + n = narrow_weight.shape[0] + with torch.no_grad(): + params_dict[name][:n].copy_(narrow_weight) + else: + if is_expert_weight: + # An expert weight that is not mapped to this rank. + continue + if "gate.e_score_correction_bias" in name: + name = name.replace( + "gate.e_score_correction_bias", "expert_bias" + ) + if "router.gate." in name: + name = name.replace("router.", "") + if "hc_fn" in name: + name = name.replace("hc_fn", "hc_fn.weight") + if "hc_head_fn" in name: + name = name.replace("hc_head_fn", "hc_head_fn.weight") + + if is_pp_missing_parameter(name, self): + continue + + if name not in params_dict: + if _should_skip_missing_param(name): + continue + logger.warning_once( + "Skipping unknown checkpoint weight: %s", + name, + ) + continue + + param = params_dict[name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + if stacked_shard_id is None: + weight_loader(param, loaded_weight) + else: + weight_loader(param, loaded_weight, stacked_shard_id) + loaded_params.add(name) + + return loaded_params + + +class HYV4ForCausalLM(nn.Module, SupportsPP, SupportsLoRA): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_a_proj": (".fused_qkv_a_proj", 0), + ".kv_a_proj_with_mqa": (".fused_qkv_a_proj", 1), + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0), + ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1), + ".indexer.weights_proj.": (".indexer.wk_weights_proj.", 1), + } + ) + packed_modules_mapping = { + "gate_up_proj": ["gate_proj", "up_proj"], + # MLA runs both latent down-projections as one GEMM. + "fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"], + # The indexer fuses wk and weights_proj into one GEMM. + "wk_weights_proj": ["wk", "weights_proj"], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + self.config = config + self.quant_config = quant_config + if quant_config is not None: + quant_config.packed_modules_mapping = self.packed_modules_mapping + + parallel_config = vllm_config.parallel_config + eplb_config = parallel_config.eplb_config + self.num_redundant_experts = eplb_config.num_redundant_experts + + self.model = HYV4Model( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + if get_pp_group().is_last_rank: + # The head stays in the model dtype; ``enable_lm_head_fp32`` is + # surfaced as ``head_dtype`` on the config (see HYV4Config), which + # makes LogitsProcessor accumulate the projection straight into + # fp32 instead of materializing an fp32 copy of the weight. + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + # With tie_word_embeddings, embed_tokens is kept on the last rank + # (see HYV4Model.__init__) so the weight can be shared here. + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + else: + self.lm_head = PPMissingLayer() + self.logits_processor = LogitsProcessor(config.vocab_size) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor | IntermediateTensors: + return self.model(input_ids, positions, intermediate_tensors, inputs_embeds) + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + logits = self.logits_processor(self.lm_head, hidden_states) + + if getattr(self.config, "soft_logits_capping", False): + soft_cap = self.config.soft_logits_capping_logits + logits = soft_cap * torch.nn.functional.tanh(logits / soft_cap) + + return logits + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + # MTP layers are loaded by the MTP head, not by the target model. They + # appear as model.mtp_layers..* and as the layer ids past the backbone. + mtp_start = self.config.num_hidden_layers + drop_prefixes: dict[str, str | None] = {"model.mtp_layers.": None} + for i in range(getattr(self.config, "num_nextn_predict_layers", 0)): + drop_prefixes[f"model.layers.{mtp_start + i}."] = None + if self.config.tie_word_embeddings: + drop_prefixes["lm_head."] = None + drop_mapper = WeightsMapper(orig_to_new_prefix=drop_prefixes) + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper | drop_mapper) + + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + return self.model.get_expert_mapping() diff --git a/vllm/models/hy_v4/nvidia/moe.py b/vllm/models/hy_v4/nvidia/moe.py new file mode 100644 index 0000000000..be8e49f21e --- /dev/null +++ b/vllm/models/hy_v4/nvidia/moe.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Dense FFN and MoE blocks for HY V4 (NVIDIA).""" + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.config import get_current_vllm_config +from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.fused_moe import FusedMoEFactory, GateLinear +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.quantization import QuantizationConfig + +logger = init_logger(__name__) + + +class HYV4FeedForward(nn.Module): + """Dense SwiGLU feed-forward block. + + Used both for the dense decoder layers and for the MoE shared experts. + The routed-expert SwiGLU clamp does not apply here, matching the reference + implementation. + """ + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: QuantizationConfig | None = None, + reduce_results: bool = True, + prefix: str = "", + ) -> None: + super().__init__() + if hidden_act != "silu": + raise ValueError( + f"Unsupported activation: {hidden_act}. Only silu is supported for now." + ) + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, + [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results, + prefix=f"{prefix}.down_proj", + ) + self.act_fn = SiluAndMul() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(x) + out = self.act_fn(gate_up) + out, _ = self.down_proj(out) + return out + + +class HYV4MoEFused(nn.Module): + """HY V4 MoE layer with optional SwiGLU clamp support. + + When ``config.swiglu_limit > 0`` the routed experts use a clamped SwiGLU:: + + gate = clamp(gate, max=limit) + up = clamp(up, -limit, limit) + output = silu(gate) * up + + Dense layers and shared experts are NOT clamped. The clamp is forwarded to + ``FusedMoEFactory`` as ``swiglu_limit`` so it travels through the quant + config to every expert backend instead of mutating global state. + + NOTE: The reference implementation additionally supports HPC gate kernels, + TPCP/MLPSP sharded shared experts and a side-stream overlap for the shared + experts. Those depend on infrastructure that is absent here, so this port + keeps only the standard path. TODO: restore once available. + """ + + def __init__( + self, + config: PretrainedConfig, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + enable_eplb: bool = False, + ): + super().__init__() + self.tp_size = get_tensor_model_parallel_world_size() + self.ep_group = get_ep_group().device_group + self.ep_rank = get_ep_group().rank_in_group + self.ep_size = self.ep_group.size() + self.n_routed_experts = config.num_experts + if self.tp_size > config.num_experts: + raise ValueError( + f"Tensor parallel size {self.tp_size} is greater than " + f"the number of experts {config.num_experts}." + ) + top_k = config.num_experts_per_tok + intermediate_size = config.expert_hidden_dim + router_scaling_factor = getattr(config, "router_scaling_factor", 1.0) + vllm_config = get_current_vllm_config() + eplb_config = vllm_config.parallel_config.eplb_config + self.enable_eplb = enable_eplb + + self.n_logical_experts = self.n_routed_experts + self.n_redundant_experts = eplb_config.num_redundant_experts + self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts + self.n_local_physical_experts = self.n_physical_experts // self.ep_size + self.physical_expert_start = self.ep_rank * self.n_local_physical_experts + self.physical_expert_end = ( + self.physical_expert_start + self.n_local_physical_experts + ) + + self.gate = GateLinear( + config.hidden_size, + config.num_experts, + bias=False, + out_dtype=torch.float32, + params_dtype=torch.float32, + prefix=f"{prefix}.gate", + ) + + self.shared_experts: HYV4FeedForward | None + if config.num_shared_experts > 0: + self.shared_experts = HYV4FeedForward( + hidden_size=config.hidden_size, + intermediate_size=config.expert_hidden_dim * config.num_shared_experts, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.shared_experts", + reduce_results=False, + ) + else: + self.shared_experts = None + + self.expert_bias = nn.Parameter( + torch.empty(config.num_experts, dtype=torch.float32) + ) + + raw_swiglu_limit = getattr(config, "swiglu_limit", 0) + use_swiglu_clamp = bool(raw_swiglu_limit) and float(raw_swiglu_limit) > 0 + moe_swiglu_limit = float(raw_swiglu_limit) if use_swiglu_clamp else None + + self.experts = FusedMoEFactory( + num_experts=self.n_routed_experts, + top_k=top_k, + hidden_size=config.hidden_size, + intermediate_size=intermediate_size, + renormalize=config.route_norm, + quant_config=quant_config, + prefix=f"{prefix}.experts", + enable_eplb=self.enable_eplb, + num_redundant_experts=self.n_redundant_experts, + scoring_func="sigmoid", + use_grouped_topk=True, + num_expert_group=1, + topk_group=1, + routed_scaling_factor=router_scaling_factor, + e_score_correction_bias=self.expert_bias, + shared_experts=self.shared_experts, + swiglu_limit=moe_swiglu_limit, + ) + self.prefix = prefix + if use_swiglu_clamp: + logger.debug_once( + "HYV4MoEFused: swiglu_limit=%.1f enabled for routed experts", + moe_swiglu_limit, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + orig_shape = hidden_states.shape + hidden_dim = hidden_states.shape[-1] + hidden_states = hidden_states.view(-1, hidden_dim) + + # router_logits: (num_tokens, n_experts); the gate stores fp32 weights + # and emits fp32 logits, so no explicit upcast is needed here. + router_logits, _ = self.gate(hidden_states) + + final_hidden_states = self.experts( + hidden_states=hidden_states, router_logits=router_logits + ) + return final_hidden_states.view(orig_shape) diff --git a/vllm/models/hy_v4/nvidia/mtp.py b/vllm/models/hy_v4/nvidia/mtp.py new file mode 100644 index 0000000000..49f1b3d7a5 --- /dev/null +++ b/vllm/models/hy_v4/nvidia/mtp.py @@ -0,0 +1,937 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Multi-token prediction (MTP) head for HY V4 (NVIDIA).""" + +import copy +import typing +from collections.abc import Callable, Iterable + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.config import CacheConfig, ModelConfig, VllmConfig +from vllm.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, + maybe_remap_kv_scale_name, +) +from vllm.model_executor.models.deepseek_v2 import _try_load_fp8_indexer_wk +from vllm.model_executor.models.utils import ( + get_pp_missing_layer_names, + is_pp_missing_parameter, + maybe_prefix, +) +from vllm.sequence import IntermediateTensors +from vllm.v1.outputs import SamplerOutput +from vllm.v1.sample.metadata import SamplingMetadata +from vllm.v1.sample.sampler import Sampler + +from .model import HYV4DecoderLayer, _normalize_hyv4_config + +logger = init_logger(__name__) + +_MTP_SCALE_ONLY_PARAM_SUBSTRINGS = ( + "_scale", + "_weight_scale", + "_weight_scale_inv", + "_input_scale", +) + +# Names of the quant-config attribute that stores the unquantized-module +# prefix list, in the order we look them up. Different QuantizationConfig +# subclasses expose the list under different names: +# * ``ignored_layers`` -- Fp8Config, GPTQ / AWQ configs. +# * ``exclude_modules`` -- ModelOpt* configs (Fp8, NvFp4, MxFp8, Mixed). +_MTP_QUANT_EXCLUSION_ATTRS = ("ignored_layers", "exclude_modules") + + +def _get_spec_layer_idx_from_weight_name( + config: PretrainedConfig, weight_name: str +) -> int | None: + """Return the MTP layer index a checkpoint weight belongs to, or None. + + Compatible with``num_nextn_predict_layers`` of 1 and 2 for a single MTP + head, matching how the HY V4 checkpoints are exported. + """ + if getattr(config, "num_nextn_predict_layers", 0) < 1: + return None + layer_idx = config.num_hidden_layers + num_mtp_layers = max(config.num_nextn_predict_layers - 1, 1) + for i in range(num_mtp_layers): + if weight_name.startswith(f"model.layers.{layer_idx + i}."): + return layer_idx + i + return None + + +def _should_skip_missing_mtp_scale_param( + quant_config: QuantizationConfig | None, + name: str, +) -> bool: + """Whether an unmatched scale parameter can be silently ignored.""" + return quant_config is None and any( + scale_name in name for scale_name in _MTP_SCALE_ONLY_PARAM_SUBSTRINGS + ) + + +def _resolve_fused_expert_param( + param_base: str, + ckpt_suffix: str, + params_dict: dict, +) -> str | None: + """Map a fused expert checkpoint suffix onto a draft parameter name. + + Fused checkpoints pack every expert into one tensor and name the companion + scales by suffixing the projection (``experts.gate_up_proj_scale_inv``), + while the draft model owns ``experts.routed_experts.w13_weight`` plus a + separate ``..._scale_inv`` parameter. Dropping the suffix would push the + scale into the weight parameter and leave the scale at its sentinel init. + + Args: + param_base: Draft parameter holding the packed weight. + ckpt_suffix: Checkpoint text following the projection name. + params_dict: The draft model's named parameters. + + Returns: + The matching draft parameter name, or None when there is none. + """ + if not ckpt_suffix: + return param_base if param_base in params_dict else None + tail = ckpt_suffix.lstrip(".").removeprefix("weight_") + for candidate in ( + f"{param_base}_{tail}", + f"{param_base}_scale_inv", + f"{param_base}_scale", + ): + if candidate in params_dict: + return candidate + return None + + +def _prepare_mtp_fp8_expert_scale( + quant_config: QuantizationConfig | None, + name: str, + loaded_weight: torch.Tensor, +) -> tuple[str, torch.Tensor]: + """Normalize block-wise FP8 expert scale names and dtypes.""" + from vllm.model_executor.layers.quantization.fp8 import Fp8Config + + if ( + isinstance(quant_config, Fp8Config) + and quant_config.weight_block_size is not None + and ".mlp.experts." in name + and name.endswith(".scale") + ): + name = name[: -len(".scale")] + ".weight_scale_inv" + if ( + getattr(quant_config, "is_scale_e8m0", False) + and loaded_weight.dtype == torch.uint8 + ): + # UE8M0 scale exponents are stored as raw uint8 bytes: reinterpret + # the bits instead of numerically converting them. + loaded_weight = loaded_weight.view(torch.float8_e8m0fnu) + return name, loaded_weight + + +def _create_mtp_quant_config( + hf_config: PretrainedConfig, + backbone_quant_config: QuantizationConfig | None = None, +) -> QuantizationConfig | None: + """Create the quantization config for the MTP layers. + + The MTP quantization algorithm is given by the ``mtp_quant_algo`` field in + ``config.json``, independently of the backbone's quantization. Supported + values are ``"FP8"``, ``"NONE"`` (inherit the backbone) and ``"BF16"`` / + ``"FP16"`` (unquantized). A missing or ``"NONE"`` value falls back to the + backbone config. + + Args: + hf_config: The draft model's HF config. + backbone_quant_config: The target model's quantization config. + + Returns: + The quantization config to use for the MTP layers, or None when the MTP + layers are unquantized. + """ + mtp_quant_algo = getattr(hf_config, "mtp_quant_algo", None) + + if mtp_quant_algo is None or mtp_quant_algo.upper() == "NONE": + logger.info_once( + "No explicit MTP quant algo (mtp_quant_algo=%s). " + "Using the backbone's quant config.", + mtp_quant_algo, + ) + return backbone_quant_config + + mtp_quant_algo = mtp_quant_algo.upper() + + if mtp_quant_algo in ("BF16", "FP16"): + logger.info_once( + "MTP layers are unquantized (mtp_quant_algo=%s).", mtp_quant_algo + ) + return None + + if mtp_quant_algo == "FP8": + from vllm.model_executor.layers.quantization.fp8 import Fp8Config + + logger.info_once("MTP uses FP8 quantization (mtp_quant_algo=FP8).") + hf_quant_config = getattr(hf_config, "quantization_config", None) or {} + weight_block_size = hf_quant_config.get("weight_block_size") + activation_scheme = hf_quant_config.get("activation_scheme", "dynamic") + if weight_block_size is not None: + activation_scheme = "dynamic" + ignored_layers = hf_quant_config.get("ignored_layers") or hf_quant_config.get( + "modules_to_not_convert" + ) + + fp8_config = Fp8Config( + is_checkpoint_fp8_serialized=True, + activation_scheme=activation_scheme, + ignored_layers=ignored_layers or [], + weight_block_size=weight_block_size, + ) + if hf_quant_config.get("scale_fmt") == "ue8m0": + # Fp8LinearMethod reads this through getattr, so it is not a + # declared Fp8Config field. + setattr(fp8_config, "is_scale_e8m0", True) # noqa: B010 + return fp8_config + + logger.warning( + "Unknown mtp_quant_algo=%s, falling back to the backbone config.", + mtp_quant_algo, + ) + return backbone_quant_config + + +def _remap_mtp_quant_exclusions( + quant_config: QuantizationConfig | None, + mtp_start_layer_idx: int, + num_mtp_layers: int, +) -> QuantizationConfig | None: + """Translate checkpoint-named MTP quant exclusions to draft prefixes. + + ``modules_to_not_convert`` / ``exclude_modules`` name MTP modules the way + the checkpoint does (``model.mtp_layers.0.self_attn.linear_gate``), while + the draft model builds them under ``model.layers.``. + ``is_layer_skipped`` compares prefixes for exact equality, so without this + translation an excluded MTP module gets a quant method even though its + checkpoint weight is BF16 with no ``weight_scale`` / ``weight_scale_inv`` + companion: the scale then keeps its ``finfo(float32).min`` sentinel and the + layer silently produces garbage. + + The list lives under a different attribute per config class, hence the + lookup over ``_MTP_QUANT_EXCLUSION_ATTRS``. + + Args: + quant_config: The MTP quantization config to widen. + mtp_start_layer_idx: Index of the first MTP layer in the draft model. + num_mtp_layers: Number of MTP layers. + + Returns: + A shallow copy with the translated exclusions, or the input unchanged. + """ + if quant_config is None: + return None + + # Different QuantizationConfig subclasses expose the exclusion list under + # different attribute names; only the one that is actually populated on + # the backbone config governs which draft modules stay unquantized. For + # ModelOpt checkpoints (``quant_method: modelopt`` and friends) this is + # ``exclude_modules``; for the legacy fp8 path it is ``ignored_layers``. + exclusion_attr: str | None = None + patterns: list[str] | None = None + for attr in _MTP_QUANT_EXCLUSION_ATTRS: + value = getattr(quant_config, attr, None) + if value: + exclusion_attr = attr + patterns = list(value) + break + if not patterns or exclusion_attr is None: + return quant_config + + extra: list[str] = [] + for offset in range(num_mtp_layers): + source_prefix = f"model.mtp_layers.{offset}." + draft_prefix = f"model.layers.{mtp_start_layer_idx + offset}." + for pattern in patterns: + if pattern.startswith(source_prefix): + extra.append(draft_prefix + pattern[len(source_prefix) :]) + # ModelOpt exclusion patterns may end with a ``*`` wildcard; the + # remap must preserve that suffix so ``is_layer_skipped`` still + # matches the draft module prefixes. + elif pattern.rstrip("*").startswith(source_prefix): + stripped = pattern.rstrip("*") + trailing_stars = pattern[len(stripped) :] + extra.append( + draft_prefix + stripped[len(source_prefix) :] + trailing_stars + ) + if not extra: + return quant_config + + # The backbone shares this object with the target model; copy before + # widening the exclusion list. The attribute is only present on the + # concrete configs that support exclusions, hence the getattr/setattr pair. + quant_config = copy.copy(quant_config) + setattr(quant_config, exclusion_attr, patterns + extra) # noqa: B010 + logger.info_once( + "HYV4 MTP: mapped %d checkpoint-named quant exclusions (%s) onto the " + "draft module prefixes: %s", + len(extra), + exclusion_attr, + ", ".join(extra), + ) + return quant_config + + +class HYV4SharedHead(nn.Module): + """Holds the draft LM head shared with the target model.""" + + def __init__( + self, + config: PretrainedConfig, + quant_config: QuantizationConfig | None = None, + ) -> None: + super().__init__() + # Use the same ``lm_head`` prefix as the backbone so that quant-config + # exclusion lists (e.g. ModelOpt's ``exclude_modules: ["lm_head", ...]``) + # also apply here: an unquantized checkpoint lm_head must not gain a + # spurious ``weight_scale`` parameter on the draft side, which would + # otherwise stay at its FP-min sentinel until the proposer swaps the + # whole ParallelLMHead for the target model's ``lm_head``. + self.head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix="lm_head", + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return hidden_states + + +class HYV4MultiTokenPredictorLayer(nn.Module): + """A single MTP draft block.""" + + def __init__( + self, + config: PretrainedConfig, + prefix: str, + vllm_config: VllmConfig, + model_config: ModelConfig, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + topk_indices_buffer: torch.Tensor | None = None, + ) -> None: + super().__init__() + del model_config # kept for signature parity with the other MTP heads + self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) + self.shared_head = HYV4SharedHead(config=config, quant_config=quant_config) + + # HYV4DecoderLayer indexes layer_types/mlp_layer_types by the numeric + # layer id parsed from the prefix. MTP layers live after the backbone + # layers, so extend the per-layer config lists to cover those ids. + layer_idx = int(prefix.rsplit(".", 1)[-1]) + mtp_config = copy.deepcopy(config) + # MTP checkpoints do not carry iHC pre/post weights for mtp_layers, so + # the draft blocks are built without iHC to match their structure. + mtp_config.enable_ihc = False + mtp_config.layer_types = _extend_layer_types( + getattr(mtp_config, "layer_types", None), layer_idx, "full_attention" + ) + mtp_config.mlp_layer_types = _extend_layer_types( + getattr(mtp_config, "mlp_layer_types", None), layer_idx, "sparse" + ) + + self.mtp_block = HYV4DecoderLayer( + config=mtp_config, + vllm_config=vllm_config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + topk_indices_buffer=topk_indices_buffer, + ) + self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + previous_hidden_states: torch.Tensor, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor: + del input_ids # embeddings are resolved by the caller + assert inputs_embeds is not None + inputs_embeds = self.enorm(inputs_embeds) + previous_hidden_states = self.hnorm(previous_hidden_states) + hidden_states = self.eh_proj( + torch.cat([inputs_embeds, previous_hidden_states], dim=-1) + ) + hidden_states, residual = self.mtp_block( + positions=positions, + hidden_states=hidden_states, + residual=None, + ) + hidden_states, _ = self.final_layernorm(hidden_states, residual) + return hidden_states + + +def _extend_layer_types( + layer_types: list[str] | None, layer_idx: int, fallback: str +) -> list[str] | None: + """Pad a per-layer config list so``layer_idx`` is addressable.""" + if layer_types is None or len(layer_types) > layer_idx: + return layer_types + fill = layer_types[-1] if layer_types else fallback + return list(layer_types) + [fill] * (layer_idx + 1 - len(layer_types)) + + +class HYV4MultiTokenPredictor(nn.Module): + """Owns the MTP draft blocks and their shared embedding / logits path.""" + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + target_config = vllm_config.model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None, ( + "HYV4MTP can only be built as a speculative draft model" + ) + draft_model_config = speculative_config.draft_model_config + assert draft_model_config is not None + config = _normalize_hyv4_config(draft_model_config.hf_config) + self.mtp_start_layer_idx = target_config.num_hidden_layers + self.num_mtp_layers = max( + getattr(target_config, "num_nextn_predict_layers", 1), 1 + ) + self.quant_config = _remap_mtp_quant_exclusions( + _create_mtp_quant_config(config, vllm_config.quant_config), + self.mtp_start_layer_idx, + self.num_mtp_layers, + ) + if self.quant_config is not None: + self.quant_config.packed_modules_mapping = HYV4MTP.packed_modules_mapping + + # Sparse DSA draft blocks must share the target model's top-k buffer. + # The proposer injects it through HYV4MTP.set_topk_indices_buffer() + # before any draft forward. + self.topk_indices_buffer: torch.Tensor | None = None + if hasattr(config, "index_topk"): + from vllm.platforms import current_platform + + self.topk_indices_buffer = torch.empty( + vllm_config.scheduler_config.max_num_batched_tokens, + config.index_topk, + dtype=torch.int32, + device=current_platform.device_type, + ) + + self.layers = nn.ModuleDict( + { + str(idx): HYV4MultiTokenPredictorLayer( + config, + f"{prefix}.layers.{idx}", + vllm_config=vllm_config, + model_config=draft_model_config, + cache_config=vllm_config.cache_config, + quant_config=self.quant_config, + topk_indices_buffer=self.topk_indices_buffer, + ) + for idx in range( + self.mtp_start_layer_idx, + self.mtp_start_layer_idx + self.num_mtp_layers, + ) + } + ) + self.requires_topk_indices_buffer = any( + layer.mtp_block.self_attn.is_sparse for layer in self.layers.values() + ) + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.logits_processor = LogitsProcessor(config.vocab_size) + self.spec_step_idx: int = 0 + + def set_skip_topk(self, skip: bool) -> None: + """Toggle the draft indexer for ``index_share_for_mtp_iteration``. + + The proposer clears the flag for draft step 0 so the MTP layer builds + its own top-k indices, then sets it for steps 1+ so they reuse what + step 0 wrote into the shared buffer instead of re-running the indexer. + """ + for layer in self.layers.values(): + layer.mtp_block.self_attn.skip_topk = skip + + def compact_topk_indices(self, slot_ids: torch.Tensor) -> None: + """Move the top-k rows at ``slot_ids`` to the front of the buffer. + + Step 0 writes one row per query token of the multi-token batch, while + steps 1+ decode a single token per request and index the buffer from 0. + Without this gather they would read another token's rows. + """ + num_slots = slot_ids.numel() + if self.topk_indices_buffer is None or num_slots == 0: + return + self.topk_indices_buffer[:num_slots] = self.topk_indices_buffer[slot_ids] + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + previous_hidden_states: torch.Tensor, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor: + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + inputs_embeds = torch.where((positions == 0).unsqueeze(-1), 0, inputs_embeds) + + current_step_idx = self.spec_step_idx % self.num_mtp_layers + return self.layers[str(self.mtp_start_layer_idx + current_step_idx)]( + input_ids, + positions, + previous_hidden_states, + inputs_embeds, + ) + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + current_step_idx = self.spec_step_idx % self.num_mtp_layers + mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] + lm_head = mtp_layer.shared_head.head + proj_input = mtp_layer.shared_head(hidden_states) + return self.logits_processor(lm_head, proj_input) + + +class HYV4MTP(nn.Module): + """HY V4 MTP draft head. + + Not a pipeline-parallel stage: the draft head always runs on a single rank, + matching `DeepseekV32MTP` / `KimiK3MTP` / `HYV3MTP`. + """ + + packed_modules_mapping = { + "gate_up_proj": ["gate_proj", "up_proj"], + # MLA runs both latent down-projections as one GEMM. + "fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"], + # The indexer fuses wk and weights_proj into one GEMM. + "wk_weights_proj": ["wk", "weights_proj"], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + self.config = vllm_config.model_config.hf_config + self.model = HYV4MultiTokenPredictor( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + self.quant_config = self.model.quant_config + self.sampler = Sampler() + + def set_topk_indices_buffer(self, topk_indices_buffer: torch.Tensor) -> None: + """Share the target sparse-index buffer with every draft consumer. + + Proposers that walk ``named_modules()`` instead of calling this reach + the same consumers via ``HYV4MLAAttention.topk_indices_buffer``. + """ + self.model.topk_indices_buffer = topk_indices_buffer + for layer in self.model.layers.values(): + self_attn = layer.mtp_block.self_attn + if not self_attn.is_sparse: + continue + + indexer = self_attn.indexer + assert indexer is not None, "Sparse HYV4 MTP attention requires an indexer" + indexer.topk_indices_buffer = topk_indices_buffer + indexer.indexer_op.topk_indices_buffer = topk_indices_buffer + + attn_impl = self_attn.mla_attn.impl + assert hasattr(attn_impl, "topk_indices_buffer"), ( + "Sparse HYV4 MTP attention backend requires a top-k indices buffer" + ) + attn_impl.topk_indices_buffer = topk_indices_buffer + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + hidden_states: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor: + del intermediate_tensors # the MTP head is single-stage + if ( + self.model.requires_topk_indices_buffer + and self.model.topk_indices_buffer is None + ): + raise RuntimeError( + "HYV4 sparse MTP requires the target model's top-k indices buffer. " + "The proposer must call HYV4MTP.set_topk_indices_buffer() before " + "the first draft forward." + ) + self.model.spec_step_idx = spec_step_idx + return self.model(input_ids, positions, hidden_states, inputs_embeds) + + def compute_logits( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor | None: + self.model.spec_step_idx = spec_step_idx + return self.model.compute_logits(hidden_states) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> SamplerOutput | None: + return self.sampler(logits, sampling_metadata) + + def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: + if f"model.layers.{spec_layer}.embed_tokens" in name: + return "__skip__" + if f"model.layers.{spec_layer}.shared_head" in name: + return "__skip__" + + spec_layer_weight_names = ["enorm", "hnorm", "eh_proj", "final_layernorm"] + spec_layer_weight = any( + weight_name in name for weight_name in spec_layer_weight_names + ) + if not spec_layer_weight: + name = name.replace( + f"model.layers.{spec_layer}.", + f"model.layers.{spec_layer}.mtp_block.", + ) + return name + + def _load_fused_expert_weights( + self, + name: str, + params_dict: dict, + loaded_weight: torch.Tensor, + shard_id: str, + num_experts: int, + ) -> bool: + if name not in params_dict: + return False + param = params_dict[name] + weight_loader = typing.cast(Callable[..., bool], param.weight_loader) + loaded_local_expert = False + for expert_id in range(num_experts): + curr_expert_weight = loaded_weight[expert_id] + success = weight_loader( + param, + curr_expert_weight, + name, + shard_id, + expert_id, + return_success=True, + ) + if success: + loaded_local_expert = True + return loaded_local_expert + + def _load_expert_weight( + self, + name: str, + loaded_weight: torch.Tensor, + params_dict: dict, + loaded_params: set[str], + split_expert_params_mapping: list[tuple[str, str, int, str]], + fused_expert_param_names: dict[tuple[str, str], str], + num_experts: int, + ) -> bool: + """Load one routed-expert weight in either checkpoint layout. + + Args: + name: Weight name already rewritten to draft-module naming. + loaded_weight: The checkpoint tensor. + params_dict: The draft model's named parameters. + loaded_params: Set updated with the parameters that received a value. + split_expert_params_mapping: Mapping for the per-expert layout. + fused_expert_param_names: ``(mlp_prefix, tag) -> param name`` for the + all-experts-packed layout. + num_experts: Total number of routed experts. + + Returns: + True when the weight was consumed (even if this rank holds none of + the addressed experts). + """ + base = name.split(".experts.")[0] + for ckpt_proj, tag in ( + (".experts.gate_up_proj", "w13_weight"), + (".experts.down_proj", "w2_weight"), + ): + if ckpt_proj not in name: + continue + param_base = fused_expert_param_names.get((base, tag)) + if param_base is None: + return False + # Keep the checkpoint suffix (e.g. `_scale_inv`) so block-scale + # tensors land in the scale parameter, not in the weight. + target = _resolve_fused_expert_param( + param_base, name.split(ckpt_proj, 1)[1], params_dict + ) + if target is None: + return False + if tag == "w13_weight": + chunks = loaded_weight.chunk(2, dim=-2) + loaded_w1 = self._load_fused_expert_weights( + target, params_dict, chunks[0], "w1", num_experts + ) + loaded_w3 = self._load_fused_expert_weights( + target, params_dict, chunks[1], "w3", num_experts + ) + loaded = loaded_w1 and loaded_w3 + else: + loaded = self._load_fused_expert_weights( + target, params_dict, loaded_weight, "w2", num_experts + ) + if loaded: + loaded_params.add(target) + # The weight belongs to the experts either way; never fall through. + return True + + consumed = False + for param_name, weight_name, expert_id, shard_id in split_expert_params_mapping: + if weight_name not in name: + continue + consumed = True + name_mapped = name.replace(weight_name, param_name) + if name_mapped not in params_dict: + continue + param = params_dict[name_mapped] + weight_loader = typing.cast(Callable[..., bool], param.weight_loader) + if weight_loader( + param, + loaded_weight, + name_mapped, + shard_id=shard_id, + expert_id=expert_id, + return_success=True, + ): + loaded_params.add(name_mapped) + return consumed + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + params_dict = dict(self.named_parameters()) + pp_missing_layer_names = get_pp_missing_layer_names(self) + loaded_params: set[str] = set() + + mtp_start = self.config.num_hidden_layers + shared_weights = { + "model.embed_tokens.weight": "model.embed_tokens.weight", + "lm_head.weight": f"model.layers.{mtp_start}.shared_head.head.weight", + } + + num_experts = getattr(self.config, "n_routed_experts", 0) + sink_tp_size = get_tensor_model_parallel_world_size() + sink_tp_rank = get_tensor_model_parallel_rank() + n_local_head = self.config.num_attention_heads // sink_tp_size + head_rank_start = n_local_head * sink_tp_rank + head_rank_end = n_local_head * (sink_tp_rank + 1) + + # Routed-expert weights come in two checkpoint layouts: + # split: mlp.experts..gate_proj.weight + # fused: mlp.experts.gate_up_proj (all experts in one tensor) + # The split layout is resolved through the shared EPLB helper so the + # target param names stay in sync with the RoutedExperts module layout; + # the fused layout is resolved from the live parameter names. + split_expert_params_mapping = fused_moe_make_expert_params_mapping( + self, + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=num_experts, + ) + fused_expert_param_names: dict[tuple[str, str], str] = {} + for param_name in params_dict: + for tag in ("w13_weight", "w2_weight"): + if param_name.endswith(tag) and ".experts." in param_name: + base = param_name.split(".experts.")[0] + fused_expert_param_names[base, tag] = param_name + stacked_mapping = [ + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + # MLA runs both latent down-projections as one GEMM. + (".fused_qkv_a_proj", ".q_a_proj", 0), + (".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1), + ] + # Sparse DSA draft blocks build an Indexer whose wk / weights_proj are + # fused into a single MergedColumnParallelLinear (wk_weights_proj). + indexer_stacked_mapping = [ + (".wk_weights_proj", ".wk", 0), + (".wk_weights_proj", ".weights_proj", 1), + ] + # FP8 indexer wk dequant buffer (weight and scale arrive separately). + pending_wk_fp8: dict[str, dict[str, torch.Tensor]] = {} + + for name, loaded_weight in weights: + if name in shared_weights: + target_name = shared_weights[name] + if target_name in params_dict: + param = params_dict[target_name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, loaded_weight) + loaded_params.add(target_name) + continue + + spec_layer = None + if name.startswith("model.mtp_layers."): + parts = name.split(".") + if len(parts) > 3 and parts[2].isdigit(): + spec_layer = mtp_start + int(parts[2]) + name = name.replace( + f"model.mtp_layers.{parts[2]}.", + f"model.layers.{spec_layer}.", + ) + else: + spec_layer = _get_spec_layer_idx_from_weight_name(self.config, name) + + if spec_layer is None: + continue + + name = self._rewrite_spec_layer_name(spec_layer, name) + if name == "__skip__": + continue + name, loaded_weight = _prepare_mtp_fp8_expert_scale( + self.quant_config, name, loaded_weight + ) + + if "mlp.gate.e_score_correction_bias" in name: + name = name.replace("gate.e_score_correction_bias", "expert_bias") + + is_loaded = False + for param_name, weight_name, shard_id in stacked_mapping: + if weight_name not in name or ".experts." in name: + continue + name_mapped = name.replace(weight_name, param_name) + if name_mapped not in params_dict: + if is_pp_missing_parameter(name_mapped, self): + is_loaded = True + break + param = params_dict[name_mapped] + param.weight_loader(param, loaded_weight, shard_id) + loaded_params.add(name_mapped) + is_loaded = True + break + if is_loaded: + continue + + # FP8 indexer wk: dequantize to BF16 and load into the fused + # wk_weights_proj. PP-aware (skips layers not held by this rank). + if _try_load_fp8_indexer_wk( + name, + loaded_weight, + pending_wk_fp8, + params_dict, + loaded_params, + pp_missing_layer_names, + ): + continue + + # BF16 indexer wk / weights_proj: merge into the fused param. + is_loaded = False + for param_name, weight_name, shard_id in indexer_stacked_mapping: + if weight_name not in name or "wk_weights" in name: + continue + name_mapped = name.replace(weight_name, param_name) + if name_mapped not in params_dict: + if is_pp_missing_parameter(name_mapped, self): + is_loaded = True + break + param = params_dict[name_mapped] + param.weight_loader(param, loaded_weight, shard_id) + loaded_params.add(name_mapped) + is_loaded = True + break + if is_loaded: + continue + + is_loaded = False + if ".experts." in name: + is_loaded = self._load_expert_weight( + name, + loaded_weight, + params_dict, + loaded_params, + split_expert_params_mapping, + fused_expert_param_names, + num_experts, + ) + if is_loaded: + continue + + if "learnable_sink_param" in name: + if name in params_dict: + narrow_weight = loaded_weight[head_rank_start:head_rank_end] + n = narrow_weight.shape[0] + with torch.no_grad(): + params_dict[name][:n].copy_(narrow_weight) + loaded_params.add(name) + continue + + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: + continue + name = remapped_name + if name not in params_dict: + if is_pp_missing_parameter(name, self): + continue + if _should_skip_missing_mtp_scale_param(self.quant_config, name): + continue + logger.warning_once("Skipping unknown MTP weight: %s", name) + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + + logger.info_once("HYV4 MTP draft model loaded: %d params", len(loaded_params)) + unassigned_all = sorted(set(params_dict) - loaded_params) + # KVCacheScaleParameter (``k_scale`` / ``v_scale`` / ``q_scale`` / + # ``prob_scale``) is created by ``BaseKVCacheMethod.create_weights`` for + # every Attention layer that runs on an fp8-family quant method, and + # its -1.0 sentinel is replaced by the runtime default (1.0) in + # ``process_weights_after_loading`` -- which runs *after* this method. + # HY V4 checkpoints intentionally omit these (``kv_cache_quant_algo: + # null``); silence them only when the parameter is actually the + # sentinel type so a genuine mismatch on a same-named tensor still + # reaches the warning. + from vllm.model_executor.layers.quantization.kv_cache import ( + KVCacheScaleParameter, + ) + + unassigned = [ + name + for name in unassigned_all + if not isinstance(params_dict.get(name), KVCacheScaleParameter) + ] + if unassigned: + # A draft parameter with no checkpoint source keeps its sentinel + # init value (FP8 scales start at finfo(float32).min), which + # silently destroys draft quality instead of failing the load. + logger.warning( + "HYV4 MTP draft model: %d parameters received no checkpoint value: %s", + len(unassigned), + ", ".join(unassigned), + ) + return loaded_params diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index 833c90d8ea..e13135293e 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -451,9 +451,10 @@ def get_harmony_structural_tag( builtin_tools: list[BuiltinToolParam], tool_choice: SimplifiedToolChoice, reasoning: bool, + token_suffix: str = "", ) -> StructuralTag: # reasoning always enabled for Harmony - del reasoning + del reasoning, token_suffix if builtin_tools: # Fallback for built-in tools diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index 8b52c4fca8..18e38e74c1 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -84,6 +84,10 @@ _REASONING_PARSERS_TO_REGISTER = { "hy_v3_reasoning_parser", "HYV3ReasoningParser", ), + "hy_v4": ( + "hy_v4_reasoning_parser", + "HYV4ReasoningParser", + ), "kimi_k2": ( "kimi_k2_reasoning_parser", "KimiK2ReasoningParser", diff --git a/vllm/reasoning/hy_v4_reasoning_parser.py b/vllm/reasoning/hy_v4_reasoning_parser.py new file mode 100644 index 0000000000..9fb8df6646 --- /dev/null +++ b/vllm/reasoning/hy_v4_reasoning_parser.py @@ -0,0 +1,324 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from typing import TypedDict + +import regex as re + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.engine.protocol import DeltaMessage +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.logger import init_logger +from vllm.reasoning import ReasoningParser +from vllm.tokenizers import TokenizerLike + +logger = init_logger(__name__) + +# Must match the chat template's own default for ``reasoning_effort``. +DEFAULT_REASONING_EFFORT = "high" + + +class ReasoningDelta(TypedDict): + """One streaming reasoning delta. Either/both fields may be None.""" + + reasoning: str | None + content: str | None + + +# NOTE: mirrored in ``vllm.tool_parsers.hy_v4_tool_parser`` (same pattern as +# ``gemma4_utils``) so neither package depends on the other. +def detect_token_suffix(tokenizer: TokenizerLike) -> str: + """Detect the per-checkpoint suffix used by Hunyuan structural tokens. + + Args: + tokenizer: Tokenizer of the served checkpoint. + + Returns: + The suffix including its leading colon (e.g. ``":6124c78e"``), or ``""`` + when the checkpoint uses unsuffixed tokens. + + Raises: + RuntimeError: The tokenizer declares the structural tokens through + ``model_specific_special_tokens``, which transformers 5 no longer + round-trips. + """ + + import transformers + + if int(transformers.__version__.split(".")[0]) >= 5: + init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} + think_begin_as_special = init_kwargs.get( + "model_specific_special_tokens", {} + ).get("think_begin_token", "") + if think_begin_as_special: + raise RuntimeError( + "This checkpoint declares HYV4 structural tokens (think_begin_token" + "/toolcalls_begin_token/argkey_begin_token) in " + "tokenizer_config.json, which transformers 5 no longer supports. " + "Remove those fields and keep the tokens in the tokenizer's own " + "token definitions so the suffix can be read from the vocab." + ) + + structural_token_re = re.compile( + r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>" + ) + for token in tokenizer.get_vocab(): + match = structural_token_re.fullmatch(token) + if match: + return match.group(1) or "" + + return "" + + +class HYV4ReasoningExtractor: + """Reasoning extraction for HYV4, on plain data. + + ``thinking`` selects the mode: + - ``True``: parse ``...``; the start token is injected at the + end of the prompt and is therefore usually absent from the output. + - ``False``: ``no_think`` -- the whole output is content, no reasoning. + """ + + def __init__(self, vocab: dict[str, int], token_suffix: str, thinking: bool = True): + self.thinking = thinking + self.start_token: str = f"" + self.end_token: str = f"" + + start_token_id = vocab.get(self.start_token) + end_token_id = vocab.get(self.end_token) + if thinking and (start_token_id is None or end_token_id is None): + raise RuntimeError( + "HYV4 reasoning extractor could not locate think " + "start/end tokens in the tokenizer!" + ) + # In no_think mode the ids are unused (kept for symmetry / debugging). + self.start_token_id: int | None = start_token_id + self.end_token_id: int | None = end_token_id + + def has_reasoning_ended(self, token_ids: Sequence[int]) -> bool: + """Idempotent STATE query: has ```` been emitted yet? + + Monotonic -- once the end token is present this stays True. Use for + "has reasoning finished, given everything so far". (no_think is always + ended.) Contrast with ``reasoning_ended_in_delta`` (edge detector). + """ + if not self.thinking: + return True + for i in range(len(token_ids) - 1, -1, -1): + if token_ids[i] == self.start_token_id: + return False + if token_ids[i] == self.end_token_id: + return True + return False + + def reasoning_ended_in_delta( + self, token_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + """Edge detector (NOT idempotent): did reasoning end *in this* step? + + True only on the step whose ``delta_ids`` contains ````. Callers + use it once behind their own "already ended" latch, so it need not be + monotonic; it only checks the new delta (cheaper than scanning the whole + sequence). Contrast with ``has_reasoning_ended`` (cumulative state). + """ + if not self.thinking: + return True + end_token_id = self.end_token_id + if end_token_id is None: + return False + return end_token_id in delta_ids + + def extract_content_ids(self, input_ids: list[int]) -> list[int]: + """Return the token ids after ```` (all of them in no_think).""" + if not self.thinking: + return input_ids + head = input_ids[: max(0, len(input_ids) - 1)] + if self.end_token_id not in head: + return [] + return input_ids[input_ids.index(self.end_token_id) + 1 :] + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + """Count reasoning tokens (0 in no_think). + + The ```` start token is injected at the END of the prompt, so it + is normally absent from ``token_ids``. Only skip a leading start token + when it actually appears first (legacy / no prompt-injection); otherwise + count every token up to ````. + """ + if not self.thinking: + return 0 + start = 1 if (token_ids and token_ids[0] == self.start_token_id) else 0 + try: + return token_ids.index(self.end_token_id, start) - start + except ValueError: + # No end token -- reasoning was truncated; everything counts. + return len(token_ids) - start + + def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]: + """Split a complete output into ``(reasoning, content)``. + + In no_think mode reasoning is None and the whole output is content. The + start token may be absent (injected in the prompt), so reasoning is + assumed to begin at the start of the output. + """ + if not self.thinking: + return None, model_output + + parts = model_output.partition(self.start_token) + model_output = parts[2] if parts[1] else parts[0] + + if self.end_token not in model_output: + return model_output, None + reasoning, _, content = model_output.partition(self.end_token) + return reasoning, content or None + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> ReasoningDelta | None: + """Classify a streaming delta as reasoning and/or content. + + In no_think mode every delta is content. Otherwise: same logic as vLLM's + generic thinking parser, except the final branch (no start token seen + anywhere) treats output as reasoning-then-content, because HYV4 injects + ```` in the prompt (absent from the output). + """ + if not self.thinking: + if not delta_text: + return None + return ReasoningDelta(reasoning=None, content=delta_text) + + # Skip a delta that is a single lone start/end special token. + if len(delta_token_ids) == 1 and delta_token_ids[0] in ( + self.start_token_id, + self.end_token_id, + ): + return None + + if self.start_token_id in previous_token_ids: + if self.end_token_id in delta_token_ids: + end_index = delta_text.find(self.end_token) + reasoning = delta_text[:end_index] + content = delta_text[end_index + len(self.end_token) :] + return ReasoningDelta(reasoning=reasoning, content=content or None) + elif self.end_token_id in previous_token_ids: + return ReasoningDelta(reasoning=None, content=delta_text) + else: + return ReasoningDelta(reasoning=delta_text, content=None) + elif self.start_token_id in delta_token_ids: + if self.end_token_id in delta_token_ids: + start_index = delta_text.find(self.start_token) + end_index = delta_text.find(self.end_token) + reasoning = delta_text[start_index + len(self.start_token) : end_index] + content = delta_text[end_index + len(self.end_token) :] + return ReasoningDelta(reasoning=reasoning, content=content or None) + else: + return ReasoningDelta(reasoning=delta_text, content=None) + else: + # HYV4: was injected at the end of the prompt, so it never + # appears in the output. Treat the stream as reasoning until + # , then content. + if self.end_token_id in delta_token_ids: + end_index = delta_text.find(self.end_token) + reasoning = delta_text[:end_index] + content = delta_text[end_index + len(self.end_token) :] + return ReasoningDelta(reasoning=reasoning, content=content or None) + elif self.end_token_id in previous_token_ids: + return ReasoningDelta(reasoning=None, content=delta_text) + else: + return ReasoningDelta(reasoning=delta_text, content=None) + + +def build_reasoning_extractor( + tokenizer: TokenizerLike, *, thinking: bool = True +) -> HYV4ReasoningExtractor: + return HYV4ReasoningExtractor( + tokenizer.get_vocab(), detect_token_suffix(tokenizer), thinking + ) + + +class HYV4ReasoningParser(ReasoningParser): + """vLLM adapter around :class:`HYV4ReasoningExtractor`. + + Decides ``thinking`` from the request's ``reasoning_effort`` and converts + the extractor's ``ReasoningDelta`` to ``DeltaMessage``. + """ + + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): + super().__init__(tokenizer, *args, **kwargs) + + # Decide thinking vs no_think from the request; the extractor handles + # both modes internally. Prefer chat_template_kwargs.reasoning_effort, + # then the top-level reasoning_effort. The default must match the chat + # template, which sets reasoning_effort='high' when the request omits + # it -- defaulting to no_think here would return the whole + # chain-of-thought (and a stray ````) as content. + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} + reasoning_effort = chat_kwargs.get("reasoning_effort", None) + if not reasoning_effort: + reasoning_effort = ( + kwargs.get("reasoning_effort") or DEFAULT_REASONING_EFFORT + ) + logger.debug("reasoning_effort for choosing parser: %s", reasoning_effort) + thinking = reasoning_effort != "no_think" + + self._extractor = build_reasoning_extractor(tokenizer, thinking=thinking) + + # Reported regardless of the per-request thinking mode: ``ReasoningConfig`` + # is engine-wide and needs the checkpoint's delimiters to derive the token + # ids used for thinking budget enforcement. + @property + def reasoning_start_str(self) -> str: + return self._extractor.start_token + + @property + def reasoning_end_str(self) -> str: + return self._extractor.end_token + + def is_reasoning_end(self, input_ids) -> bool: + return self._extractor.has_reasoning_ended(input_ids) + + def is_reasoning_end_streaming(self, input_ids, delta_ids) -> bool: + return self._extractor.reasoning_ended_in_delta(input_ids, delta_ids) + + def extract_content_ids(self, input_ids) -> list[int]: + return self._extractor.extract_content_ids(input_ids) + + def count_reasoning_tokens(self, token_ids) -> int: + return self._extractor.count_reasoning_tokens(token_ids) + + def extract_reasoning( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> tuple[str | None, str | None]: + return self._extractor.extract_reasoning(model_output) + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids, + current_token_ids, + delta_token_ids, + ) -> DeltaMessage | None: + delta = self._extractor.extract_reasoning_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + ) + if delta is None: + return None + return DeltaMessage(reasoning=delta["reasoning"], content=delta["content"]) diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index 9c218a5eeb..06ed049beb 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -94,6 +94,10 @@ _TOOL_PARSERS_TO_REGISTER = { "hy_v3_tool_parser", "HYV3ToolParser", ), + "hy_v4": ( + "hy_v4_tool_parser", + "HYV4ToolParser", + ), "internlm": ( "internlm2_tool_parser", "Internlm2ToolParser", diff --git a/vllm/tool_parsers/hy_v4_tool_parser.py b/vllm/tool_parsers/hy_v4_tool_parser.py new file mode 100644 index 0000000000..c3d64a56f9 --- /dev/null +++ b/vllm/tool_parsers/hy_v4_tool_parser.py @@ -0,0 +1,1166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import ast +import json +from typing import Any, TypedDict + +import regex as re + +import vllm.envs as envs +from vllm.entrypoints.chat_utils import make_tool_call_id +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.engine.protocol import ( + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, + ToolCall, +) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.logger import init_logger +from vllm.tokenizers import TokenizerLike +from vllm.tool_parsers.abstract_tool_parser import ToolParser +from vllm.tool_parsers.utils import partial_tag_overlap + +logger = init_logger(__name__) + +# ============================================================================= +# Plain-dict contract with the vLLM-side wrapper +# ============================================================================= + + +class ToolSchema(TypedDict): + """A tool as seen by this parser (built by the wrapper from ``request.tools``).""" + + name: str + parameters: dict[str, Any] | None # JSON Schema, or None + + +class ToolCallDict(TypedDict): + """One extracted tool call. ``arguments`` is a JSON-encoded string.""" + + name: str + arguments: str + + +class ExtractResult(TypedDict): + """Return of ``extract_tool_calls`` (non-streaming).""" + + tools_called: bool + content: str | None + tool_calls: list[ToolCallDict] + + +class StreamToolCall(TypedDict): + """One streaming tool-call delta. + + ``new`` is True only on the chunk that first emits this tool's name -- the + wrapper mints an id + sets type on that chunk only. ``arguments`` is the + incremental JSON diff (or None). + """ + + index: int + new: bool + name: str | None + arguments: str | None + + +class StreamDelta(TypedDict): + """Return of ``extract_tool_calls_streaming``. + + A single engine delta can carry content plus every tool call that was + drained from the buffer (batched decode), so ``tool_calls`` is not limited + to one entry. + """ + + content: str | None + tool_calls: list[StreamToolCall] + + +# NOTE: mirrored in ``vllm.reasoning.hy_v4_reasoning_parser`` (same pattern as +# ``gemma4_utils``) so neither package depends on the other. +def detect_token_suffix(tokenizer: TokenizerLike) -> str: + """Detect the per-checkpoint suffix used by Hunyuan structural tokens. + + Args: + tokenizer: Tokenizer of the served checkpoint. + + Returns: + The suffix including its leading colon (e.g. ``":6124c78e"``), or ``""`` + when the checkpoint uses unsuffixed tokens. + + Raises: + RuntimeError: The tokenizer declares the structural tokens through + ``model_specific_special_tokens``, which transformers 5 no longer + round-trips. + """ + + import transformers + + if int(transformers.__version__.split(".")[0]) >= 5: + init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} + think_begin_as_special = init_kwargs.get( + "model_specific_special_tokens", {} + ).get("think_begin_token", "") + if think_begin_as_special: + raise RuntimeError( + "This checkpoint declares HYV4 structural tokens (think_begin_token" + "/toolcalls_begin_token/argkey_begin_token) in " + "tokenizer_config.json, which transformers 5 no longer supports. " + "Remove those fields and keep the tokens in the tokenizer's own " + "token definitions so the suffix can be read from the vocab." + ) + + structural_token_re = re.compile( + r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>" + ) + for token in tokenizer.get_vocab(): + match = structural_token_re.fullmatch(token) + if match: + return match.group(1) or "" + + return "" + + +# ============================================================================= +# Argument-value parsing utilities (stateless, pure functions) +# +# These map the model's raw ```` text to typed Python values using +# the tool's JSON Schema. They do not touch any parser state, so they live at +# module level instead of tangling up HYV4ToolExtractor. +# ============================================================================= + +_TYPE_ALIASES: dict[str, str] = { + "str": "string", + "text": "string", + "varchar": "string", + "char": "string", + "enum": "string", + "bool": "boolean", + "binary": "boolean", + "int": "integer", + "float": "number", + "double": "number", + "list": "array", + "dict": "object", + "map": "object", +} + +# Prefix-based wildcard matching for non-standard type names. +# Following the same approach as qwen3coder_tool_parser._convert_param_value +# which uses param_type.startswith("int"), startswith("uint"), etc. +_INTEGER_PREFIXES: tuple[str, ...] = ("int", "uint", "long", "short", "unsigned") +_NUMBER_PREFIXES: tuple[str, ...] = ("num", "float") + + +def _normalize_type(raw_type: str) -> str: + """Map non-standard type aliases to JSON Schema standard names. + + First performs exact lookup in _TYPE_ALIASES. On miss, falls back to + prefix-based matching using startswith() + - int*/uint*/long*/short*/unsigned* → "integer" + - num*/float* → "number" + """ + exact = _TYPE_ALIASES.get(raw_type) + if exact is not None: + return exact + lower = raw_type.lower() + if any(lower.startswith(p) for p in _INTEGER_PREFIXES): + return "integer" + if any(lower.startswith(p) for p in _NUMBER_PREFIXES): + return "number" + return raw_type + + +def _get_arg_schema( + function_name: str, + arg_key: str, + tools: list[ToolSchema] | None, +) -> dict[str, Any]: + """Look up a specific argument's property schema from the tools list.""" + if tools is None: + return {} + for tool in tools: + if tool.get("name") == function_name: + parameters = tool.get("parameters") + if parameters is None: + return {} + return parameters.get("properties", {}).get(arg_key, {}) + logger.warning_once("No tool named '%s'.", function_name) + return {} + + +def _get_schema_options(arg_schema: dict[str, Any]) -> list[dict[str, Any]]: + """Normalize any property schema into a list of sub-schemas. + - has type (single type) → return [arg_schema] + - anyOf → return the anyOf list + - oneOf → return the oneOf list + - fallback → [{"type": "string"}] + + Note: single ``type`` has the highest priority. + """ + if "type" in arg_schema: + type_val = arg_schema["type"] + # JSON Schema allows "type" to be an array to represent union types, + # e.g. "type": ["string", "object"]. + # Expand it into an anyOf-equivalent format: + # [{"type": "string"}, {"type": "object"}] + # so that _get_types / _parse_value can handle it uniformly later. + if isinstance(type_val, list): + return [{"type": t} for t in type_val] + return [arg_schema] + if "anyOf" in arg_schema: + return arg_schema["anyOf"] + if "oneOf" in arg_schema: + return arg_schema["oneOf"] + + return [{"type": "string"}] + + +def _get_types(arg_schema: dict[str, Any]) -> set[str]: + """Extract normalized, non-null type set from a property schema.""" + schemas = _get_schema_options(arg_schema) + return {_normalize_type(s.get("type", "string")) for s in schemas} - {"null"} + + +def _is_only_string_type( + function_name: str, + arg_key: str, + tools: list[ToolSchema] | None, +) -> bool: + """Return True if the parameter's type set is exactly {"string"}. + + Only pure string types get partial value streaming; compound types like + anyOf(string | array) do not, since the partial value might end up being a + JSON array or object. + """ + types = _get_types(_get_arg_schema(function_name, arg_key, tools)) + return types == {"string"} + + +def _try_parse_bool(value: str) -> bool | None: + """Try to parse a string as bool; return None on failure.""" + lower = value.lower() + if lower == "true": + return True + elif lower == "false": + return False + return None + + +def _try_parse_int(value: str) -> int | None: + """Try to parse a string as int; return None on failure.""" + try: + return int(value) + except (ValueError, TypeError): + return None + + +def _try_parse_wildcard_number(value: str) -> int | float | None: + """Try to parse a string as a number (int or float). + + Decision rule: if the string contains '.' or 'e'/'E' (scientific notation), + parse as float; otherwise parse as int. + + Examples: "5" → 5, "5.0" → 5.0, "1e3" → 1000.0, "-3" → -3. + Return None on failure. + """ + try: + if "." in value or "e" in value or "E" in value: + return float(value) + return int(value) + except (ValueError, TypeError): + return None + + +def _deserialize(value: str) -> Any: + """Deserialize a string value using json.loads then ast.literal_eval.""" + try: + return json.loads(value) + except Exception: + pass + try: + return ast.literal_eval(value) + except Exception: + pass + return value + + +def _parse_value( + value: str, + function_name: str, + arg_key: str, + tools: list[ToolSchema] | None, +) -> Any: + """Unified argument value parser with anyOf/oneOf support. + + Fallthrough chain: + bool → int → number(wildcard_number) + → json.loads for array/object → string → _deserialize + """ + types = _get_types(_get_arg_schema(function_name, arg_key, tools)) + + # 1. Try bool + if "boolean" in types: + result_bool = _try_parse_bool(value) + if result_bool is not None: + return result_bool + + # 2. Try int + if "integer" in types: + result_int = _try_parse_int(value) + if result_int is not None: + return result_int + + # 3. Try number (wildcard_number: int if no '.'/e/E, float otherwise) + if "number" in types: + result_number = _try_parse_wildcard_number(value) + if result_number is not None: + return result_number + + # 4. Try json.loads (covers array/object and other unlisted types) + if types - {"string", "boolean", "integer", "number"}: + try: + return json.loads(value) + except (json.JSONDecodeError, ValueError): + pass + + # 5. String fallback + if "string" in types: + return value + + # 6. Final fallback + return _deserialize(value) + + +# ============================================================================= +# Stateful tool-call parser +# ============================================================================= + + +class HYV4ToolExtractor: + """Pure tool-call parsing logic for HYV4. Returns plain dicts; no vLLM types. + + Holds only parser state: the structural token strings/ids, the compiled + regexes, and the streaming incremental state. Stateless argument parsing is + delegated to the module-level utilities above. + + Streaming has two entry paths sharing one incremental argument parser, + selected by the ``guided`` flag of :meth:`extract_tool_calls_streaming`: + normal / auto tool choice detects markers on atomic token ids so ordinary + ``<`` in content is never withheld, while structural-tag guided decoding + detects them on the string level because the grammar may split a marker + across sub-word tokens. + """ + + def __init__(self, vocab: dict[str, int], token_suffix: str, strict: bool): + self._strict = strict + + # Kept so the wrapper can build a matching structural tag for + # constrained decoding without re-scanning the tokenizer vocab. + self.token_suffix: str = token_suffix + + # Running state read by the vLLM serving layer (mirrored by the wrapper). + self.prev_tool_call_arr: list[dict] = [] + self.current_tool_id: int = -1 + self.streamed_args_for_tool: list[str] = [] + + # Streaming state: send tool name first, then return arguments at once + self._streaming_tool_name: str | None = None # tool name being streamed + + # State fields for incremental argument streaming + self._completed_args: dict = {} # closed {key: parsed_value} + self._current_arg_key: str | None = None # key being collected + self._current_arg_is_string: bool = False # is current arg pure string? + self._streamed_json_len: int = 0 # bytes of JSON already sent + + self.tool_calls_start_token: str = f"" + self.tool_calls_end_token: str = f"" + + self.tool_call_start_token: str = f"" + self.tool_call_end_token: str = f"" + + self.arg_key_start_token: str = f"" + self.arg_key_end_token: str = f"" + + self.arg_value_start_token: str = f"" + self.arg_value_end_token: str = f"" + + logger.debug( + "HYV4ToolExtractor structural tokens: %s ... %s, suffix=%r, strict=%s", + self.tool_calls_start_token, + self.tool_calls_end_token, + token_suffix, + strict, + ) + + # The suffix comes from the tokenizer vocab, so it must not be treated + # as a pattern. + self.tool_call_regex = re.compile( + rf"{re.escape(self.tool_call_start_token)}(.*?)" + rf"{re.escape(self.tool_call_end_token)}", + re.DOTALL, + ) + + self.func_args_regex = re.compile( + rf"{re.escape(self.arg_key_start_token)}(.*?)" + rf"{re.escape(self.arg_key_end_token)}" + rf"{re.escape(self.arg_value_start_token)}(.*?)" + rf"{re.escape(self.arg_value_end_token)}", + re.DOTALL, + ) + + self.tool_calls_start_token_id = vocab.get(self.tool_calls_start_token) + self.tool_calls_end_token_id = vocab.get(self.tool_calls_end_token) + + self.tool_call_start_token_id = vocab.get(self.tool_call_start_token) + self.tool_call_end_token_id = vocab.get(self.tool_call_end_token) + self._buffer = "" + + if ( + self.tool_calls_start_token_id is None + or self.tool_calls_end_token_id is None + ): + raise RuntimeError( + "HYV4 tool extractor could not locate tool call " + "start/end tokens in the tokenizer!" + ) + + def _extract_tool_calls( + self, + model_output: str, + tools: list[ToolSchema] | None, + ) -> list[ToolCallDict]: + try: + tool_calls: list[ToolCallDict] = [] + function_calls = self.tool_call_regex.findall(model_output) + for tool_call_body in function_calls: + arg_start = tool_call_body.find(self.arg_key_start_token) + if arg_start == -1: + function_name = tool_call_body.strip() + function_args = "" + else: + function_name = tool_call_body[:arg_start].strip() + function_args = tool_call_body[arg_start:].strip() + + arg_pairs = self.func_args_regex.findall(function_args) + arg_dict = {} + for key, value in arg_pairs: + arg_dict[key] = _parse_value(value, function_name, key, tools) + tool_calls.append( + ToolCallDict( + name=function_name, + arguments=json.dumps(arg_dict, ensure_ascii=False), + ) + ) + return tool_calls + except Exception: + logger.exception("Error in extracting tool call from response.") + return [] + + def _extract_tool_calls_strict( + self, + model_output: str, + tools: list[ToolSchema] | None, + ) -> list[ToolCallDict]: + start_idx = model_output.find(self.tool_calls_start_token) + end_idx = model_output.find(self.tool_calls_end_token) + if start_idx == -1 or end_idx == -1 or end_idx < start_idx: + raise ValueError("Malformed tool block wrapper.") + + tool_block = model_output[ + start_idx + len(self.tool_calls_start_token) : end_idx + ] + if tool_block.count(self.tool_call_start_token) != tool_block.count( + self.tool_call_end_token + ): + raise ValueError("Malformed tool_call wrapper count.") + + tool_calls: list[ToolCallDict] = [] + function_calls = self.tool_call_regex.findall(tool_block) + if tool_block.count(self.tool_call_start_token) != len(function_calls): + raise ValueError("Malformed tool_call body.") + + for tool_call_body in function_calls: + arg_start = tool_call_body.find(self.arg_key_start_token) + if arg_start == -1: + function_name = tool_call_body.strip() + function_args = "" + else: + function_name = tool_call_body[:arg_start].strip() + function_args = tool_call_body[arg_start:].strip() + + if not function_name: + raise ValueError("Empty function name in tool call.") + + if function_args.count(self.arg_key_start_token) != function_args.count( + self.arg_key_end_token + ) or function_args.count(self.arg_value_start_token) != function_args.count( + self.arg_value_end_token + ): + raise ValueError("Malformed argument tag count.") + + arg_pairs = self.func_args_regex.findall(function_args) + remainder = self.func_args_regex.sub("", function_args) + if remainder.strip(): + raise ValueError("Unparsed argument payload remains.") + + arg_dict = {} + for key, value in arg_pairs: + arg_dict[key] = _parse_value(value, function_name, key, tools) + + tool_calls.append( + ToolCallDict( + name=function_name, + arguments=json.dumps(arg_dict, ensure_ascii=False), + ) + ) + + if not tool_calls: + raise ValueError("No valid tool calls extracted.") + + return tool_calls + + def extract_tool_calls( + self, + model_output: str, + tools: list[ToolSchema] | None, + ) -> ExtractResult: + # sanity check; avoid unnecessary processing + if self.tool_calls_start_token not in model_output: + return ExtractResult( + tools_called=False, content=model_output, tool_calls=[] + ) + else: + try: + if self._strict: + tool_calls = self._extract_tool_calls_strict(model_output, tools) + else: + tool_calls = self._extract_tool_calls(model_output, tools) + if not tool_calls: + raise ValueError("No valid tool calls extracted.") + + s_index = model_output.find(self.tool_calls_start_token) + content = model_output[:s_index] if s_index != -1 else model_output + return ExtractResult( + tools_called=True, + content=content if content else None, + tool_calls=tool_calls, + ) + + except Exception: + logger.warning( + "HYV4ToolExtractor detected malformed tool output; " + "falling back to raw content." + ) + return ExtractResult( + tools_called=False, + content=model_output, + tool_calls=[], + ) + + def _reset_streaming_tool_state(self): + """Reset the streaming state for a single tool call.""" + self._streaming_tool_name = None + self._completed_args = {} + self._current_arg_key = None + self._current_arg_is_string = False + self._streamed_json_len = 0 + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids, + current_token_ids, + delta_token_ids, + tools: list[ToolSchema] | None, + *, + guided: bool = False, + ) -> StreamDelta | None: + """Consume one engine delta and emit content / tool-call deltas. + + Two entry paths share the same incremental argument parser: + - ``guided=False`` (normal / auto tool choice): the structural markers + arrive as atomic tokenizer tokens, so detection is done on token ids + and ordinary ``<`` in model content is never withheld. + - ``guided=True`` (structural-tag constrained decoding): the grammar + only constrains strings, so ```` may be split across + sub-word tokens and must be detected on the string level. + + Args: + previous_text: Text decoded before this delta. + current_text: Full text decoded so far. + delta_text: Text decoded in this delta. + previous_token_ids: Token ids before this delta. + current_token_ids: Token ids decoded so far. + delta_token_ids: Token ids in this delta. + tools: Plain tool schemas used for argument typing. + guided: Whether guided decoding may split markers into sub-word + tokens. + + Returns: + A streaming delta carrying content and/or the tool calls drained + from the buffer, or None when nothing can be emitted yet. + """ + content_delta: str | None = None + tool_calls: list[StreamToolCall] = [] + + if guided: + # Guided path: a structural-tag grammar constrains strings, not + # token ids, so the atomic ``tool_calls_start_token_id`` never + # appears in the token stream and an id-based check would + # misclassify every tool delta as plain content. + marker = self.tool_calls_start_token + marker_pos = current_text.find(marker) + + if marker_pos == -1: + # Marker not seen yet. The tail of ``current_text`` might still + # be a partial prefix of the marker split across deltas; hold + # that tail back and only stream the safe portion as content. + safe_end = len(current_text) - partial_tag_overlap(current_text, marker) + # Emit only the newly-arrived, safe content (delta relative to + # what was previously streamed). + content = current_text[len(previous_text) : safe_end] + return StreamDelta( + content=content if content else None, + tool_calls=[], + ) + + # Marker present. Stream any content that precedes it (only the + # part we have not already emitted), then buffer the tool payload. + # + # The payload is everything after the marker in the *full* text. We + # keep the buffer in sync with it by only appending the portion that + # arrived in this delta, mirroring the incremental ``+=`` semantics + # while remaining correct when the marker itself was split across + # sub-word tokens. + content_end = marker_pos + content = current_text[len(previous_text) : content_end] + + payload_start = marker_pos + len(marker) + # How much of the payload had already been buffered before this + # delta. + prev_payload_len = max(len(previous_text) - payload_start, 0) + new_payload = current_text[payload_start + prev_payload_len :] + self._buffer += new_payload + + if content: + content_delta = content + else: + # Atomic-token path: normal / auto generation emits the HYV4 + # structural markers as single tokenizer tokens. Pass ordinary + # content through exactly as decoded, including comparison + # operators such as ``<``. + if self.tool_calls_start_token_id not in current_token_ids: + return StreamDelta(content=delta_text, tool_calls=[]) + + if self.tool_calls_start_token in delta_text: + text_parts = delta_text.split(self.tool_calls_start_token) + self._buffer += text_parts[-1] + if text_parts[0]: + content_delta = text_parts[0] + else: + self._buffer += delta_text + + # Encountered finish, extract valid arguments + if ( + current_text.find(self.tool_call_end_token + self.tool_calls_end_token) + != -1 + and self._buffer.find(self.tool_call_end_token) == -1 + ): + self._buffer += self.tool_call_end_token + self.tool_calls_end_token + + # Drain every complete tool call that is already buffered so a single + # engine delta can carry content + N tool calls (batched decode). + while True: + cur_text = self._buffer + + # Haven't encountered tool_call start tag yet; drop the inter-tag + # padding but keep any tail that may be a partial tool_call start + # tag (guided decoding can split it across deltas). + start_idx = cur_text.find(self.tool_call_start_token) + if start_idx == -1 and self._streaming_tool_name is None: + keep = partial_tag_overlap(cur_text, self.tool_call_start_token) + self._buffer = cur_text[len(cur_text) - keep :] if keep else "" + break + + # === Phase 1: Detect tool name (send when first arg_key is seen) = + # ``name_new`` mirrors the old ``name_delta is not None``: it marks + # the chunk that first emits this tool's name, which is the only + # chunk the wrapper mints an id + type for. + name_new = False + pending_name: str | None = None + if self._streaming_tool_name is None: + arg_idx = cur_text.find(self.arg_key_start_token, start_idx) + end_idx = cur_text.find(self.tool_call_end_token, start_idx) + if arg_idx == -1 and end_idx == -1: + # tool name not yet complete; keep buffering from + # tool_call_start + self._buffer = cur_text[start_idx:] + break + + name_start = start_idx + len(self.tool_call_start_token) + name_end = arg_idx if arg_idx != -1 else end_idx + + tool_name = cur_text[name_start:name_end].strip() + self._streaming_tool_name = tool_name + + if arg_idx != -1: + self._buffer = cur_text[arg_idx:] + else: + self._buffer = cur_text[end_idx:] + + # Increment tool_id and mark that a name chunk should be emitted + self.current_tool_id += 1 + name_new = True + pending_name = tool_name + + # Check if buffer already has complete arguments + # (all-in-one-delta); otherwise only the name is ready now. + if self.tool_call_end_token not in self._buffer: + tool_calls.append( + StreamToolCall( + index=self.current_tool_id, + new=True, + name=tool_name, + arguments=None, + ) + ) + break + # Buffer already has a complete tool call; fall through to + # phase 2 below. + + # === Phase 2: Incremental argument streaming === + result = self._extract_streaming_incremental(name_new, pending_name, tools) + if result: + tool_calls.extend(result["tool_calls"]) + + # Continue only after the current tool call has completed and the + # remaining structural payload starts with another tool call. + # A literal ```` may legitimately appear inside an open + # string argument; in that case the current tool name is still set + # and the buffer has not been consumed, so continuing would spin on + # the same buffer forever. + if self._streaming_tool_name is None and self._buffer.startswith( + self.tool_call_start_token + ): + continue + break + + if content_delta is not None or tool_calls: + return StreamDelta(content=content_delta, tool_calls=tool_calls) + return None + + def _make_args_delta(self, argument_diff: str) -> StreamDelta: + """Build an args-only streaming delta (no id/type -> new=False).""" + return StreamDelta( + content=None, + tool_calls=[ + StreamToolCall( + index=self.current_tool_id, + new=False, + name=None, + arguments=argument_diff, + ) + ], + ) + + def _extract_streaming_incremental( + self, + name_new: bool, + pending_name: str | None, + tools: list[ToolSchema] | None, + ) -> StreamDelta | None: + """Incremental phase-2: scan tags in buffer, emit JSON diffs. + + Strategy: + - Track completed args and emit each one as a JSON fragment. + - For string-typed args, stream the value character-by-character. + - Withhold the closing ``}`` until ```` is seen. + + We build JSON manually via fragments rather than using json.dumps + with a cursor, because json.dumps of partial-vs-full string values + produces incompatible prefixes (e.g. ``""}`` vs ``"Hello"}``). + """ + buf = self._buffer + is_complete = self.tool_call_end_token in buf + + if is_complete: + end_idx = buf.find(self.tool_call_end_token) + args_text = buf[:end_idx] + remaining = buf[end_idx + len(self.tool_call_end_token) :] + else: + args_text = buf + remaining = "" + + # --- scan all fully closed kv pairs --- + arg_pairs = self.func_args_regex.findall(args_text) + for key, value in arg_pairs: + key = key.strip() + if key not in self._completed_args: + self._completed_args[key] = _parse_value( + value, self._streaming_tool_name or "", key, tools + ) + + # --- detect partial (unclosed) kv at the tail --- + last_closed_end = 0 + for m in self.func_args_regex.finditer(args_text): + last_closed_end = m.end() + tail = args_text[last_closed_end:] + + partial_key: str | None = None + partial_value: str | None = None + + ak_start = tail.find(self.arg_key_start_token) + if ak_start != -1: + ak_end = tail.find( + self.arg_key_end_token, + ak_start + len(self.arg_key_start_token), + ) + if ak_end != -1: + partial_key = tail[ + ak_start + len(self.arg_key_start_token) : ak_end + ].strip() + self._current_arg_key = partial_key + self._current_arg_is_string = _is_only_string_type( + self._streaming_tool_name or "", + partial_key, + tools, + ) + + av_start = tail.find(self.arg_value_start_token, ak_end) + if av_start != -1: + val_content_start = av_start + len(self.arg_value_start_token) + if self._current_arg_is_string: + raw_value = tail[val_content_start:] + # Guided decoding can split ```` across + # deltas; hold back a partial closing tag so it is not + # streamed as part of the value. + keep = len(raw_value) - partial_tag_overlap( + raw_value, self.arg_value_end_token + ) + partial_value = raw_value[:keep] + else: + # key not yet closed + self._current_arg_key = None + self._current_arg_is_string = False + + # --- build the current JSON snapshot as a string --- + # We construct JSON manually so we can precisely control + # what gets sent incrementally. + snapshot_parts: list[str] = [] + for k, v in self._completed_args.items(): + k_json = json.dumps(k, ensure_ascii=False) + v_json = json.dumps(v, ensure_ascii=False) + snapshot_parts.append(f"{k_json}: {v_json}") + + if partial_key is not None and partial_value is not None: + k_json = json.dumps(partial_key, ensure_ascii=False) + # For string partial value, we build the JSON string + # WITHOUT the closing quote, so the prefix stays stable + # as the value grows. The closing `"` and `}` will be + # sent when the value or tool_call closes. + escaped_val = json.dumps(partial_value, ensure_ascii=False)[1:-1] + # Note: no closing " here - it's appended only on close + snapshot_parts.append(f'{k_json}: "{escaped_val}') + + snapshot = "{" + ", ".join(snapshot_parts) + "}" + + # --- compute diff --- + argument_diff: str | None = None + + if is_complete: + # Tool call finished - send everything remaining. + # Build final snapshot with proper JSON (all values closed). + final_args = dict(self._completed_args) + final_json = json.dumps(final_args, ensure_ascii=False) + if self._streamed_json_len < len(final_json): + argument_diff = final_json[self._streamed_json_len :] + self._streamed_json_len = len(final_json) + + # Record into prev_tool_call_arr + self.prev_tool_call_arr.append( + { + "name": self._streaming_tool_name, + "arguments": final_args, + } + ) + self.streamed_args_for_tool.append(final_json) + + self._reset_streaming_tool_state() + self._buffer = remaining + else: + # Still in progress - withhold the tail. + # For open strings: snapshot ends with ...partial_val} + # we withhold "}" (1 char) - the missing closing " will + # be sent when the value closes. + # For no open string: snapshot ends with ...value"} + # we withhold "}" (1 char). + end = len(snapshot) - 1 # exclude trailing "}" + if end > self._streamed_json_len: + argument_diff = snapshot[self._streamed_json_len : end] + self._streamed_json_len = end + + # --- construct return dict --- + if name_new and argument_diff: + return StreamDelta( + content=None, + tool_calls=[ + StreamToolCall( + index=self.current_tool_id, + new=True, + name=pending_name, + arguments=argument_diff, + ) + ], + ) + elif name_new: + return StreamDelta( + content=None, + tool_calls=[ + StreamToolCall( + index=self.current_tool_id, + new=True, + name=pending_name, + arguments=None, + ) + ], + ) + elif argument_diff: + return self._make_args_delta(argument_diff) + else: + return None + + +def build_tool_extractor( + tokenizer: TokenizerLike, *, strict: bool +) -> HYV4ToolExtractor: + return HYV4ToolExtractor( + tokenizer.get_vocab(), detect_token_suffix(tokenizer), strict + ) + + +def _tools_to_plain(tools) -> list[ToolSchema] | None: + """Convert tool-schema objects into plain dicts for the extractor.""" + if not tools: + return None + return [ + ToolSchema(name=t.function.name, parameters=t.function.parameters) + for t in tools + ] + + +class HYV4ToolParser(ToolParser): + """vLLM adapter around :class:`HYV4ToolExtractor`. + + Converts the extractor's plain dicts to vLLM protocol objects, mints + tool-call ids, and mirrors ``prev_tool_call_arr`` / + ``streamed_args_for_tool`` so the serving layer sees live state. + """ + + # HYV4 emits its own /// + # structural tokens, not a plain JSON array. The generic required/named + # JSON streaming path (streaming.extract_required_tool_call_streaming) + # cannot parse this format and produces misaligned multi-tool deltas in + # streaming mode. Fall back to our own (auto-path) parser for + # required/named too, so streaming and non-streaming stay consistent. + supports_required_and_named: bool = False + structural_tag_model: str = "hy_v4" + + def __init__(self, tokenizer: TokenizerLike, tools=None): + super().__init__(tokenizer, tools) + + # Tools are supplied at construction (``tool_parser(tokenizer, + # request.tools)``); keep a plain-dict copy for the extractor. + self._plain_tools = _tools_to_plain(tools) + # Malformed tag structure is rejected and returned as raw content + # instead of a half-parsed tool call. + self._extractor = build_tool_extractor(tokenizer, strict=True) + self._mirror_state() + + def get_structural_tag( + self, + request: ChatCompletionRequest | ResponsesRequest, + *, + reasoning: bool = False, + ): + """Build a structural tag matching HYV4's tool tokens. + + Overridden only to pass the per-checkpoint ``token_suffix`` through to + the builder; everything else follows the base implementation. + + Named (forced) tool choice is constrained with the same structural tag: + because ``supports_required_and_named`` is False, required *and* named + requests are parsed by the HYV4 extractor, which only recognizes the HYV4 + structural tokens -- the default JSON guided-decoding path would make + the model emit JSON the extractor silently drops. + + Args: + request: The request being adjusted. + reasoning: Whether the grammar also covers the reasoning phase. + + Returns: + The structural tag, or None when structural tagging does not apply. + """ + if not envs.VLLM_ENFORCE_STRICT_TOOL_CALLING: + return None + + # Only constrain required / forced tool choice with the structural tag. + # For "auto", tool calling is optional and most requests never emit a + # tool call, so applying the grammar to every auto request would make + # the whole stream pay the output inflation and the per-token + # bitmask/accept overhead for a minority benefit. Skip it here so auto + # falls through to free generation; the HYV4 parser still reads back + # any tool calls the model emits. + tool_choice = request.tool_choice + if ( + tool_choice is None + or tool_choice == "auto" + or getattr(tool_choice, "mode", None) == "auto" + ): + return None + + from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag + + try: + return get_model_structural_tag( + model=self.structural_tag_model, + tools=request.tools, + tool_choice=request.tool_choice, + reasoning=reasoning, + token_suffix=self._extractor.token_suffix, + ) + except Exception: + logger.warning( + "HYV4ToolParser failed to build a structural tag; falling back " + "to the default guided-decoding path.", + exc_info=True, + ) + return None + + def _mirror_state(self) -> None: + # Alias the SAME list objects the extractor appends to, so the serving + # layer reads live state. ``current_tool_id`` is an int, so re-read it. + self.prev_tool_call_arr = self._extractor.prev_tool_call_arr + self.streamed_args_for_tool = self._extractor.streamed_args_for_tool + self.current_tool_id = self._extractor.current_tool_id + + @staticmethod + def _is_guided(request: ChatCompletionRequest) -> bool: + """Whether a structural tag constrains this request's decoding. + + ``get_structural_tag`` only applies to required / named tool choice and + the serving layer records the result on the request. A structural tag + constrains strings rather than token ids, so the ```` marker + can arrive split across sub-word tokens and must be detected on the + string level. + + Args: + request: The request being streamed. + + Returns: + True when the streaming parser must use the string-marker path. + """ + structured_outputs = getattr(request, "structured_outputs", None) + return ( + structured_outputs is not None + and structured_outputs.structural_tag is not None + ) + + def _tools(self, request: ChatCompletionRequest) -> list[ToolSchema] | None: + # Prefer the tools passed at construction; fall back to the request. + if self._plain_tools is not None: + return self._plain_tools + return _tools_to_plain(getattr(request, "tools", None)) + + def extract_tool_calls( + self, + model_output: str, + request: ChatCompletionRequest, + ) -> ExtractedToolCallInformation: + result = self._extractor.extract_tool_calls(model_output, self._tools(request)) + self._mirror_state() + return ExtractedToolCallInformation( + tools_called=result["tools_called"], + content=result["content"], + tool_calls=[ + ToolCall( + type="function", + function=FunctionCall( + name=tc["name"], + arguments=tc["arguments"], + ), + ) + for tc in result["tool_calls"] + ], + ) + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids, + current_token_ids, + delta_token_ids, + request: ChatCompletionRequest, + ) -> DeltaMessage | None: + delta = self._extractor.extract_tool_calls_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + self._tools(request), + guided=self._is_guided(request), + ) + self._mirror_state() + if delta is None: + return None + + tool_calls: list[DeltaToolCall] = [] + for tc in delta.get("tool_calls", []): + # Only set the fields that are present, leaving the rest UNSET (not + # None). Streaming serialization uses exclude_unset, so an explicit + # ``arguments=None`` would emit ``"arguments": null`` and break + # clients that concatenate argument deltas. Matches hy_v3. + fn_kwargs: dict[str, str] = {} + name = tc.get("name") + if name is not None: + fn_kwargs["name"] = name + arguments = tc.get("arguments") + if arguments is not None: + fn_kwargs["arguments"] = arguments + function = DeltaFunctionCall(**fn_kwargs) + if tc.get("new"): + # First chunk for this tool: mint an id + set type here only. + tool_calls.append( + DeltaToolCall( + index=tc["index"], + id=make_tool_call_id(), + type="function", + function=function, + ) + ) + else: + tool_calls.append(DeltaToolCall(index=tc["index"], function=function)) + + # A single engine delta can carry content plus N drained tool calls. + # Only set the fields that are present so streaming serialization + # (exclude_unset) never emits ``content: null`` on a tool-call chunk. + content = delta.get("content") + kwargs: dict[str, Any] = {} + if content is not None: + kwargs["content"] = content + if tool_calls: + kwargs["tool_calls"] = tool_calls + if not kwargs: + return None + return DeltaMessage(**kwargs) diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 7710082b3b..413a97715f 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -49,6 +49,7 @@ StructuralTagBuilder: TypeAlias = Callable[ list[BuiltinToolParam], SimplifiedToolChoice, bool, + str, ], StructuralTag, ] @@ -70,7 +71,7 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset( "deepseek_v4", } ) -VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "kimi_k3"}) +VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"}) SUPPORTED_STRUCTURAL_TAG_MODELS = ( XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS ) @@ -78,7 +79,9 @@ SUPPORTED_STRUCTURAL_TAG_MODELS = ( _VLLM_STRUCTURAL_TAG_REGISTRY: dict[str, StructuralTagBuilder] = {} -def register_vllm_structural_tag(model: str): +def register_vllm_structural_tag( + model: str, +) -> Callable[[StructuralTagBuilder], StructuralTagBuilder]: """Register a vLLM-owned structural tag builder.""" def decorator(func: StructuralTagBuilder) -> StructuralTagBuilder: @@ -104,6 +107,7 @@ def get_model_structural_tag( tools: Sequence[ChatCompletionToolsParam | ResponsesTool] | None, tool_choice: ToolChoice, reasoning: bool, + token_suffix: str = "", ) -> StructuralTag | None: """Build a structural tag with xgrammar's builtin model templates.""" @@ -126,12 +130,19 @@ def get_model_structural_tag( builtin_tools, simplified_tool_choice, reasoning, + token_suffix, ) if model not in XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS: supported = sorted(SUPPORTED_STRUCTURAL_TAG_MODELS) raise ValueError(f"Unknown format type: {model}, supported types: {supported}") + if token_suffix: + raise ValueError( + f"Structural tag model {model!r} is an xgrammar builtin with fixed " + f"tokens and cannot apply token_suffix={token_suffix!r}" + ) + return get_xgrammar_model_structural_tag( model=model, tools=dumped_tools, @@ -244,8 +255,9 @@ def get_hermes_structural_tag( builtin_tools: list[BuiltinToolParam], tool_choice: SimplifiedToolChoice, reasoning: bool, + token_suffix: str = "", ) -> StructuralTag: - del builtin_tools, reasoning + del builtin_tools, reasoning, token_suffix tool_call_trigger = "" @@ -293,8 +305,9 @@ def get_minimax_structural_tag( builtin_tools: list[BuiltinToolParam], tool_choice: SimplifiedToolChoice, reasoning: bool, + token_suffix: str = "", ) -> StructuralTag: - del builtin_tools, reasoning + del builtin_tools, reasoning, token_suffix tool_call_begin = "\n" tool_call_end = "" @@ -589,8 +602,9 @@ def get_kimi_k3_structural_tag( builtin_tools: list[BuiltinToolParam], tool_choice: SimplifiedToolChoice, reasoning: bool, + token_suffix: str = "", ) -> StructuralTag: - del builtin_tools, reasoning + del builtin_tools, reasoning, token_suffix trailer = OptionalFormat(content=ConstStringFormat(value=_K3_MESSAGE_CLOSE)) @@ -611,3 +625,142 @@ def get_kimi_k3_structural_tag( return StructuralTag( format=SequenceFormat(elements=[*_k3_response_prefix(), tools_part, trailer]) ) + + +# --------------------------------------------------------------------------- +# HYV4 (/// structural tokens) +# --------------------------------------------------------------------------- +# HYV4 assistant output after the reasoning gate (````): +# +# NAME +# KV +# +# +# Adjacent blocks have no separator, and argument values are emitted +# verbatim (no surrounding quotes), so the value body cannot be described by +# JSONSchemaFormat. Only the skeleton (tool names, argument keys, tag order) is +# constrained; each body stays free text. + + +def _hy_v4_argument_keys(parameters: dict[str, Any]) -> tuple[list[str], list[str]]: + """Split a tool's argument keys into ``(required, optional)``. + + Both lists keep declaration order. Required keys missing from + ``properties`` are still treated as required. + """ + properties = parameters.get("properties") or {} + required = parameters.get("required") or [] + required_set = set(required) + required_keys = [k for k in properties if k in required_set] + required_keys += [k for k in required if k not in properties] + optional_keys = [k for k in properties if k not in required_set] + return required_keys, optional_keys + + +@register_vllm_structural_tag("hy_v4") +def get_hy_v4_structural_tag( + tools: list[FunctionToolParam], + builtin_tools: list[BuiltinToolParam], + tool_choice: SimplifiedToolChoice, + reasoning: bool, + token_suffix: str = "", +) -> StructuralTag: + """Build HYV4 /// structural tags. + + Args: + tools: Normalized function tools the model may call. + builtin_tools: Unused; HYV4 has no builtin tools. + tool_choice: Simplified tool choice. + reasoning: Whether the grammar also covers the reasoning phase. + token_suffix: Per-checkpoint structural-token suffix including the + leading colon (e.g. ``":6124c78e"``), or ``""`` when the checkpoint + uses unsuffixed tokens. The HYV4 tool parser reads it off the + tokenizer vocab and passes it to ``get_model_structural_tag``. + """ + del builtin_tools + + think_end = f"" + tool_calls_begin = f"" + tool_calls_end = f"" + tool_call_begin = f"" + tool_call_end = f"" + arg_key_begin = f"" + arg_key_end = f"" + arg_value_begin = f"" + arg_value_end = f"" + + def arg_pair(key: str) -> SequenceFormat: + return SequenceFormat( + elements=[ + ConstStringFormat(value=arg_key_begin), + ConstStringFormat(value=key), + ConstStringFormat(value=arg_key_end), + ConstStringFormat(value=arg_value_begin), + AnyTextFormat(), + ConstStringFormat(value=arg_value_end), + ] + ) + + def single_tool_call(tool: FunctionToolParam) -> TagFormat: + function = tool.function + parameters = ( + function.parameters if isinstance(function.parameters, dict) else {} + ) + required_keys, optional_keys = _hy_v4_argument_keys(parameters) + elements: list[Any] = [arg_pair(k) for k in required_keys] + if optional_keys: + # Optional keys may appear in any order, or not at all. + optional_pairs = [arg_pair(k) for k in optional_keys] + elements.append( + StarFormat( + content=optional_pairs[0] + if len(optional_pairs) == 1 + else OrFormat(elements=optional_pairs) + ) + ) + content: Any = ( + SequenceFormat(elements=elements) if elements else AnyTextFormat() + ) + return TagFormat( + begin=tool_call_begin + function.name, + content=content, + end=tool_call_end, + ) + + tags = [single_tool_call(tool) for tool in tools] + + if tool_choice == "auto": + block = TagFormat( + begin=tool_calls_begin, + content=TagsWithSeparatorFormat( + tags=tags, + separator="", + at_least_one=True, + ), + end=tool_calls_end, + ) + suffix_tag: Any = TriggeredTagsFormat(triggers=[tool_calls_begin], tags=[block]) + else: + # HYV4 places tool_call blocks back-to-back with no separator. "forced" + # has already been filtered down to the single named tool and must emit + # exactly one block. + suffix_tag = SequenceFormat( + elements=[ + ConstStringFormat(value=tool_calls_begin), + TagsWithSeparatorFormat( + tags=tags, + separator="", + at_least_one=True, + stop_after_first=tool_choice == "forced", + ), + ConstStringFormat(value=tool_calls_end), + ] + ) + + if not reasoning: + return StructuralTag(format=suffix_tag) + + # The reasoning phase is constrained too, so the tag must explicitly skip + # the ``...`` prefix. + prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=think_end) + return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag])) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index aa11ded482..0ed71feb22 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -91,6 +91,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( granite4_vision="Granite4VisionConfig", hyperclovax="HyperCLOVAXConfig", hy_v3="HYV3Config", + hy_v4="HYV4Config", isaac="IsaacConfig", kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3 kimi_linear="KimiLinearConfig", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 776eb642bd..cfc6d1c3bf 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -41,6 +41,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat", "Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision", "HYV3Config": "vllm.transformers_utils.configs.hy_v3", + "HYV4Config": "vllm.transformers_utils.configs.hy_v4", "HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax", "IsaacConfig": "vllm.transformers_utils.configs.isaac", # RWConfig is for the original tiiuae/falcon-40b(-instruct) and @@ -130,6 +131,7 @@ __all__ = [ "FunAudioChatAudioEncoderConfig", "Granite4VisionConfig", "HYV3Config", + "HYV4Config", "HyperCLOVAXConfig", "IsaacConfig", "RWConfig", diff --git a/vllm/transformers_utils/configs/hy_v4.py b/vllm/transformers_utils/configs/hy_v4.py new file mode 100644 index 0000000000..beca23ee74 --- /dev/null +++ b/vllm/transformers_utils/configs/hy_v4.py @@ -0,0 +1,172 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from huggingface_hub.dataclasses import strict +from transformers.configuration_utils import PreTrainedConfig +from transformers.modeling_rope_utils import RopeParameters + + +@strict +class HYV4Config(PreTrainedConfig): + r"""Configuration class for the HY V4 (`HYV4ForCausalLM`) model. + + Kept field-for-field in sync with + `transformers.models.hy_v4.configuration_hy_v4.HYV4Config` so that a + checkpoint resolves to the same values through either class. Registering it + in vLLM's `_CONFIG_REGISTRY` lets the architecture load without + `trust_remote_code` and without an `auto_map` module next to the weights. + + HYV4 is a mixture-of-experts causal language model using Multi-head Latent + Attention (MLA), DeepSeek-style sparse attention (DSA), gated MLA, + learnable attention sinks, and independent Hyper-Connections (iHC). + + The `layer_types`, `num_nextn_predict_layers` and `mtp_loss_factor` fields + below are vLLM-only additions: upstream leaves them to `**kwargs`, but + vLLM reads them directly (sparse-attention selection and MTP speculative + decoding), so they are declared to give checkpoints that omit them a + working default instead of a `None`. + + Args: + mlp_layer_types: Per-layer MLP kind, `"dense"` or `"sparse"`. Defaults + to one dense layer followed by sparse MoE layers. + layer_types: Per-layer attention kind, e.g. `"full_attention"` or + `"deepseek_sparse_attention"`. + indexer_types: Per-layer DSA indexer kind, `"full"` or `"shared"`. A + shared layer reuses the most recent full indexer in the same + forward request. + index_topk: Maximum number of key positions selected by each DSA query. + index_head_dim: Hidden dimension of each DSA indexer head. + index_n_heads: Number of DSA indexer heads. + enable_lm_head_fp32: Whether the language-model head emits float32 + logits. Surfaced as ``head_dtype`` so `LogitsProcessor` accumulates + the projection into fp32 while the weight stays in the model dtype. + enable_ihc: Whether independent Hyper-Connections are enabled. + hc_mult: Number of hidden-state channels maintained by iHC. + hc_magnitude: Scale applied to the iHC post-gating branch. + hc_eps: Numerical epsilon added to iHC sigmoid gates. + gated_mla: Whether to gate the MLA output. + gating_type: MLA gate granularity, `"elementwise"` or `"headwise"`. + learnable_sink: Whether to add a learned per-head attention sink. + learnable_sink_init: Initial value of each learned attention-sink + logit. + swiglu_limit: Magnitude of the routed-expert SwiGLU clamp. Values at or + below zero disable the clamp. + num_nextn_predict_layers: Number of MTP layers in the checkpoint. + mtp_loss_factor: Training-time MTP loss weight (unused at inference). + """ + + model_type = "hy_v4" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = { + "num_local_experts": "n_routed_experts", + } + base_model_tp_plan = { + "layers.*.self_attn.q_b_proj": "colwise", + "layers.*.self_attn.kv_a_proj_with_mqa": "mla_kv_a_proj", + "layers.*.self_attn.kv_b_proj": "colwise", + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.self_attn.linear_gate": "colwise", + "layers.*.mlp.experts.gate_up_proj": "packed_colwise", + "layers.*.mlp.experts.down_proj": "rowwise", + "layers.*.mlp.experts": "moe_tp_experts", + "layers.*.mlp.shared_experts.gate_proj": "colwise", + "layers.*.mlp.shared_experts.up_proj": "colwise", + "layers.*.mlp.shared_experts.down_proj": "rowwise", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + + vocab_size: int = 120832 + hidden_size: int = 2816 + intermediate_size: int = 6912 + moe_intermediate_size: int = 768 + num_hidden_layers: int = 34 + num_attention_heads: int = 32 + num_key_value_heads: int = 32 + head_dim: int = 256 + hidden_act: str = "silu" + max_position_embeddings: int = 262144 + initializer_range: float = 0.006 + rms_norm_eps: float = 1e-5 + use_cache: bool = True + pad_token_id: int | None = 120002 + bos_token_id: int | None = 120000 + eos_token_id: int | list[int] | None = 120025 + tie_word_embeddings: bool = False + attention_bias: bool = False + attention_dropout: float = 0.0 + n_routed_experts: int = 256 + n_shared_experts: int = 1 + num_experts_per_tok: int = 8 + routed_scaling_factor: float = 2.827 + norm_topk_prob: bool = True + q_lora_rank: int = 1536 + kv_lora_rank: int = 512 + qk_nope_head_dim: int = 192 + qk_rope_head_dim: int = 64 + v_head_dim: int = 256 + mlp_layer_types: list[str] | None = None + layer_types: list[str] | None = None + index_topk: int = 2048 + index_head_dim: int = 128 + index_n_heads: int = 16 + indexer_types: list[str] | None = None + enable_lm_head_fp32: bool = True + enable_ihc: bool = True + hc_mult: int = 4 + hc_magnitude: float = 2.0 + hc_eps: float = 1e-6 + gated_mla: bool = True + gating_type: str = "elementwise" + learnable_sink: bool = True + learnable_sink_init: float = 0.0 + swiglu_limit: float = 10.0 + rope_parameters: RopeParameters | dict | None = None + num_nextn_predict_layers: int = 1 + mtp_loss_factor: float = 0.1 + + def __post_init__(self, **kwargs): + self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim + # RoPE applies only to the rope slice, so point `head_dim` at it: the + # inherited rotary embedding reads `config.head_dim` and then computes + # the right frequencies with no override. + self.head_dim = self.qk_rope_head_dim + + if self.mlp_layer_types is None: + self.mlp_layer_types = ["dense"] * min(1, self.num_hidden_layers) + [ + "sparse" + ] * max(self.num_hidden_layers - 1, 0) + if self.indexer_types is None: + self.indexer_types = [ + "full" if layer_idx == 0 or (layer_idx - 1) % 4 == 0 else "shared" + for layer_idx in range(self.num_hidden_layers) + ] + # `PreTrainedConfig` validates `layer_types` against a fixed allow-list + # in which "deepseek_sparse_attention" is the only accepted spelling of + # a DSA layer. The model treats "sparse_attention" and "sparse" as + # equivalent, so fold them in before the base class validates. + if self.layer_types is not None: + self.layer_types = [ + "deepseek_sparse_attention" + if lt in ("sparse_attention", "sparse") + else lt + for lt in self.layer_types + ] + + # `ModelConfig.head_dtype` reads this off the HF config, so an fp32 head + # needs no model-side dtype juggling: LogitsProcessor picks the + # `torch.mm(out_dtype=float32)` path and leaves the weight in the model + # dtype. Only fill it in when the checkpoint (or --hf-overrides) has not + # already pinned a value. + if self.enable_lm_head_fp32 and getattr(self, "head_dtype", None) is None: + self.head_dtype = "float32" + + super().__post_init__(**kwargs) + + +__all__ = ["HYV4Config"] diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index b6d720b7af..01613911f0 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -332,6 +332,8 @@ class ModelArchConfigConvertorBase: "bailing_hybrid", "bailing_hybrid_mtp", "bailing_hybrid_v3_mtp", + "hy_v4", + "hy_v4_mtp", ): # check is deepseek_v4 model if hasattr(self.hf_text_config, "compress_ratios"): diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 303f16b79f..382132e61e 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -30,6 +30,19 @@ from vllm.v1.worker.utils import AttentionGroup logger = init_logger(__name__) +def _target_feeds_hc_residual(vllm_config: VllmConfig) -> bool: + """Whether the target replaces the drafter's input with its HC residual. + + Keyed on the same hook the model runner calls to perform that swap. It is + resolved from the target model class because speculators are built before + the target model is instantiated. + """ + from vllm.model_executor.model_loader.utils import get_model_cls + + target_cls = get_model_cls(vllm_config.model_config) + return hasattr(target_cls, "get_mtp_target_hidden_states") + + class BaseSpeculator(ABC): @abstractmethod def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: @@ -90,11 +103,16 @@ class DraftModelSpeculator(BaseSpeculator): # the draft model's hidden size can be different from the target model's # hidden size (e.g., Llama 3.3 70B). self.hidden_size = self.draft_model_config.get_hidden_size() - # Widen for HC-multiplexed residuals (e.g. DeepSeek V4 feeds the MTP - # draft the target's pre-hc_head (T, hc_mult * hidden_size) residual). - # Non-HC models default to hc_mult=1 and are unaffected. - hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1) - self.hidden_size = self.hidden_size * hc_mult + # Widen for HC-multiplexed residuals: a target that implements + # get_mtp_target_hidden_states() (e.g. DeepSeek V4) hands the drafter + # its pre-hc_head (T, hc_mult * hidden_size) residual in place of the + # collapsed hidden states, so the drafter's buffers must match. Key off + # that hook rather than hc_mult alone -- HY V4 runs iHC in its backbone + # (hc_mult=4) but its MTP head consumes the collapsed states, so + # widening it feeds propose() a 4x-too-wide buffer. + if _target_feeds_hc_residual(vllm_config): + hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1) + self.hidden_size = self.hidden_size * hc_mult self.vocab_size = self.draft_model_config.get_vocab_size() self.dtype = vllm_config.model_config.dtype self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel