[Hy4] support Hy4-preview model (#54160)

Signed-off-by: chengvjiang <chengvjiang@tencent.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: chengvjiang <chengvjiang@tencent.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: russellfeng <russellfeng@tencent.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Cheng Jiang
2026-08-29 12:07:28 +08:00
committed by GitHub
co-authored by chengvjiang Jee Jee Li Bugen Zhao russellfeng Jee Jee Li
parent 6c18a54648
commit b2f685834a
47 changed files with 6260 additions and 217 deletions
+2 -1
View File
@@ -405,7 +405,8 @@ th {
| `GraniteMoeSWAForCausalLM` | Granite MoE SWA | `ibm-granite/granite-swash-3b-a600m` | ✅︎ | ✅︎ |
| `GraniteSWAForCausalLM` | Granite SWA | `ibm-granite/granite-swash-2b` | ✅︎ | ✅︎ |
| `HrmTextForCausalLM` | HRM-Text | `sapientinc/HRM-Text-1B`, etc. | | |
| `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview` | ✅︎ | ✅︎ |
| `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview`, `tencent/Hy3`, `tencent/Hy3-FP8` | ✅︎ | ✅︎ |
| `HYV4ForCausalLM` | HY4 | `tencent/Hy4-preview`, `tencent/Hy4-preview-FP8` | ✅︎ | ✅︎ |
| `HyperCLOVAXForCausalLM` | HyperCLOVAX-SEED-Think-14B | `naver-hyperclovax/HyperCLOVAX-SEED-Think-14B` | ✅︎ | ✅︎ |
| `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ |
| `InternLM3ForCausalLM` | InternLM3 | `internlm/internlm3-8b-instruct`, etc. | ✅︎ | ✅︎ |
+2 -2
View File
@@ -405,7 +405,7 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
}
#[test]
@@ -416,6 +416,6 @@ mod tests {
)
.unwrap_err();
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
}
}
@@ -26,6 +26,7 @@ pub mod names {
pub const INKLING: &str = "inkling";
pub const GLM45: &str = "glm45";
pub const HY_V3: &str = "hy_v3";
pub const HY_V4: &str = "hy_v4";
pub const KIMI: &str = "kimi";
pub const KIMI_K2: &str = "kimi_k2";
pub const KIMI_K3: &str = "kimi_k3";
@@ -69,6 +70,7 @@ impl ReasoningParserFactory {
.register_unified_dummy(names::INKLING)
.register_parser::<Glm45ReasoningParser>(names::GLM45)
.register_unified_dummy(names::HY_V3)
.register_unified_dummy(names::HY_V4)
.register_parser::<KimiReasoningParser>(names::KIMI)
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
.register_unified_dummy(names::KIMI_K3)
@@ -94,6 +96,7 @@ impl ReasoningParserFactory {
.register_pattern("glm-4.6", names::GLM45)
.register_pattern("glm-4.5", names::GLM45)
.register_pattern("hy3", names::HY_V3)
.register_pattern("hy4", names::HY_V4)
.register_pattern("kimi-k2", names::KIMI_K2)
.register_pattern("kimi", names::KIMI)
// step3p5 patterns must precede `step3`: substring matching would
+3
View File
@@ -29,6 +29,7 @@ pub mod names {
pub const GRANITE4: &str = "granite4";
pub const HERMES: &str = "hermes";
pub const HY_V3: &str = "hy_v3";
pub const HY_V4: &str = "hy_v4";
// Matches the Python CLI name `--tool-call-parser internlm`, which Python
// also routes to `Internlm2ToolParser` despite the version-agnostic name.
pub const INTERNLM: &str = "internlm";
@@ -77,6 +78,7 @@ impl ToolParserFactory {
.register_parser::<Granite4ToolParser>(names::GRANITE4)
.register_parser::<HermesToolParser>(names::HERMES)
.register_unified_dummy(names::HY_V3)
.register_unified_dummy(names::HY_V4)
.register_parser::<Internlm2ToolParser>(names::INTERNLM)
.register_parser::<KimiK2ToolParser>(names::KIMI_K2)
.register_unified_dummy(names::KIMI_K3)
@@ -100,6 +102,7 @@ impl ToolParserFactory {
.register_pattern("qwen3", names::QWEN3_XML)
.register_pattern("hermes", names::HERMES)
.register_pattern("hy3", names::HY_V3)
.register_pattern("hy4", names::HY_V4)
// Narrow to `internlm2` substring so it matches `internlm2-chat-7b`
// and `internlm2_5-7b-chat` but NOT `internlm-chat-7b` (InternLM v1,
// routes to Llama), `internlm3-*` (also Llama-architecture per
+4
View File
@@ -183,6 +183,10 @@ fn factory_new_resolves_default_patterns() {
factory.resolve_name_for_model("tencent/Hy3-preview"),
Some(names::HY_V3)
);
assert_eq!(
factory.resolve_name_for_model("tencent/Hy4-preview"),
Some(names::HY_V4)
);
assert_eq!(
factory.resolve_name_for_model("MiniMax/MiniMax-M3-Text"),
Some(names::MINIMAX_M3)
+34 -2
View File
@@ -6,8 +6,8 @@
use std::sync::LazyLock;
pub use vllm_parser::unified::{
Gemma4UnifiedParser, HyV3UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser,
UnifiedParser,
Gemma4UnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser, InklingUnifiedParser,
KimiK3UnifiedParser, UnifiedParser,
};
use vllm_tokenizer::DynTokenizer;
@@ -18,6 +18,7 @@ use crate::request::ChatTool;
pub mod names {
pub const GEMMA4: &str = "gemma4";
pub const HY_V3: &str = "hy_v3";
pub const HY_V4: &str = "hy_v4";
pub const INKLING: &str = "inkling";
pub const KIMI_K3: &str = "kimi_k3";
}
@@ -44,6 +45,7 @@ impl UnifiedParserFactory {
factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
factory.register_parser::<HyV3UnifiedParser>(names::HY_V3);
factory.register_parser::<HyV4UnifiedParser>(names::HY_V4);
factory.register_parser::<InklingUnifiedParser>(names::INKLING);
factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
@@ -51,6 +53,7 @@ impl UnifiedParserFactory {
.register_pattern("gemma-4", names::GEMMA4)
.register_pattern("gemma4", names::GEMMA4)
.register_pattern("hy3", names::HY_V3)
.register_pattern("hy4", names::HY_V4)
.register_pattern("inkling", names::INKLING)
.register_pattern("kimi-k3", names::KIMI_K3)
.register_pattern("kimi_k3", names::KIMI_K3);
@@ -147,4 +150,33 @@ mod tests {
);
factory.create(names::KIMI_K3, &[], Arc::new(tokenizer)).unwrap();
}
#[test]
fn factory_registers_hy_v4() {
let factory = UnifiedParserFactory::new();
let tokenizer = [
"<think:opensource>",
"</think:opensource>",
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>",
]
.into_iter()
.enumerate()
.fold(TestTokenizer::new(), |tokenizer, (index, token)| {
tokenizer.with_regular_token(token, 1000 + index as u32)
});
assert!(factory.contains(names::HY_V4));
assert_eq!(
factory.resolve_name_for_model("tencent/Hy4-preview"),
Some(names::HY_V4)
);
factory.create(names::HY_V4, &[], Arc::new(tokenizer)).unwrap();
}
}
@@ -5,13 +5,13 @@ use vllm_tokenizer::DynTokenizer;
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, Result};
/// Internal HY3 reasoning stage used by the unified HY3 parser.
pub(crate) struct HyV3ReasoningParser {
/// Internal HY reasoning stage used by the unified HY parsers.
pub(crate) struct HyReasoningParser {
inner: DelimitedReasoningParser,
}
impl HyV3ReasoningParser {
/// Create a HY3 reasoning parser for the tokenizer-specific marker suffix.
impl HyReasoningParser {
/// Create a HY reasoning parser for the tokenizer-specific marker suffix.
pub(crate) fn new(tokenizer: DynTokenizer, suffix: &str) -> Result<Self> {
Ok(Self {
inner: DelimitedReasoningParser::new(
@@ -24,15 +24,15 @@ impl HyV3ReasoningParser {
}
}
impl ReasoningParser for HyV3ReasoningParser {
// Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and
impl ReasoningParser for HyReasoningParser {
// Suffix discovery belongs to the unified HY parser so its reasoning and
// tool delimiters always use the same tokenizer-derived value.
fn create(_tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
where
Self: Sized + 'static,
{
Err(ReasoningError::DummyUnifiedParser {
name: "hy_v3".to_string(),
name: "hy".to_string(),
})
}
+2 -2
View File
@@ -20,7 +20,7 @@
mod cohere_cmd;
mod deepseek_r1;
mod delimited;
mod hy_v3;
mod hy;
mod kimi;
mod minimax_m3;
mod qwen3;
@@ -33,7 +33,7 @@ use vllm_tokenizer::DynTokenizer;
pub use self::cohere_cmd::CohereCmdReasoningParser;
pub use self::deepseek_r1::DeepSeekR1ReasoningParser;
pub(crate) use self::delimited::{DelimitedReasoningParser, last_reasoning_boundary};
pub(crate) use self::hy_v3::HyV3ReasoningParser;
pub(crate) use self::hy::HyReasoningParser;
pub use self::kimi::KimiReasoningParser;
pub use self::minimax_m3::MiniMaxM3ReasoningParser;
pub use self::qwen3::Qwen3ReasoningParser;
@@ -11,33 +11,56 @@ use winnow::prelude::*;
use winnow::stream::Partial;
use winnow::token::{literal, rest, take_until};
use self::structural_tag::HyV3StructuralTagBuilder;
use self::structural_tag::HyStructuralTagBuilder;
use super::parameters::ToolSchemas;
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
use super::{Result, ToolCallDelta, ToolParser, ToolParserError, ToolParserOutput};
use crate::tool::{StructuralTagBuilder, Tool};
/// Wire-level HY tool-call dialect.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum HyDialect {
V3,
V4,
}
impl HyDialect {
fn separator(self) -> &'static str {
match self {
Self::V3 => "\n",
Self::V4 => "",
}
}
fn label(self) -> &'static str {
match self {
Self::V3 => "HY3",
Self::V4 => "HY4",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct HyV3ToolMarkers {
pub(crate) struct HyToolMarkers {
tool_calls_start: String,
tool_calls_end: String,
tool_call_start: String,
tool_call_end: String,
tool_sep: String,
tool_sep: Option<String>,
arg_key_start: String,
arg_key_end: String,
arg_value_start: String,
arg_value_end: String,
}
impl HyV3ToolMarkers {
pub(crate) fn new(suffix: &str) -> Self {
impl HyToolMarkers {
pub(crate) fn new(suffix: &str, dialect: HyDialect) -> Self {
Self {
tool_calls_start: format!("<tool_calls{suffix}>"),
tool_calls_end: format!("</tool_calls{suffix}>"),
tool_call_start: format!("<tool_call{suffix}>"),
tool_call_end: format!("</tool_call{suffix}>"),
tool_sep: format!("<tool_sep{suffix}>"),
tool_sep: (dialect == HyDialect::V3).then(|| format!("<tool_sep{suffix}>")),
arg_key_start: format!("<arg_key{suffix}>"),
arg_key_end: format!("</arg_key{suffix}>"),
arg_value_start: format!("<arg_value{suffix}>"),
@@ -47,31 +70,32 @@ impl HyV3ToolMarkers {
pub(crate) fn iter(&self) -> impl Iterator<Item = &str> {
[
self.tool_calls_start.as_str(),
self.tool_calls_end.as_str(),
self.tool_call_start.as_str(),
self.tool_call_end.as_str(),
self.tool_sep.as_str(),
self.arg_key_start.as_str(),
self.arg_key_end.as_str(),
self.arg_value_start.as_str(),
self.arg_value_end.as_str(),
Some(self.tool_calls_start.as_str()),
Some(self.tool_calls_end.as_str()),
Some(self.tool_call_start.as_str()),
Some(self.tool_call_end.as_str()),
self.tool_sep.as_deref(),
Some(self.arg_key_start.as_str()),
Some(self.arg_key_end.as_str()),
Some(self.arg_value_start.as_str()),
Some(self.arg_value_end.as_str()),
]
.into_iter()
.flatten()
}
}
type HyV3Input<'i> = Partial<&'i str>;
type HyInput<'i> = Partial<&'i str>;
#[derive(Debug, Clone, PartialEq, Eq)]
enum HyV3Mode {
enum HyMode {
Text,
ToolBlock { tool_call_end_scan: MarkerScanState },
Done,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum HyV3Event {
enum HyEvent {
Text {
len: usize,
},
@@ -84,7 +108,7 @@ enum HyV3Event {
IgnoredRest,
}
/// Tool parser for HY3 XML-style tool calls.
/// Tool parser for HY XML-style tool calls.
///
/// Example tool call content:
///
@@ -97,43 +121,45 @@ enum HyV3Event {
/// ```
///
/// Arguments are emitted only after a full `<tool_call>` block is parsed.
/// HY3 marker tokens are added-vocabulary tokens rather than tokenizer special
/// HY marker tokens are added-vocabulary tokens rather than tokenizer special
/// tokens, so the default `preserve_special_tokens() == false` is sufficient.
pub(crate) struct HyV3ToolParser {
pub(crate) struct HyToolParser {
buffer: String,
mode: HyV3Mode,
mode: HyMode,
emitted_tool_count: usize,
tool_parameters: ToolSchemas,
markers: Arc<HyV3ToolMarkers>,
structural_tag_builder: HyV3StructuralTagBuilder,
dialect: HyDialect,
markers: Arc<HyToolMarkers>,
structural_tag_builder: HyStructuralTagBuilder,
}
impl HyV3ToolParser {
/// Create a HY3 tool parser.
pub(crate) fn new(tools: &[Tool], suffix: &str) -> Self {
let markers = Arc::new(HyV3ToolMarkers::new(suffix));
impl HyToolParser {
/// Create a HY tool parser for one wire dialect.
pub(crate) fn new(tools: &[Tool], suffix: &str, dialect: HyDialect) -> Self {
let markers = Arc::new(HyToolMarkers::new(suffix, dialect));
Self {
buffer: String::new(),
mode: HyV3Mode::Text,
mode: HyMode::Text,
emitted_tool_count: 0,
tool_parameters: ToolSchemas::from_tools(tools),
dialect,
markers: Arc::clone(&markers),
structural_tag_builder: HyV3StructuralTagBuilder::new(markers),
structural_tag_builder: HyStructuralTagBuilder::new(markers, dialect),
}
}
/// Apply one parsed HY3 event to parser state and output.
fn apply_event(&mut self, event: HyV3Event, output: &mut ToolParserOutput) -> Result<()> {
/// Apply one parsed HY event to parser state and output.
fn apply_event(&mut self, event: HyEvent, output: &mut ToolParserOutput) -> Result<()> {
match event {
HyV3Event::Text { len: consumed_len } => {
HyEvent::Text { len: consumed_len } => {
output.push_text(&self.buffer[..consumed_len]);
}
HyV3Event::ToolBlockStart => {
self.mode = HyV3Mode::ToolBlock {
HyEvent::ToolBlockStart => {
self.mode = HyMode::ToolBlock {
tool_call_end_scan: MarkerScanState::default(),
};
}
HyV3Event::ToolCall { name, raw_params } => {
HyEvent::ToolCall { name, raw_params } => {
let arguments = self.tool_parameters.convert_params_with_schema(&name, raw_params);
let arguments = serde_json::to_string(&arguments)
.map_err(|error| parsing_failed!("failed to serialize arguments: {}", error))?;
@@ -145,22 +171,22 @@ impl HyV3ToolParser {
});
self.emitted_tool_count += 1;
}
HyV3Event::ToolBlockEnd => self.mode = HyV3Mode::Done,
HyV3Event::IgnoredRest => {}
HyEvent::ToolBlockEnd => self.mode = HyMode::Done,
HyEvent::IgnoredRest => {}
}
Ok(())
}
}
impl ToolParser for HyV3ToolParser {
// Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and
impl ToolParser for HyToolParser {
// Suffix discovery belongs to the unified HY parser so its reasoning and
// tool delimiters always use the same tokenizer-derived value.
fn create(_tools: &[Tool]) -> Result<Box<dyn ToolParser>>
where
Self: Sized + 'static,
{
Err(ToolParserError::DummyUnifiedParser {
name: "hy_v3".to_string(),
name: "hy".to_string(),
})
}
@@ -172,7 +198,7 @@ impl ToolParser for HyV3ToolParser {
self.buffer.push_str(chunk);
while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
parse_next_hy_v3_event(input, &mut self.mode, &self.markers)
parse_next_hy_event(input, &mut self.mode, &self.markers, self.dialect)
})? {
self.apply_event(event, output)?;
self.buffer.drain(..consumed_len);
@@ -184,113 +210,145 @@ impl ToolParser for HyV3ToolParser {
fn finish(&mut self) -> Result<ToolParserOutput> {
let mut output = ToolParserOutput::default();
match self.mode {
HyV3Mode::Text => output.push_text(&self.buffer),
HyV3Mode::ToolBlock { .. } => return Err(parsing_failed!("incomplete HY3 tool call")),
HyV3Mode::Done => {}
HyMode::Text => output.push_text(&self.buffer),
HyMode::ToolBlock { .. } => {
return Err(parsing_failed!(
"incomplete {} tool call",
self.dialect.label()
));
}
HyMode::Done => {}
}
let _ = self.reset();
Ok(output)
}
fn reset(&mut self) -> String {
self.mode = HyV3Mode::Text;
self.mode = HyMode::Text;
self.emitted_tool_count = 0;
std::mem::take(&mut self.buffer)
}
}
/// Parse a HY3 event for the current parser mode.
fn parse_next_hy_v3_event(
input: &mut HyV3Input<'_>,
mode: &mut HyV3Mode,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
/// Parse a HY event for the current parser mode.
fn parse_next_hy_event(
input: &mut HyInput<'_>,
mode: &mut HyMode,
markers: &HyToolMarkers,
dialect: HyDialect,
) -> ModalResult<HyEvent> {
match mode {
HyV3Mode::Text => parse_text_event(input, markers),
HyV3Mode::ToolBlock { tool_call_end_scan } => {
parse_tool_block_event(input, tool_call_end_scan, markers)
HyMode::Text => parse_text_event(input, markers),
HyMode::ToolBlock { tool_call_end_scan } => {
parse_tool_block_event(input, tool_call_end_scan, markers, dialect)
}
HyV3Mode::Done => ignored_rest_event(input),
HyMode::Done => ignored_rest_event(input),
}
}
/// Parse a text-mode HY3 event.
fn parse_text_event(
input: &mut HyV3Input<'_>,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
/// Parse a text-mode HY event.
fn parse_text_event(input: &mut HyInput<'_>, markers: &HyToolMarkers) -> ModalResult<HyEvent> {
alt((
|input: &mut HyV3Input<'_>| tool_block_start_event(input, markers),
|input: &mut HyV3Input<'_>| safe_text_event(input, markers),
|input: &mut HyInput<'_>| tool_block_start_event(input, markers),
|input: &mut HyInput<'_>| safe_text_event(input, markers),
))
.parse_next(input)
}
/// Parse a HY3 tool-block start marker.
/// Parse a HY tool-block start marker.
fn tool_block_start_event(
input: &mut HyV3Input<'_>,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
input: &mut HyInput<'_>,
markers: &HyToolMarkers,
) -> ModalResult<HyEvent> {
literal(markers.tool_calls_start.as_str())
.value(HyV3Event::ToolBlockStart)
.value(HyEvent::ToolBlockStart)
.parse_next(input)
}
/// Parse a safe text run before the next HY3 marker.
fn safe_text_event(input: &mut HyV3Input<'_>, markers: &HyV3ToolMarkers) -> ModalResult<HyV3Event> {
safe_text_len(input, &markers.tool_calls_start).map(|len| HyV3Event::Text { len })
/// Parse a safe text run before the next HY marker.
fn safe_text_event(input: &mut HyInput<'_>, markers: &HyToolMarkers) -> ModalResult<HyEvent> {
safe_text_len(input, &markers.tool_calls_start).map(|len| HyEvent::Text { len })
}
/// Parse one event inside a HY3 tool block.
/// Parse one event inside a HY tool block.
fn parse_tool_block_event(
input: &mut HyV3Input<'_>,
input: &mut HyInput<'_>,
tool_call_end_scan: &mut MarkerScanState,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
markers: &HyToolMarkers,
dialect: HyDialect,
) -> ModalResult<HyEvent> {
alt((
|input: &mut HyV3Input<'_>| tool_block_end_event(input, markers),
|input: &mut HyV3Input<'_>| tool_call_event(input, tool_call_end_scan, markers),
|input: &mut HyInput<'_>| tool_block_end_event(input, markers),
|input: &mut HyInput<'_>| tool_call_event(input, tool_call_end_scan, markers, dialect),
))
.parse_next(input)
}
/// Parse a HY3 tool-block end marker.
fn tool_block_end_event(
input: &mut HyV3Input<'_>,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
/// Parse a HY tool-block end marker.
fn tool_block_end_event(input: &mut HyInput<'_>, markers: &HyToolMarkers) -> ModalResult<HyEvent> {
(ws0, literal(markers.tool_calls_end.as_str()))
.value(HyV3Event::ToolBlockEnd)
.value(HyEvent::ToolBlockEnd)
.parse_next(input)
}
/// Parse a complete HY3 tool-call block.
/// Parse a complete HY tool-call block.
fn tool_call_event(
input: &mut HyV3Input<'_>,
input: &mut HyInput<'_>,
tool_call_end_scan: &mut MarkerScanState,
markers: &HyV3ToolMarkers,
) -> ModalResult<HyV3Event> {
let (name, body) = seq!(
markers: &HyToolMarkers,
dialect: HyDialect,
) -> ModalResult<HyEvent> {
let (body,) = seq!(
_: ws0,
_: literal(markers.tool_call_start.as_str()),
take_until(0.., markers.tool_sep.as_str()),
_: literal(markers.tool_sep.as_str()),
take_until_marker(markers.tool_call_end.as_str(), tool_call_end_scan),
_: literal(markers.tool_call_end.as_str()),
)
.parse_next(input)?;
let raw_params = parse_tool_call_params(body, markers)?;
let mut body_input = body;
let (name, params) = parse_tool_call_body(&mut body_input, markers, dialect)?;
let raw_params = parse_tool_call_params(params, markers)?;
Ok(HyV3Event::ToolCall {
Ok(HyEvent::ToolCall {
name: name.trim().to_string(),
raw_params,
})
}
/// Parse all parameter blocks inside a complete HY3 tool call.
/// Parse a complete HY tool-call body according to its dialect.
fn parse_tool_call_body<'i>(
input: &mut &'i str,
markers: &HyToolMarkers,
dialect: HyDialect,
) -> ModalResult<(&'i str, &'i str)> {
match dialect {
HyDialect::V3 => {
let tool_sep = markers.tool_sep.as_deref().expect("HY3 has a tool separator");
terminated(
seq!(
take_until(0.., tool_sep),
_: literal(tool_sep),
rest,
),
eof,
)
.parse_next(input)
}
HyDialect::V4 => terminated(
alt((
seq!(take_until(0.., markers.arg_key_start.as_str()), rest,),
rest.map(|name| (name, "")),
)),
eof,
)
.parse_next(input),
}
}
/// Parse all parameter blocks inside a complete HY tool call.
fn parse_tool_call_params(
tool_call_body: &str,
markers: &HyV3ToolMarkers,
markers: &HyToolMarkers,
) -> ModalResult<Vec<(String, String)>> {
let mut input = tool_call_body;
delimited(
@@ -304,8 +362,8 @@ fn parse_tool_call_params(
.parse_next(&mut input)
}
/// Parse a HY3 argument key/value block.
fn parameter(input: &mut &str, markers: &HyV3ToolMarkers) -> ModalResult<(String, String)> {
/// Parse a HY argument key/value block.
fn parameter(input: &mut &str, markers: &HyToolMarkers) -> ModalResult<(String, String)> {
let (name, value) = seq!(
_: literal(markers.arg_key_start.as_str()),
take_until(0.., markers.arg_key_end.as_str()),
@@ -320,9 +378,9 @@ fn parameter(input: &mut &str, markers: &HyV3ToolMarkers) -> ModalResult<(String
Ok((name.trim().to_string(), value.to_string()))
}
/// Parse ignored rest after the HY3 tool block ends.
fn ignored_rest_event(input: &mut HyV3Input<'_>) -> ModalResult<HyV3Event> {
rest.value(HyV3Event::IgnoredRest).parse_next(input)
/// Parse ignored rest after the HY tool block ends.
fn ignored_rest_event(input: &mut HyInput<'_>) -> ModalResult<HyEvent> {
rest.value(HyEvent::IgnoredRest).parse_next(input)
}
#[cfg(test)]
@@ -331,7 +389,7 @@ mod tests {
use serde_json::{Value, json};
use thiserror_ext::AsReport;
use super::{HyV3ToolParser, ToolParser};
use super::{HyDialect, HyToolParser, ToolParser};
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
@@ -354,14 +412,14 @@ mod tests {
#[test]
fn hy_v3_does_not_preserve_special_tokens() {
let parser = HyV3ToolParser::new(&test_tools(), "");
let parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
assert!(!parser.preserve_special_tokens());
}
#[test]
fn hy_v3_parse_complete_without_tool_call_keeps_text() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let output = parser.parse_complete("This is a plain response.").unwrap();
assert_eq!(output.normal_text(), "This is a plain response.");
@@ -370,7 +428,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_extracts_zero_arg_inline_tool_call() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let output = parser
.parse_complete(
"<tool_calls><tool_call>get_current_date<tool_sep></tool_call></tool_calls>",
@@ -385,7 +443,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_extracts_zero_arg_newline_tool_call() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let output = parser
.parse_complete(
"<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>",
@@ -398,7 +456,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_extracts_arguments_on_same_line() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let output = parser
.parse_complete(
"<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing</arg_value><arg_key>date</arg_key><arg_value>2026-03-30</arg_value></tool_call></tool_calls>",
@@ -413,7 +471,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_extracts_arguments_with_newlines() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let output = parser
.parse_complete(&build_tool_calls(&[build_tool_call(
"get_weather",
@@ -429,7 +487,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_preserves_prefix_and_ignores_trailing_text() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let output = parser
.parse_complete(&format!(
"Checking.{} trailing text",
@@ -443,7 +501,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_extracts_multiple_tool_calls_in_one_block() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let output = parser
.parse_complete(&build_tool_calls(&[
build_tool_call(
@@ -486,7 +544,7 @@ mod tests {
#[test]
fn hy_v3_parse_complete_converts_schema_types() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let output = parser
.parse_complete(&build_tool_calls(&[build_tool_call(
"convert",
@@ -512,7 +570,7 @@ mod tests {
#[test]
fn hy_v3_streaming_without_tool_call_emits_text_incrementally() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let mut output = ToolParserOutput::default();
output.append(parser.parse_chunk("This is ").unwrap());
@@ -526,7 +584,7 @@ mod tests {
#[test]
fn hy_v3_streaming_extracts_zero_arg_tool_call() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let chunks = [
"<tool_calls>",
"\n<tool_call>",
@@ -545,7 +603,7 @@ mod tests {
#[test]
fn hy_v3_streaming_extracts_arguments() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let chunks = [
"<tool_calls>",
"\n<tool_call>",
@@ -571,7 +629,7 @@ mod tests {
#[test]
fn hy_v3_streaming_preserves_prefix_text() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let chunks = [
"Checking.",
"<tool_calls>",
@@ -601,7 +659,7 @@ mod tests {
),
]);
let chunks = split_by_chars(&input, 9);
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let output = collect_stream(&mut parser, &chunks);
@@ -617,7 +675,7 @@ mod tests {
build_tool_calls(&[build_tool_call("get_weather", &[("city", "Beijing")])])
);
let chunks = split_by_chars(&input, 5);
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let output = collect_stream(&mut parser, &chunks);
@@ -628,7 +686,7 @@ mod tests {
#[test]
fn hy_v3_streaming_does_not_emit_incomplete_tool_call() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let mut output = ToolParserOutput::default();
parser
@@ -644,7 +702,7 @@ mod tests {
#[test]
fn hy_v3_finish_fails_incomplete_tool_call() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
parser.parse_chunk("<tool_calls><tool_call>get_weather<tool_sep>").unwrap();
let error = parser.finish().unwrap_err();
@@ -655,7 +713,7 @@ mod tests {
#[test]
fn hy_v3_malformed_tool_call_fails_fast() {
let mut parser = HyV3ToolParser::new(&test_tools(), "");
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
let error = parser
.parse_complete(
"<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing</tool_call></tool_calls>",
@@ -664,4 +722,78 @@ mod tests {
assert!(error.to_report_string().starts_with("tool parser parsing failed:"));
}
#[test]
fn hy_v4_parse_complete_extracts_compact_calls_with_and_without_arguments() {
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4);
let output = parser
.parse_complete(
"<tool_calls><tool_call>get_weather<arg_key>city</arg_key><arg_value>Beijing</arg_value><arg_key>date</arg_key><arg_value>2026-03-30</arg_value></tool_call><tool_call>get_current_date</tool_call></tool_calls>",
)
.unwrap();
expect![[r#"
ToolParserOutput {
events: [
ToolCall(
ToolCallDelta {
tool_index: 0,
name: Some(
"get_weather",
),
arguments: "{\"city\":\"Beijing\",\"date\":\"2026-03-30\"}",
},
),
ToolCall(
ToolCallDelta {
tool_index: 1,
name: Some(
"get_current_date",
),
arguments: "{}",
},
),
],
}
"#]]
.assert_debug_eq(&output);
}
#[test]
fn hy_v4_streaming_handles_markers_split_across_chunks() {
let input = "prefix<tool_calls><tool_call>get_weather<arg_key>city</arg_key><arg_value>上海</arg_value></tool_call><tool_call>get_current_date</tool_call></tool_calls>";
let chunks = split_by_chars(input, 7);
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4);
let output = collect_stream(&mut parser, &chunks);
assert_eq!(output.normal_text(), "prefix");
assert_eq!(output.calls().len(), 2);
assert_eq!(parsed_arguments(&output, 0), json!({ "city": "上海" }));
assert_eq!(output.calls()[1].name.as_deref(), Some("get_current_date"));
assert_eq!(parsed_arguments(&output, 1), json!({}));
}
#[test]
fn hy_v4_finish_reports_its_dialect_for_incomplete_calls() {
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4);
parser.parse_chunk("<tool_calls><tool_call>get_weather").unwrap();
let error = parser.finish().unwrap_err();
expect!["tool parser parsing failed: incomplete HY4 tool call"]
.assert_eq(&error.to_report_string());
}
#[test]
fn hy_v4_malformed_argument_pair_fails_fast() {
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4);
let error = parser
.parse_complete(
"<tool_calls><tool_call>get_weather<arg_key>city</arg_key>Beijing</tool_call></tool_calls>",
)
.unwrap_err();
assert!(error.to_report_string().starts_with("tool parser parsing failed:"));
}
}
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Structural-tag grammar for HY3 XML-style tool calls.
//! Structural-tag grammar for HY XML-style tool calls.
use std::collections::HashSet;
use std::sync::Arc;
@@ -12,31 +12,40 @@ use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagConte
use xgrammar_structural_tag::format::{Format, StructuralTag, TagFormat};
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam};
use super::HyV3ToolMarkers;
use super::{HyDialect, HyToolMarkers};
/// HY3 structural-tag builder using tokenizer-specific structural markers.
/// HY structural-tag builder using tokenizer-specific structural markers.
#[derive(Debug, Clone)]
pub(super) struct HyV3StructuralTagBuilder {
markers: Arc<HyV3ToolMarkers>,
pub(super) struct HyStructuralTagBuilder {
markers: Arc<HyToolMarkers>,
dialect: HyDialect,
}
impl HyV3StructuralTagBuilder {
pub(super) fn new(markers: Arc<HyV3ToolMarkers>) -> Self {
Self { markers }
impl HyStructuralTagBuilder {
pub(super) fn new(markers: Arc<HyToolMarkers>, dialect: HyDialect) -> Self {
Self { markers, dialect }
}
fn argument_pair(&self, key: &str) -> Format {
let excludes = self.markers.iter().collect::<Vec<_>>();
Format::sequence(vec![
let separator = self.dialect.separator();
let mut elements = vec![
Format::const_string(&self.markers.arg_key_start),
Format::const_string(key),
Format::const_string(&self.markers.arg_key_end),
Format::const_string("\n"),
];
if !separator.is_empty() {
elements.push(Format::const_string(separator));
}
elements.extend([
Format::const_string(&self.markers.arg_value_start),
Format::any_text_excluding(&excludes),
Format::const_string(&self.markers.arg_value_end),
Format::const_string("\n"),
])
]);
if !separator.is_empty() {
elements.push(Format::const_string(separator));
}
Format::sequence(elements)
}
fn tool_call(&self, tool: &FunctionToolParam) -> TagFormat {
@@ -56,14 +65,21 @@ impl HyV3StructuralTagBuilder {
}
let content = if elements.is_empty() {
Format::any_text()
match self.dialect {
HyDialect::V3 => Format::any_text(),
HyDialect::V4 => Format::const_string(""),
}
} else {
Format::sequence(elements)
};
let tool_sep = self.markers.tool_sep.as_deref().unwrap_or_default();
TagFormat::new(
format!(
"{}{}{}\n",
self.markers.tool_call_start, tool.function.name, self.markers.tool_sep
"{}{}{}{}",
self.markers.tool_call_start,
tool.function.name,
tool_sep,
self.dialect.separator(),
),
content,
self.markers.tool_call_end.clone(),
@@ -72,15 +88,16 @@ impl HyV3StructuralTagBuilder {
fn tool_calls(&self, tools: &[FunctionToolParam], choice: BuilderToolChoice) -> Format {
let mut calls = tools.iter().map(|tool| self.tool_call(tool)).collect::<Vec<_>>();
let begin = format!("{}\n", self.markers.tool_calls_start);
let end = format!("\n{}", self.markers.tool_calls_end);
let separator = self.dialect.separator();
let begin = format!("{}{separator}", self.markers.tool_calls_start);
let end = format!("{separator}{}", self.markers.tool_calls_end);
match choice {
BuilderToolChoice::Auto if calls.is_empty() => Format::any_text(),
BuilderToolChoice::Auto => {
let outer = TagFormat::new(
begin,
Format::tags_with_separator(calls, "\n", true, false),
Format::tags_with_separator(calls, separator, true, false),
end,
);
Format::triggered_tags(&[&self.markers.tool_calls_start], vec![outer])
@@ -92,14 +109,14 @@ impl HyV3StructuralTagBuilder {
]),
BuilderToolChoice::Required => Format::sequence(vec![
Format::const_string(begin),
Format::tags_with_separator(calls, "\n", true, false),
Format::tags_with_separator(calls, separator, true, false),
Format::const_string(end),
]),
}
}
}
impl StructuralTagBuilder for HyV3StructuralTagBuilder {
impl StructuralTagBuilder for HyStructuralTagBuilder {
fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
Ok(StructuralTag::new(
self.tool_calls(ctx.function_tools, ctx.tool_choice),
@@ -152,8 +169,8 @@ mod tests {
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
};
use super::HyV3StructuralTagBuilder;
use crate::tool::HyV3ToolMarkers;
use super::HyStructuralTagBuilder;
use crate::tool::{HyDialect, HyToolMarkers};
fn tool(name: &str, parameters: Value) -> ToolParam {
ToolParam::Function(FunctionToolParam::new(
@@ -165,9 +182,18 @@ mod tests {
suffix: &str,
tools: &[ToolParam],
choice: ToolChoice,
) -> xgrammar_structural_tag::format::StructuralTag {
build_for_dialect(HyDialect::V3, suffix, tools, choice)
}
fn build_for_dialect(
dialect: HyDialect,
suffix: &str,
tools: &[ToolParam],
choice: ToolChoice,
) -> xgrammar_structural_tag::format::StructuralTag {
build_structural_tag(
HyV3StructuralTagBuilder::new(Arc::new(HyV3ToolMarkers::new(suffix))),
HyStructuralTagBuilder::new(Arc::new(HyToolMarkers::new(suffix, dialect)), dialect),
tools,
choice,
StructuralTagOptions::default().with_reasoning(false),
@@ -196,6 +222,25 @@ mod tests {
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>\n"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather<tool_sep:opensource>\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"days"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]}}]},"end":"</tool_call:opensource>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
}
#[test]
fn required_uses_suffixed_compact_hy4_skeleton_and_bounded_values() {
let tag = build_for_dialect(
HyDialect::V4,
":opensource",
&[tool(
"get_weather",
json!({
"type": "object",
"properties": { "city": { "type": "string" } },
"required": ["city"]
}),
)],
ToolChoice::required(),
);
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"}]}]},"end":"</tool_call:opensource>"}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
}
#[test]
fn optional_only_schema_keeps_declared_key_alternatives() {
let tag = build(
@@ -235,6 +280,24 @@ mod tests {
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls>\n"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"type":"const_string","value":"\n</tool_calls>"}]}}"#]].assert_eq(&forced);
}
#[test]
fn hy_v4_zero_argument_tool_has_empty_compact_body() {
let tag = build_for_dialect(
HyDialect::V4,
"",
&[tool(
"get_current_date",
json!({ "type": "object", "properties": {} }),
)],
ToolChoice::function("get_current_date"),
);
let value = serde_json::to_value(tag).unwrap();
expect![[r#"{"type":"tag","begin":"<tool_call>get_current_date","content":{"type":"const_string","value":""},"end":"</tool_call>"}"#]].assert_eq(
&serde_json::to_string(&value["format"]["elements"][1]).unwrap(),
);
}
#[test]
fn missing_required_property_is_still_emitted() {
let tag = build(
+2 -2
View File
@@ -8,7 +8,7 @@ pub(crate) mod error;
mod deepseek_dsml;
pub(crate) mod deepseek_json;
mod glm_xml;
mod hy_v3;
mod hy;
pub(crate) mod json;
mod kimi_k2;
mod minimax_m2;
@@ -24,7 +24,7 @@ pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser};
pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser};
pub use error::{Result, ToolParserError};
pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser};
pub(crate) use hy_v3::{HyV3ToolMarkers, HyV3ToolParser};
pub(crate) use hy::{HyDialect, HyToolMarkers, HyToolParser};
pub use json::{
Granite4ToolParser, HermesToolParser, Internlm2ToolParser, Llama3JsonToolParser,
MistralToolParser, Phi4MiniJsonToolParser, Qwen3XmlToolParser,
@@ -1,20 +1,12 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use vllm_tokenizer::{DynTokenizer, Tokenizer};
use vllm_tokenizer::DynTokenizer;
use super::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
use crate::reasoning::HyV3ReasoningParser;
use crate::tool::{HyV3ToolMarkers, HyV3ToolParser, StructuralTagBuilder, Tool};
const HY_V3_MARKER_STEMS: &[&str] = &[
"think",
"tool_calls",
"tool_call",
"tool_sep",
"arg_key",
"arg_value",
];
use super::detect_hy_token_suffix;
use crate::reasoning::HyReasoningParser;
use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, StructuralTagBuilder, Tool};
use crate::unified::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
/// Unified reasoning and tool parser for HY3 output.
pub struct HyV3UnifiedParser {
@@ -24,14 +16,14 @@ pub struct HyV3UnifiedParser {
impl HyV3UnifiedParser {
/// Create a HY3 parser using the suffix encoded in tokenizer added tokens.
pub fn new(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
let suffix = detect_token_suffix(tokenizer.as_ref());
let markers = HyV3ToolMarkers::new(&suffix);
let suffix = detect_hy_token_suffix(tokenizer.as_ref());
let markers = HyToolMarkers::new(&suffix, HyDialect::V3);
for marker in markers.iter() {
token_id(tokenizer.as_ref(), marker)?;
}
let reasoning = HyV3ReasoningParser::new(tokenizer, &suffix)?;
let tool = HyV3ToolParser::new(tools, &suffix);
let reasoning = HyReasoningParser::new(tokenizer, &suffix)?;
let tool = HyToolParser::new(tools, &suffix, HyDialect::V3);
Ok(Self {
inner: CombinedParser::new(Some(Box::new(reasoning)), Some(Box::new(tool))),
})
@@ -75,28 +67,6 @@ impl UnifiedParser for HyV3UnifiedParser {
}
}
/// Detect the HY3 structural-token suffix from tokenizer added vocabulary.
fn detect_token_suffix(tokenizer: &dyn Tokenizer) -> String {
tokenizer
.added_vocab()
.iter()
.filter_map(|(token, id)| marker_suffix(token).map(|suffix| (*id, suffix)))
.min_by_key(|(id, _)| *id)
.map(|(_, suffix)| suffix.to_string())
.unwrap_or_default()
}
/// Extract the suffix from one opening or closing HY3 structural token.
fn marker_suffix(token: &str) -> Option<&str> {
let body = token.strip_prefix('<')?.strip_suffix('>')?;
let body = body.strip_prefix('/').unwrap_or(body);
HY_V3_MARKER_STEMS.iter().find_map(|stem| {
let suffix = body.strip_prefix(stem)?;
(suffix.is_empty() || suffix.starts_with(':')).then_some(suffix)
})
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
+194
View File
@@ -0,0 +1,194 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use vllm_tokenizer::DynTokenizer;
use super::detect_hy_token_suffix;
use crate::reasoning::HyReasoningParser;
use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, StructuralTagBuilder, Tool};
use crate::unified::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
/// Unified reasoning and tool parser for HY4 output.
pub struct HyV4UnifiedParser {
inner: CombinedParser,
}
impl HyV4UnifiedParser {
/// Create a HY4 parser using the suffix encoded in tokenizer added tokens.
pub fn new(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
let suffix = detect_hy_token_suffix(tokenizer.as_ref());
let markers = HyToolMarkers::new(&suffix, HyDialect::V4);
for marker in markers.iter() {
token_id(tokenizer.as_ref(), marker)?;
}
let reasoning = HyReasoningParser::new(tokenizer, &suffix)?;
let tool = HyToolParser::new(tools, &suffix, HyDialect::V4);
Ok(Self {
inner: CombinedParser::new(Some(Box::new(reasoning)), Some(Box::new(tool))),
})
}
}
impl UnifiedParser for HyV4UnifiedParser {
fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
where
Self: Sized + 'static,
{
Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box<dyn UnifiedParser>)
}
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
self.inner.initialize(prompt_token_ids)
}
fn preserve_special_tokens(&self) -> bool {
self.inner.preserve_special_tokens()
}
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
self.inner.structural_tag_builder()
}
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
self.inner.tool_call_id(tool_index)
}
fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()> {
self.inner.parse_into(delta, output)
}
fn finish(&mut self) -> Result<UnifiedParserOutput> {
self.inner.finish()
}
fn reset(&mut self) -> String {
self.inner.reset()
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use serde_json::json;
use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer};
use xgrammar_structural_tag::builders::StructuralTagOptions;
use xgrammar_structural_tag::{
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
};
use super::{HyV4UnifiedParser, UnifiedParser};
use crate::tool::Tool;
use crate::unified::{UnifiedParserEvent, UnifiedParserOutput};
fn tokenizer() -> TestTokenizer {
[
"<think:opensource>",
"</think:opensource>",
"<tool_calls:opensource>",
"</tool_calls:opensource>",
"<tool_call:opensource>",
"</tool_call:opensource>",
"<arg_key:opensource>",
"</arg_key:opensource>",
"<arg_value:opensource>",
"</arg_value:opensource>",
]
.into_iter()
.enumerate()
.fold(TestTokenizer::new(), |tokenizer, (index, token)| {
tokenizer.with_regular_token(token, 1000 + index as u32)
})
}
fn tools() -> Vec<Tool> {
vec![
Tool {
name: "get_weather".to_string(),
description: None,
parameters: json!({
"type": "object",
"properties": { "city": { "type": "string" } },
}),
strict: None,
},
Tool {
name: "get_current_date".to_string(),
description: None,
parameters: json!({ "type": "object", "properties": {} }),
strict: None,
},
]
}
#[test]
fn parses_suffixed_reasoning_and_compact_tool_calls_as_one_stream() {
let tokenizer = Arc::new(tokenizer());
let think_start_id = tokenizer.token_to_id("<think:opensource>").unwrap();
let mut parser = HyV4UnifiedParser::new(&tools(), tokenizer).unwrap();
parser.initialize(&[think_start_id]).unwrap();
let chunks = [
"reasoning</think:open",
"source>answer<tool_calls:opensource><tool_call:opensource>get_weather",
"<arg_key:opensource>city</arg_key:opensource>",
"<arg_value:opensource>Beijing</arg_value:opensource>",
"</tool_call:opensource><tool_call:opensource>get_current_date",
"</tool_call:opensource></tool_calls:opensource>",
];
let mut output = UnifiedParserOutput::default();
for chunk in chunks {
parser.parse_into(chunk, &mut output).unwrap();
}
output.append(parser.finish().unwrap());
assert_eq!(
output.events,
vec![
UnifiedParserEvent::Reasoning("reasoning".to_string()),
UnifiedParserEvent::Text("answer".to_string()),
UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
tool_index: 0,
name: Some("get_weather".to_string()),
arguments: r#"{"city":"Beijing"}"#.to_string(),
}),
UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
tool_index: 1,
name: Some("get_current_date".to_string()),
arguments: "{}".to_string(),
}),
]
);
}
#[test]
fn structural_tag_uses_compact_hy4_skeleton() {
let parser = HyV4UnifiedParser::new(&tools(), Arc::new(tokenizer())).unwrap();
let structural_tools = [ToolParam::Function(FunctionToolParam::new(
FunctionDefinition::new("get_weather").with_parameters(json!({
"type": "object",
"properties": { "city": { "type": "string" } },
"required": ["city"]
})),
))];
let tag = build_structural_tag(
parser.structural_tag_builder().unwrap(),
&structural_tools,
ToolChoice::required(),
StructuralTagOptions::default().with_reasoning(false),
)
.unwrap()
.to_json_string()
.unwrap();
assert!(tag.contains("<tool_calls:opensource>"));
assert!(tag.contains("<tool_call:opensource>get_weather"));
assert!(tag.contains("<arg_key:opensource>"));
assert!(tag.contains("</tool_call:opensource>"));
assert!(tag.contains("</tool_calls:opensource>"));
assert!(!tag.contains("tool_sep"));
assert!(!tag.contains(r#"\n"#));
}
}
+39
View File
@@ -0,0 +1,39 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
mod hy_v3;
mod hy_v4;
pub use hy_v3::HyV3UnifiedParser;
pub use hy_v4::HyV4UnifiedParser;
const HY_MARKER_STEMS: &[&str] = &[
"think",
"tool_calls",
"tool_call",
"tool_sep",
"arg_key",
"arg_value",
];
/// Detect the HY structural-token suffix from tokenizer added vocabulary.
fn detect_hy_token_suffix(tokenizer: &dyn vllm_tokenizer::Tokenizer) -> String {
tokenizer
.added_vocab()
.iter()
.filter_map(|(token, id)| hy_marker_suffix(token).map(|suffix| (*id, suffix)))
.min_by_key(|(id, _)| *id)
.map(|(_, suffix)| suffix.to_string())
.unwrap_or_default()
}
/// Extract the suffix from one opening or closing HY structural token.
fn hy_marker_suffix(token: &str) -> Option<&str> {
let body = token.strip_prefix('<')?.strip_suffix('>')?;
let body = body.strip_prefix('/').unwrap_or(body);
HY_MARKER_STEMS.iter().find_map(|stem| {
let suffix = body.strip_prefix(stem)?;
(suffix.is_empty() || suffix.starts_with(':')).then_some(suffix)
})
}
+2 -2
View File
@@ -5,13 +5,13 @@
mod combined;
mod gemma4;
mod hy_v3;
mod hy;
mod inkling;
mod kimi_k3;
pub use combined::CombinedParser;
pub use gemma4::Gemma4UnifiedParser;
pub use hy_v3::HyV3UnifiedParser;
pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser};
pub use inkling::InklingUnifiedParser;
pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser};
use thiserror::Error;
+7
View File
@@ -331,6 +331,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"),
"HYV3ForCausalLM": _HfExamplesInfo("tencent/Hy3-preview", trust_remote_code=True),
# Internal checkpoint, not publicly available yet.
"HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True),
"HyperCLOVAXForCausalLM": _HfExamplesInfo(
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
min_transformers_version="5.9.0",
@@ -1723,6 +1725,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"tencent/Hy3-preview",
speculative_model="tencent/Hy3-preview",
),
# Internal checkpoint, not publicly available yet.
"HYV4MTPModel": _HfExamplesInfo(
"tencent/Hy4-preview",
speculative_model="tencent/Hy4-preview",
),
"InklingMTPModel": _HfExamplesInfo(
"thinkingmachines/Inkling-NVFP4",
speculative_model="thinkingmachines/Inkling-NVFP4",
+5
View File
@@ -59,6 +59,11 @@ def test_registry_imports(model_arch):
):
pytest.skip("Dots3 NOTE is only supported on CUDA")
if model_arch in ("HYV4ForCausalLM", "HYV4MTPModel") and not (
current_platform.is_cuda()
):
pytest.skip("HY V4 is only supported on CUDA")
# Ensure all model classes can be imported successfully
model_cls = ModelRegistry._try_load_model_cls(model_arch)
assert model_cls is not None
+5
View File
@@ -552,6 +552,11 @@ def dummy_hf_overrides(
text_config.update(update_dict)
# Keep per-layer metadata consistent with the reduced layer count.
# HY V4 uses this to decide which sparse-attention indexers are shared.
if indexer_types := getattr(text_config, "indexer_types", None):
text_config.update({"indexer_types": indexer_types[:num_hidden_layers]})
# Update n_layers and moe configs for Moondream3 model
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
text_config.update(
+8
View File
@@ -57,6 +57,7 @@ MTPModelTypes = Literal[
"pangu_ultra_moe_mtp",
"step3p5_mtp",
"hy_v3_mtp",
"hy_v4_mtp",
"gemma4_mtp",
"inkling_mtp",
]
@@ -917,6 +918,13 @@ class SpeculativeConfig:
{"n_predict": n_predict, "architectures": ["HYV3MTPModel"]}
)
if hf_config.model_type == "hy_v4":
hf_config.model_type = "hy_v4_mtp"
n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
hf_config.update(
{"n_predict": n_predict, "architectures": ["HYV4MTPModel"]}
)
if hf_config.model_type in ("inkling_mm_model", "inkling_model"):
mtp_config = getattr(hf_config, "mtp_config", None) or {}
hf_config = getattr(hf_config, "text_config", hf_config)
+2
View File
@@ -79,6 +79,8 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
"DeepseekV4ForCausalLM",
"DeepSeekV4MTPModel",
"GlmMoeDsaForCausalLM",
"HYV4ForCausalLM",
"HYV4MTPModel",
"InklingForCausalLM",
"InklingForConditionalGeneration",
"KimiK3ForConditionalGeneration",
+10
View File
@@ -321,6 +321,7 @@ if TYPE_CHECKING:
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
VLLM_NIC_SELECTION_VARS: str = ""
VLLM_PREFIX_CACHE_RETENTION_INTERVAL: int | None = None
VLLM_ENABLE_HPC_OPS: bool = False
def get_default_cache_root():
@@ -2155,6 +2156,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
# Each entry is VAR_NAME or VAR_NAME:<suffix> (suffix appended to
# RDMA device name). Must be set together with VLLM_GPU_NIC_PCIE_MAPPING.
"VLLM_NIC_SELECTION_VARS": lambda: os.getenv("VLLM_NIC_SELECTION_VARS", ""),
# If set to 1, enable the HPC fused kernels (requires the hpc package
# (.so) and an sm100/sm103 device). Covers:
# * the HY V4 iHC ops -- each of the eager HYV4HCPreLayer /
# HYV4HCPostLayer / HYV4HCHeadLayer bodies becomes one kernel launch;
# * the gated-MLA output gating (attn_out * sigmoid(gate projection)),
# fused into gated_mla_gemm; elementwise gating only.
# Each op additionally checks its own shape / dtype constraints and falls
# back to the eager path when they do not hold.
"VLLM_ENABLE_HPC_OPS": lambda: bool(int(os.getenv("VLLM_ENABLE_HPC_OPS", "0"))),
}
@@ -79,12 +79,27 @@ class TrtLlmFp8ExpertsBase:
activation_key,
activation_format,
)
if not supported or moe_config.num_experts <= 2048:
if not supported:
return supported, reason
return False, (
"FlashInfer TRTLLM routing supports at most 2048 experts, "
f"but got {moe_config.num_experts}"
)
# FlashInfer accepts gemm1_alpha/beta/clamp_limit only for MXFP8; the
# DeepSeek-FP8 block-scale kernel rejects them and the per-tensor kernel
# has no clamp at all, so a model-requested SwiGLU clamp cannot be
# honored on those paths.
if (
moe_config.swiglu_limit is not None
or moe_config.swiglu_alpha is not None
or moe_config.swiglu_beta is not None
) and (weight_key, activation_key) != (kMxfp8Static, kMxfp8Dynamic):
return False, (
"the TRTLLM FP8 kernels apply the SwiGLU alpha/beta/clamp "
"parameters only for MXFP8 weights"
)
if moe_config.num_experts > 2048:
return False, (
"FlashInfer TRTLLM routing supports at most 2048 experts, "
f"but got {moe_config.num_experts}"
)
return True, None
def __init__(
self,
@@ -1,11 +1,25 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.layers.hpc.gated_mla import (
hpc_gated_mla_gemm,
hpc_gated_mla_supported,
)
from vllm.model_executor.layers.hpc.hpc_ihc import (
HpcIHCHead,
HpcIHCPost,
HpcIHCPre,
)
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
from vllm.model_executor.layers.hpc.rope_norm import HpcRopeNorm, QkNormPolicy
__all__ = [
"HpcIHCHead",
"HpcIHCPost",
"HpcIHCPre",
"HpcModule",
"HpcRopeNorm",
"QkNormPolicy",
"hpc_gated_mla_gemm",
"hpc_gated_mla_supported",
]
+113
View File
@@ -0,0 +1,113 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Enablement gate for the HPC fused gated-MLA GEMM.
``hpc.gated_mla_gemm`` fuses the MLA output gating -- the gate projection, the
sigmoid and the elementwise product -- into a single kernel launch, replacing
the eager ``attn_out * sigmoid(hidden_states @ weight.T)`` sequence.
Kernel constraints:
- Requires VLLM_ENABLE_HPC_OPS=1
- Requires the hpc package (.so) built for the current arch
- Only sm100 / sm103 (compute capability 100, 103)
- All three operands must be bfloat16 and contiguous
- The gate width must be a multiple of 256
- Only elementwise gating (headwise has no fused form)
"""
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.utils.hpc import has_hpc
logger = init_logger(__name__)
# hpc.gated_mla_gemm fuses `attn_out * sigmoid(x @ weight.T)` into one launch.
# It is built for sm100/sm103 and needs all three operands bf16 + contiguous,
# with the gate width a multiple of 256.
_HPC_GATED_GEMM_N_ALIGN = 256
_HPC_GATED_GEMM_CAPABILITIES: frozenset[int] = frozenset({100, 103})
def hpc_gated_mla_supported(
gating_type: str | None, attn_output_gate: torch.nn.Module | None
) -> bool:
"""Gate for running the MLA output gating through hpc.gated_mla_gemm."""
if not envs.VLLM_ENABLE_HPC_OPS:
return False
if attn_output_gate is None:
return False
if not has_hpc():
logger.warning_once(
"HPC gated MLA disabled: 'hpc' package is not installed. "
"Please install the HPC library (.so) to enable the fused kernel."
)
return False
from vllm.platforms import current_platform
if not current_platform.is_cuda():
logger.warning_once("HPC gated MLA disabled: only CUDA is supported.")
return False
capability = current_platform.get_device_capability()
if capability is None or capability.to_int() not in _HPC_GATED_GEMM_CAPABILITIES:
logger.warning_once(
"HPC gated MLA disabled: compute capability %s not in %s.",
capability,
_HPC_GATED_GEMM_CAPABILITIES,
)
return False
# Headwise gating broadcasts one scalar per head over v_head_dim; the
# kernel only implements the elementwise product.
if gating_type != "elementwise":
logger.warning_once(
"HPC gated MLA disabled: gating_type '%s' has no fused form.",
gating_type,
)
return False
weight = getattr(attn_output_gate, "weight", None)
if weight is None or weight.dtype != torch.bfloat16:
logger.warning_once(
"HPC gated MLA disabled: gate weight dtype is %s, the kernel needs "
"bfloat16 (keep the layer out of quantization).",
None if weight is None else weight.dtype,
)
return False
if weight.shape[0] % _HPC_GATED_GEMM_N_ALIGN != 0:
logger.warning_once(
"HPC gated MLA disabled: gate width %d is not a multiple of %d.",
weight.shape[0],
_HPC_GATED_GEMM_N_ALIGN,
)
return False
logger.info_once("HPC gated MLA enabled by set VLLM_ENABLE_HPC_OPS.")
return True
def hpc_gated_mla_gemm(
x: torch.Tensor,
weight: torch.Tensor,
atten_output: torch.Tensor,
) -> torch.Tensor:
"""Fused ``atten_output * sigmoid(x @ weight.T)``.
Args:
x: ``[m, k]`` bfloat16, contiguous.
weight: ``[n, k]`` bfloat16, contiguous and row-major (the kernel
transposes it). ``n`` must be a multiple of 256.
atten_output: ``[m, n]`` bfloat16, contiguous.
Returns:
``[m, n]`` bfloat16.
"""
from hpc.gemm import gated_mla_gemm
return gated_mla_gemm(x, weight, atten_output)
+261
View File
@@ -0,0 +1,261 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HPC fused iHC (independent Hyper-Connections) kernels for HY V4.
Replaces the eager HYV4HCPreLayer / HYV4HCPostLayer / HYV4HCHeadLayer bodies
with single-kernel HPC implementations:
pre : rms square sum + w projection + sigmoid gates + hc-dim weighted sum
post : y[n, i, :] = H_post[n, i] * x[n, :] + residual[n, i, :]
head : same as pre but hc_mult projection rows and no H_post output
The eager path issues 20 / 5 / 15 kernels for pre / post / head; each HPC op is
a single kernel.
Constraints:
- Requires VLLM_ENABLE_HPC_OPS=1
- Requires the hpc package (.so) built for the current arch
- Only sm100 / sm103 (compute capability 100, 103)
- Only hc_mult == 4 and hidden_size in {4096, 6144}
- x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint
keeps them out of fp8 quantization via modules_to_not_convert)
The fused modules own no parameters: they hold references to the weights of the
eager layer they replace, bound in ``process_weights_after_loading`` (invoked
generically for every ``HpcModule`` by the model loader).
NOTE: The reference implementation also offers a cross-layer post+pre fusion
(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre.
It requires reworking the decoder-layer forward scheduling and is not ported
here. TODO: add it once the decoder dataflow is restructured for it.
"""
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
from vllm.utils.hpc import has_hpc
logger = init_logger(__name__)
# Shapes the HPC kernels are instantiated for. Other shapes launch nothing,
# so gate on this rather than relying on a silent no-op.
_SUPPORTED_HIDDEN_SIZES: frozenset[int] = frozenset({4096, 6144})
_SUPPORTED_HC_MULTS: frozenset[int] = frozenset({4})
_SUPPORTED_CAPABILITIES: frozenset[int] = frozenset({100, 103})
def _ihc_supported(hc_mult: int, hidden_size: int) -> bool:
"""Shared gate for all three iHC ops."""
if not envs.VLLM_ENABLE_HPC_OPS:
return False
if not has_hpc():
logger.warning_once(
"HPC iHC disabled: 'hpc' package is not installed. "
"Please install the HPC library (.so) to enable fused kernels."
)
return False
from vllm.platforms import current_platform
if not current_platform.is_cuda():
logger.warning_once("HPC iHC disabled: only CUDA is supported.")
return False
capability = current_platform.get_device_capability()
if capability is None or capability.to_int() not in _SUPPORTED_CAPABILITIES:
logger.warning_once(
"HPC iHC disabled: compute capability %s not in %s.",
capability,
_SUPPORTED_CAPABILITIES,
)
return False
if hc_mult not in _SUPPORTED_HC_MULTS:
logger.warning_once(
"HPC iHC disabled: hc_mult=%d not in %s.", hc_mult, _SUPPORTED_HC_MULTS
)
return False
if hidden_size not in _SUPPORTED_HIDDEN_SIZES:
logger.warning_once(
"HPC iHC disabled: hidden_size=%d not in %s.",
hidden_size,
_SUPPORTED_HIDDEN_SIZES,
)
return False
if envs.VLLM_BATCH_INVARIANT:
# Batch-invariant mode relies on torch's deterministic reductions; the
# HPC kernel reduces in a different order, so keep the two regimes
# apart.
logger.warning_once(
"HPC iHC disabled: not supported under VLLM_BATCH_INVARIANT."
)
return False
logger.info_once("HPC iHC enabled by set VLLM_ENABLE_HPC_OPS.")
return True
class HpcIHCPre(HpcModule):
"""Fused iHC pre block.
Computes, in one kernel:
x_flat = x.flatten(1)
r = rsqrt(x_flat.square().mean(-1) + rms_norm_eps)
mixes = (x_flat @ w.T) * r
H_pre = sigmoid(mixes[:, :hc] * hc_scale[0] + hc_base[:hc]) + hc_eps
H_post = magnitude * sigmoid(
mixes[:, hc:] * hc_scale[1] + hc_base[hc:]) + hc_eps
y = sum_i H_pre[:, i] * x[:, i, :]
Args:
hc_mult: HC expand ratio.
hidden_size: Model hidden dimension.
magnitude: H_post multiplier (config.hc_magnitude).
hc_eps: Epsilon added to both gates (config.hc_eps).
norm_eps: Epsilon inside the rsqrt (config.rms_norm_eps).
fallback_op: The eager HYV4HCPreLayer to source weights from.
norm_owner: Optional RMSNorm that immediately follows this pre block.
When given, its weight/eps are folded into the kernel so a single
launch covers pre + RMSNorm (the caller then skips the separate
layernorm). None keeps the original pre-only behaviour.
"""
def __init__(
self,
hc_mult: int,
hidden_size: int,
magnitude: float,
hc_eps: float,
norm_eps: float,
fallback_op: torch.nn.Module,
norm_owner: torch.nn.Module | None = None,
) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
self.magnitude = magnitude
self.hc_eps = hc_eps
self.norm_eps = norm_eps
# Stash the owner outside nn.Module's attribute machinery: callers pass
# their own `self` here, and registering that as a submodule would make
# the module tree cyclic (state_dict() then hits RecursionError).
object.__setattr__(self, "_fallback_op", fallback_op)
# Same anti-cycle idiom for the fused RMSNorm owner (a sibling module).
object.__setattr__(self, "_norm_owner", norm_owner)
self.weight = None
self.hc_scale = None
self.hc_base = None
self.rms_weight = None
self.rms_eps = 0.0
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def process_weights_after_loading(self, model=None):
# hc_fn is a ReplicatedLinear whose weight is [2 * hc_mult, hc_mult * d].
# It stays float32 -- the HPC kernel consumes fp32 weights directly, so
# there is no conversion (and no precision loss) here.
owner = self._fallback_op
self.weight = owner.hc_fn.weight
self.hc_scale = owner.hc_scale
self.hc_base = owner.hc_base
if self._norm_owner is not None:
self.rms_weight = self._norm_owner.weight
self.rms_eps = self._norm_owner.variance_epsilon
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
import hpc
return hpc.fuse_ihc_pre(
x,
self.weight,
self.hc_scale,
self.hc_base,
self.norm_eps,
self.hc_eps,
self.magnitude,
self.rms_weight,
self.rms_eps,
)
class HpcIHCPost(HpcModule):
"""Fused iHC post block: H_post gating plus multi-channel residual add.
Unlike mHC there is no comb matrix, so each output channel only needs its
own residual channel and the whole thing is one fused multiply-add per
element.
"""
def __init__(self, hc_mult: int, hidden_size: int) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def forward(
self, x: torch.Tensor, residual: torch.Tensor, H_post: torch.Tensor
) -> torch.Tensor:
import hpc
return hpc.fuse_ihc_post(x, residual, H_post)
class HpcIHCHead(HpcModule):
"""Fused iHC head block: merge the hc channels back into one hidden state.
Same structure as HpcIHCPre but the projection emits only hc_mult gate
logits and there is no H_post output. Called once per forward, after the
last decoder layer.
"""
def __init__(
self,
hc_mult: int,
hidden_size: int,
hc_eps: float,
norm_eps: float,
fallback_op: torch.nn.Module,
) -> None:
super().__init__()
self.hc_mult = hc_mult
self.hidden_size = hidden_size
self.hc_eps = hc_eps
self.norm_eps = norm_eps
# See HpcIHCPre: keep the owner out of the module tree.
object.__setattr__(self, "_fallback_op", fallback_op)
self.weight = None
self.hc_scale = None
self.hc_base = None
@classmethod
def support(cls, hc_mult: int, hidden_size: int) -> bool:
return _ihc_supported(hc_mult, hidden_size)
def process_weights_after_loading(self, model=None):
# hc_head_fn weight is [hc_mult, hc_mult * d], float32.
owner = self._fallback_op
self.weight = owner.hc_head_fn.weight
self.hc_scale = owner.hc_head_scale
self.hc_base = owner.hc_head_base
def forward(self, x: torch.Tensor) -> torch.Tensor:
import hpc
return hpc.fuse_ihc_head(
x,
self.weight,
self.hc_scale,
self.hc_base,
self.norm_eps,
self.hc_eps,
)
@@ -15,6 +15,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.vocab_parallel_embedding import (
UnquantizedEmbeddingMethod,
VocabParallelEmbedding,
@@ -144,7 +145,12 @@ class LogitsProcessor(PluggableLayer):
lm_head, hidden_states, bias=embedding_bias
)
if not isinstance(lm_head.quant_method, UnquantizedEmbeddingMethod):
# A quant config that excludes lm_head hands out UnquantizedLinearMethod
# rather than UnquantizedEmbeddingMethod, so accept both: either way the
# weight is plain and `lm_head.weight` can be cast directly.
if not isinstance(
lm_head.quant_method, (UnquantizedEmbeddingMethod, UnquantizedLinearMethod)
):
raise ValueError(
"A head_dtype different from the model dtype is only "
"supported for an unquantized lm_head."
+10 -2
View File
@@ -874,12 +874,20 @@ def _try_load_fp8_indexer_wk(
# We have both weight and scale: dequantize FP8 to BF16.
weight_fp8, scale_inv = entry["weight"], entry["scale"]
del buf[layer_prefix]
if scale_inv.dtype in (torch.uint8, torch.float8_e8m0fnu):
# MXFP8 checkpoints store power-of-two E8M0 scales, decode them to float.
scale_inv = scale_inv.view(torch.float8_e8m0fnu).to(torch.float32)
if scale_inv.ndim == 1:
# Per-channel scale: one scale per row of [out, in]
group_shape = GroupShape(1, weight_fp8.shape[1])
else:
block_size = weight_fp8.shape[1] // scale_inv.shape[1]
group_shape = GroupShape(block_size, block_size)
# Derive the block size independently per dim so that per-channel
# ([out, 1]), MX ([out, in / 32]) and 2D block ([out / 128, in / 128])
# scale layouts are all handled.
group_shape = GroupShape(
weight_fp8.shape[0] // scale_inv.shape[0],
weight_fp8.shape[1] // scale_inv.shape[1],
)
weight_bf16 = scaled_dequantize(
weight_fp8,
scale_inv,
+2
View File
@@ -128,6 +128,7 @@ _TEXT_GENERATION_MODELS = {
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
"HYV4ForCausalLM": ("vllm.models.hy_v4", "HYV4ForCausalLM"),
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
@@ -663,6 +664,7 @@ _SPECULATIVE_DECODING_MODELS = {
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
"InternS2MobiusMTP": ("interns2_mobius", "InternS2MobiusMTP"),
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
"HYV4MTPModel": ("vllm.models.hy_v4", "HYV4MTP"),
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
# Temporarily disabled.
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
+36
View File
@@ -0,0 +1,36 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HY V4 (``hy_v4``) model — hardware-isolated entry point.
HY V4 combines three architectural pieces:
- **iHC** (independent Hyper-Connections): the single residual stream is
replaced by ``hc_mult`` parallel residual channels, gated per sub-block.
- **MLA + lightning indexer**: multi-head latent attention with an optional
DSA-style sparse top-k selection, plus an output gate and a learnable sink.
- **MoE**: sigmoid-routed experts with a clamped SwiGLU and shared experts.
The package is organized like `vllm.models.deepseek_v32`: this module is the
only public entry point and dispatches on the current platform, so registry
entries never reach into a platform subpackage.
Only NVIDIA is supported for now. The port also drops the reference
implementation's HPC/TPCP fusion paths, which depend on infrastructure that
does not exist in this tree.
"""
from vllm.platforms import current_platform
if current_platform.is_rocm():
raise NotImplementedError("hy_v4 does not yet support ROCm.")
elif current_platform.is_xpu():
raise NotImplementedError("hy_v4 does not yet support XPU.")
else:
# Covers Blackwell (sm100) and all other CUDA devices.
from .nvidia.model import HYV4ForCausalLM
from .nvidia.mtp import HYV4MTP
__all__ = [
"HYV4ForCausalLM",
"HYV4MTP",
]
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+765
View File
@@ -0,0 +1,765 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MLA attention and lightning indexer for HY V4 (NVIDIA).
NOTE: The reference implementation also offers a fused MLA preprocessing
wrapper, which is not available here, so this port keeps the eager MLA path
with an FP8 indexer cache. TODO: restore the fused preprocessing once the
kernel lands. The HPC gated-MLA GEMM *is* wired up: it replaces the eager
output gating when the hpc package is installed, ``VLLM_ENABLE_HPC_OPS=1``
and the elementwise/dtype/alignment constraints hold.
The per-head learnable sink is supported through `.flashmla_sparse`, which
subclasses the platform's sparse MLA backend to forward ``attn_sink``.
"""
from typing import cast
import regex as re
import torch
from torch import nn
from transformers import DeepseekV2Config, DeepseekV3Config, PretrainedConfig
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
from vllm.config.cache import CacheDType
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.hpc import hpc_gated_mla_gemm, hpc_gated_mla_supported
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.platforms import current_platform
from vllm.v1.attention.backend import AttentionBackend, AttentionType
from vllm.v1.attention.selector import get_attn_backend
logger = init_logger(__name__)
_SPARSE_LAYER_TYPES = ("sparse_attention", "sparse", "deepseek_sparse_attention")
_WEIGHT_LAYER_INDEX_RE = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)")
def compute_skip_topk_layers(config: PretrainedConfig) -> set[int]:
"""Return the backbone layers that reuse a previous layer's top-k indices.
A "shared" indexer layer performs sparse attention with the indices computed
by the closest preceding "full" indexer layer, so it does not build its own
indexer and its checkpoint indexer weights must be skipped.
Args:
config: The model config.
Returns:
The set of layer indices that share another layer's top-k indices.
Raises:
ValueError: If ``indexer_types`` has the wrong length or an unknown
entry, or if ``index_topk_freq`` is not a positive integer.
"""
if not hasattr(config, "index_topk"):
return set()
num_hidden_layers = config.num_hidden_layers
indexer_types = getattr(config, "indexer_types", None)
if indexer_types is not None:
if len(indexer_types) != num_hidden_layers:
raise ValueError(
"indexer_types must contain one entry per hidden layer: "
f"expected {num_hidden_layers}, got {len(indexer_types)}."
)
invalid_types = sorted(set(indexer_types) - {"full", "shared"})
if invalid_types:
raise ValueError(
f"indexer_types only supports 'full' and 'shared', got {invalid_types}."
)
return {
layer_idx
for layer_idx, indexer_type in enumerate(indexer_types)
if indexer_type == "shared"
}
freq = getattr(config, "index_topk_freq", 1)
if not isinstance(freq, int) or freq <= 0:
raise ValueError(f"index_topk_freq must be a positive integer, got {freq!r}.")
pattern = getattr(config, "index_topk_pattern", None)
offset = getattr(config, "index_skip_topk_offset", 2)
skip_layers: set[int] = set()
for layer_idx in range(num_hidden_layers):
if pattern is None:
if max(layer_idx - offset + 1, 0) % freq != 0:
skip_layers.add(layer_idx)
elif 0 <= layer_idx < len(pattern) and pattern[layer_idx] == "S":
skip_layers.add(layer_idx)
return skip_layers
def is_skip_topk_indexer_weight(weight_name: str, skip_topk_layers: set[int]) -> bool:
"""Return whether an indexer weight belongs to a top-k sharing layer.
Args:
weight_name: Checkpoint weight name.
skip_topk_layers: Result of `compute_skip_topk_layers`.
Returns:
True when the weight is an indexer weight of a layer that has no
indexer module and therefore must be dropped.
"""
if ".indexer." not in weight_name or not skip_topk_layers:
return False
match = _WEIGHT_LAYER_INDEX_RE.search(weight_name)
return match is not None and int(match.group(1)) in skip_topk_layers
class Indexer(nn.Module):
"""Lightning indexer selecting the top-k tokens for sparse MLA."""
def __init__(
self,
vllm_config: VllmConfig,
config: DeepseekV2Config | DeepseekV3Config,
hidden_size: int,
q_lora_rank: int,
quant_config: QuantizationConfig | None,
cache_config: CacheConfig | None,
topk_indices_buffer: torch.Tensor | None,
prefix: str = "",
):
super().__init__()
self.vllm_config = vllm_config
self.config = config
self.quant_config = quant_config
self.topk_tokens = config.index_topk
self.n_head = config.index_n_heads
self.head_dim = config.index_head_dim
self.rope_dim = config.qk_rope_head_dim
self.q_lora_rank = q_lora_rank
# No tensor parallelism, just replicated.
self.wq_b = ReplicatedLinear(
self.q_lora_rank,
self.head_dim * self.n_head,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.wq_b",
)
# Fused wk + weights_proj: single GEMM producing [head_dim + n_head].
# FP8 wk weights are upcast to BF16 while loading to keep the fusion.
self.wk_weights_proj = MergedColumnParallelLinear(
hidden_size,
[self.head_dim, self.n_head],
bias=False,
quant_config=None,
disable_tp=True,
prefix=f"{prefix}.wk_weights_proj",
)
self.k_norm = LayerNorm(self.head_dim, eps=1e-6)
self.softmax_scale = self.head_dim**-0.5
self.scale_fmt = "ue8m0"
self.quant_block_size = 128
self.topk_indices_buffer = topk_indices_buffer
# FP8 naive cache: values in fp8 plus one fp32 scale per
# ``quant_block_size`` elements.
assert cache_config is not None, "HYV4 indexer requires cache_config"
self.k_cache = DeepseekV32IndexerCache(
head_dim=self.head_dim + self.head_dim // self.quant_block_size * 4,
dtype=torch.uint8,
prefix=f"{prefix}.k_cache",
cache_config=cache_config,
)
self.max_model_len = vllm_config.model_config.max_model_len
self.prefix = prefix
from vllm.v1.attention.backends.mla.indexer import get_max_prefill_buffer_size
self.max_total_seq_len = get_max_prefill_buffer_size(vllm_config)
self.indexer_op = SparseAttnIndexer(
self.k_cache,
self.quant_block_size,
self.scale_fmt,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.max_total_seq_len,
self.topk_indices_buffer,
)
def forward(
self,
hidden_states: torch.Tensor,
qr: torch.Tensor,
positions: torch.Tensor,
rotary_emb: nn.Module,
) -> torch.Tensor:
hidden_states, q_quant, k, weights = self.prepare_inputs(
hidden_states, qr, positions, rotary_emb
)
return self.indexer_op(hidden_states, q_quant, k, weights)
def prepare_inputs(
self,
hidden_states: torch.Tensor,
qr: torch.Tensor,
positions: torch.Tensor,
rotary_emb: nn.Module,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build the quantized query, key and per-head weights of the indexer."""
q, _ = self.wq_b(qr)
q = q.view(-1, self.n_head, self.head_dim)
# Checkpoint (PTM) layout: pe occupies the LAST rope_dim dims.
q_nope, q_pe = torch.split(
q, [self.head_dim - self.rope_dim, self.rope_dim], dim=-1
)
kw, _ = self.wk_weights_proj(hidden_states)
k = kw[:, : self.head_dim]
weights = kw[:, self.head_dim :]
k = self.k_norm(k)
k_nope, k_pe = torch.split(
k, [self.head_dim - self.rope_dim, self.rope_dim], dim=-1
)
q_pe, k_pe = rotary_emb(positions, q_pe, k_pe.unsqueeze(1))
# RoPE (NeoX) can introduce extra leading dims, so flatten back to the
# token-major shapes.
q_pe = q_pe.reshape(-1, self.n_head, self.rope_dim)
k_pe = k_pe.reshape(-1, 1, self.rope_dim)
# Reassemble with the original physical layout: no_pe first, pe last.
q = torch.cat([q_nope, q_pe], dim=-1)
# ``k_pe`` is [num_tokens, 1, rope_dim] (MQA).
k = torch.cat([k_nope, k_pe.squeeze(-2)], dim=-1)
# Only q is quantized here; k quantization is fused with cache insertion.
q = q.view(-1, self.head_dim)
q_fp8, q_scale = per_token_group_quant_fp8(
q,
self.quant_block_size,
column_major_scales=False,
use_ue8m0=self.scale_fmt is not None,
)
q_fp8 = q_fp8.view(-1, self.n_head, self.head_dim)
q_scale = q_scale.view(-1, self.n_head, 1)
weights = (
weights.unsqueeze(-1) * q_scale * self.softmax_scale * self.n_head**-0.5
)
weights = weights.squeeze(-1)
return hidden_states, q_fp8, k, weights
class HYV4MLAAttention(nn.Module):
"""Multi-head latent attention with optional sparse lightning indexer.
Main reference: the DeepSeek-V2 paper and the FlashInfer implementation
(https://arxiv.org/abs/2405.04434). HY V4 additionally supports an output
gate (``gated_mla``) and a per-head learnable attention sink.
The sink is applied by binding the sink-capable backend from
`.flashmla_sparse`; if no backend on this platform can consume sinks, the
weight is still loaded but the bias is disabled with a warning.
"""
def __init__(
self,
vllm_config: VllmConfig,
config: PretrainedConfig,
hidden_size: int,
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
q_lora_rank: int | None,
kv_lora_rank: int,
max_position_embeddings: int = 8192,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
topk_indices_buffer: torch.Tensor | None = None,
layer_idx: int = 0,
) -> None:
super().__init__()
self.config = config
self.hidden_size = hidden_size
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
self.v_head_dim = v_head_dim
self.layer_idx = layer_idx
self.q_lora_rank = q_lora_rank
self.kv_lora_rank = kv_lora_rank
self.num_heads = num_heads
tp_size = get_tensor_model_parallel_world_size()
assert num_heads % tp_size == 0
self.num_local_heads = num_heads // tp_size
self.layer_id = int(prefix.split(".")[-2])
layer_types = getattr(config, "layer_types", None)
requested_sparse = (
hasattr(config, "index_topk")
and layer_types is not None
and self.layer_id < len(layer_types)
and layer_types[self.layer_id] in _SPARSE_LAYER_TYPES
)
# Only actual sparse layers may share another layer's top-k indices.
self.skip_topk = requested_sparse and self.layer_id in compute_skip_topk_layers(
config
)
# The skip pattern only governs backbone layers. MTP/nextn layers
# (layer_id >= num_hidden_layers) always build a full indexer: they
# compute indices at draft step 0 and toggle at runtime.
num_hidden_layers = getattr(config, "num_hidden_layers", None)
is_mtp_layer = (
num_hidden_layers is not None and self.layer_id >= num_hidden_layers
)
self.create_indexer = requested_sparse and (not self.skip_topk or is_mtp_layer)
self.is_sparse = requested_sparse
# Do not silently degrade sparse layers into dense attention. Probe the
# sparse MLA backend directly and fail fast with the real error.
kv_cache_dtype = cache_config.cache_dtype if cache_config else "auto"
if self.is_sparse:
try:
get_attn_backend(
head_size=self.kv_lora_rank + self.qk_rope_head_dim,
dtype=torch.get_default_dtype(),
kv_cache_dtype=kv_cache_dtype,
use_mla=True,
has_sink=False,
use_sparse=True,
num_heads=self.num_local_heads,
)
except Exception as exc:
raise RuntimeError(
"HYV4 sparse attention was requested, but no valid sparse MLA "
"backend is available for current runtime/config. "
"Refusing to fall back to dense attention."
) from exc
self.scaling = self.qk_head_dim**-0.5
self.max_position_embeddings = max_position_embeddings
self.fused_qkv_a_proj = None
self.kv_a_proj_with_mqa = None
if self.q_lora_rank is not None:
# ``q_a_proj`` and ``kv_a_proj_with_mqa`` read the same
# ``hidden_states`` and are both TP-replicated, so they run as one
# GEMM. The checkpoint keeps them separate; `load_weights` merges
# them through ``stacked_params_mapping``.
self.fused_qkv_a_proj = MergedColumnParallelLinear(
self.hidden_size,
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.fused_qkv_a_proj",
disable_tp=True,
)
else:
self.kv_a_proj_with_mqa = ReplicatedLinear(
self.hidden_size,
self.kv_lora_rank + self.qk_rope_head_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.kv_a_proj_with_mqa",
)
self.q_a_layernorm = None
self.q_b_proj = None
self.q_proj = None
if self.q_lora_rank is not None:
self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
self.q_b_proj = ColumnParallelLinear(
self.q_lora_rank,
self.num_heads * self.qk_head_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.q_b_proj",
)
else:
self.q_proj = ColumnParallelLinear(
self.hidden_size,
self.num_heads * self.qk_head_dim,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.q_proj",
)
self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps)
self.kv_b_proj = ColumnParallelLinear(
self.kv_lora_rank,
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.kv_b_proj",
)
self.o_proj = RowParallelLinear(
self.num_heads * self.v_head_dim,
self.hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
qk_rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
is_neox_style=False,
)
self.indexer_rope_emb: nn.Module | None
self.indexer: Indexer | None
if self.create_indexer:
# The checkpoint stores indexer q_pe/k_pe in interleaved
# (Megatron/PTM) layout, so the indexer must use interleaved RoPE
# (is_neox_style=False) like the main attention path. Using NeoX
# here loses the relative-position dependence and corrupts the DSA
# top-k selection.
self.indexer_rope_emb = get_rope(
qk_rope_head_dim,
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
is_neox_style=False,
)
# The indexer projects its queries from the MLA q_lora activations,
# so a sparse layer requires a query down-projection.
assert q_lora_rank is not None, (
"HYV4 sparse attention requires q_lora_rank to be set"
)
self.indexer = Indexer(
vllm_config,
config,
hidden_size,
q_lora_rank,
quant_config,
cache_config,
topk_indices_buffer,
f"{prefix}.indexer",
)
else:
self.indexer_rope_emb = None
self.indexer = None
self.gated_mla = bool(getattr(config, "gated_mla", False))
self.linear_gate: ColumnParallelLinear | None
if self.gated_mla:
if config.gating_type == "headwise":
self.gate_projection_size_per_head = 1
elif config.gating_type == "elementwise":
self.gate_projection_size_per_head = self.v_head_dim
else:
raise ValueError(f"Unknown gating type: {config.gating_type}")
self.linear_gate = ColumnParallelLinear(
self.hidden_size,
self.num_heads * self.gate_projection_size_per_head,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.linear_gate",
)
self.use_hpc_gated_mla = hpc_gated_mla_supported(
config.gating_type, self.linear_gate
)
else:
self.linear_gate = None
self.use_hpc_gated_mla = False
self.prefix = prefix
# Per-head learnable attention sink. Created BEFORE ``MLAAttention`` so
# it can be forwarded as the ``sinks`` impl kwarg. The parameter always
# holds the local TP shard.
self.learnable_sink = bool(getattr(config, "learnable_sink", False))
sinks = None
sink_backend: type[AttentionBackend] | None = None
if self.learnable_sink:
sink_backend = self._resolve_sink_backend(kv_cache_dtype)
enable_sink = sink_backend is not None
self.learnable_sink_param = nn.Parameter(
torch.empty(
self.num_local_heads,
# The kernels require fp32 sinks; the disabled path keeps
# the checkpoint dtype since the value is never consumed.
dtype=torch.float32 if enable_sink else torch.bfloat16,
)
)
if enable_sink:
sinks = self.learnable_sink_param
self._force_sparse_mqa()
extra_impl_args = {} if sinks is None else {"sinks": sinks}
self.mla_attn = MLAAttention(
num_heads=self.num_local_heads,
scale=self.scaling,
qk_nope_head_dim=self.qk_nope_head_dim,
qk_rope_head_dim=self.qk_rope_head_dim,
v_head_dim=self.v_head_dim,
q_lora_rank=self.q_lora_rank,
kv_lora_rank=self.kv_lora_rank,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
kv_b_proj=self.kv_b_proj,
use_sparse=self.is_sparse,
indexer=self.indexer,
topk_indices_buffer=topk_indices_buffer,
attn_backend=sink_backend,
**extra_impl_args,
)
@property
def topk_indices_buffer(self) -> torch.Tensor | None:
"""The sparse top-k index buffer this layer reads from.
The real consumer is ``mla_attn.impl``, an ``AttentionImpl`` rather than
an ``nn.Module``, so a plain ``named_modules()`` walk cannot reach it.
The MTP proposer rebinds the draft layers onto the target model's buffer
through such a walk, so expose it here: without this the draft attention
would keep reading its own buffer while the indices it needs are written
to the target's.
"""
return getattr(self.mla_attn.impl, "topk_indices_buffer", None)
@topk_indices_buffer.setter
def topk_indices_buffer(self, buffer: torch.Tensor) -> None:
self.mla_attn.impl.topk_indices_buffer = buffer # type: ignore[attr-defined]
def _resolve_sink_backend(
self, kv_cache_dtype: str
) -> type[AttentionBackend] | None:
"""Return an MLA backend that can apply this layer's learnable sink.
The sink is part of the architecture, so a backend that cannot apply it
changes the model's output. Resolution order:
1. If the backend the selector would pick already advertises
`supports_sink`, keep it — this also honours an explicit
``--attention-backend`` choice.
2. Otherwise fall back to the sink-capable ``FLASHMLA_SPARSE`` subclass
in `.flashmla_sparse`, whose kernels accept ``attn_sink``, provided
it validates against the current runtime configuration.
3. Otherwise give up on the bias rather than failing the load.
Args:
kv_cache_dtype: The layer's KV cache dtype string.
Returns:
The backend class to bind, or None when no sink-capable backend is
available; the caller then loads the sink weight but disables the
bias.
"""
head_size = self.kv_lora_rank + self.qk_rope_head_dim
dtype = torch.get_default_dtype()
try:
selected_cls = get_attn_backend(
head_size=head_size,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
use_mla=True,
use_sparse=self.is_sparse,
num_heads=self.num_local_heads,
)
except Exception as exc:
# Stringify before logging: warning_once dedupes on the arguments,
# and a fresh exception object per layer would defeat it.
logger.warning_once(
"HYV4 failed to select an MLA backend for the learnable sink "
"(%s); the sink parameter is loaded but the sink bias is "
"disabled.",
str(exc),
)
return None
if selected_cls.supports_sink():
return selected_cls
from .flashmla_sparse import HYV4FlashMLASparseBackend
# Mirror how the selector derives the configuration-dependent inputs so
# this check accepts exactly what the backend would accept at runtime.
capability = current_platform.get_device_capability()
if capability is None:
logger.warning_once(
"HYV4 learnable sink is unavailable: the device compute "
"capability is unknown. The sink parameter is loaded but the "
"sink bias is disabled."
)
return None
cache_config = get_current_vllm_config().cache_config
block_size = (
cache_config.block_size
if cache_config is not None and cache_config.user_specified_block_size
else None
)
invalid_reasons = HYV4FlashMLASparseBackend.validate_configuration(
head_size=head_size,
dtype=dtype,
kv_cache_dtype=cast(CacheDType, kv_cache_dtype),
block_size=block_size,
use_mla=True,
has_sink=True,
use_sparse=self.is_sparse,
use_mm_prefix=False,
use_per_head_quant_scales=False,
device_capability=capability,
attn_type=AttentionType.DECODER,
)
if invalid_reasons:
logger.warning_once(
"HYV4 learnable sink is unavailable: the selected backend %s "
"cannot apply sinks and the sink-capable FLASHMLA_SPARSE path "
"is invalid here (%s). The sink parameter is loaded but the "
"sink bias is disabled.",
selected_cls.get_name(),
", ".join(invalid_reasons),
)
return None
logger.info_once(
"HYV4 learnable sink enabled: using the sink-capable "
"FLASHMLA_SPARSE impl instead of %s, which cannot apply sinks.",
selected_cls.get_name(),
)
return HYV4FlashMLASparseBackend
def _force_sparse_mqa(self) -> None:
"""Keep every token on the sink-capable sparse MQA path.
``_resolve_sink_backend`` only binds the backend that serves decode.
``MLAAttention`` additionally routes short prefills to a separate dense
MLA prefill backend, and none of those accept ``attn_sink``, so prefill
would silently drop the sink while decode applies it. Such a partially
applied sink is not the trained architecture and corrupts the output, so
opt out of the dense split instead.
Prefills up to ``index_topk`` keep every token inside the sparse top-k,
making the sparse path numerically equivalent to the dense one apart from
also applying the sink.
"""
attention_config = get_current_vllm_config().attention_config
if attention_config.sparse_mla_force_mqa:
return
attention_config.sparse_mla_force_mqa = True
logger.info_once(
"HYV4 learnable sink enabled: forcing sparse MQA for prefill too, "
"as the dense MLA prefill backends cannot apply sinks."
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
llama_4_scaling: torch.Tensor | None = None,
) -> torch.Tensor:
q_c = None
if self.q_lora_rank is not None:
assert self.fused_qkv_a_proj is not None
assert self.q_a_layernorm is not None
assert self.q_b_proj is not None
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
q_c, kv_lora = qkv_lora.split(
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
dim=-1,
)
q_c = self.q_a_layernorm(q_c)
q = self.q_b_proj(q_c)[0]
else:
assert self.q_proj is not None
assert self.kv_a_proj_with_mqa is not None
q = self.q_proj(hidden_states)[0]
kv_lora = self.kv_a_proj_with_mqa(hidden_states)[0]
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
kv_c_normed = self.kv_a_layernorm(kv_c)
q = q.view(-1, self.num_local_heads, self.qk_head_dim)
# Add a head dim of 1 to k_pe.
k_pe = k_pe.unsqueeze(1)
q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb(
positions, q[..., self.qk_nope_head_dim :], k_pe
)
if llama_4_scaling is not None:
q *= llama_4_scaling
output_shape = (
hidden_states.shape[0],
self.num_local_heads * self.v_head_dim,
)
# Single coarse eager break covering the indexer and MLA attention, as
# the breakable cudagraph contract requires: everything that reads
# per-batch metadata runs in one eager segment, so no tensor has to stay
# alive across a capture-segment boundary.
attn_out = torch.empty(
output_shape, dtype=hidden_states.dtype, device=hidden_states.device
)
self._indexer_and_attn(
hidden_states, q_c, positions, q, kv_c_normed, k_pe, attn_out
)
if self.gated_mla and self.linear_gate is not None:
if self.use_hpc_gated_mla:
# Projection, sigmoid and the product in one launch. The gate
# is column-parallel and unbiased, so its local weight shard
# maps straight onto the local attn_out columns.
assert hidden_states.is_contiguous() and attn_out.is_contiguous()
attn_out = hpc_gated_mla_gemm(
hidden_states,
self.linear_gate.weight,
attn_out,
)
else:
gate_score = self.linear_gate(hidden_states)[0]
if self.config.gating_type == "headwise":
gate_score = gate_score.unsqueeze(-1)
attn_out = attn_out.reshape(
*attn_out.shape[:-1], -1, self.v_head_dim
)
attn_out = attn_out * torch.sigmoid(gate_score)
attn_out = attn_out.reshape(*attn_out.shape[:-2], -1)
else:
attn_out = attn_out * torch.sigmoid(gate_score)
out, _ = self.o_proj(attn_out)
return out
@eager_break_during_capture
def _indexer_and_attn(
self,
hidden_states: torch.Tensor,
q_c: torch.Tensor | None,
positions: torch.Tensor,
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
out: torch.Tensor, # [num_tokens, heads * v_head_dim], written in place
) -> None:
"""Run the lightning indexer and MLA attention in one eager segment.
Both read per-batch attention metadata, so under the breakable cudagraph
they must not be captured. Keeping them in a single break (instead of one
break each) also means the attention inputs never have to survive a
capture-segment boundary. The nested ``sparse_attn_indexer`` and
``unified_mla_attention_with_output`` breaks short-circuit here, since
the capture is no longer active inside an eager segment.
"""
if self.indexer is not None and self.is_sparse and not self.skip_topk:
self.indexer(hidden_states, q_c, positions, self.indexer_rope_emb)
out.copy_(
self.mla_attn(
q,
kv_c_normed,
k_pe,
output_shape=out.shape,
)
)
+283
View File
@@ -0,0 +1,283 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sink-capable FlashMLA sparse backend for HY V4 (NVIDIA).
HY V4 adds a per-head learnable attention sink on top of sparse MLA. The
vendored FlashMLA kernels already accept an ``attn_sink`` argument, but vLLM's
shared ``FLASHMLA_SPARSE`` backend neither advertises sink support nor forwards
the tensor, so the bias would be silently dropped.
This module supplies the missing wiring inside the model package, mirroring how
`vllm.models.deepseek_v4.nvidia.flashinfer_sparse` hands ``sinks`` to the
FlashInfer sparse MLA kernels: subclass the platform backend, declare
`supports_sink`, and thread the sink into every kernel call.
The subclass intentionally keeps the inherited ``get_name()``
(``"FLASHMLA_SPARSE"``). Several shared code paths key off that exact string —
`_canonicalize_sparse_mla_kv_cache_dtype` promotes a quantized KV cache to
``fp8_ds_mla`` for it, and `FlashMLASparseImpl` asserts that layout — so a new
name would silently change KV cache behaviour. Only``supports_sink`` and the
two kernel wrappers differ from the parent.
"""
from typing import TYPE_CHECKING
import torch
from vllm.logger import init_logger
from vllm.v1.attention.backends.mla.flashmla_sparse import (
FlashMLASparseBackend,
FlashMLASparseImpl,
FlashMLASparseMetadata,
)
from vllm.v1.attention.ops.flashmla import (
flash_mla_sparse_fwd,
flash_mla_with_kvcache,
)
if TYPE_CHECKING:
from vllm.model_executor.models.deepseek_v2 import Indexer
logger = init_logger(__name__)
class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
"""FlashMLA sparse impl that applies HY V4's per-head learnable sink.
The sink enters as the ``sinks`` impl kwarg of
`vllm.model_executor.layers.attention.MLAAttention` and is consumed by the
FlashMLA kernels, which fold it into the softmax denominator:
``out *= exp(lse) / (exp(lse) + exp(sink))``.
"""
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int,
alibi_slopes: list[float] | None,
sliding_window: int | None,
kv_cache_dtype: str,
logits_soft_cap: float | None,
attn_type: str,
kv_sharing_target_layer_name: str | None,
# MLA Specific Arguments
topk_indices_buffer: torch.Tensor | None = None,
indexer: "Indexer | None" = None,
**mla_args,
) -> None:
# ``SparseMLACommonImpl`` takes explicit keyword arguments only, so the
# sink has to be removed before the base classes see``mla_args``.
sinks: torch.Tensor | None = mla_args.pop("sinks", None)
super().__init__(
num_heads,
head_size,
scale,
num_kv_heads,
alibi_slopes,
sliding_window,
kv_cache_dtype,
logits_soft_cap,
attn_type,
kv_sharing_target_layer_name,
topk_indices_buffer=topk_indices_buffer,
indexer=indexer,
**mla_args,
)
self._validate_sinks(sinks, num_heads)
self.sinks = sinks
@staticmethod
def _validate_sinks(sinks: torch.Tensor | None, num_heads: int) -> None:
"""Reject sink tensors the FlashMLA kernels cannot consume.
Args:
sinks: Candidate sink tensor, or None when the layer has no sink.
num_heads: Local (TP-sharded) query head count of this layer.
Raises:
ValueError: If the dtype is not float32 or the shape is not
``(num_heads,)``.
"""
if sinks is None:
return
if sinks.dtype != torch.float32:
raise ValueError(
"HYV4 FlashMLA sparse attention sinks must have dtype "
f"torch.float32, but got {sinks.dtype}."
)
if sinks.ndim != 1 or sinks.shape[0] != num_heads:
raise ValueError(
"HYV4 FlashMLA sparse attention sinks must have shape "
f"({num_heads},), but got {tuple(sinks.shape)}."
)
def _sinks_for_query(
self,
q: torch.Tensor,
head_dim: int,
kernel_heads: int,
) -> torch.Tensor | None:
"""Return the sink laid out for the kernel's query head count.
Args:
q: Query tensor, before any head padding.
head_dim: Axis of ``q`` holding the query heads.
kernel_heads: Head count the kernel is invoked with, which may
exceed the query head count because of padding.
Returns:
The sink tensor padded to ``kernel_heads`` with ``-inf`` (a no-op
sink) for the padded lanes, or None when the layer has no sink.
Raises:
ValueError: If the sink and query head layouts disagree, or if they
live on different devices.
"""
sinks = self.sinks
if sinks is None:
return None
query_heads = q.shape[head_dim]
if sinks.shape[0] != query_heads:
raise ValueError(
"HYV4 FlashMLA sparse attention sink head count must match the "
f"runtime query layout: sinks={sinks.shape[0]}, "
f"query_heads={query_heads}. The sink must use the same "
"unpadded head layout as the query."
)
if sinks.device != q.device:
raise ValueError(
"HYV4 FlashMLA sparse attention sinks and query must be on the "
f"same device, but got sinks={sinks.device}, query={q.device}."
)
if kernel_heads < query_heads:
raise ValueError(
"HYV4 FlashMLA sparse kernel head count cannot be smaller than "
f"the runtime query layout: query_heads={query_heads}, "
f"kernel_heads={kernel_heads}."
)
if kernel_heads == query_heads:
return sinks
# Mirror the query padding the kernels require. Reading ``sinks`` here
# (rather than caching a padded copy at construction time) keeps the
# values correct for weights loaded after the module is built, and the
# allocation plus copy are captured in the CUDA graph.
padded_sinks = sinks.new_full((kernel_heads,), float("-inf"))
padded_sinks[:query_heads] = sinks
return padded_sinks
def _fp8_flash_mla_kernel(
self,
q: torch.Tensor,
kv_c_and_k_pe_cache: torch.Tensor,
topk_indices: torch.Tensor,
kernel_metadata: FlashMLASparseMetadata.FP8KernelMetadata,
) -> tuple[torch.Tensor, torch.Tensor]:
# q shape: (batch, seq_len, num_heads, head_dim)
actual_num_heads = q.size(2)
padded_num_heads = self.fp8_decode_padded_heads
attn_sink = self._sinks_for_query(q, head_dim=2, kernel_heads=padded_num_heads)
# Pad query if needed (kernel only supports h_q = 64 or 128)
if actual_num_heads < padded_num_heads:
logger.warning_once(
f"Padding num_heads from {actual_num_heads} to "
f"{padded_num_heads} for FP8 sparse decode kernel"
)
q_padded = q.new_zeros((q.size(0), q.size(1), padded_num_heads, q.size(3)))
q_padded[:, :, :actual_num_heads, :] = q
q = q_padded
out, lse = flash_mla_with_kvcache(
q=q,
k_cache=kv_c_and_k_pe_cache.view(torch.uint8).unsqueeze(-2),
block_table=kernel_metadata.dummy_block_table,
head_dim_v=512,
cache_seqlens=kernel_metadata.cache_lens,
tile_scheduler_metadata=kernel_metadata.scheduler_metadata,
is_fp8_kvcache=True,
indices=topk_indices,
softmax_scale=self.softmax_scale,
attn_sink=attn_sink,
)
# Slice output and lse back to actual head count if we padded
if actual_num_heads < padded_num_heads:
out = out[:, :, :actual_num_heads, :]
lse = lse[:, :actual_num_heads, :]
return out, lse
def _bf16_flash_mla_kernel(
self,
q: torch.Tensor,
kv_c_and_k_pe_cache: torch.Tensor,
topk_indices: torch.Tensor,
topk_length: torch.Tensor | None = None,
actual_num_heads: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens = q.shape[0]
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(
-1, 1, kv_c_and_k_pe_cache.shape[-1]
)
# NOTE(Chen): kernel requires num_local_head to be a multiple of
# 64 on hopper and 128 on blackwell. Pad from q's head count, not
# self.num_heads: under DCP the heads are all-gathered before this.
if actual_num_heads is None:
actual_num_heads = q.shape[1]
padded_num_heads = (
(actual_num_heads + self.prefill_padding - 1)
// self.prefill_padding
* self.prefill_padding
)
# q_concat_buffer may already include the kernel-required padding.
# The sink remains defined only for the real query heads.
attn_sink = self._sinks_for_query(
q[:, :actual_num_heads], head_dim=1, kernel_heads=padded_num_heads
)
if q.shape[1] < padded_num_heads:
logger.warning_once(
f"Padding num_heads from {actual_num_heads} to "
f"{padded_num_heads} for BF16 sparse prefill kernel"
)
# Zero (not new_empty) the padded lanes: topk_indices is shared by
# all heads, so the kernel reduces across the head group and NaNs
# from uninitialized memory would leak into the real heads.
q_padded = q.new_zeros((q.shape[0], padded_num_heads, q.shape[2]))
q_padded[:, :actual_num_heads, :] = q
q = q_padded
topk_indices = topk_indices.view(num_tokens, 1, -1)
output, _, lse = flash_mla_sparse_fwd(
q,
kv_c_and_k_pe_cache,
topk_indices,
self.softmax_scale,
attn_sink=attn_sink,
topk_length=topk_length,
)
output = output[:, :actual_num_heads, :]
lse = lse[:, :actual_num_heads]
return output, lse
class HYV4FlashMLASparseBackend(FlashMLASparseBackend):
"""``FLASHMLA_SPARSE`` with attention-sink support for HY V4.
Keeps the parent's name, metadata and builder; only the impl class and the
sink capability differ. See the module docstring for why the name is reused.
"""
@staticmethod
def get_impl_cls() -> type[HYV4FlashMLASparseImpl]:
return HYV4FlashMLASparseImpl
@classmethod
def supports_sink(cls) -> bool:
return True
+370
View File
@@ -0,0 +1,370 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""iHC (independent Hyper-Connections) layers for HY V4 (NVIDIA).
iHC replaces the single residual stream of a standard transformer with
``hc_mult`` parallel residual channels. Each decoder sub-block reduces the
channels to one hidden state (``HYV4HCPreLayer``), runs the sub-block, then
scatters the result back over the channels (``HYV4HCPostLayer``). The final
``HYV4HCHeadLayer`` merges the channels before the model's output norm.
NOTE: Each of the three steps has an optional single-kernel HPC replacement
(``HpcIHCPre`` / ``HpcIHCPost`` / ``HpcIHCHead``). They are only constructed
when the hpc package is installed, ``VLLM_ENABLE_HPC_OPS=1`` and the shape /
device constraints hold; otherwise the eager path below runs unchanged.
TODO: port the cross-layer post+pre fusion (``HpcIHCPostPre``) as well; it
requires restructuring the decoder-layer forward scheduling.
"""
import torch
from torch import nn
from transformers import PretrainedConfig
from vllm.model_executor.layers.hpc import HpcIHCHead, HpcIHCPost, HpcIHCPre
from vllm.model_executor.layers.linear import ReplicatedLinear
class HYV4HCPreLayer(nn.Module):
"""iHC pre-processing layer (2D-activation adaptation).
Steps:
1. RMS-normalize the flattened ``[num_tokens, hc * d]`` input.
2. Project to the pre/post gating logits.
3. Turn the logits into sigmoid gates.
4. Reduce over the channel dim with the pre gates.
"""
def __init__(
self,
config: PretrainedConfig,
hidden_dim: int,
hc_mult: int = 4,
magnitude: float = 2.0,
init_std: float = 6e-3,
base_noise_std: float = 0.0,
hc_eps: float = 1e-6,
layernorm_epsilon: float = 1e-5,
prefix: str = "",
):
super().__init__()
self.config = config
self.hidden_dim = hidden_dim
self.hc_mult = hc_mult
self.magnitude = magnitude
self.hc_eps = hc_eps
self.layernorm_epsilon = layernorm_epsilon
hc_in_dim = hc_mult * hidden_dim
mix_hc = 2 * hc_mult # pre + post only, no comb
self.hc_fn = ReplicatedLinear(
input_size=hc_in_dim,
output_size=mix_hc,
params_dtype=torch.float32,
bias=False,
prefix=f"{prefix}.hc_fn",
)
self.hc_scale = nn.Parameter(torch.empty(2, dtype=torch.float32))
self.hc_base = nn.Parameter(torch.empty(mix_hc, dtype=torch.float32))
self.reset_parameters(init_std, base_noise_std)
# Optional single-kernel HPC replacement for the whole forward below.
# ``norm_owner`` is left unset: the RMSNorm that follows the pre block
# lives in the decoder layer and is not folded in yet.
self.hpc_op: HpcIHCPre | None = None
if HpcIHCPre.support(hc_mult, hidden_dim):
self.hpc_op = HpcIHCPre(
hc_mult=hc_mult,
hidden_size=hidden_dim,
magnitude=magnitude,
hc_eps=hc_eps,
norm_eps=layernorm_epsilon,
fallback_op=self,
)
def reset_parameters(self, init_std: float, base_noise_std: float = 0.0) -> None:
"""Initialize the gate scale and per-channel gate bias."""
del init_std # hc_fn is initialized by ReplicatedLinear
nn.init.constant_(self.hc_scale, 0.01)
with torch.no_grad():
self.hc_base[: self.hc_mult].fill_(
-torch.log(torch.tensor(self.hc_mult - 1.0, dtype=self.hc_base.dtype))
)
self.hc_base[self.hc_mult : 2 * self.hc_mult].fill_(0.0)
if base_noise_std > 0.0:
self.hc_base.add_(torch.randn_like(self.hc_base) * base_noise_std)
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Reduce the iHC channels and emit the post gates.
Args:
x: Input of shape ``[num_tokens, hc, d]``.
Returns:
A tuple of the pre-gated reduction ``[num_tokens, d]`` and the post
gates ``[num_tokens, hc]`` consumed by `HYV4HCPostLayer`.
"""
if self.hpc_op is not None:
return self.hpc_op(x)
shape = x.size() # [num_tokens, hc, d]
hc = self.hc_mult
hc_eps = self.hc_eps
x_flat = x.flatten(1).float() # [num_tokens, hc*d]
rsqrt = torch.rsqrt(
x_flat.square().mean(-1, keepdim=True) + self.layernorm_epsilon
)
mixes = self.hc_fn(x_flat)[0] * rsqrt # [num_tokens, 2*hc]
pre_raw = mixes[..., :hc]
post_raw = mixes[..., hc : 2 * hc]
pre = (
torch.sigmoid(
pre_raw * self.hc_scale[0].float() + self.hc_base[:hc].float()
)
+ hc_eps
)
post = (
self.magnitude
* torch.sigmoid(
post_raw * self.hc_scale[1].float() + self.hc_base[hc : 2 * hc].float()
)
+ hc_eps
)
y = torch.sum(pre.unsqueeze(-1) * x.reshape(shape), dim=1) # [num_tokens, d]
return y.to(x.dtype), post
class HYV4HCPostLayer(nn.Module):
"""iHC post-processing layer (2D-activation adaptation).
Applies post-gating to the sub-block output and adds the multi-channel
residual (no comb mixing)::
y[n, i, d] = post[n, i] * x[n, d] + residual[n, i, d]
"""
def __init__(self, config: PretrainedConfig):
super().__init__()
self.config = config
# Optional single-kernel HPC replacement for the whole forward below.
# Only constructed under enable_ihc (see HYV4HCLayer), so hc_mult is
# present; getattr keeps this robust if that ever changes.
self.hpc_op: HpcIHCPost | None = None
hc_mult = getattr(config, "hc_mult", 0)
if HpcIHCPost.support(hc_mult, config.hidden_size):
self.hpc_op = HpcIHCPost(hc_mult=hc_mult, hidden_size=config.hidden_size)
def forward(
self, x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor
) -> torch.Tensor:
"""Scatter the sub-block output back onto the iHC channels.
Args:
x: Attention/MLP output of shape ``[num_tokens, d]``.
residual: Multi-channel residual ``[num_tokens, hc, d]``.
post: Post gates ``[num_tokens, hc]`` from `HYV4HCPreLayer`.
Returns:
The updated residual channels ``[num_tokens, hc, d]``.
"""
if self.hpc_op is not None:
return self.hpc_op(x, residual, post)
dtype = x.dtype
x = x.float()
residual = residual.float()
post = post.float()
post_gated = post.unsqueeze(-1) * x.unsqueeze(-2) # [num_tokens, hc, d]
y = post_gated + residual
return y.to(dtype)
class HYV4HCHeadLayer(nn.Module):
"""iHC head layer (2D-activation adaptation).
Merges the iHC channels back into a single hidden state before the final
layer norm, using an RMS-normed projection plus sigmoid-gated reduction.
"""
def __init__(
self,
config: PretrainedConfig,
hidden_size: int,
hc_mult: int = 4,
hc_eps: float = 1e-6,
init_std: float = 6e-3,
base_noise_std: float = 0.0,
prefix: str = "",
):
super().__init__()
self.config = config
self.hidden_size = hidden_size
self.hc_mult = hc_mult
self.hc_eps = hc_eps
self.hc_head_fn = ReplicatedLinear(
input_size=hc_mult * hidden_size,
output_size=hc_mult,
params_dtype=torch.float32,
bias=False,
prefix=f"{prefix}.hc_head_fn",
)
self.hc_head_base = nn.Parameter(torch.empty(hc_mult, dtype=torch.float32))
self.hc_head_scale = nn.Parameter(torch.empty(1, dtype=torch.float32))
self.reset_parameters(init_std, base_noise_std)
# Optional single-kernel HPC replacement for the whole forward below.
self.hpc_op: HpcIHCHead | None = None
if HpcIHCHead.support(hc_mult, hidden_size):
self.hpc_op = HpcIHCHead(
hc_mult=hc_mult,
hidden_size=hidden_size,
hc_eps=hc_eps,
norm_eps=config.rms_norm_eps,
fallback_op=self,
)
def reset_parameters(
self, init_std: float = 6e-3, base_noise_std: float = 0.0
) -> None:
"""Initialize the head gate scale and per-channel gate bias."""
del init_std # hc_head_fn is initialized by ReplicatedLinear
nn.init.constant_(self.hc_head_scale, 0.01)
with torch.no_grad():
self.hc_head_base.fill_(
-torch.log(
torch.tensor(self.hc_mult - 1.0, dtype=self.hc_head_base.dtype)
)
)
if base_noise_std > 0.0:
self.hc_head_base.add_(
torch.randn_like(self.hc_head_base) * base_noise_std
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""Merge the iHC channels into a single hidden state.
Args:
x: Input of shape ``[num_tokens, hc, d]``.
Returns:
The merged hidden state ``[num_tokens, d]``.
"""
if self.hpc_op is not None:
return self.hpc_op(x)
shape, x_dtype = x.size(), x.dtype
x = x.flatten(1).float() # [num_tokens, hc*d]
rsqrt = torch.rsqrt(
x.square().mean(-1, keepdim=True) + self.config.rms_norm_eps
)
mixes = self.hc_head_fn(x)[0] * rsqrt # [num_tokens, hc]
pre = (
torch.sigmoid(
mixes * self.hc_head_scale.float() + self.hc_head_base.float()
)
+ self.hc_eps
)
y = torch.sum(pre.unsqueeze(-1) * x.reshape(shape), dim=1) # [num_tokens, d]
return y.to(x_dtype)
class HYV4HCLayer(nn.Module):
"""Wrapper owning one iHC boundary (pre + post) of a decoder sub-block."""
def __init__(
self,
config: PretrainedConfig,
layer_idx: int,
init_std: float = 6e-3,
base_noise_std: float = 0.0,
prefix: str = "",
):
super().__init__()
self.config = config
self.layer_idx = layer_idx
self.enable_ihc = getattr(config, "enable_ihc", False)
if self.enable_ihc:
self.hc_pre = HYV4HCPreLayer(
config,
config.hidden_size,
config.hc_mult,
config.hc_magnitude,
init_std,
base_noise_std,
config.hc_eps,
config.rms_norm_eps,
prefix=f"{prefix}.hc_pre",
)
self.hc_post = HYV4HCPostLayer(config)
def prepare_input(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Normalize the sub-block input to 3D when iHC is enabled."""
if not self.enable_ihc:
return hidden_states
return self._prepare_input_to_3d(hidden_states)
def _prepare_input_to_3d(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Reshape the iHC input to ``[num_tokens, hc, h]``.
Accepted inputs are ``[num_tokens, hc, h]`` (no-op), ``[num_tokens, h]``
(broadcast over the channels) and ``[num_tokens, hc * h]`` (reshape).
"""
if hidden_states.dim() == 3:
return hidden_states
if hidden_states.dim() != 2:
raise RuntimeError(
f"HC expects a 2D/3D tensor, got shape={tuple(hidden_states.shape)}"
)
d0, d1 = hidden_states.shape
h = self.config.hidden_size
hc = self.config.hc_mult
if d1 == h:
return hidden_states.unsqueeze(1).repeat(1, hc, 1)
expected = hc * h
if d1 == expected:
return hidden_states.reshape(d0, hc, h)
raise RuntimeError(
f"HC expects last dim to be hidden_size ({h})"
f"or hc_mult*hidden_size ({expected}), got {d1}. "
f"hc_mult={hc}, hidden_size={h}."
)
def pre(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
"""Reduce the iHC channels and produce the post gates.
Returns:
A tuple of the reduced hidden states ``[num_tokens, d]``, the post
gates ``[num_tokens, hc]`` (``None`` when iHC is disabled) and the
residual (the untouched input).
"""
if not self.enable_ihc:
return hidden_states, None, hidden_states
reduced, post_gates = self.hc_pre(hidden_states)
return reduced, post_gates, hidden_states
def post(
self,
output_with_bias: torch.Tensor,
residual: torch.Tensor,
post_gates: torch.Tensor | None,
) -> torch.Tensor:
"""Apply post-gating and add the residual."""
if not self.enable_ihc:
return output_with_bias + residual
assert post_gates is not None
return self.hc_post(output_with_bias, residual, post_gates)
+716
View File
@@ -0,0 +1,716 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only HY V4 model compatible with HuggingFace weights (NVIDIA)."""
import typing
from collections.abc import Callable, Iterable, MutableSequence, Sequence
from itertools import islice
import regex as re
import torch
from torch import nn
from transformers import PretrainedConfig
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.deepseek_v2 import _try_load_fp8_indexer_wk
from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
get_pp_missing_layer_names,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .attention import (
HYV4MLAAttention,
compute_skip_topk_layers,
is_skip_topk_indexer_weight,
)
from .hc import HYV4HCHeadLayer, HYV4HCLayer
from .moe import HYV4FeedForward, HYV4MoEFused
logger = init_logger(__name__)
def _normalize_hyv4_config(config: PretrainedConfig) -> PretrainedConfig:
"""Populate the aliases consumed by the shared MoE implementation."""
config.router_scaling_factor = config.routed_scaling_factor
config.num_experts = config.n_routed_experts
config.expert_hidden_dim = config.moe_intermediate_size
config.num_shared_experts = config.n_shared_experts
config.route_norm = config.norm_topk_prob
return config
class HYV4DecoderLayer(nn.Module):
"""One HY V4 decoder layer: MLA attention plus a dense or MoE MLP.
When``config.enable_ihc`` is set the layer runs on ``hc_mult`` residual
channels and each sub-block is wrapped by an `HYV4HCLayer` boundary;
otherwise it uses the standard single-stream residual.
"""
def __init__(
self,
config: PretrainedConfig,
vllm_config: VllmConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
topk_indices_buffer: torch.Tensor | None = None,
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
self.config = config
self.enable_ihc = getattr(config, "enable_ihc", False)
layer_idx = int(prefix.split(".")[-1])
self.layer_idx = layer_idx
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
self.self_attn = HYV4MLAAttention(
vllm_config=vllm_config,
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
qk_nope_head_dim=config.qk_nope_head_dim,
qk_rope_head_dim=config.qk_rope_head_dim,
v_head_dim=config.v_head_dim,
q_lora_rank=config.q_lora_rank,
kv_lora_rank=config.kv_lora_rank,
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
layer_idx=layer_idx,
topk_indices_buffer=topk_indices_buffer,
)
self.input_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps)
if config.mlp_layer_types[layer_idx] == "dense":
self.mlp = HYV4FeedForward(
hidden_size=config.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.block_type = "feedforward"
else:
self.mlp = HYV4MoEFused(
config=config, quant_config=quant_config, prefix=f"{prefix}.mlp"
)
self.block_type = "moe"
self.post_attention_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps)
self.hc_attn_layer = HYV4HCLayer(
config, layer_idx, prefix=f"{prefix}.hc_attn_layer"
)
self.hc_mlp_layer = HYV4HCLayer(
config, layer_idx, prefix=f"{prefix}.hc_mlp_layer"
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if self.enable_ihc:
return self._forward_ihc(positions, hidden_states)
return self._forward_normal(positions, hidden_states, residual)
def _forward_normal(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Standard single-residual-stream forward (iHC disabled)."""
if residual is not None:
hidden_states = hidden_states + residual
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
)
hidden_states = hidden_states + residual
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
def _forward_ihc(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, None]:
"""iHC forward: each sub-block reduces and re-scatters the channels."""
hidden_states = self.hc_attn_layer.prepare_input(hidden_states)
hidden_states, post_gates, residual = self.hc_attn_layer.pre(hidden_states)
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
)
hidden_states = self.hc_attn_layer.post(hidden_states, residual, post_gates)
hidden_states = self.hc_mlp_layer.prepare_input(hidden_states)
hidden_states, post_gates, residual = self.hc_mlp_layer.pre(hidden_states)
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = self.hc_mlp_layer.post(hidden_states, residual, post_gates)
# Under iHC the residual is carried inside hidden_states.
return hidden_states, None
class HYV4Model(nn.Module):
"""HY V4 backbone."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = _normalize_hyv4_config(vllm_config.model_config.hf_config)
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
parallel_config = vllm_config.parallel_config
eplb_config = parallel_config.eplb_config
self.num_redundant_experts = eplb_config.num_redundant_experts
self.device = current_platform.device_type
self.vocab_size = config.vocab_size
self.is_sparse = hasattr(config, "index_topk")
if self.is_sparse:
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
device=self.device,
)
else:
self.topk_indices_buffer = None
self.config = config
self.quant_config = quant_config
self.enable_ihc = getattr(config, "enable_ihc", False)
if get_pp_group().is_first_rank or (
self.config.tie_word_embeddings and get_pp_group().is_last_rank
):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
)
else:
self.embed_tokens = PPMissingLayer()
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: HYV4DecoderLayer(
config=config,
vllm_config=vllm_config,
cache_config=cache_config,
quant_config=quant_config,
prefix=prefix,
topk_indices_buffer=self.topk_indices_buffer,
),
prefix=f"{prefix}.layers",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
else:
self.norm = PPMissingLayer()
if self.enable_ihc:
# iHC head layer: merge the residual channels into one stream.
if get_pp_group().is_last_rank:
self.hc_head = HYV4HCHeadLayer(
config,
hidden_size=config.hidden_size,
hc_mult=config.hc_mult,
hc_eps=config.hc_eps,
prefix=f"{prefix}.hc_head",
)
else:
self.hc_head = PPMissingLayer()
self.make_empty_intermediate_tensors = (
make_empty_intermediate_tensors_factory(
["hidden_states"], config.hc_mult * config.hidden_size
)
)
else:
self.make_empty_intermediate_tensors = (
make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
)
# MoE hyperparameters (consumed by EPLB).
self.expert_weights: MutableSequence[Sequence[torch.Tensor]] = []
self.num_expert_groups = 1
self.moe_layers: list[nn.Module] = []
example_layer: HYV4MoEFused | None = None
for layer in self.layers:
if isinstance(layer, PPMissingLayer):
continue
assert isinstance(layer, HYV4DecoderLayer)
if layer.block_type == "moe":
assert isinstance(layer.mlp, HYV4MoEFused)
example_layer = layer.mlp
self.moe_layers.append(layer.mlp.experts)
if example_layer is None:
self.num_moe_layers = 0
self.num_logical_experts = 0
self.num_physical_experts = 0
self.num_local_physical_experts = 0
self.num_routed_experts = 0
self.num_redundant_experts = 0
return
self.num_moe_layers = len(self.moe_layers)
self.num_logical_experts = example_layer.n_logical_experts
self.num_physical_experts = example_layer.n_physical_experts
self.num_local_physical_experts = example_layer.n_local_physical_experts
self.num_routed_experts = example_layer.n_routed_experts
self.num_redundant_experts = example_layer.n_redundant_experts
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def update_physical_experts_metadata(
self,
num_physical_experts: int,
num_local_physical_experts: int,
) -> None:
assert self.num_local_physical_experts == num_local_physical_experts
self.num_physical_experts = num_physical_experts
self.num_local_physical_experts = num_local_physical_experts
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
for layer in self.layers:
if isinstance(layer, PPMissingLayer):
continue
if isinstance(layer.mlp, HYV4MoEFused):
moe = layer.mlp
moe.n_local_physical_experts = num_local_physical_experts
moe.n_physical_experts = num_physical_experts
moe.n_redundant_experts = self.num_redundant_experts
moe.experts.update_expert_map()
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# (param_name, weight_name, expert_id, shard_id) for weights, fp8
# weight scales and fp8 activation scales.
if not hasattr(self, "_cached_expert_params_mapping"):
self._cached_expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_redundant_experts=self.num_redundant_experts,
)
return self._cached_expert_params_mapping
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
# In iHC mode the flattened [num_tokens, hc*h] tensor from the
# previous PP stage is reshaped back to 3D by the first layer's
# prepare_input, and residual is unused.
residual = None if self.enable_ihc else intermediate_tensors["residual"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states, residual = layer(positions, hidden_states, residual)
if not get_pp_group().is_last_rank:
if self.enable_ihc:
# hidden_states is [num_tokens, hc, h]; flatten the channel dim
# for PP transfer (matches the 2D receive buffer).
return IntermediateTensors({"hidden_states": hidden_states.flatten(1)})
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
if self.enable_ihc:
hidden_states = self.hc_head(hidden_states)
else:
hidden_states = hidden_states + residual
return self.norm(hidden_states)
def load_fused_expert_weights(
self,
name: str,
params_dict: dict,
loaded_weight: torch.Tensor,
shard_id: str,
num_experts: int,
) -> bool:
param = params_dict[name]
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
loaded_local_expert = False
for expert_id in range(num_experts):
curr_expert_weight = loaded_weight[expert_id]
success = weight_loader(
param,
curr_expert_weight,
name,
shard_id,
expert_id,
return_success=True,
)
if success:
loaded_local_expert = True
return loaded_local_expert
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# FP8 indexer wk dequant buffer (weight and scale arrive separately).
pending_wk_fp8: dict[str, dict[str, torch.Tensor]] = {}
pp_missing_layer_names = get_pp_missing_layer_names(self)
skip_topk_layers = compute_skip_topk_layers(self.config)
# Must not be cached on `self`: `process_weights_after_loading` swaps in
# kernel-specific expert weights via `replace_parameter`, and a cache
# outliving this call would pin the pre-shuffle storage (OOM on large
# MoE) and make a later reload target orphaned tensors.
params_dict = dict(self.named_parameters())
# Split per-expert mapping (V3 style): experts.0.gate_proj.weight
split_expert_params_mapping = self.get_expert_mapping()
loaded_params: set[str] = set()
# Sink weights are sharded like the q/k/v linears.
sink_tp_size = get_tensor_model_parallel_world_size()
sink_tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // sink_tp_size
head_rank_start = n_local_head * sink_tp_rank
head_rank_end = n_local_head * (sink_tp_rank + 1)
base_layer = (
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
)
# Fused expert mapping: experts.gate_up_proj (all experts in one tensor).
# The packed weights are owned by the RoutedExperts submodule, so the
# targets are experts.routed_experts.[base_layer.]w{13,2}_weight.
fused_expert_prefix = f".experts.routed_experts.{base_layer}"
fused_expert_params_mapping = [
(f"{fused_expert_prefix}w13_weight", ".experts.gate_up_proj", 0, "w1"),
(f"{fused_expert_prefix}w2_weight", ".experts.down_proj", 0, "w2"),
]
num_experts = getattr(self.config, "num_experts", 0)
def _should_skip_missing_param(param_name: str) -> bool:
# Sparse checkpoints may contain indexer weights for layers that
# fall back to dense MLA on unsupported GPUs.
return ".indexer." in param_name
def _is_split_expert_weight(weight_name: str) -> bool:
"""Whether this weight is in split (per-expert) format."""
# Split format: mlp.experts.<id>.gate_proj.weight
return bool(re.search(r"\.experts\.\d+\.", weight_name))
def _is_fused_expert_weight(weight_name: str) -> bool:
"""Whether this weight is in fused (all-experts-packed) format."""
return ".experts.gate_up_proj" in weight_name or (
".experts.down_proj" in weight_name
and not _is_split_expert_weight(weight_name)
)
for name, loaded_weight in weights:
if is_skip_topk_indexer_weight(name, skip_topk_layers):
continue
if _try_load_fp8_indexer_wk(
name,
loaded_weight,
pending_wk_fp8,
params_dict,
loaded_params,
pp_missing_layer_names,
):
continue
if self.config.tie_word_embeddings and "lm_head.weight" in name:
continue
# Names and stacked shard ids are normalized upstream by
# AutoWeightsLoader, through quant_config.get_cache_scale_mapper()
# and hf_to_vllm_mapper.
stacked_shard_id = getattr(loaded_weight, "shard_id", None)
if ".indexer.wk." in name:
name = name.replace(".indexer.wk.", ".indexer.wk_weights_proj.")
stacked_shard_id = 0
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Determine per-weight whether this is fused or split format.
is_fused_expert = _is_fused_expert_weight(name)
expert_params_mapping = (
fused_expert_params_mapping
if is_fused_expert
else split_expert_params_mapping
)
is_expert_weight = False
loaded_expert_param_names: set[str] = set()
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
# This is an expert weight and must not be attempted as any
# other kind of weight later on.
is_expert_weight = True
# Do not modify `name`: the loop may continue past this point.
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
if is_fused_expert:
if "experts.gate_up_proj" in name:
chunks = loaded_weight.chunk(2, dim=-2)
success_w1 = self.load_fused_expert_weights(
name_mapped, params_dict, chunks[0], "w1", num_experts
)
success_w3 = self.load_fused_expert_weights(
name_mapped, params_dict, chunks[1], "w3", num_experts
)
success = success_w1 and success_w3
else:
success = self.load_fused_expert_weights(
name_mapped,
params_dict,
loaded_weight,
shard_id,
num_experts,
)
if success:
name = name_mapped
break
else:
# Split per-expert format (V3 style).
if name_mapped not in params_dict:
if _should_skip_missing_param(name_mapped):
continue
logger.warning_once(
"Skipping unknown checkpoint weight: %s",
name_mapped,
)
continue
param = params_dict[name_mapped]
# Ask the weight loader whether it succeeded, otherwise we
# may skip experts that have other available replicas.
weight_loader: Callable[..., typing.Any] = param.weight_loader
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
if not is_fused_expert:
loaded_expert_param_names.add(name_mapped)
continue
name = name_mapped
break
else:
if loaded_expert_param_names:
loaded_params.update(loaded_expert_param_names)
continue
if "learnable_sink_param" in name:
if is_pp_missing_parameter(name, self):
continue
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
with torch.no_grad():
params_dict[name][:n].copy_(narrow_weight)
else:
if is_expert_weight:
# An expert weight that is not mapped to this rank.
continue
if "gate.e_score_correction_bias" in name:
name = name.replace(
"gate.e_score_correction_bias", "expert_bias"
)
if "router.gate." in name:
name = name.replace("router.", "")
if "hc_fn" in name:
name = name.replace("hc_fn", "hc_fn.weight")
if "hc_head_fn" in name:
name = name.replace("hc_head_fn", "hc_head_fn.weight")
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
if _should_skip_missing_param(name):
continue
logger.warning_once(
"Skipping unknown checkpoint weight: %s",
name,
)
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
if stacked_shard_id is None:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(name)
return loaded_params
class HYV4ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_a_proj": (".fused_qkv_a_proj", 0),
".kv_a_proj_with_mqa": (".fused_qkv_a_proj", 1),
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
".indexer.weights_proj.": (".indexer.wk_weights_proj.", 1),
}
)
packed_modules_mapping = {
"gate_up_proj": ["gate_proj", "up_proj"],
# MLA runs both latent down-projections as one GEMM.
"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"],
# The indexer fuses wk and weights_proj into one GEMM.
"wk_weights_proj": ["wk", "weights_proj"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
if quant_config is not None:
quant_config.packed_modules_mapping = self.packed_modules_mapping
parallel_config = vllm_config.parallel_config
eplb_config = parallel_config.eplb_config
self.num_redundant_experts = eplb_config.num_redundant_experts
self.model = HYV4Model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if get_pp_group().is_last_rank:
# The head stays in the model dtype; ``enable_lm_head_fp32`` is
# surfaced as ``head_dtype`` on the config (see HYV4Config), which
# makes LogitsProcessor accumulate the projection straight into
# fp32 instead of materializing an fp32 copy of the weight.
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
# With tie_word_embeddings, embed_tokens is kept on the last rank
# (see HYV4Model.__init__) so the weight can be shared here.
if self.config.tie_word_embeddings:
self.lm_head.weight = self.model.embed_tokens.weight
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
if getattr(self.config, "soft_logits_capping", False):
soft_cap = self.config.soft_logits_capping_logits
logits = soft_cap * torch.nn.functional.tanh(logits / soft_cap)
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# MTP layers are loaded by the MTP head, not by the target model. They
# appear as model.mtp_layers.<i>.* and as the layer ids past the backbone.
mtp_start = self.config.num_hidden_layers
drop_prefixes: dict[str, str | None] = {"model.mtp_layers.": None}
for i in range(getattr(self.config, "num_nextn_predict_layers", 0)):
drop_prefixes[f"model.layers.{mtp_start + i}."] = None
if self.config.tie_word_embeddings:
drop_prefixes["lm_head."] = None
drop_mapper = WeightsMapper(orig_to_new_prefix=drop_prefixes)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper | drop_mapper)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+190
View File
@@ -0,0 +1,190 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Dense FFN and MoE blocks for HY V4 (NVIDIA)."""
import torch
from torch import nn
from transformers import PretrainedConfig
from vllm.config import get_current_vllm_config
from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, GateLinear
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
class HYV4FeedForward(nn.Module):
"""Dense SwiGLU feed-forward block.
Used both for the dense decoder layers and for the MoE shared experts.
The routed-expert SwiGLU clamp does not apply here, matching the reference
implementation.
"""
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
) -> None:
super().__init__()
if hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.down_proj",
)
self.act_fn = SiluAndMul()
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(x)
out = self.act_fn(gate_up)
out, _ = self.down_proj(out)
return out
class HYV4MoEFused(nn.Module):
"""HY V4 MoE layer with optional SwiGLU clamp support.
When ``config.swiglu_limit > 0`` the routed experts use a clamped SwiGLU::
gate = clamp(gate, max=limit)
up = clamp(up, -limit, limit)
output = silu(gate) * up
Dense layers and shared experts are NOT clamped. The clamp is forwarded to
``FusedMoEFactory`` as ``swiglu_limit`` so it travels through the quant
config to every expert backend instead of mutating global state.
NOTE: The reference implementation additionally supports HPC gate kernels,
TPCP/MLPSP sharded shared experts and a side-stream overlap for the shared
experts. Those depend on infrastructure that is absent here, so this port
keeps only the standard path. TODO: restore once available.
"""
def __init__(
self,
config: PretrainedConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
enable_eplb: bool = False,
):
super().__init__()
self.tp_size = get_tensor_model_parallel_world_size()
self.ep_group = get_ep_group().device_group
self.ep_rank = get_ep_group().rank_in_group
self.ep_size = self.ep_group.size()
self.n_routed_experts = config.num_experts
if self.tp_size > config.num_experts:
raise ValueError(
f"Tensor parallel size {self.tp_size} is greater than "
f"the number of experts {config.num_experts}."
)
top_k = config.num_experts_per_tok
intermediate_size = config.expert_hidden_dim
router_scaling_factor = getattr(config, "router_scaling_factor", 1.0)
vllm_config = get_current_vllm_config()
eplb_config = vllm_config.parallel_config.eplb_config
self.enable_eplb = enable_eplb
self.n_logical_experts = self.n_routed_experts
self.n_redundant_experts = eplb_config.num_redundant_experts
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
self.physical_expert_end = (
self.physical_expert_start + self.n_local_physical_experts
)
self.gate = GateLinear(
config.hidden_size,
config.num_experts,
bias=False,
out_dtype=torch.float32,
params_dtype=torch.float32,
prefix=f"{prefix}.gate",
)
self.shared_experts: HYV4FeedForward | None
if config.num_shared_experts > 0:
self.shared_experts = HYV4FeedForward(
hidden_size=config.hidden_size,
intermediate_size=config.expert_hidden_dim * config.num_shared_experts,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.shared_experts",
reduce_results=False,
)
else:
self.shared_experts = None
self.expert_bias = nn.Parameter(
torch.empty(config.num_experts, dtype=torch.float32)
)
raw_swiglu_limit = getattr(config, "swiglu_limit", 0)
use_swiglu_clamp = bool(raw_swiglu_limit) and float(raw_swiglu_limit) > 0
moe_swiglu_limit = float(raw_swiglu_limit) if use_swiglu_clamp else None
self.experts = FusedMoEFactory(
num_experts=self.n_routed_experts,
top_k=top_k,
hidden_size=config.hidden_size,
intermediate_size=intermediate_size,
renormalize=config.route_norm,
quant_config=quant_config,
prefix=f"{prefix}.experts",
enable_eplb=self.enable_eplb,
num_redundant_experts=self.n_redundant_experts,
scoring_func="sigmoid",
use_grouped_topk=True,
num_expert_group=1,
topk_group=1,
routed_scaling_factor=router_scaling_factor,
e_score_correction_bias=self.expert_bias,
shared_experts=self.shared_experts,
swiglu_limit=moe_swiglu_limit,
)
self.prefix = prefix
if use_swiglu_clamp:
logger.debug_once(
"HYV4MoEFused: swiglu_limit=%.1f enabled for routed experts",
moe_swiglu_limit,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
orig_shape = hidden_states.shape
hidden_dim = hidden_states.shape[-1]
hidden_states = hidden_states.view(-1, hidden_dim)
# router_logits: (num_tokens, n_experts); the gate stores fp32 weights
# and emits fp32 logits, so no explicit upcast is needed here.
router_logits, _ = self.gate(hidden_states)
final_hidden_states = self.experts(
hidden_states=hidden_states, router_logits=router_logits
)
return final_hidden_states.view(orig_shape)
+937
View File
@@ -0,0 +1,937 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Multi-token prediction (MTP) head for HY V4 (NVIDIA)."""
import copy
import typing
from collections.abc import Callable, Iterable
import torch
from torch import nn
from transformers import PretrainedConfig
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.deepseek_v2 import _try_load_fp8_indexer_wk
from vllm.model_executor.models.utils import (
get_pp_missing_layer_names,
is_pp_missing_parameter,
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from .model import HYV4DecoderLayer, _normalize_hyv4_config
logger = init_logger(__name__)
_MTP_SCALE_ONLY_PARAM_SUBSTRINGS = (
"_scale",
"_weight_scale",
"_weight_scale_inv",
"_input_scale",
)
# Names of the quant-config attribute that stores the unquantized-module
# prefix list, in the order we look them up. Different QuantizationConfig
# subclasses expose the list under different names:
# * ``ignored_layers`` -- Fp8Config, GPTQ / AWQ configs.
# * ``exclude_modules`` -- ModelOpt* configs (Fp8, NvFp4, MxFp8, Mixed).
_MTP_QUANT_EXCLUSION_ATTRS = ("ignored_layers", "exclude_modules")
def _get_spec_layer_idx_from_weight_name(
config: PretrainedConfig, weight_name: str
) -> int | None:
"""Return the MTP layer index a checkpoint weight belongs to, or None.
Compatible with``num_nextn_predict_layers`` of 1 and 2 for a single MTP
head, matching how the HY V4 checkpoints are exported.
"""
if getattr(config, "num_nextn_predict_layers", 0) < 1:
return None
layer_idx = config.num_hidden_layers
num_mtp_layers = max(config.num_nextn_predict_layers - 1, 1)
for i in range(num_mtp_layers):
if weight_name.startswith(f"model.layers.{layer_idx + i}."):
return layer_idx + i
return None
def _should_skip_missing_mtp_scale_param(
quant_config: QuantizationConfig | None,
name: str,
) -> bool:
"""Whether an unmatched scale parameter can be silently ignored."""
return quant_config is None and any(
scale_name in name for scale_name in _MTP_SCALE_ONLY_PARAM_SUBSTRINGS
)
def _resolve_fused_expert_param(
param_base: str,
ckpt_suffix: str,
params_dict: dict,
) -> str | None:
"""Map a fused expert checkpoint suffix onto a draft parameter name.
Fused checkpoints pack every expert into one tensor and name the companion
scales by suffixing the projection (``experts.gate_up_proj_scale_inv``),
while the draft model owns ``experts.routed_experts.w13_weight`` plus a
separate ``..._scale_inv`` parameter. Dropping the suffix would push the
scale into the weight parameter and leave the scale at its sentinel init.
Args:
param_base: Draft parameter holding the packed weight.
ckpt_suffix: Checkpoint text following the projection name.
params_dict: The draft model's named parameters.
Returns:
The matching draft parameter name, or None when there is none.
"""
if not ckpt_suffix:
return param_base if param_base in params_dict else None
tail = ckpt_suffix.lstrip(".").removeprefix("weight_")
for candidate in (
f"{param_base}_{tail}",
f"{param_base}_scale_inv",
f"{param_base}_scale",
):
if candidate in params_dict:
return candidate
return None
def _prepare_mtp_fp8_expert_scale(
quant_config: QuantizationConfig | None,
name: str,
loaded_weight: torch.Tensor,
) -> tuple[str, torch.Tensor]:
"""Normalize block-wise FP8 expert scale names and dtypes."""
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
if (
isinstance(quant_config, Fp8Config)
and quant_config.weight_block_size is not None
and ".mlp.experts." in name
and name.endswith(".scale")
):
name = name[: -len(".scale")] + ".weight_scale_inv"
if (
getattr(quant_config, "is_scale_e8m0", False)
and loaded_weight.dtype == torch.uint8
):
# UE8M0 scale exponents are stored as raw uint8 bytes: reinterpret
# the bits instead of numerically converting them.
loaded_weight = loaded_weight.view(torch.float8_e8m0fnu)
return name, loaded_weight
def _create_mtp_quant_config(
hf_config: PretrainedConfig,
backbone_quant_config: QuantizationConfig | None = None,
) -> QuantizationConfig | None:
"""Create the quantization config for the MTP layers.
The MTP quantization algorithm is given by the ``mtp_quant_algo`` field in
``config.json``, independently of the backbone's quantization. Supported
values are ``"FP8"``, ``"NONE"`` (inherit the backbone) and ``"BF16"`` /
``"FP16"`` (unquantized). A missing or ``"NONE"`` value falls back to the
backbone config.
Args:
hf_config: The draft model's HF config.
backbone_quant_config: The target model's quantization config.
Returns:
The quantization config to use for the MTP layers, or None when the MTP
layers are unquantized.
"""
mtp_quant_algo = getattr(hf_config, "mtp_quant_algo", None)
if mtp_quant_algo is None or mtp_quant_algo.upper() == "NONE":
logger.info_once(
"No explicit MTP quant algo (mtp_quant_algo=%s). "
"Using the backbone's quant config.",
mtp_quant_algo,
)
return backbone_quant_config
mtp_quant_algo = mtp_quant_algo.upper()
if mtp_quant_algo in ("BF16", "FP16"):
logger.info_once(
"MTP layers are unquantized (mtp_quant_algo=%s).", mtp_quant_algo
)
return None
if mtp_quant_algo == "FP8":
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
logger.info_once("MTP uses FP8 quantization (mtp_quant_algo=FP8).")
hf_quant_config = getattr(hf_config, "quantization_config", None) or {}
weight_block_size = hf_quant_config.get("weight_block_size")
activation_scheme = hf_quant_config.get("activation_scheme", "dynamic")
if weight_block_size is not None:
activation_scheme = "dynamic"
ignored_layers = hf_quant_config.get("ignored_layers") or hf_quant_config.get(
"modules_to_not_convert"
)
fp8_config = Fp8Config(
is_checkpoint_fp8_serialized=True,
activation_scheme=activation_scheme,
ignored_layers=ignored_layers or [],
weight_block_size=weight_block_size,
)
if hf_quant_config.get("scale_fmt") == "ue8m0":
# Fp8LinearMethod reads this through getattr, so it is not a
# declared Fp8Config field.
setattr(fp8_config, "is_scale_e8m0", True) # noqa: B010
return fp8_config
logger.warning(
"Unknown mtp_quant_algo=%s, falling back to the backbone config.",
mtp_quant_algo,
)
return backbone_quant_config
def _remap_mtp_quant_exclusions(
quant_config: QuantizationConfig | None,
mtp_start_layer_idx: int,
num_mtp_layers: int,
) -> QuantizationConfig | None:
"""Translate checkpoint-named MTP quant exclusions to draft prefixes.
``modules_to_not_convert`` / ``exclude_modules`` name MTP modules the way
the checkpoint does (``model.mtp_layers.0.self_attn.linear_gate``), while
the draft model builds them under ``model.layers.<num_hidden_layers + i>``.
``is_layer_skipped`` compares prefixes for exact equality, so without this
translation an excluded MTP module gets a quant method even though its
checkpoint weight is BF16 with no ``weight_scale`` / ``weight_scale_inv``
companion: the scale then keeps its ``finfo(float32).min`` sentinel and the
layer silently produces garbage.
The list lives under a different attribute per config class, hence the
lookup over ``_MTP_QUANT_EXCLUSION_ATTRS``.
Args:
quant_config: The MTP quantization config to widen.
mtp_start_layer_idx: Index of the first MTP layer in the draft model.
num_mtp_layers: Number of MTP layers.
Returns:
A shallow copy with the translated exclusions, or the input unchanged.
"""
if quant_config is None:
return None
# Different QuantizationConfig subclasses expose the exclusion list under
# different attribute names; only the one that is actually populated on
# the backbone config governs which draft modules stay unquantized. For
# ModelOpt checkpoints (``quant_method: modelopt`` and friends) this is
# ``exclude_modules``; for the legacy fp8 path it is ``ignored_layers``.
exclusion_attr: str | None = None
patterns: list[str] | None = None
for attr in _MTP_QUANT_EXCLUSION_ATTRS:
value = getattr(quant_config, attr, None)
if value:
exclusion_attr = attr
patterns = list(value)
break
if not patterns or exclusion_attr is None:
return quant_config
extra: list[str] = []
for offset in range(num_mtp_layers):
source_prefix = f"model.mtp_layers.{offset}."
draft_prefix = f"model.layers.{mtp_start_layer_idx + offset}."
for pattern in patterns:
if pattern.startswith(source_prefix):
extra.append(draft_prefix + pattern[len(source_prefix) :])
# ModelOpt exclusion patterns may end with a ``*`` wildcard; the
# remap must preserve that suffix so ``is_layer_skipped`` still
# matches the draft module prefixes.
elif pattern.rstrip("*").startswith(source_prefix):
stripped = pattern.rstrip("*")
trailing_stars = pattern[len(stripped) :]
extra.append(
draft_prefix + stripped[len(source_prefix) :] + trailing_stars
)
if not extra:
return quant_config
# The backbone shares this object with the target model; copy before
# widening the exclusion list. The attribute is only present on the
# concrete configs that support exclusions, hence the getattr/setattr pair.
quant_config = copy.copy(quant_config)
setattr(quant_config, exclusion_attr, patterns + extra) # noqa: B010
logger.info_once(
"HYV4 MTP: mapped %d checkpoint-named quant exclusions (%s) onto the "
"draft module prefixes: %s",
len(extra),
exclusion_attr,
", ".join(extra),
)
return quant_config
class HYV4SharedHead(nn.Module):
"""Holds the draft LM head shared with the target model."""
def __init__(
self,
config: PretrainedConfig,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
# Use the same ``lm_head`` prefix as the backbone so that quant-config
# exclusion lists (e.g. ModelOpt's ``exclude_modules: ["lm_head", ...]``)
# also apply here: an unquantized checkpoint lm_head must not gain a
# spurious ``weight_scale`` parameter on the draft side, which would
# otherwise stay at its FP-min sentinel until the proposer swaps the
# whole ParallelLMHead for the target model's ``lm_head``.
self.head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix="lm_head",
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
return hidden_states
class HYV4MultiTokenPredictorLayer(nn.Module):
"""A single MTP draft block."""
def __init__(
self,
config: PretrainedConfig,
prefix: str,
vllm_config: VllmConfig,
model_config: ModelConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
topk_indices_buffer: torch.Tensor | None = None,
) -> None:
super().__init__()
del model_config # kept for signature parity with the other MTP heads
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
self.shared_head = HYV4SharedHead(config=config, quant_config=quant_config)
# HYV4DecoderLayer indexes layer_types/mlp_layer_types by the numeric
# layer id parsed from the prefix. MTP layers live after the backbone
# layers, so extend the per-layer config lists to cover those ids.
layer_idx = int(prefix.rsplit(".", 1)[-1])
mtp_config = copy.deepcopy(config)
# MTP checkpoints do not carry iHC pre/post weights for mtp_layers, so
# the draft blocks are built without iHC to match their structure.
mtp_config.enable_ihc = False
mtp_config.layer_types = _extend_layer_types(
getattr(mtp_config, "layer_types", None), layer_idx, "full_attention"
)
mtp_config.mlp_layer_types = _extend_layer_types(
getattr(mtp_config, "mlp_layer_types", None), layer_idx, "sparse"
)
self.mtp_block = HYV4DecoderLayer(
config=mtp_config,
vllm_config=vllm_config,
cache_config=cache_config,
quant_config=quant_config,
prefix=prefix,
topk_indices_buffer=topk_indices_buffer,
)
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
del input_ids # embeddings are resolved by the caller
assert inputs_embeds is not None
inputs_embeds = self.enorm(inputs_embeds)
previous_hidden_states = self.hnorm(previous_hidden_states)
hidden_states = self.eh_proj(
torch.cat([inputs_embeds, previous_hidden_states], dim=-1)
)
hidden_states, residual = self.mtp_block(
positions=positions,
hidden_states=hidden_states,
residual=None,
)
hidden_states, _ = self.final_layernorm(hidden_states, residual)
return hidden_states
def _extend_layer_types(
layer_types: list[str] | None, layer_idx: int, fallback: str
) -> list[str] | None:
"""Pad a per-layer config list so``layer_idx`` is addressable."""
if layer_types is None or len(layer_types) > layer_idx:
return layer_types
fill = layer_types[-1] if layer_types else fallback
return list(layer_types) + [fill] * (layer_idx + 1 - len(layer_types))
class HYV4MultiTokenPredictor(nn.Module):
"""Owns the MTP draft blocks and their shared embedding / logits path."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
target_config = vllm_config.model_config.hf_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None, (
"HYV4MTP can only be built as a speculative draft model"
)
draft_model_config = speculative_config.draft_model_config
assert draft_model_config is not None
config = _normalize_hyv4_config(draft_model_config.hf_config)
self.mtp_start_layer_idx = target_config.num_hidden_layers
self.num_mtp_layers = max(
getattr(target_config, "num_nextn_predict_layers", 1), 1
)
self.quant_config = _remap_mtp_quant_exclusions(
_create_mtp_quant_config(config, vllm_config.quant_config),
self.mtp_start_layer_idx,
self.num_mtp_layers,
)
if self.quant_config is not None:
self.quant_config.packed_modules_mapping = HYV4MTP.packed_modules_mapping
# Sparse DSA draft blocks must share the target model's top-k buffer.
# The proposer injects it through HYV4MTP.set_topk_indices_buffer()
# before any draft forward.
self.topk_indices_buffer: torch.Tensor | None = None
if hasattr(config, "index_topk"):
from vllm.platforms import current_platform
self.topk_indices_buffer = torch.empty(
vllm_config.scheduler_config.max_num_batched_tokens,
config.index_topk,
dtype=torch.int32,
device=current_platform.device_type,
)
self.layers = nn.ModuleDict(
{
str(idx): HYV4MultiTokenPredictorLayer(
config,
f"{prefix}.layers.{idx}",
vllm_config=vllm_config,
model_config=draft_model_config,
cache_config=vllm_config.cache_config,
quant_config=self.quant_config,
topk_indices_buffer=self.topk_indices_buffer,
)
for idx in range(
self.mtp_start_layer_idx,
self.mtp_start_layer_idx + self.num_mtp_layers,
)
}
)
self.requires_topk_indices_buffer = any(
layer.mtp_block.self_attn.is_sparse for layer in self.layers.values()
)
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
)
self.logits_processor = LogitsProcessor(config.vocab_size)
self.spec_step_idx: int = 0
def set_skip_topk(self, skip: bool) -> None:
"""Toggle the draft indexer for ``index_share_for_mtp_iteration``.
The proposer clears the flag for draft step 0 so the MTP layer builds
its own top-k indices, then sets it for steps 1+ so they reuse what
step 0 wrote into the shared buffer instead of re-running the indexer.
"""
for layer in self.layers.values():
layer.mtp_block.self_attn.skip_topk = skip
def compact_topk_indices(self, slot_ids: torch.Tensor) -> None:
"""Move the top-k rows at ``slot_ids`` to the front of the buffer.
Step 0 writes one row per query token of the multi-token batch, while
steps 1+ decode a single token per request and index the buffer from 0.
Without this gather they would read another token's rows.
"""
num_slots = slot_ids.numel()
if self.topk_indices_buffer is None or num_slots == 0:
return
self.topk_indices_buffer[:num_slots] = self.topk_indices_buffer[slot_ids]
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if inputs_embeds is None:
inputs_embeds = self.embed_tokens(input_ids)
inputs_embeds = torch.where((positions == 0).unsqueeze(-1), 0, inputs_embeds)
current_step_idx = self.spec_step_idx % self.num_mtp_layers
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
input_ids,
positions,
previous_hidden_states,
inputs_embeds,
)
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor:
current_step_idx = self.spec_step_idx % self.num_mtp_layers
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
lm_head = mtp_layer.shared_head.head
proj_input = mtp_layer.shared_head(hidden_states)
return self.logits_processor(lm_head, proj_input)
class HYV4MTP(nn.Module):
"""HY V4 MTP draft head.
Not a pipeline-parallel stage: the draft head always runs on a single rank,
matching `DeepseekV32MTP` / `KimiK3MTP` / `HYV3MTP`.
"""
packed_modules_mapping = {
"gate_up_proj": ["gate_proj", "up_proj"],
# MLA runs both latent down-projections as one GEMM.
"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"],
# The indexer fuses wk and weights_proj into one GEMM.
"wk_weights_proj": ["wk", "weights_proj"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
self.model = HYV4MultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.quant_config = self.model.quant_config
self.sampler = Sampler()
def set_topk_indices_buffer(self, topk_indices_buffer: torch.Tensor) -> None:
"""Share the target sparse-index buffer with every draft consumer.
Proposers that walk ``named_modules()`` instead of calling this reach
the same consumers via ``HYV4MLAAttention.topk_indices_buffer``.
"""
self.model.topk_indices_buffer = topk_indices_buffer
for layer in self.model.layers.values():
self_attn = layer.mtp_block.self_attn
if not self_attn.is_sparse:
continue
indexer = self_attn.indexer
assert indexer is not None, "Sparse HYV4 MTP attention requires an indexer"
indexer.topk_indices_buffer = topk_indices_buffer
indexer.indexer_op.topk_indices_buffer = topk_indices_buffer
attn_impl = self_attn.mla_attn.impl
assert hasattr(attn_impl, "topk_indices_buffer"), (
"Sparse HYV4 MTP attention backend requires a top-k indices buffer"
)
attn_impl.topk_indices_buffer = topk_indices_buffer
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
del intermediate_tensors # the MTP head is single-stage
if (
self.model.requires_topk_indices_buffer
and self.model.topk_indices_buffer is None
):
raise RuntimeError(
"HYV4 sparse MTP requires the target model's top-k indices buffer. "
"The proposer must call HYV4MTP.set_topk_indices_buffer() before "
"the first draft forward."
)
self.model.spec_step_idx = spec_step_idx
return self.model(input_ids, positions, hidden_states, inputs_embeds)
def compute_logits(
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> torch.Tensor | None:
self.model.spec_step_idx = spec_step_idx
return self.model.compute_logits(hidden_states)
def sample(
self,
logits: torch.Tensor,
sampling_metadata: SamplingMetadata,
) -> SamplerOutput | None:
return self.sampler(logits, sampling_metadata)
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
if f"model.layers.{spec_layer}.embed_tokens" in name:
return "__skip__"
if f"model.layers.{spec_layer}.shared_head" in name:
return "__skip__"
spec_layer_weight_names = ["enorm", "hnorm", "eh_proj", "final_layernorm"]
spec_layer_weight = any(
weight_name in name for weight_name in spec_layer_weight_names
)
if not spec_layer_weight:
name = name.replace(
f"model.layers.{spec_layer}.",
f"model.layers.{spec_layer}.mtp_block.",
)
return name
def _load_fused_expert_weights(
self,
name: str,
params_dict: dict,
loaded_weight: torch.Tensor,
shard_id: str,
num_experts: int,
) -> bool:
if name not in params_dict:
return False
param = params_dict[name]
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
loaded_local_expert = False
for expert_id in range(num_experts):
curr_expert_weight = loaded_weight[expert_id]
success = weight_loader(
param,
curr_expert_weight,
name,
shard_id,
expert_id,
return_success=True,
)
if success:
loaded_local_expert = True
return loaded_local_expert
def _load_expert_weight(
self,
name: str,
loaded_weight: torch.Tensor,
params_dict: dict,
loaded_params: set[str],
split_expert_params_mapping: list[tuple[str, str, int, str]],
fused_expert_param_names: dict[tuple[str, str], str],
num_experts: int,
) -> bool:
"""Load one routed-expert weight in either checkpoint layout.
Args:
name: Weight name already rewritten to draft-module naming.
loaded_weight: The checkpoint tensor.
params_dict: The draft model's named parameters.
loaded_params: Set updated with the parameters that received a value.
split_expert_params_mapping: Mapping for the per-expert layout.
fused_expert_param_names: ``(mlp_prefix, tag) -> param name`` for the
all-experts-packed layout.
num_experts: Total number of routed experts.
Returns:
True when the weight was consumed (even if this rank holds none of
the addressed experts).
"""
base = name.split(".experts.")[0]
for ckpt_proj, tag in (
(".experts.gate_up_proj", "w13_weight"),
(".experts.down_proj", "w2_weight"),
):
if ckpt_proj not in name:
continue
param_base = fused_expert_param_names.get((base, tag))
if param_base is None:
return False
# Keep the checkpoint suffix (e.g. `_scale_inv`) so block-scale
# tensors land in the scale parameter, not in the weight.
target = _resolve_fused_expert_param(
param_base, name.split(ckpt_proj, 1)[1], params_dict
)
if target is None:
return False
if tag == "w13_weight":
chunks = loaded_weight.chunk(2, dim=-2)
loaded_w1 = self._load_fused_expert_weights(
target, params_dict, chunks[0], "w1", num_experts
)
loaded_w3 = self._load_fused_expert_weights(
target, params_dict, chunks[1], "w3", num_experts
)
loaded = loaded_w1 and loaded_w3
else:
loaded = self._load_fused_expert_weights(
target, params_dict, loaded_weight, "w2", num_experts
)
if loaded:
loaded_params.add(target)
# The weight belongs to the experts either way; never fall through.
return True
consumed = False
for param_name, weight_name, expert_id, shard_id in split_expert_params_mapping:
if weight_name not in name:
continue
consumed = True
name_mapped = name.replace(weight_name, param_name)
if name_mapped not in params_dict:
continue
param = params_dict[name_mapped]
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
if weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
):
loaded_params.add(name_mapped)
return consumed
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
pp_missing_layer_names = get_pp_missing_layer_names(self)
loaded_params: set[str] = set()
mtp_start = self.config.num_hidden_layers
shared_weights = {
"model.embed_tokens.weight": "model.embed_tokens.weight",
"lm_head.weight": f"model.layers.{mtp_start}.shared_head.head.weight",
}
num_experts = getattr(self.config, "n_routed_experts", 0)
sink_tp_size = get_tensor_model_parallel_world_size()
sink_tp_rank = get_tensor_model_parallel_rank()
n_local_head = self.config.num_attention_heads // sink_tp_size
head_rank_start = n_local_head * sink_tp_rank
head_rank_end = n_local_head * (sink_tp_rank + 1)
# Routed-expert weights come in two checkpoint layouts:
# split: mlp.experts.<id>.gate_proj.weight
# fused: mlp.experts.gate_up_proj (all experts in one tensor)
# The split layout is resolved through the shared EPLB helper so the
# target param names stay in sync with the RoutedExperts module layout;
# the fused layout is resolved from the live parameter names.
split_expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=num_experts,
)
fused_expert_param_names: dict[tuple[str, str], str] = {}
for param_name in params_dict:
for tag in ("w13_weight", "w2_weight"):
if param_name.endswith(tag) and ".experts." in param_name:
base = param_name.split(".experts.")[0]
fused_expert_param_names[base, tag] = param_name
stacked_mapping = [
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
# MLA runs both latent down-projections as one GEMM.
(".fused_qkv_a_proj", ".q_a_proj", 0),
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
]
# Sparse DSA draft blocks build an Indexer whose wk / weights_proj are
# fused into a single MergedColumnParallelLinear (wk_weights_proj).
indexer_stacked_mapping = [
(".wk_weights_proj", ".wk", 0),
(".wk_weights_proj", ".weights_proj", 1),
]
# FP8 indexer wk dequant buffer (weight and scale arrive separately).
pending_wk_fp8: dict[str, dict[str, torch.Tensor]] = {}
for name, loaded_weight in weights:
if name in shared_weights:
target_name = shared_weights[name]
if target_name in params_dict:
param = params_dict[target_name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(target_name)
continue
spec_layer = None
if name.startswith("model.mtp_layers."):
parts = name.split(".")
if len(parts) > 3 and parts[2].isdigit():
spec_layer = mtp_start + int(parts[2])
name = name.replace(
f"model.mtp_layers.{parts[2]}.",
f"model.layers.{spec_layer}.",
)
else:
spec_layer = _get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is None:
continue
name = self._rewrite_spec_layer_name(spec_layer, name)
if name == "__skip__":
continue
name, loaded_weight = _prepare_mtp_fp8_expert_scale(
self.quant_config, name, loaded_weight
)
if "mlp.gate.e_score_correction_bias" in name:
name = name.replace("gate.e_score_correction_bias", "expert_bias")
is_loaded = False
for param_name, weight_name, shard_id in stacked_mapping:
if weight_name not in name or ".experts." in name:
continue
name_mapped = name.replace(weight_name, param_name)
if name_mapped not in params_dict:
if is_pp_missing_parameter(name_mapped, self):
is_loaded = True
break
param = params_dict[name_mapped]
param.weight_loader(param, loaded_weight, shard_id)
loaded_params.add(name_mapped)
is_loaded = True
break
if is_loaded:
continue
# FP8 indexer wk: dequantize to BF16 and load into the fused
# wk_weights_proj. PP-aware (skips layers not held by this rank).
if _try_load_fp8_indexer_wk(
name,
loaded_weight,
pending_wk_fp8,
params_dict,
loaded_params,
pp_missing_layer_names,
):
continue
# BF16 indexer wk / weights_proj: merge into the fused param.
is_loaded = False
for param_name, weight_name, shard_id in indexer_stacked_mapping:
if weight_name not in name or "wk_weights" in name:
continue
name_mapped = name.replace(weight_name, param_name)
if name_mapped not in params_dict:
if is_pp_missing_parameter(name_mapped, self):
is_loaded = True
break
param = params_dict[name_mapped]
param.weight_loader(param, loaded_weight, shard_id)
loaded_params.add(name_mapped)
is_loaded = True
break
if is_loaded:
continue
is_loaded = False
if ".experts." in name:
is_loaded = self._load_expert_weight(
name,
loaded_weight,
params_dict,
loaded_params,
split_expert_params_mapping,
fused_expert_param_names,
num_experts,
)
if is_loaded:
continue
if "learnable_sink_param" in name:
if name in params_dict:
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
n = narrow_weight.shape[0]
with torch.no_grad():
params_dict[name][:n].copy_(narrow_weight)
loaded_params.add(name)
continue
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
if name not in params_dict:
if is_pp_missing_parameter(name, self):
continue
if _should_skip_missing_mtp_scale_param(self.quant_config, name):
continue
logger.warning_once("Skipping unknown MTP weight: %s", name)
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
logger.info_once("HYV4 MTP draft model loaded: %d params", len(loaded_params))
unassigned_all = sorted(set(params_dict) - loaded_params)
# KVCacheScaleParameter (``k_scale`` / ``v_scale`` / ``q_scale`` /
# ``prob_scale``) is created by ``BaseKVCacheMethod.create_weights`` for
# every Attention layer that runs on an fp8-family quant method, and
# its -1.0 sentinel is replaced by the runtime default (1.0) in
# ``process_weights_after_loading`` -- which runs *after* this method.
# HY V4 checkpoints intentionally omit these (``kv_cache_quant_algo:
# null``); silence them only when the parameter is actually the
# sentinel type so a genuine mismatch on a same-named tensor still
# reaches the warning.
from vllm.model_executor.layers.quantization.kv_cache import (
KVCacheScaleParameter,
)
unassigned = [
name
for name in unassigned_all
if not isinstance(params_dict.get(name), KVCacheScaleParameter)
]
if unassigned:
# A draft parameter with no checkpoint source keeps its sentinel
# init value (FP8 scales start at finfo(float32).min), which
# silently destroys draft quality instead of failing the load.
logger.warning(
"HYV4 MTP draft model: %d parameters received no checkpoint value: %s",
len(unassigned),
", ".join(unassigned),
)
return loaded_params
+2 -1
View File
@@ -451,9 +451,10 @@ def get_harmony_structural_tag(
builtin_tools: list[BuiltinToolParam],
tool_choice: SimplifiedToolChoice,
reasoning: bool,
token_suffix: str = "",
) -> StructuralTag:
# reasoning always enabled for Harmony
del reasoning
del reasoning, token_suffix
if builtin_tools:
# Fallback for built-in tools
+4
View File
@@ -84,6 +84,10 @@ _REASONING_PARSERS_TO_REGISTER = {
"hy_v3_reasoning_parser",
"HYV3ReasoningParser",
),
"hy_v4": (
"hy_v4_reasoning_parser",
"HYV4ReasoningParser",
),
"kimi_k2": (
"kimi_k2_reasoning_parser",
"KimiK2ReasoningParser",
+324
View File
@@ -0,0 +1,324 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from collections.abc import Iterable, Sequence
from typing import TypedDict
import regex as re
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParser
from vllm.tokenizers import TokenizerLike
logger = init_logger(__name__)
# Must match the chat template's own default for ``reasoning_effort``.
DEFAULT_REASONING_EFFORT = "high"
class ReasoningDelta(TypedDict):
"""One streaming reasoning delta. Either/both fields may be None."""
reasoning: str | None
content: str | None
# NOTE: mirrored in ``vllm.tool_parsers.hy_v4_tool_parser`` (same pattern as
# ``gemma4_utils``) so neither package depends on the other.
def detect_token_suffix(tokenizer: TokenizerLike) -> str:
"""Detect the per-checkpoint suffix used by Hunyuan structural tokens.
Args:
tokenizer: Tokenizer of the served checkpoint.
Returns:
The suffix including its leading colon (e.g. ``":6124c78e"``), or ``""``
when the checkpoint uses unsuffixed tokens.
Raises:
RuntimeError: The tokenizer declares the structural tokens through
``model_specific_special_tokens``, which transformers 5 no longer
round-trips.
"""
import transformers
if int(transformers.__version__.split(".")[0]) >= 5:
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
think_begin_as_special = init_kwargs.get(
"model_specific_special_tokens", {}
).get("think_begin_token", "")
if think_begin_as_special:
raise RuntimeError(
"This checkpoint declares HYV4 structural tokens (think_begin_token"
"/toolcalls_begin_token/argkey_begin_token) in "
"tokenizer_config.json, which transformers 5 no longer supports. "
"Remove those fields and keep the tokens in the tokenizer's own "
"token definitions so the suffix can be read from the vocab."
)
structural_token_re = re.compile(
r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>"
)
for token in tokenizer.get_vocab():
match = structural_token_re.fullmatch(token)
if match:
return match.group(1) or ""
return ""
class HYV4ReasoningExtractor:
"""Reasoning extraction for HYV4, on plain data.
``thinking`` selects the mode:
- ``True``: parse ``<think>...</think>``; the start token is injected at the
end of the prompt and is therefore usually absent from the output.
- ``False``: ``no_think`` -- the whole output is content, no reasoning.
"""
def __init__(self, vocab: dict[str, int], token_suffix: str, thinking: bool = True):
self.thinking = thinking
self.start_token: str = f"<think{token_suffix}>"
self.end_token: str = f"</think{token_suffix}>"
start_token_id = vocab.get(self.start_token)
end_token_id = vocab.get(self.end_token)
if thinking and (start_token_id is None or end_token_id is None):
raise RuntimeError(
"HYV4 reasoning extractor could not locate think "
"start/end tokens in the tokenizer!"
)
# In no_think mode the ids are unused (kept for symmetry / debugging).
self.start_token_id: int | None = start_token_id
self.end_token_id: int | None = end_token_id
def has_reasoning_ended(self, token_ids: Sequence[int]) -> bool:
"""Idempotent STATE query: has ``</think>`` been emitted yet?
Monotonic -- once the end token is present this stays True. Use for
"has reasoning finished, given everything so far". (no_think is always
ended.) Contrast with ``reasoning_ended_in_delta`` (edge detector).
"""
if not self.thinking:
return True
for i in range(len(token_ids) - 1, -1, -1):
if token_ids[i] == self.start_token_id:
return False
if token_ids[i] == self.end_token_id:
return True
return False
def reasoning_ended_in_delta(
self, token_ids: Sequence[int], delta_ids: Iterable[int]
) -> bool:
"""Edge detector (NOT idempotent): did reasoning end *in this* step?
True only on the step whose ``delta_ids`` contains ``</think>``. Callers
use it once behind their own "already ended" latch, so it need not be
monotonic; it only checks the new delta (cheaper than scanning the whole
sequence). Contrast with ``has_reasoning_ended`` (cumulative state).
"""
if not self.thinking:
return True
end_token_id = self.end_token_id
if end_token_id is None:
return False
return end_token_id in delta_ids
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
"""Return the token ids after ``</think>`` (all of them in no_think)."""
if not self.thinking:
return input_ids
head = input_ids[: max(0, len(input_ids) - 1)]
if self.end_token_id not in head:
return []
return input_ids[input_ids.index(self.end_token_id) + 1 :]
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
"""Count reasoning tokens (0 in no_think).
The ``<think>`` start token is injected at the END of the prompt, so it
is normally absent from ``token_ids``. Only skip a leading start token
when it actually appears first (legacy / no prompt-injection); otherwise
count every token up to ``</think>``.
"""
if not self.thinking:
return 0
start = 1 if (token_ids and token_ids[0] == self.start_token_id) else 0
try:
return token_ids.index(self.end_token_id, start) - start
except ValueError:
# No end token -- reasoning was truncated; everything counts.
return len(token_ids) - start
def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]:
"""Split a complete output into ``(reasoning, content)``.
In no_think mode reasoning is None and the whole output is content. The
start token may be absent (injected in the prompt), so reasoning is
assumed to begin at the start of the output.
"""
if not self.thinking:
return None, model_output
parts = model_output.partition(self.start_token)
model_output = parts[2] if parts[1] else parts[0]
if self.end_token not in model_output:
return model_output, None
reasoning, _, content = model_output.partition(self.end_token)
return reasoning, content or None
def extract_reasoning_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
) -> ReasoningDelta | None:
"""Classify a streaming delta as reasoning and/or content.
In no_think mode every delta is content. Otherwise: same logic as vLLM's
generic thinking parser, except the final branch (no start token seen
anywhere) treats output as reasoning-then-content, because HYV4 injects
``<think>`` in the prompt (absent from the output).
"""
if not self.thinking:
if not delta_text:
return None
return ReasoningDelta(reasoning=None, content=delta_text)
# Skip a delta that is a single lone start/end special token.
if len(delta_token_ids) == 1 and delta_token_ids[0] in (
self.start_token_id,
self.end_token_id,
):
return None
if self.start_token_id in previous_token_ids:
if self.end_token_id in delta_token_ids:
end_index = delta_text.find(self.end_token)
reasoning = delta_text[:end_index]
content = delta_text[end_index + len(self.end_token) :]
return ReasoningDelta(reasoning=reasoning, content=content or None)
elif self.end_token_id in previous_token_ids:
return ReasoningDelta(reasoning=None, content=delta_text)
else:
return ReasoningDelta(reasoning=delta_text, content=None)
elif self.start_token_id in delta_token_ids:
if self.end_token_id in delta_token_ids:
start_index = delta_text.find(self.start_token)
end_index = delta_text.find(self.end_token)
reasoning = delta_text[start_index + len(self.start_token) : end_index]
content = delta_text[end_index + len(self.end_token) :]
return ReasoningDelta(reasoning=reasoning, content=content or None)
else:
return ReasoningDelta(reasoning=delta_text, content=None)
else:
# HYV4: <think> was injected at the end of the prompt, so it never
# appears in the output. Treat the stream as reasoning until
# </think>, then content.
if self.end_token_id in delta_token_ids:
end_index = delta_text.find(self.end_token)
reasoning = delta_text[:end_index]
content = delta_text[end_index + len(self.end_token) :]
return ReasoningDelta(reasoning=reasoning, content=content or None)
elif self.end_token_id in previous_token_ids:
return ReasoningDelta(reasoning=None, content=delta_text)
else:
return ReasoningDelta(reasoning=delta_text, content=None)
def build_reasoning_extractor(
tokenizer: TokenizerLike, *, thinking: bool = True
) -> HYV4ReasoningExtractor:
return HYV4ReasoningExtractor(
tokenizer.get_vocab(), detect_token_suffix(tokenizer), thinking
)
class HYV4ReasoningParser(ReasoningParser):
"""vLLM adapter around :class:`HYV4ReasoningExtractor`.
Decides ``thinking`` from the request's ``reasoning_effort`` and converts
the extractor's ``ReasoningDelta`` to ``DeltaMessage``.
"""
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
super().__init__(tokenizer, *args, **kwargs)
# Decide thinking vs no_think from the request; the extractor handles
# both modes internally. Prefer chat_template_kwargs.reasoning_effort,
# then the top-level reasoning_effort. The default must match the chat
# template, which sets reasoning_effort='high' when the request omits
# it -- defaulting to no_think here would return the whole
# chain-of-thought (and a stray ``</think:SUF>``) as content.
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
reasoning_effort = chat_kwargs.get("reasoning_effort", None)
if not reasoning_effort:
reasoning_effort = (
kwargs.get("reasoning_effort") or DEFAULT_REASONING_EFFORT
)
logger.debug("reasoning_effort for choosing parser: %s", reasoning_effort)
thinking = reasoning_effort != "no_think"
self._extractor = build_reasoning_extractor(tokenizer, thinking=thinking)
# Reported regardless of the per-request thinking mode: ``ReasoningConfig``
# is engine-wide and needs the checkpoint's delimiters to derive the token
# ids used for thinking budget enforcement.
@property
def reasoning_start_str(self) -> str:
return self._extractor.start_token
@property
def reasoning_end_str(self) -> str:
return self._extractor.end_token
def is_reasoning_end(self, input_ids) -> bool:
return self._extractor.has_reasoning_ended(input_ids)
def is_reasoning_end_streaming(self, input_ids, delta_ids) -> bool:
return self._extractor.reasoning_ended_in_delta(input_ids, delta_ids)
def extract_content_ids(self, input_ids) -> list[int]:
return self._extractor.extract_content_ids(input_ids)
def count_reasoning_tokens(self, token_ids) -> int:
return self._extractor.count_reasoning_tokens(token_ids)
def extract_reasoning(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
) -> tuple[str | None, str | None]:
return self._extractor.extract_reasoning(model_output)
def extract_reasoning_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids,
current_token_ids,
delta_token_ids,
) -> DeltaMessage | None:
delta = self._extractor.extract_reasoning_streaming(
previous_text,
current_text,
delta_text,
previous_token_ids,
current_token_ids,
delta_token_ids,
)
if delta is None:
return None
return DeltaMessage(reasoning=delta["reasoning"], content=delta["content"])
+4
View File
@@ -94,6 +94,10 @@ _TOOL_PARSERS_TO_REGISTER = {
"hy_v3_tool_parser",
"HYV3ToolParser",
),
"hy_v4": (
"hy_v4_tool_parser",
"HYV4ToolParser",
),
"internlm": (
"internlm2_tool_parser",
"Internlm2ToolParser",
File diff suppressed because it is too large Load Diff
+158 -5
View File
@@ -49,6 +49,7 @@ StructuralTagBuilder: TypeAlias = Callable[
list[BuiltinToolParam],
SimplifiedToolChoice,
bool,
str,
],
StructuralTag,
]
@@ -70,7 +71,7 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
"deepseek_v4",
}
)
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "kimi_k3"})
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"})
SUPPORTED_STRUCTURAL_TAG_MODELS = (
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
)
@@ -78,7 +79,9 @@ SUPPORTED_STRUCTURAL_TAG_MODELS = (
_VLLM_STRUCTURAL_TAG_REGISTRY: dict[str, StructuralTagBuilder] = {}
def register_vllm_structural_tag(model: str):
def register_vllm_structural_tag(
model: str,
) -> Callable[[StructuralTagBuilder], StructuralTagBuilder]:
"""Register a vLLM-owned structural tag builder."""
def decorator(func: StructuralTagBuilder) -> StructuralTagBuilder:
@@ -104,6 +107,7 @@ def get_model_structural_tag(
tools: Sequence[ChatCompletionToolsParam | ResponsesTool] | None,
tool_choice: ToolChoice,
reasoning: bool,
token_suffix: str = "",
) -> StructuralTag | None:
"""Build a structural tag with xgrammar's builtin model templates."""
@@ -126,12 +130,19 @@ def get_model_structural_tag(
builtin_tools,
simplified_tool_choice,
reasoning,
token_suffix,
)
if model not in XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS:
supported = sorted(SUPPORTED_STRUCTURAL_TAG_MODELS)
raise ValueError(f"Unknown format type: {model}, supported types: {supported}")
if token_suffix:
raise ValueError(
f"Structural tag model {model!r} is an xgrammar builtin with fixed "
f"tokens and cannot apply token_suffix={token_suffix!r}"
)
return get_xgrammar_model_structural_tag(
model=model,
tools=dumped_tools,
@@ -244,8 +255,9 @@ def get_hermes_structural_tag(
builtin_tools: list[BuiltinToolParam],
tool_choice: SimplifiedToolChoice,
reasoning: bool,
token_suffix: str = "",
) -> StructuralTag:
del builtin_tools, reasoning
del builtin_tools, reasoning, token_suffix
tool_call_trigger = "<tool_call>"
@@ -293,8 +305,9 @@ def get_minimax_structural_tag(
builtin_tools: list[BuiltinToolParam],
tool_choice: SimplifiedToolChoice,
reasoning: bool,
token_suffix: str = "",
) -> StructuralTag:
del builtin_tools, reasoning
del builtin_tools, reasoning, token_suffix
tool_call_begin = "<minimax:tool_call>\n"
tool_call_end = "</minimax:tool_call>"
@@ -589,8 +602,9 @@ def get_kimi_k3_structural_tag(
builtin_tools: list[BuiltinToolParam],
tool_choice: SimplifiedToolChoice,
reasoning: bool,
token_suffix: str = "",
) -> StructuralTag:
del builtin_tools, reasoning
del builtin_tools, reasoning, token_suffix
trailer = OptionalFormat(content=ConstStringFormat(value=_K3_MESSAGE_CLOSE))
@@ -611,3 +625,142 @@ def get_kimi_k3_structural_tag(
return StructuralTag(
format=SequenceFormat(elements=[*_k3_response_prefix(), tools_part, trailer])
)
# ---------------------------------------------------------------------------
# HYV4 (<tool_calls>/<tool_call>/<arg_key>/<arg_value> structural tokens)
# ---------------------------------------------------------------------------
# HYV4 assistant output after the reasoning gate (``</think:SUF>``):
# <tool_calls:SUF>
# <tool_call:SUF>NAME
# <arg_key:SUF>K</arg_key:SUF><arg_value:SUF>V</arg_value:SUF>
# </tool_call:SUF>
# </tool_calls:SUF>
# Adjacent <tool_call> blocks have no separator, and argument values are emitted
# verbatim (no surrounding quotes), so the value body cannot be described by
# JSONSchemaFormat. Only the skeleton (tool names, argument keys, tag order) is
# constrained; each <arg_value> body stays free text.
def _hy_v4_argument_keys(parameters: dict[str, Any]) -> tuple[list[str], list[str]]:
"""Split a tool's argument keys into ``(required, optional)``.
Both lists keep declaration order. Required keys missing from
``properties`` are still treated as required.
"""
properties = parameters.get("properties") or {}
required = parameters.get("required") or []
required_set = set(required)
required_keys = [k for k in properties if k in required_set]
required_keys += [k for k in required if k not in properties]
optional_keys = [k for k in properties if k not in required_set]
return required_keys, optional_keys
@register_vllm_structural_tag("hy_v4")
def get_hy_v4_structural_tag(
tools: list[FunctionToolParam],
builtin_tools: list[BuiltinToolParam],
tool_choice: SimplifiedToolChoice,
reasoning: bool,
token_suffix: str = "",
) -> StructuralTag:
"""Build HYV4 <tool_calls>/<tool_call>/<arg_key>/<arg_value> structural tags.
Args:
tools: Normalized function tools the model may call.
builtin_tools: Unused; HYV4 has no builtin tools.
tool_choice: Simplified tool choice.
reasoning: Whether the grammar also covers the reasoning phase.
token_suffix: Per-checkpoint structural-token suffix including the
leading colon (e.g. ``":6124c78e"``), or ``""`` when the checkpoint
uses unsuffixed tokens. The HYV4 tool parser reads it off the
tokenizer vocab and passes it to ``get_model_structural_tag``.
"""
del builtin_tools
think_end = f"</think{token_suffix}>"
tool_calls_begin = f"<tool_calls{token_suffix}>"
tool_calls_end = f"</tool_calls{token_suffix}>"
tool_call_begin = f"<tool_call{token_suffix}>"
tool_call_end = f"</tool_call{token_suffix}>"
arg_key_begin = f"<arg_key{token_suffix}>"
arg_key_end = f"</arg_key{token_suffix}>"
arg_value_begin = f"<arg_value{token_suffix}>"
arg_value_end = f"</arg_value{token_suffix}>"
def arg_pair(key: str) -> SequenceFormat:
return SequenceFormat(
elements=[
ConstStringFormat(value=arg_key_begin),
ConstStringFormat(value=key),
ConstStringFormat(value=arg_key_end),
ConstStringFormat(value=arg_value_begin),
AnyTextFormat(),
ConstStringFormat(value=arg_value_end),
]
)
def single_tool_call(tool: FunctionToolParam) -> TagFormat:
function = tool.function
parameters = (
function.parameters if isinstance(function.parameters, dict) else {}
)
required_keys, optional_keys = _hy_v4_argument_keys(parameters)
elements: list[Any] = [arg_pair(k) for k in required_keys]
if optional_keys:
# Optional keys may appear in any order, or not at all.
optional_pairs = [arg_pair(k) for k in optional_keys]
elements.append(
StarFormat(
content=optional_pairs[0]
if len(optional_pairs) == 1
else OrFormat(elements=optional_pairs)
)
)
content: Any = (
SequenceFormat(elements=elements) if elements else AnyTextFormat()
)
return TagFormat(
begin=tool_call_begin + function.name,
content=content,
end=tool_call_end,
)
tags = [single_tool_call(tool) for tool in tools]
if tool_choice == "auto":
block = TagFormat(
begin=tool_calls_begin,
content=TagsWithSeparatorFormat(
tags=tags,
separator="",
at_least_one=True,
),
end=tool_calls_end,
)
suffix_tag: Any = TriggeredTagsFormat(triggers=[tool_calls_begin], tags=[block])
else:
# HYV4 places tool_call blocks back-to-back with no separator. "forced"
# has already been filtered down to the single named tool and must emit
# exactly one block.
suffix_tag = SequenceFormat(
elements=[
ConstStringFormat(value=tool_calls_begin),
TagsWithSeparatorFormat(
tags=tags,
separator="",
at_least_one=True,
stop_after_first=tool_choice == "forced",
),
ConstStringFormat(value=tool_calls_end),
]
)
if not reasoning:
return StructuralTag(format=suffix_tag)
# The reasoning phase is constrained too, so the tag must explicitly skip
# the ``<think>...</think:SUF>`` prefix.
prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=think_end)
return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag]))
+1
View File
@@ -91,6 +91,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
granite4_vision="Granite4VisionConfig",
hyperclovax="HyperCLOVAXConfig",
hy_v3="HYV3Config",
hy_v4="HYV4Config",
isaac="IsaacConfig",
kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3
kimi_linear="KimiLinearConfig",
@@ -41,6 +41,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
"Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
"HYV4Config": "vllm.transformers_utils.configs.hy_v4",
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
@@ -130,6 +131,7 @@ __all__ = [
"FunAudioChatAudioEncoderConfig",
"Granite4VisionConfig",
"HYV3Config",
"HYV4Config",
"HyperCLOVAXConfig",
"IsaacConfig",
"RWConfig",
+172
View File
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from huggingface_hub.dataclasses import strict
from transformers.configuration_utils import PreTrainedConfig
from transformers.modeling_rope_utils import RopeParameters
@strict
class HYV4Config(PreTrainedConfig):
r"""Configuration class for the HY V4 (`HYV4ForCausalLM`) model.
Kept field-for-field in sync with
`transformers.models.hy_v4.configuration_hy_v4.HYV4Config` so that a
checkpoint resolves to the same values through either class. Registering it
in vLLM's `_CONFIG_REGISTRY` lets the architecture load without
`trust_remote_code` and without an `auto_map` module next to the weights.
HYV4 is a mixture-of-experts causal language model using Multi-head Latent
Attention (MLA), DeepSeek-style sparse attention (DSA), gated MLA,
learnable attention sinks, and independent Hyper-Connections (iHC).
The `layer_types`, `num_nextn_predict_layers` and `mtp_loss_factor` fields
below are vLLM-only additions: upstream leaves them to `**kwargs`, but
vLLM reads them directly (sparse-attention selection and MTP speculative
decoding), so they are declared to give checkpoints that omit them a
working default instead of a `None`.
Args:
mlp_layer_types: Per-layer MLP kind, `"dense"` or `"sparse"`. Defaults
to one dense layer followed by sparse MoE layers.
layer_types: Per-layer attention kind, e.g. `"full_attention"` or
`"deepseek_sparse_attention"`.
indexer_types: Per-layer DSA indexer kind, `"full"` or `"shared"`. A
shared layer reuses the most recent full indexer in the same
forward request.
index_topk: Maximum number of key positions selected by each DSA query.
index_head_dim: Hidden dimension of each DSA indexer head.
index_n_heads: Number of DSA indexer heads.
enable_lm_head_fp32: Whether the language-model head emits float32
logits. Surfaced as ``head_dtype`` so `LogitsProcessor` accumulates
the projection into fp32 while the weight stays in the model dtype.
enable_ihc: Whether independent Hyper-Connections are enabled.
hc_mult: Number of hidden-state channels maintained by iHC.
hc_magnitude: Scale applied to the iHC post-gating branch.
hc_eps: Numerical epsilon added to iHC sigmoid gates.
gated_mla: Whether to gate the MLA output.
gating_type: MLA gate granularity, `"elementwise"` or `"headwise"`.
learnable_sink: Whether to add a learned per-head attention sink.
learnable_sink_init: Initial value of each learned attention-sink
logit.
swiglu_limit: Magnitude of the routed-expert SwiGLU clamp. Values at or
below zero disable the clamp.
num_nextn_predict_layers: Number of MTP layers in the checkpoint.
mtp_loss_factor: Training-time MTP loss weight (unused at inference).
"""
model_type = "hy_v4"
keys_to_ignore_at_inference = ["past_key_values"]
attribute_map = {
"num_local_experts": "n_routed_experts",
}
base_model_tp_plan = {
"layers.*.self_attn.q_b_proj": "colwise",
"layers.*.self_attn.kv_a_proj_with_mqa": "mla_kv_a_proj",
"layers.*.self_attn.kv_b_proj": "colwise",
"layers.*.self_attn.o_proj": "rowwise",
"layers.*.self_attn.linear_gate": "colwise",
"layers.*.mlp.experts.gate_up_proj": "packed_colwise",
"layers.*.mlp.experts.down_proj": "rowwise",
"layers.*.mlp.experts": "moe_tp_experts",
"layers.*.mlp.shared_experts.gate_proj": "colwise",
"layers.*.mlp.shared_experts.up_proj": "colwise",
"layers.*.mlp.shared_experts.down_proj": "rowwise",
"layers.*.mlp.gate_proj": "colwise",
"layers.*.mlp.up_proj": "colwise",
"layers.*.mlp.down_proj": "rowwise",
}
base_model_pp_plan = {
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
"norm": (["hidden_states"], ["hidden_states"]),
}
vocab_size: int = 120832
hidden_size: int = 2816
intermediate_size: int = 6912
moe_intermediate_size: int = 768
num_hidden_layers: int = 34
num_attention_heads: int = 32
num_key_value_heads: int = 32
head_dim: int = 256
hidden_act: str = "silu"
max_position_embeddings: int = 262144
initializer_range: float = 0.006
rms_norm_eps: float = 1e-5
use_cache: bool = True
pad_token_id: int | None = 120002
bos_token_id: int | None = 120000
eos_token_id: int | list[int] | None = 120025
tie_word_embeddings: bool = False
attention_bias: bool = False
attention_dropout: float = 0.0
n_routed_experts: int = 256
n_shared_experts: int = 1
num_experts_per_tok: int = 8
routed_scaling_factor: float = 2.827
norm_topk_prob: bool = True
q_lora_rank: int = 1536
kv_lora_rank: int = 512
qk_nope_head_dim: int = 192
qk_rope_head_dim: int = 64
v_head_dim: int = 256
mlp_layer_types: list[str] | None = None
layer_types: list[str] | None = None
index_topk: int = 2048
index_head_dim: int = 128
index_n_heads: int = 16
indexer_types: list[str] | None = None
enable_lm_head_fp32: bool = True
enable_ihc: bool = True
hc_mult: int = 4
hc_magnitude: float = 2.0
hc_eps: float = 1e-6
gated_mla: bool = True
gating_type: str = "elementwise"
learnable_sink: bool = True
learnable_sink_init: float = 0.0
swiglu_limit: float = 10.0
rope_parameters: RopeParameters | dict | None = None
num_nextn_predict_layers: int = 1
mtp_loss_factor: float = 0.1
def __post_init__(self, **kwargs):
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
# RoPE applies only to the rope slice, so point `head_dim` at it: the
# inherited rotary embedding reads `config.head_dim` and then computes
# the right frequencies with no override.
self.head_dim = self.qk_rope_head_dim
if self.mlp_layer_types is None:
self.mlp_layer_types = ["dense"] * min(1, self.num_hidden_layers) + [
"sparse"
] * max(self.num_hidden_layers - 1, 0)
if self.indexer_types is None:
self.indexer_types = [
"full" if layer_idx == 0 or (layer_idx - 1) % 4 == 0 else "shared"
for layer_idx in range(self.num_hidden_layers)
]
# `PreTrainedConfig` validates `layer_types` against a fixed allow-list
# in which "deepseek_sparse_attention" is the only accepted spelling of
# a DSA layer. The model treats "sparse_attention" and "sparse" as
# equivalent, so fold them in before the base class validates.
if self.layer_types is not None:
self.layer_types = [
"deepseek_sparse_attention"
if lt in ("sparse_attention", "sparse")
else lt
for lt in self.layer_types
]
# `ModelConfig.head_dtype` reads this off the HF config, so an fp32 head
# needs no model-side dtype juggling: LogitsProcessor picks the
# `torch.mm(out_dtype=float32)` path and leaves the weight in the model
# dtype. Only fill it in when the checkpoint (or --hf-overrides) has not
# already pinned a value.
if self.enable_lm_head_fp32 and getattr(self, "head_dtype", None) is None:
self.head_dtype = "float32"
super().__post_init__(**kwargs)
__all__ = ["HYV4Config"]
@@ -332,6 +332,8 @@ class ModelArchConfigConvertorBase:
"bailing_hybrid",
"bailing_hybrid_mtp",
"bailing_hybrid_v3_mtp",
"hy_v4",
"hy_v4_mtp",
):
# check is deepseek_v4 model
if hasattr(self.hf_text_config, "compress_ratios"):
+23 -5
View File
@@ -30,6 +30,19 @@ from vllm.v1.worker.utils import AttentionGroup
logger = init_logger(__name__)
def _target_feeds_hc_residual(vllm_config: VllmConfig) -> bool:
"""Whether the target replaces the drafter's input with its HC residual.
Keyed on the same hook the model runner calls to perform that swap. It is
resolved from the target model class because speculators are built before
the target model is instantiated.
"""
from vllm.model_executor.model_loader.utils import get_model_cls
target_cls = get_model_cls(vllm_config.model_config)
return hasattr(target_cls, "get_mtp_target_hidden_states")
class BaseSpeculator(ABC):
@abstractmethod
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
@@ -90,11 +103,16 @@ class DraftModelSpeculator(BaseSpeculator):
# the draft model's hidden size can be different from the target model's
# hidden size (e.g., Llama 3.3 70B).
self.hidden_size = self.draft_model_config.get_hidden_size()
# Widen for HC-multiplexed residuals (e.g. DeepSeek V4 feeds the MTP
# draft the target's pre-hc_head (T, hc_mult * hidden_size) residual).
# Non-HC models default to hc_mult=1 and are unaffected.
hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1)
self.hidden_size = self.hidden_size * hc_mult
# Widen for HC-multiplexed residuals: a target that implements
# get_mtp_target_hidden_states() (e.g. DeepSeek V4) hands the drafter
# its pre-hc_head (T, hc_mult * hidden_size) residual in place of the
# collapsed hidden states, so the drafter's buffers must match. Key off
# that hook rather than hc_mult alone -- HY V4 runs iHC in its backbone
# (hc_mult=4) but its MTP head consumes the collapsed states, so
# widening it feeds propose() a 4x-too-wide buffer.
if _target_feeds_hc_residual(vllm_config):
hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1)
self.hidden_size = self.hidden_size * hc_mult
self.vocab_size = self.draft_model_config.get_vocab_size()
self.dtype = vllm_config.model_config.dtype
self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel