[Hy4] support Hy4-preview model (#54160)
Signed-off-by: chengvjiang <chengvjiang@tencent.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: russellfeng <russellfeng@tencent.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
chengvjiang
Jee Jee Li
Bugen Zhao
russellfeng
Jee Jee Li
parent
6c18a54648
commit
b2f685834a
@@ -405,7 +405,8 @@ th {
|
||||
| `GraniteMoeSWAForCausalLM` | Granite MoE SWA | `ibm-granite/granite-swash-3b-a600m` | ✅︎ | ✅︎ |
|
||||
| `GraniteSWAForCausalLM` | Granite SWA | `ibm-granite/granite-swash-2b` | ✅︎ | ✅︎ |
|
||||
| `HrmTextForCausalLM` | HRM-Text | `sapientinc/HRM-Text-1B`, etc. | | |
|
||||
| `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview` | ✅︎ | ✅︎ |
|
||||
| `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview`, `tencent/Hy3`, `tencent/Hy3-FP8` | ✅︎ | ✅︎ |
|
||||
| `HYV4ForCausalLM` | HY4 | `tencent/Hy4-preview`, `tencent/Hy4-preview-FP8` | ✅︎ | ✅︎ |
|
||||
| `HyperCLOVAXForCausalLM` | HyperCLOVAX-SEED-Think-14B | `naver-hyperclovax/HyperCLOVAX-SEED-Think-14B` | ✅︎ | ✅︎ |
|
||||
| `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ |
|
||||
| `InternLM3ForCausalLM` | InternLM3 | `internlm/internlm3-8b-instruct`, etc. | ✅︎ | ✅︎ |
|
||||
|
||||
@@ -405,7 +405,7 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -416,6 +416,6 @@ mod tests {
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ pub mod names {
|
||||
pub const INKLING: &str = "inkling";
|
||||
pub const GLM45: &str = "glm45";
|
||||
pub const HY_V3: &str = "hy_v3";
|
||||
pub const HY_V4: &str = "hy_v4";
|
||||
pub const KIMI: &str = "kimi";
|
||||
pub const KIMI_K2: &str = "kimi_k2";
|
||||
pub const KIMI_K3: &str = "kimi_k3";
|
||||
@@ -69,6 +70,7 @@ impl ReasoningParserFactory {
|
||||
.register_unified_dummy(names::INKLING)
|
||||
.register_parser::<Glm45ReasoningParser>(names::GLM45)
|
||||
.register_unified_dummy(names::HY_V3)
|
||||
.register_unified_dummy(names::HY_V4)
|
||||
.register_parser::<KimiReasoningParser>(names::KIMI)
|
||||
.register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
|
||||
.register_unified_dummy(names::KIMI_K3)
|
||||
@@ -94,6 +96,7 @@ impl ReasoningParserFactory {
|
||||
.register_pattern("glm-4.6", names::GLM45)
|
||||
.register_pattern("glm-4.5", names::GLM45)
|
||||
.register_pattern("hy3", names::HY_V3)
|
||||
.register_pattern("hy4", names::HY_V4)
|
||||
.register_pattern("kimi-k2", names::KIMI_K2)
|
||||
.register_pattern("kimi", names::KIMI)
|
||||
// step3p5 patterns must precede `step3`: substring matching would
|
||||
|
||||
@@ -29,6 +29,7 @@ pub mod names {
|
||||
pub const GRANITE4: &str = "granite4";
|
||||
pub const HERMES: &str = "hermes";
|
||||
pub const HY_V3: &str = "hy_v3";
|
||||
pub const HY_V4: &str = "hy_v4";
|
||||
// Matches the Python CLI name `--tool-call-parser internlm`, which Python
|
||||
// also routes to `Internlm2ToolParser` despite the version-agnostic name.
|
||||
pub const INTERNLM: &str = "internlm";
|
||||
@@ -77,6 +78,7 @@ impl ToolParserFactory {
|
||||
.register_parser::<Granite4ToolParser>(names::GRANITE4)
|
||||
.register_parser::<HermesToolParser>(names::HERMES)
|
||||
.register_unified_dummy(names::HY_V3)
|
||||
.register_unified_dummy(names::HY_V4)
|
||||
.register_parser::<Internlm2ToolParser>(names::INTERNLM)
|
||||
.register_parser::<KimiK2ToolParser>(names::KIMI_K2)
|
||||
.register_unified_dummy(names::KIMI_K3)
|
||||
@@ -100,6 +102,7 @@ impl ToolParserFactory {
|
||||
.register_pattern("qwen3", names::QWEN3_XML)
|
||||
.register_pattern("hermes", names::HERMES)
|
||||
.register_pattern("hy3", names::HY_V3)
|
||||
.register_pattern("hy4", names::HY_V4)
|
||||
// Narrow to `internlm2` substring so it matches `internlm2-chat-7b`
|
||||
// and `internlm2_5-7b-chat` but NOT `internlm-chat-7b` (InternLM v1,
|
||||
// routes to Llama), `internlm3-*` (also Llama-architecture per
|
||||
|
||||
@@ -183,6 +183,10 @@ fn factory_new_resolves_default_patterns() {
|
||||
factory.resolve_name_for_model("tencent/Hy3-preview"),
|
||||
Some(names::HY_V3)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("tencent/Hy4-preview"),
|
||||
Some(names::HY_V4)
|
||||
);
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("MiniMax/MiniMax-M3-Text"),
|
||||
Some(names::MINIMAX_M3)
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub use vllm_parser::unified::{
|
||||
Gemma4UnifiedParser, HyV3UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser,
|
||||
UnifiedParser,
|
||||
Gemma4UnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser, InklingUnifiedParser,
|
||||
KimiK3UnifiedParser, UnifiedParser,
|
||||
};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::request::ChatTool;
|
||||
pub mod names {
|
||||
pub const GEMMA4: &str = "gemma4";
|
||||
pub const HY_V3: &str = "hy_v3";
|
||||
pub const HY_V4: &str = "hy_v4";
|
||||
pub const INKLING: &str = "inkling";
|
||||
pub const KIMI_K3: &str = "kimi_k3";
|
||||
}
|
||||
@@ -44,6 +45,7 @@ impl UnifiedParserFactory {
|
||||
|
||||
factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
|
||||
factory.register_parser::<HyV3UnifiedParser>(names::HY_V3);
|
||||
factory.register_parser::<HyV4UnifiedParser>(names::HY_V4);
|
||||
factory.register_parser::<InklingUnifiedParser>(names::INKLING);
|
||||
factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
|
||||
|
||||
@@ -51,6 +53,7 @@ impl UnifiedParserFactory {
|
||||
.register_pattern("gemma-4", names::GEMMA4)
|
||||
.register_pattern("gemma4", names::GEMMA4)
|
||||
.register_pattern("hy3", names::HY_V3)
|
||||
.register_pattern("hy4", names::HY_V4)
|
||||
.register_pattern("inkling", names::INKLING)
|
||||
.register_pattern("kimi-k3", names::KIMI_K3)
|
||||
.register_pattern("kimi_k3", names::KIMI_K3);
|
||||
@@ -147,4 +150,33 @@ mod tests {
|
||||
);
|
||||
factory.create(names::KIMI_K3, &[], Arc::new(tokenizer)).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn factory_registers_hy_v4() {
|
||||
let factory = UnifiedParserFactory::new();
|
||||
let tokenizer = [
|
||||
"<think:opensource>",
|
||||
"</think:opensource>",
|
||||
"<tool_calls:opensource>",
|
||||
"</tool_calls:opensource>",
|
||||
"<tool_call:opensource>",
|
||||
"</tool_call:opensource>",
|
||||
"<arg_key:opensource>",
|
||||
"</arg_key:opensource>",
|
||||
"<arg_value:opensource>",
|
||||
"</arg_value:opensource>",
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.fold(TestTokenizer::new(), |tokenizer, (index, token)| {
|
||||
tokenizer.with_regular_token(token, 1000 + index as u32)
|
||||
});
|
||||
|
||||
assert!(factory.contains(names::HY_V4));
|
||||
assert_eq!(
|
||||
factory.resolve_name_for_model("tencent/Hy4-preview"),
|
||||
Some(names::HY_V4)
|
||||
);
|
||||
factory.create(names::HY_V4, &[], Arc::new(tokenizer)).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,13 +5,13 @@ use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, Result};
|
||||
|
||||
/// Internal HY3 reasoning stage used by the unified HY3 parser.
|
||||
pub(crate) struct HyV3ReasoningParser {
|
||||
/// Internal HY reasoning stage used by the unified HY parsers.
|
||||
pub(crate) struct HyReasoningParser {
|
||||
inner: DelimitedReasoningParser,
|
||||
}
|
||||
|
||||
impl HyV3ReasoningParser {
|
||||
/// Create a HY3 reasoning parser for the tokenizer-specific marker suffix.
|
||||
impl HyReasoningParser {
|
||||
/// Create a HY reasoning parser for the tokenizer-specific marker suffix.
|
||||
pub(crate) fn new(tokenizer: DynTokenizer, suffix: &str) -> Result<Self> {
|
||||
Ok(Self {
|
||||
inner: DelimitedReasoningParser::new(
|
||||
@@ -24,15 +24,15 @@ impl HyV3ReasoningParser {
|
||||
}
|
||||
}
|
||||
|
||||
impl ReasoningParser for HyV3ReasoningParser {
|
||||
// Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and
|
||||
impl ReasoningParser for HyReasoningParser {
|
||||
// Suffix discovery belongs to the unified HY parser so its reasoning and
|
||||
// tool delimiters always use the same tokenizer-derived value.
|
||||
fn create(_tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Err(ReasoningError::DummyUnifiedParser {
|
||||
name: "hy_v3".to_string(),
|
||||
name: "hy".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
mod cohere_cmd;
|
||||
mod deepseek_r1;
|
||||
mod delimited;
|
||||
mod hy_v3;
|
||||
mod hy;
|
||||
mod kimi;
|
||||
mod minimax_m3;
|
||||
mod qwen3;
|
||||
@@ -33,7 +33,7 @@ use vllm_tokenizer::DynTokenizer;
|
||||
pub use self::cohere_cmd::CohereCmdReasoningParser;
|
||||
pub use self::deepseek_r1::DeepSeekR1ReasoningParser;
|
||||
pub(crate) use self::delimited::{DelimitedReasoningParser, last_reasoning_boundary};
|
||||
pub(crate) use self::hy_v3::HyV3ReasoningParser;
|
||||
pub(crate) use self::hy::HyReasoningParser;
|
||||
pub use self::kimi::KimiReasoningParser;
|
||||
pub use self::minimax_m3::MiniMaxM3ReasoningParser;
|
||||
pub use self::qwen3::Qwen3ReasoningParser;
|
||||
|
||||
@@ -11,33 +11,56 @@ use winnow::prelude::*;
|
||||
use winnow::stream::Partial;
|
||||
use winnow::token::{literal, rest, take_until};
|
||||
|
||||
use self::structural_tag::HyV3StructuralTagBuilder;
|
||||
use self::structural_tag::HyStructuralTagBuilder;
|
||||
use super::parameters::ToolSchemas;
|
||||
use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
|
||||
use super::{Result, ToolCallDelta, ToolParser, ToolParserError, ToolParserOutput};
|
||||
use crate::tool::{StructuralTagBuilder, Tool};
|
||||
|
||||
/// Wire-level HY tool-call dialect.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum HyDialect {
|
||||
V3,
|
||||
V4,
|
||||
}
|
||||
|
||||
impl HyDialect {
|
||||
fn separator(self) -> &'static str {
|
||||
match self {
|
||||
Self::V3 => "\n",
|
||||
Self::V4 => "",
|
||||
}
|
||||
}
|
||||
|
||||
fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::V3 => "HY3",
|
||||
Self::V4 => "HY4",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct HyV3ToolMarkers {
|
||||
pub(crate) struct HyToolMarkers {
|
||||
tool_calls_start: String,
|
||||
tool_calls_end: String,
|
||||
tool_call_start: String,
|
||||
tool_call_end: String,
|
||||
tool_sep: String,
|
||||
tool_sep: Option<String>,
|
||||
arg_key_start: String,
|
||||
arg_key_end: String,
|
||||
arg_value_start: String,
|
||||
arg_value_end: String,
|
||||
}
|
||||
|
||||
impl HyV3ToolMarkers {
|
||||
pub(crate) fn new(suffix: &str) -> Self {
|
||||
impl HyToolMarkers {
|
||||
pub(crate) fn new(suffix: &str, dialect: HyDialect) -> Self {
|
||||
Self {
|
||||
tool_calls_start: format!("<tool_calls{suffix}>"),
|
||||
tool_calls_end: format!("</tool_calls{suffix}>"),
|
||||
tool_call_start: format!("<tool_call{suffix}>"),
|
||||
tool_call_end: format!("</tool_call{suffix}>"),
|
||||
tool_sep: format!("<tool_sep{suffix}>"),
|
||||
tool_sep: (dialect == HyDialect::V3).then(|| format!("<tool_sep{suffix}>")),
|
||||
arg_key_start: format!("<arg_key{suffix}>"),
|
||||
arg_key_end: format!("</arg_key{suffix}>"),
|
||||
arg_value_start: format!("<arg_value{suffix}>"),
|
||||
@@ -47,31 +70,32 @@ impl HyV3ToolMarkers {
|
||||
|
||||
pub(crate) fn iter(&self) -> impl Iterator<Item = &str> {
|
||||
[
|
||||
self.tool_calls_start.as_str(),
|
||||
self.tool_calls_end.as_str(),
|
||||
self.tool_call_start.as_str(),
|
||||
self.tool_call_end.as_str(),
|
||||
self.tool_sep.as_str(),
|
||||
self.arg_key_start.as_str(),
|
||||
self.arg_key_end.as_str(),
|
||||
self.arg_value_start.as_str(),
|
||||
self.arg_value_end.as_str(),
|
||||
Some(self.tool_calls_start.as_str()),
|
||||
Some(self.tool_calls_end.as_str()),
|
||||
Some(self.tool_call_start.as_str()),
|
||||
Some(self.tool_call_end.as_str()),
|
||||
self.tool_sep.as_deref(),
|
||||
Some(self.arg_key_start.as_str()),
|
||||
Some(self.arg_key_end.as_str()),
|
||||
Some(self.arg_value_start.as_str()),
|
||||
Some(self.arg_value_end.as_str()),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
}
|
||||
}
|
||||
|
||||
type HyV3Input<'i> = Partial<&'i str>;
|
||||
type HyInput<'i> = Partial<&'i str>;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum HyV3Mode {
|
||||
enum HyMode {
|
||||
Text,
|
||||
ToolBlock { tool_call_end_scan: MarkerScanState },
|
||||
Done,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum HyV3Event {
|
||||
enum HyEvent {
|
||||
Text {
|
||||
len: usize,
|
||||
},
|
||||
@@ -84,7 +108,7 @@ enum HyV3Event {
|
||||
IgnoredRest,
|
||||
}
|
||||
|
||||
/// Tool parser for HY3 XML-style tool calls.
|
||||
/// Tool parser for HY XML-style tool calls.
|
||||
///
|
||||
/// Example tool call content:
|
||||
///
|
||||
@@ -97,43 +121,45 @@ enum HyV3Event {
|
||||
/// ```
|
||||
///
|
||||
/// Arguments are emitted only after a full `<tool_call>` block is parsed.
|
||||
/// HY3 marker tokens are added-vocabulary tokens rather than tokenizer special
|
||||
/// HY marker tokens are added-vocabulary tokens rather than tokenizer special
|
||||
/// tokens, so the default `preserve_special_tokens() == false` is sufficient.
|
||||
pub(crate) struct HyV3ToolParser {
|
||||
pub(crate) struct HyToolParser {
|
||||
buffer: String,
|
||||
mode: HyV3Mode,
|
||||
mode: HyMode,
|
||||
emitted_tool_count: usize,
|
||||
tool_parameters: ToolSchemas,
|
||||
markers: Arc<HyV3ToolMarkers>,
|
||||
structural_tag_builder: HyV3StructuralTagBuilder,
|
||||
dialect: HyDialect,
|
||||
markers: Arc<HyToolMarkers>,
|
||||
structural_tag_builder: HyStructuralTagBuilder,
|
||||
}
|
||||
|
||||
impl HyV3ToolParser {
|
||||
/// Create a HY3 tool parser.
|
||||
pub(crate) fn new(tools: &[Tool], suffix: &str) -> Self {
|
||||
let markers = Arc::new(HyV3ToolMarkers::new(suffix));
|
||||
impl HyToolParser {
|
||||
/// Create a HY tool parser for one wire dialect.
|
||||
pub(crate) fn new(tools: &[Tool], suffix: &str, dialect: HyDialect) -> Self {
|
||||
let markers = Arc::new(HyToolMarkers::new(suffix, dialect));
|
||||
Self {
|
||||
buffer: String::new(),
|
||||
mode: HyV3Mode::Text,
|
||||
mode: HyMode::Text,
|
||||
emitted_tool_count: 0,
|
||||
tool_parameters: ToolSchemas::from_tools(tools),
|
||||
dialect,
|
||||
markers: Arc::clone(&markers),
|
||||
structural_tag_builder: HyV3StructuralTagBuilder::new(markers),
|
||||
structural_tag_builder: HyStructuralTagBuilder::new(markers, dialect),
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply one parsed HY3 event to parser state and output.
|
||||
fn apply_event(&mut self, event: HyV3Event, output: &mut ToolParserOutput) -> Result<()> {
|
||||
/// Apply one parsed HY event to parser state and output.
|
||||
fn apply_event(&mut self, event: HyEvent, output: &mut ToolParserOutput) -> Result<()> {
|
||||
match event {
|
||||
HyV3Event::Text { len: consumed_len } => {
|
||||
HyEvent::Text { len: consumed_len } => {
|
||||
output.push_text(&self.buffer[..consumed_len]);
|
||||
}
|
||||
HyV3Event::ToolBlockStart => {
|
||||
self.mode = HyV3Mode::ToolBlock {
|
||||
HyEvent::ToolBlockStart => {
|
||||
self.mode = HyMode::ToolBlock {
|
||||
tool_call_end_scan: MarkerScanState::default(),
|
||||
};
|
||||
}
|
||||
HyV3Event::ToolCall { name, raw_params } => {
|
||||
HyEvent::ToolCall { name, raw_params } => {
|
||||
let arguments = self.tool_parameters.convert_params_with_schema(&name, raw_params);
|
||||
let arguments = serde_json::to_string(&arguments)
|
||||
.map_err(|error| parsing_failed!("failed to serialize arguments: {}", error))?;
|
||||
@@ -145,22 +171,22 @@ impl HyV3ToolParser {
|
||||
});
|
||||
self.emitted_tool_count += 1;
|
||||
}
|
||||
HyV3Event::ToolBlockEnd => self.mode = HyV3Mode::Done,
|
||||
HyV3Event::IgnoredRest => {}
|
||||
HyEvent::ToolBlockEnd => self.mode = HyMode::Done,
|
||||
HyEvent::IgnoredRest => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolParser for HyV3ToolParser {
|
||||
// Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and
|
||||
impl ToolParser for HyToolParser {
|
||||
// Suffix discovery belongs to the unified HY parser so its reasoning and
|
||||
// tool delimiters always use the same tokenizer-derived value.
|
||||
fn create(_tools: &[Tool]) -> Result<Box<dyn ToolParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Err(ToolParserError::DummyUnifiedParser {
|
||||
name: "hy_v3".to_string(),
|
||||
name: "hy".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -172,7 +198,7 @@ impl ToolParser for HyV3ToolParser {
|
||||
self.buffer.push_str(chunk);
|
||||
|
||||
while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
|
||||
parse_next_hy_v3_event(input, &mut self.mode, &self.markers)
|
||||
parse_next_hy_event(input, &mut self.mode, &self.markers, self.dialect)
|
||||
})? {
|
||||
self.apply_event(event, output)?;
|
||||
self.buffer.drain(..consumed_len);
|
||||
@@ -184,113 +210,145 @@ impl ToolParser for HyV3ToolParser {
|
||||
fn finish(&mut self) -> Result<ToolParserOutput> {
|
||||
let mut output = ToolParserOutput::default();
|
||||
match self.mode {
|
||||
HyV3Mode::Text => output.push_text(&self.buffer),
|
||||
HyV3Mode::ToolBlock { .. } => return Err(parsing_failed!("incomplete HY3 tool call")),
|
||||
HyV3Mode::Done => {}
|
||||
HyMode::Text => output.push_text(&self.buffer),
|
||||
HyMode::ToolBlock { .. } => {
|
||||
return Err(parsing_failed!(
|
||||
"incomplete {} tool call",
|
||||
self.dialect.label()
|
||||
));
|
||||
}
|
||||
HyMode::Done => {}
|
||||
}
|
||||
let _ = self.reset();
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> String {
|
||||
self.mode = HyV3Mode::Text;
|
||||
self.mode = HyMode::Text;
|
||||
self.emitted_tool_count = 0;
|
||||
std::mem::take(&mut self.buffer)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a HY3 event for the current parser mode.
|
||||
fn parse_next_hy_v3_event(
|
||||
input: &mut HyV3Input<'_>,
|
||||
mode: &mut HyV3Mode,
|
||||
markers: &HyV3ToolMarkers,
|
||||
) -> ModalResult<HyV3Event> {
|
||||
/// Parse a HY event for the current parser mode.
|
||||
fn parse_next_hy_event(
|
||||
input: &mut HyInput<'_>,
|
||||
mode: &mut HyMode,
|
||||
markers: &HyToolMarkers,
|
||||
dialect: HyDialect,
|
||||
) -> ModalResult<HyEvent> {
|
||||
match mode {
|
||||
HyV3Mode::Text => parse_text_event(input, markers),
|
||||
HyV3Mode::ToolBlock { tool_call_end_scan } => {
|
||||
parse_tool_block_event(input, tool_call_end_scan, markers)
|
||||
HyMode::Text => parse_text_event(input, markers),
|
||||
HyMode::ToolBlock { tool_call_end_scan } => {
|
||||
parse_tool_block_event(input, tool_call_end_scan, markers, dialect)
|
||||
}
|
||||
HyV3Mode::Done => ignored_rest_event(input),
|
||||
HyMode::Done => ignored_rest_event(input),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a text-mode HY3 event.
|
||||
fn parse_text_event(
|
||||
input: &mut HyV3Input<'_>,
|
||||
markers: &HyV3ToolMarkers,
|
||||
) -> ModalResult<HyV3Event> {
|
||||
/// Parse a text-mode HY event.
|
||||
fn parse_text_event(input: &mut HyInput<'_>, markers: &HyToolMarkers) -> ModalResult<HyEvent> {
|
||||
alt((
|
||||
|input: &mut HyV3Input<'_>| tool_block_start_event(input, markers),
|
||||
|input: &mut HyV3Input<'_>| safe_text_event(input, markers),
|
||||
|input: &mut HyInput<'_>| tool_block_start_event(input, markers),
|
||||
|input: &mut HyInput<'_>| safe_text_event(input, markers),
|
||||
))
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse a HY3 tool-block start marker.
|
||||
/// Parse a HY tool-block start marker.
|
||||
fn tool_block_start_event(
|
||||
input: &mut HyV3Input<'_>,
|
||||
markers: &HyV3ToolMarkers,
|
||||
) -> ModalResult<HyV3Event> {
|
||||
input: &mut HyInput<'_>,
|
||||
markers: &HyToolMarkers,
|
||||
) -> ModalResult<HyEvent> {
|
||||
literal(markers.tool_calls_start.as_str())
|
||||
.value(HyV3Event::ToolBlockStart)
|
||||
.value(HyEvent::ToolBlockStart)
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse a safe text run before the next HY3 marker.
|
||||
fn safe_text_event(input: &mut HyV3Input<'_>, markers: &HyV3ToolMarkers) -> ModalResult<HyV3Event> {
|
||||
safe_text_len(input, &markers.tool_calls_start).map(|len| HyV3Event::Text { len })
|
||||
/// Parse a safe text run before the next HY marker.
|
||||
fn safe_text_event(input: &mut HyInput<'_>, markers: &HyToolMarkers) -> ModalResult<HyEvent> {
|
||||
safe_text_len(input, &markers.tool_calls_start).map(|len| HyEvent::Text { len })
|
||||
}
|
||||
|
||||
/// Parse one event inside a HY3 tool block.
|
||||
/// Parse one event inside a HY tool block.
|
||||
fn parse_tool_block_event(
|
||||
input: &mut HyV3Input<'_>,
|
||||
input: &mut HyInput<'_>,
|
||||
tool_call_end_scan: &mut MarkerScanState,
|
||||
markers: &HyV3ToolMarkers,
|
||||
) -> ModalResult<HyV3Event> {
|
||||
markers: &HyToolMarkers,
|
||||
dialect: HyDialect,
|
||||
) -> ModalResult<HyEvent> {
|
||||
alt((
|
||||
|input: &mut HyV3Input<'_>| tool_block_end_event(input, markers),
|
||||
|input: &mut HyV3Input<'_>| tool_call_event(input, tool_call_end_scan, markers),
|
||||
|input: &mut HyInput<'_>| tool_block_end_event(input, markers),
|
||||
|input: &mut HyInput<'_>| tool_call_event(input, tool_call_end_scan, markers, dialect),
|
||||
))
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse a HY3 tool-block end marker.
|
||||
fn tool_block_end_event(
|
||||
input: &mut HyV3Input<'_>,
|
||||
markers: &HyV3ToolMarkers,
|
||||
) -> ModalResult<HyV3Event> {
|
||||
/// Parse a HY tool-block end marker.
|
||||
fn tool_block_end_event(input: &mut HyInput<'_>, markers: &HyToolMarkers) -> ModalResult<HyEvent> {
|
||||
(ws0, literal(markers.tool_calls_end.as_str()))
|
||||
.value(HyV3Event::ToolBlockEnd)
|
||||
.value(HyEvent::ToolBlockEnd)
|
||||
.parse_next(input)
|
||||
}
|
||||
|
||||
/// Parse a complete HY3 tool-call block.
|
||||
/// Parse a complete HY tool-call block.
|
||||
fn tool_call_event(
|
||||
input: &mut HyV3Input<'_>,
|
||||
input: &mut HyInput<'_>,
|
||||
tool_call_end_scan: &mut MarkerScanState,
|
||||
markers: &HyV3ToolMarkers,
|
||||
) -> ModalResult<HyV3Event> {
|
||||
let (name, body) = seq!(
|
||||
markers: &HyToolMarkers,
|
||||
dialect: HyDialect,
|
||||
) -> ModalResult<HyEvent> {
|
||||
let (body,) = seq!(
|
||||
_: ws0,
|
||||
_: literal(markers.tool_call_start.as_str()),
|
||||
take_until(0.., markers.tool_sep.as_str()),
|
||||
_: literal(markers.tool_sep.as_str()),
|
||||
take_until_marker(markers.tool_call_end.as_str(), tool_call_end_scan),
|
||||
_: literal(markers.tool_call_end.as_str()),
|
||||
)
|
||||
.parse_next(input)?;
|
||||
let raw_params = parse_tool_call_params(body, markers)?;
|
||||
let mut body_input = body;
|
||||
let (name, params) = parse_tool_call_body(&mut body_input, markers, dialect)?;
|
||||
let raw_params = parse_tool_call_params(params, markers)?;
|
||||
|
||||
Ok(HyV3Event::ToolCall {
|
||||
Ok(HyEvent::ToolCall {
|
||||
name: name.trim().to_string(),
|
||||
raw_params,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse all parameter blocks inside a complete HY3 tool call.
|
||||
/// Parse a complete HY tool-call body according to its dialect.
|
||||
fn parse_tool_call_body<'i>(
|
||||
input: &mut &'i str,
|
||||
markers: &HyToolMarkers,
|
||||
dialect: HyDialect,
|
||||
) -> ModalResult<(&'i str, &'i str)> {
|
||||
match dialect {
|
||||
HyDialect::V3 => {
|
||||
let tool_sep = markers.tool_sep.as_deref().expect("HY3 has a tool separator");
|
||||
terminated(
|
||||
seq!(
|
||||
take_until(0.., tool_sep),
|
||||
_: literal(tool_sep),
|
||||
rest,
|
||||
),
|
||||
eof,
|
||||
)
|
||||
.parse_next(input)
|
||||
}
|
||||
HyDialect::V4 => terminated(
|
||||
alt((
|
||||
seq!(take_until(0.., markers.arg_key_start.as_str()), rest,),
|
||||
rest.map(|name| (name, "")),
|
||||
)),
|
||||
eof,
|
||||
)
|
||||
.parse_next(input),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse all parameter blocks inside a complete HY tool call.
|
||||
fn parse_tool_call_params(
|
||||
tool_call_body: &str,
|
||||
markers: &HyV3ToolMarkers,
|
||||
markers: &HyToolMarkers,
|
||||
) -> ModalResult<Vec<(String, String)>> {
|
||||
let mut input = tool_call_body;
|
||||
delimited(
|
||||
@@ -304,8 +362,8 @@ fn parse_tool_call_params(
|
||||
.parse_next(&mut input)
|
||||
}
|
||||
|
||||
/// Parse a HY3 argument key/value block.
|
||||
fn parameter(input: &mut &str, markers: &HyV3ToolMarkers) -> ModalResult<(String, String)> {
|
||||
/// Parse a HY argument key/value block.
|
||||
fn parameter(input: &mut &str, markers: &HyToolMarkers) -> ModalResult<(String, String)> {
|
||||
let (name, value) = seq!(
|
||||
_: literal(markers.arg_key_start.as_str()),
|
||||
take_until(0.., markers.arg_key_end.as_str()),
|
||||
@@ -320,9 +378,9 @@ fn parameter(input: &mut &str, markers: &HyV3ToolMarkers) -> ModalResult<(String
|
||||
Ok((name.trim().to_string(), value.to_string()))
|
||||
}
|
||||
|
||||
/// Parse ignored rest after the HY3 tool block ends.
|
||||
fn ignored_rest_event(input: &mut HyV3Input<'_>) -> ModalResult<HyV3Event> {
|
||||
rest.value(HyV3Event::IgnoredRest).parse_next(input)
|
||||
/// Parse ignored rest after the HY tool block ends.
|
||||
fn ignored_rest_event(input: &mut HyInput<'_>) -> ModalResult<HyEvent> {
|
||||
rest.value(HyEvent::IgnoredRest).parse_next(input)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -331,7 +389,7 @@ mod tests {
|
||||
use serde_json::{Value, json};
|
||||
use thiserror_ext::AsReport;
|
||||
|
||||
use super::{HyV3ToolParser, ToolParser};
|
||||
use super::{HyDialect, HyToolParser, ToolParser};
|
||||
use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
|
||||
use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
|
||||
|
||||
@@ -354,14 +412,14 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_does_not_preserve_special_tokens() {
|
||||
let parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
|
||||
assert!(!parser.preserve_special_tokens());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hy_v3_parse_complete_without_tool_call_keeps_text() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let output = parser.parse_complete("This is a plain response.").unwrap();
|
||||
|
||||
assert_eq!(output.normal_text(), "This is a plain response.");
|
||||
@@ -370,7 +428,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_parse_complete_extracts_zero_arg_inline_tool_call() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let output = parser
|
||||
.parse_complete(
|
||||
"<tool_calls><tool_call>get_current_date<tool_sep></tool_call></tool_calls>",
|
||||
@@ -385,7 +443,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_parse_complete_extracts_zero_arg_newline_tool_call() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let output = parser
|
||||
.parse_complete(
|
||||
"<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>",
|
||||
@@ -398,7 +456,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_parse_complete_extracts_arguments_on_same_line() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let output = parser
|
||||
.parse_complete(
|
||||
"<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing</arg_value><arg_key>date</arg_key><arg_value>2026-03-30</arg_value></tool_call></tool_calls>",
|
||||
@@ -413,7 +471,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_parse_complete_extracts_arguments_with_newlines() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let output = parser
|
||||
.parse_complete(&build_tool_calls(&[build_tool_call(
|
||||
"get_weather",
|
||||
@@ -429,7 +487,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_parse_complete_preserves_prefix_and_ignores_trailing_text() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let output = parser
|
||||
.parse_complete(&format!(
|
||||
"Checking.{} trailing text",
|
||||
@@ -443,7 +501,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_parse_complete_extracts_multiple_tool_calls_in_one_block() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let output = parser
|
||||
.parse_complete(&build_tool_calls(&[
|
||||
build_tool_call(
|
||||
@@ -486,7 +544,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_parse_complete_converts_schema_types() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let output = parser
|
||||
.parse_complete(&build_tool_calls(&[build_tool_call(
|
||||
"convert",
|
||||
@@ -512,7 +570,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_streaming_without_tool_call_emits_text_incrementally() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let mut output = ToolParserOutput::default();
|
||||
|
||||
output.append(parser.parse_chunk("This is ").unwrap());
|
||||
@@ -526,7 +584,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_streaming_extracts_zero_arg_tool_call() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let chunks = [
|
||||
"<tool_calls>",
|
||||
"\n<tool_call>",
|
||||
@@ -545,7 +603,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_streaming_extracts_arguments() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let chunks = [
|
||||
"<tool_calls>",
|
||||
"\n<tool_call>",
|
||||
@@ -571,7 +629,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_streaming_preserves_prefix_text() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let chunks = [
|
||||
"Checking.",
|
||||
"<tool_calls>",
|
||||
@@ -601,7 +659,7 @@ mod tests {
|
||||
),
|
||||
]);
|
||||
let chunks = split_by_chars(&input, 9);
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
|
||||
let output = collect_stream(&mut parser, &chunks);
|
||||
|
||||
@@ -617,7 +675,7 @@ mod tests {
|
||||
build_tool_calls(&[build_tool_call("get_weather", &[("city", "Beijing")])])
|
||||
);
|
||||
let chunks = split_by_chars(&input, 5);
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
|
||||
let output = collect_stream(&mut parser, &chunks);
|
||||
|
||||
@@ -628,7 +686,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_streaming_does_not_emit_incomplete_tool_call() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let mut output = ToolParserOutput::default();
|
||||
|
||||
parser
|
||||
@@ -644,7 +702,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_finish_fails_incomplete_tool_call() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
parser.parse_chunk("<tool_calls><tool_call>get_weather<tool_sep>").unwrap();
|
||||
|
||||
let error = parser.finish().unwrap_err();
|
||||
@@ -655,7 +713,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn hy_v3_malformed_tool_call_fails_fast() {
|
||||
let mut parser = HyV3ToolParser::new(&test_tools(), "");
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V3);
|
||||
let error = parser
|
||||
.parse_complete(
|
||||
"<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing</tool_call></tool_calls>",
|
||||
@@ -664,4 +722,78 @@ mod tests {
|
||||
|
||||
assert!(error.to_report_string().starts_with("tool parser parsing failed:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hy_v4_parse_complete_extracts_compact_calls_with_and_without_arguments() {
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4);
|
||||
let output = parser
|
||||
.parse_complete(
|
||||
"<tool_calls><tool_call>get_weather<arg_key>city</arg_key><arg_value>Beijing</arg_value><arg_key>date</arg_key><arg_value>2026-03-30</arg_value></tool_call><tool_call>get_current_date</tool_call></tool_calls>",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
expect![[r#"
|
||||
ToolParserOutput {
|
||||
events: [
|
||||
ToolCall(
|
||||
ToolCallDelta {
|
||||
tool_index: 0,
|
||||
name: Some(
|
||||
"get_weather",
|
||||
),
|
||||
arguments: "{\"city\":\"Beijing\",\"date\":\"2026-03-30\"}",
|
||||
},
|
||||
),
|
||||
ToolCall(
|
||||
ToolCallDelta {
|
||||
tool_index: 1,
|
||||
name: Some(
|
||||
"get_current_date",
|
||||
),
|
||||
arguments: "{}",
|
||||
},
|
||||
),
|
||||
],
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&output);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hy_v4_streaming_handles_markers_split_across_chunks() {
|
||||
let input = "prefix<tool_calls><tool_call>get_weather<arg_key>city</arg_key><arg_value>上海</arg_value></tool_call><tool_call>get_current_date</tool_call></tool_calls>";
|
||||
let chunks = split_by_chars(input, 7);
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4);
|
||||
|
||||
let output = collect_stream(&mut parser, &chunks);
|
||||
|
||||
assert_eq!(output.normal_text(), "prefix");
|
||||
assert_eq!(output.calls().len(), 2);
|
||||
assert_eq!(parsed_arguments(&output, 0), json!({ "city": "上海" }));
|
||||
assert_eq!(output.calls()[1].name.as_deref(), Some("get_current_date"));
|
||||
assert_eq!(parsed_arguments(&output, 1), json!({}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hy_v4_finish_reports_its_dialect_for_incomplete_calls() {
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4);
|
||||
parser.parse_chunk("<tool_calls><tool_call>get_weather").unwrap();
|
||||
|
||||
let error = parser.finish().unwrap_err();
|
||||
|
||||
expect!["tool parser parsing failed: incomplete HY4 tool call"]
|
||||
.assert_eq(&error.to_report_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hy_v4_malformed_argument_pair_fails_fast() {
|
||||
let mut parser = HyToolParser::new(&test_tools(), "", HyDialect::V4);
|
||||
let error = parser
|
||||
.parse_complete(
|
||||
"<tool_calls><tool_call>get_weather<arg_key>city</arg_key>Beijing</tool_call></tool_calls>",
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.to_report_string().starts_with("tool parser parsing failed:"));
|
||||
}
|
||||
}
|
||||
+86
-23
@@ -1,7 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Structural-tag grammar for HY3 XML-style tool calls.
|
||||
//! Structural-tag grammar for HY XML-style tool calls.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
@@ -12,31 +12,40 @@ use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagConte
|
||||
use xgrammar_structural_tag::format::{Format, StructuralTag, TagFormat};
|
||||
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam};
|
||||
|
||||
use super::HyV3ToolMarkers;
|
||||
use super::{HyDialect, HyToolMarkers};
|
||||
|
||||
/// HY3 structural-tag builder using tokenizer-specific structural markers.
|
||||
/// HY structural-tag builder using tokenizer-specific structural markers.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct HyV3StructuralTagBuilder {
|
||||
markers: Arc<HyV3ToolMarkers>,
|
||||
pub(super) struct HyStructuralTagBuilder {
|
||||
markers: Arc<HyToolMarkers>,
|
||||
dialect: HyDialect,
|
||||
}
|
||||
|
||||
impl HyV3StructuralTagBuilder {
|
||||
pub(super) fn new(markers: Arc<HyV3ToolMarkers>) -> Self {
|
||||
Self { markers }
|
||||
impl HyStructuralTagBuilder {
|
||||
pub(super) fn new(markers: Arc<HyToolMarkers>, dialect: HyDialect) -> Self {
|
||||
Self { markers, dialect }
|
||||
}
|
||||
|
||||
fn argument_pair(&self, key: &str) -> Format {
|
||||
let excludes = self.markers.iter().collect::<Vec<_>>();
|
||||
Format::sequence(vec![
|
||||
let separator = self.dialect.separator();
|
||||
let mut elements = vec![
|
||||
Format::const_string(&self.markers.arg_key_start),
|
||||
Format::const_string(key),
|
||||
Format::const_string(&self.markers.arg_key_end),
|
||||
Format::const_string("\n"),
|
||||
];
|
||||
if !separator.is_empty() {
|
||||
elements.push(Format::const_string(separator));
|
||||
}
|
||||
elements.extend([
|
||||
Format::const_string(&self.markers.arg_value_start),
|
||||
Format::any_text_excluding(&excludes),
|
||||
Format::const_string(&self.markers.arg_value_end),
|
||||
Format::const_string("\n"),
|
||||
])
|
||||
]);
|
||||
if !separator.is_empty() {
|
||||
elements.push(Format::const_string(separator));
|
||||
}
|
||||
Format::sequence(elements)
|
||||
}
|
||||
|
||||
fn tool_call(&self, tool: &FunctionToolParam) -> TagFormat {
|
||||
@@ -56,14 +65,21 @@ impl HyV3StructuralTagBuilder {
|
||||
}
|
||||
|
||||
let content = if elements.is_empty() {
|
||||
Format::any_text()
|
||||
match self.dialect {
|
||||
HyDialect::V3 => Format::any_text(),
|
||||
HyDialect::V4 => Format::const_string(""),
|
||||
}
|
||||
} else {
|
||||
Format::sequence(elements)
|
||||
};
|
||||
let tool_sep = self.markers.tool_sep.as_deref().unwrap_or_default();
|
||||
TagFormat::new(
|
||||
format!(
|
||||
"{}{}{}\n",
|
||||
self.markers.tool_call_start, tool.function.name, self.markers.tool_sep
|
||||
"{}{}{}{}",
|
||||
self.markers.tool_call_start,
|
||||
tool.function.name,
|
||||
tool_sep,
|
||||
self.dialect.separator(),
|
||||
),
|
||||
content,
|
||||
self.markers.tool_call_end.clone(),
|
||||
@@ -72,15 +88,16 @@ impl HyV3StructuralTagBuilder {
|
||||
|
||||
fn tool_calls(&self, tools: &[FunctionToolParam], choice: BuilderToolChoice) -> Format {
|
||||
let mut calls = tools.iter().map(|tool| self.tool_call(tool)).collect::<Vec<_>>();
|
||||
let begin = format!("{}\n", self.markers.tool_calls_start);
|
||||
let end = format!("\n{}", self.markers.tool_calls_end);
|
||||
let separator = self.dialect.separator();
|
||||
let begin = format!("{}{separator}", self.markers.tool_calls_start);
|
||||
let end = format!("{separator}{}", self.markers.tool_calls_end);
|
||||
|
||||
match choice {
|
||||
BuilderToolChoice::Auto if calls.is_empty() => Format::any_text(),
|
||||
BuilderToolChoice::Auto => {
|
||||
let outer = TagFormat::new(
|
||||
begin,
|
||||
Format::tags_with_separator(calls, "\n", true, false),
|
||||
Format::tags_with_separator(calls, separator, true, false),
|
||||
end,
|
||||
);
|
||||
Format::triggered_tags(&[&self.markers.tool_calls_start], vec![outer])
|
||||
@@ -92,14 +109,14 @@ impl HyV3StructuralTagBuilder {
|
||||
]),
|
||||
BuilderToolChoice::Required => Format::sequence(vec![
|
||||
Format::const_string(begin),
|
||||
Format::tags_with_separator(calls, "\n", true, false),
|
||||
Format::tags_with_separator(calls, separator, true, false),
|
||||
Format::const_string(end),
|
||||
]),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl StructuralTagBuilder for HyV3StructuralTagBuilder {
|
||||
impl StructuralTagBuilder for HyStructuralTagBuilder {
|
||||
fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
|
||||
Ok(StructuralTag::new(
|
||||
self.tool_calls(ctx.function_tools, ctx.tool_choice),
|
||||
@@ -152,8 +169,8 @@ mod tests {
|
||||
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
|
||||
};
|
||||
|
||||
use super::HyV3StructuralTagBuilder;
|
||||
use crate::tool::HyV3ToolMarkers;
|
||||
use super::HyStructuralTagBuilder;
|
||||
use crate::tool::{HyDialect, HyToolMarkers};
|
||||
|
||||
fn tool(name: &str, parameters: Value) -> ToolParam {
|
||||
ToolParam::Function(FunctionToolParam::new(
|
||||
@@ -165,9 +182,18 @@ mod tests {
|
||||
suffix: &str,
|
||||
tools: &[ToolParam],
|
||||
choice: ToolChoice,
|
||||
) -> xgrammar_structural_tag::format::StructuralTag {
|
||||
build_for_dialect(HyDialect::V3, suffix, tools, choice)
|
||||
}
|
||||
|
||||
fn build_for_dialect(
|
||||
dialect: HyDialect,
|
||||
suffix: &str,
|
||||
tools: &[ToolParam],
|
||||
choice: ToolChoice,
|
||||
) -> xgrammar_structural_tag::format::StructuralTag {
|
||||
build_structural_tag(
|
||||
HyV3StructuralTagBuilder::new(Arc::new(HyV3ToolMarkers::new(suffix))),
|
||||
HyStructuralTagBuilder::new(Arc::new(HyToolMarkers::new(suffix, dialect)), dialect),
|
||||
tools,
|
||||
choice,
|
||||
StructuralTagOptions::default().with_reasoning(false),
|
||||
@@ -196,6 +222,25 @@ mod tests {
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>\n"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather<tool_sep:opensource>\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"days"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]}}]},"end":"</tool_call:opensource>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_uses_suffixed_compact_hy4_skeleton_and_bounded_values() {
|
||||
let tag = build_for_dialect(
|
||||
HyDialect::V4,
|
||||
":opensource",
|
||||
&[tool(
|
||||
"get_weather",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": { "city": { "type": "string" } },
|
||||
"required": ["city"]
|
||||
}),
|
||||
)],
|
||||
ToolChoice::required(),
|
||||
);
|
||||
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"}]}]},"end":"</tool_call:opensource>"}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optional_only_schema_keeps_declared_key_alternatives() {
|
||||
let tag = build(
|
||||
@@ -235,6 +280,24 @@ mod tests {
|
||||
expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls>\n"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"type":"const_string","value":"\n</tool_calls>"}]}}"#]].assert_eq(&forced);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hy_v4_zero_argument_tool_has_empty_compact_body() {
|
||||
let tag = build_for_dialect(
|
||||
HyDialect::V4,
|
||||
"",
|
||||
&[tool(
|
||||
"get_current_date",
|
||||
json!({ "type": "object", "properties": {} }),
|
||||
)],
|
||||
ToolChoice::function("get_current_date"),
|
||||
);
|
||||
let value = serde_json::to_value(tag).unwrap();
|
||||
|
||||
expect![[r#"{"type":"tag","begin":"<tool_call>get_current_date","content":{"type":"const_string","value":""},"end":"</tool_call>"}"#]].assert_eq(
|
||||
&serde_json::to_string(&value["format"]["elements"][1]).unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_required_property_is_still_emitted() {
|
||||
let tag = build(
|
||||
@@ -8,7 +8,7 @@ pub(crate) mod error;
|
||||
mod deepseek_dsml;
|
||||
pub(crate) mod deepseek_json;
|
||||
mod glm_xml;
|
||||
mod hy_v3;
|
||||
mod hy;
|
||||
pub(crate) mod json;
|
||||
mod kimi_k2;
|
||||
mod minimax_m2;
|
||||
@@ -24,7 +24,7 @@ pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser};
|
||||
pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser};
|
||||
pub use error::{Result, ToolParserError};
|
||||
pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser};
|
||||
pub(crate) use hy_v3::{HyV3ToolMarkers, HyV3ToolParser};
|
||||
pub(crate) use hy::{HyDialect, HyToolMarkers, HyToolParser};
|
||||
pub use json::{
|
||||
Granite4ToolParser, HermesToolParser, Internlm2ToolParser, Llama3JsonToolParser,
|
||||
MistralToolParser, Phi4MiniJsonToolParser, Qwen3XmlToolParser,
|
||||
|
||||
@@ -1,20 +1,12 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use vllm_tokenizer::{DynTokenizer, Tokenizer};
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use super::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
|
||||
use crate::reasoning::HyV3ReasoningParser;
|
||||
use crate::tool::{HyV3ToolMarkers, HyV3ToolParser, StructuralTagBuilder, Tool};
|
||||
|
||||
const HY_V3_MARKER_STEMS: &[&str] = &[
|
||||
"think",
|
||||
"tool_calls",
|
||||
"tool_call",
|
||||
"tool_sep",
|
||||
"arg_key",
|
||||
"arg_value",
|
||||
];
|
||||
use super::detect_hy_token_suffix;
|
||||
use crate::reasoning::HyReasoningParser;
|
||||
use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, StructuralTagBuilder, Tool};
|
||||
use crate::unified::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
|
||||
|
||||
/// Unified reasoning and tool parser for HY3 output.
|
||||
pub struct HyV3UnifiedParser {
|
||||
@@ -24,14 +16,14 @@ pub struct HyV3UnifiedParser {
|
||||
impl HyV3UnifiedParser {
|
||||
/// Create a HY3 parser using the suffix encoded in tokenizer added tokens.
|
||||
pub fn new(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
|
||||
let suffix = detect_token_suffix(tokenizer.as_ref());
|
||||
let markers = HyV3ToolMarkers::new(&suffix);
|
||||
let suffix = detect_hy_token_suffix(tokenizer.as_ref());
|
||||
let markers = HyToolMarkers::new(&suffix, HyDialect::V3);
|
||||
for marker in markers.iter() {
|
||||
token_id(tokenizer.as_ref(), marker)?;
|
||||
}
|
||||
|
||||
let reasoning = HyV3ReasoningParser::new(tokenizer, &suffix)?;
|
||||
let tool = HyV3ToolParser::new(tools, &suffix);
|
||||
let reasoning = HyReasoningParser::new(tokenizer, &suffix)?;
|
||||
let tool = HyToolParser::new(tools, &suffix, HyDialect::V3);
|
||||
Ok(Self {
|
||||
inner: CombinedParser::new(Some(Box::new(reasoning)), Some(Box::new(tool))),
|
||||
})
|
||||
@@ -75,28 +67,6 @@ impl UnifiedParser for HyV3UnifiedParser {
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect the HY3 structural-token suffix from tokenizer added vocabulary.
|
||||
fn detect_token_suffix(tokenizer: &dyn Tokenizer) -> String {
|
||||
tokenizer
|
||||
.added_vocab()
|
||||
.iter()
|
||||
.filter_map(|(token, id)| marker_suffix(token).map(|suffix| (*id, suffix)))
|
||||
.min_by_key(|(id, _)| *id)
|
||||
.map(|(_, suffix)| suffix.to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Extract the suffix from one opening or closing HY3 structural token.
|
||||
fn marker_suffix(token: &str) -> Option<&str> {
|
||||
let body = token.strip_prefix('<')?.strip_suffix('>')?;
|
||||
let body = body.strip_prefix('/').unwrap_or(body);
|
||||
|
||||
HY_V3_MARKER_STEMS.iter().find_map(|stem| {
|
||||
let suffix = body.strip_prefix(stem)?;
|
||||
(suffix.is_empty() || suffix.starts_with(':')).then_some(suffix)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
@@ -0,0 +1,194 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use vllm_tokenizer::DynTokenizer;
|
||||
|
||||
use super::detect_hy_token_suffix;
|
||||
use crate::reasoning::HyReasoningParser;
|
||||
use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, StructuralTagBuilder, Tool};
|
||||
use crate::unified::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
|
||||
|
||||
/// Unified reasoning and tool parser for HY4 output.
|
||||
pub struct HyV4UnifiedParser {
|
||||
inner: CombinedParser,
|
||||
}
|
||||
|
||||
impl HyV4UnifiedParser {
|
||||
/// Create a HY4 parser using the suffix encoded in tokenizer added tokens.
|
||||
pub fn new(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
|
||||
let suffix = detect_hy_token_suffix(tokenizer.as_ref());
|
||||
let markers = HyToolMarkers::new(&suffix, HyDialect::V4);
|
||||
for marker in markers.iter() {
|
||||
token_id(tokenizer.as_ref(), marker)?;
|
||||
}
|
||||
|
||||
let reasoning = HyReasoningParser::new(tokenizer, &suffix)?;
|
||||
let tool = HyToolParser::new(tools, &suffix, HyDialect::V4);
|
||||
Ok(Self {
|
||||
inner: CombinedParser::new(Some(Box::new(reasoning)), Some(Box::new(tool))),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl UnifiedParser for HyV4UnifiedParser {
|
||||
fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
|
||||
where
|
||||
Self: Sized + 'static,
|
||||
{
|
||||
Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box<dyn UnifiedParser>)
|
||||
}
|
||||
|
||||
fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
|
||||
self.inner.initialize(prompt_token_ids)
|
||||
}
|
||||
|
||||
fn preserve_special_tokens(&self) -> bool {
|
||||
self.inner.preserve_special_tokens()
|
||||
}
|
||||
|
||||
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
|
||||
self.inner.structural_tag_builder()
|
||||
}
|
||||
|
||||
fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
|
||||
self.inner.tool_call_id(tool_index)
|
||||
}
|
||||
|
||||
fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()> {
|
||||
self.inner.parse_into(delta, output)
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<UnifiedParserOutput> {
|
||||
self.inner.finish()
|
||||
}
|
||||
|
||||
fn reset(&mut self) -> String {
|
||||
self.inner.reset()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::json;
|
||||
use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer};
|
||||
use xgrammar_structural_tag::builders::StructuralTagOptions;
|
||||
use xgrammar_structural_tag::{
|
||||
FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
|
||||
};
|
||||
|
||||
use super::{HyV4UnifiedParser, UnifiedParser};
|
||||
use crate::tool::Tool;
|
||||
use crate::unified::{UnifiedParserEvent, UnifiedParserOutput};
|
||||
|
||||
fn tokenizer() -> TestTokenizer {
|
||||
[
|
||||
"<think:opensource>",
|
||||
"</think:opensource>",
|
||||
"<tool_calls:opensource>",
|
||||
"</tool_calls:opensource>",
|
||||
"<tool_call:opensource>",
|
||||
"</tool_call:opensource>",
|
||||
"<arg_key:opensource>",
|
||||
"</arg_key:opensource>",
|
||||
"<arg_value:opensource>",
|
||||
"</arg_value:opensource>",
|
||||
]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.fold(TestTokenizer::new(), |tokenizer, (index, token)| {
|
||||
tokenizer.with_regular_token(token, 1000 + index as u32)
|
||||
})
|
||||
}
|
||||
|
||||
fn tools() -> Vec<Tool> {
|
||||
vec![
|
||||
Tool {
|
||||
name: "get_weather".to_string(),
|
||||
description: None,
|
||||
parameters: json!({
|
||||
"type": "object",
|
||||
"properties": { "city": { "type": "string" } },
|
||||
}),
|
||||
strict: None,
|
||||
},
|
||||
Tool {
|
||||
name: "get_current_date".to_string(),
|
||||
description: None,
|
||||
parameters: json!({ "type": "object", "properties": {} }),
|
||||
strict: None,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_suffixed_reasoning_and_compact_tool_calls_as_one_stream() {
|
||||
let tokenizer = Arc::new(tokenizer());
|
||||
let think_start_id = tokenizer.token_to_id("<think:opensource>").unwrap();
|
||||
let mut parser = HyV4UnifiedParser::new(&tools(), tokenizer).unwrap();
|
||||
parser.initialize(&[think_start_id]).unwrap();
|
||||
|
||||
let chunks = [
|
||||
"reasoning</think:open",
|
||||
"source>answer<tool_calls:opensource><tool_call:opensource>get_weather",
|
||||
"<arg_key:opensource>city</arg_key:opensource>",
|
||||
"<arg_value:opensource>Beijing</arg_value:opensource>",
|
||||
"</tool_call:opensource><tool_call:opensource>get_current_date",
|
||||
"</tool_call:opensource></tool_calls:opensource>",
|
||||
];
|
||||
let mut output = UnifiedParserOutput::default();
|
||||
for chunk in chunks {
|
||||
parser.parse_into(chunk, &mut output).unwrap();
|
||||
}
|
||||
output.append(parser.finish().unwrap());
|
||||
|
||||
assert_eq!(
|
||||
output.events,
|
||||
vec![
|
||||
UnifiedParserEvent::Reasoning("reasoning".to_string()),
|
||||
UnifiedParserEvent::Text("answer".to_string()),
|
||||
UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
|
||||
tool_index: 0,
|
||||
name: Some("get_weather".to_string()),
|
||||
arguments: r#"{"city":"Beijing"}"#.to_string(),
|
||||
}),
|
||||
UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
|
||||
tool_index: 1,
|
||||
name: Some("get_current_date".to_string()),
|
||||
arguments: "{}".to_string(),
|
||||
}),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn structural_tag_uses_compact_hy4_skeleton() {
|
||||
let parser = HyV4UnifiedParser::new(&tools(), Arc::new(tokenizer())).unwrap();
|
||||
let structural_tools = [ToolParam::Function(FunctionToolParam::new(
|
||||
FunctionDefinition::new("get_weather").with_parameters(json!({
|
||||
"type": "object",
|
||||
"properties": { "city": { "type": "string" } },
|
||||
"required": ["city"]
|
||||
})),
|
||||
))];
|
||||
|
||||
let tag = build_structural_tag(
|
||||
parser.structural_tag_builder().unwrap(),
|
||||
&structural_tools,
|
||||
ToolChoice::required(),
|
||||
StructuralTagOptions::default().with_reasoning(false),
|
||||
)
|
||||
.unwrap()
|
||||
.to_json_string()
|
||||
.unwrap();
|
||||
|
||||
assert!(tag.contains("<tool_calls:opensource>"));
|
||||
assert!(tag.contains("<tool_call:opensource>get_weather"));
|
||||
assert!(tag.contains("<arg_key:opensource>"));
|
||||
assert!(tag.contains("</tool_call:opensource>"));
|
||||
assert!(tag.contains("</tool_calls:opensource>"));
|
||||
assert!(!tag.contains("tool_sep"));
|
||||
assert!(!tag.contains(r#"\n"#));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
mod hy_v3;
|
||||
mod hy_v4;
|
||||
|
||||
pub use hy_v3::HyV3UnifiedParser;
|
||||
pub use hy_v4::HyV4UnifiedParser;
|
||||
|
||||
const HY_MARKER_STEMS: &[&str] = &[
|
||||
"think",
|
||||
"tool_calls",
|
||||
"tool_call",
|
||||
"tool_sep",
|
||||
"arg_key",
|
||||
"arg_value",
|
||||
];
|
||||
|
||||
/// Detect the HY structural-token suffix from tokenizer added vocabulary.
|
||||
fn detect_hy_token_suffix(tokenizer: &dyn vllm_tokenizer::Tokenizer) -> String {
|
||||
tokenizer
|
||||
.added_vocab()
|
||||
.iter()
|
||||
.filter_map(|(token, id)| hy_marker_suffix(token).map(|suffix| (*id, suffix)))
|
||||
.min_by_key(|(id, _)| *id)
|
||||
.map(|(_, suffix)| suffix.to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Extract the suffix from one opening or closing HY structural token.
|
||||
fn hy_marker_suffix(token: &str) -> Option<&str> {
|
||||
let body = token.strip_prefix('<')?.strip_suffix('>')?;
|
||||
let body = body.strip_prefix('/').unwrap_or(body);
|
||||
|
||||
HY_MARKER_STEMS.iter().find_map(|stem| {
|
||||
let suffix = body.strip_prefix(stem)?;
|
||||
(suffix.is_empty() || suffix.starts_with(':')).then_some(suffix)
|
||||
})
|
||||
}
|
||||
@@ -5,13 +5,13 @@
|
||||
|
||||
mod combined;
|
||||
mod gemma4;
|
||||
mod hy_v3;
|
||||
mod hy;
|
||||
mod inkling;
|
||||
mod kimi_k3;
|
||||
|
||||
pub use combined::CombinedParser;
|
||||
pub use gemma4::Gemma4UnifiedParser;
|
||||
pub use hy_v3::HyV3UnifiedParser;
|
||||
pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser};
|
||||
pub use inkling::InklingUnifiedParser;
|
||||
pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser};
|
||||
use thiserror::Error;
|
||||
|
||||
@@ -331,6 +331,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
|
||||
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"),
|
||||
"HYV3ForCausalLM": _HfExamplesInfo("tencent/Hy3-preview", trust_remote_code=True),
|
||||
# Internal checkpoint, not publicly available yet.
|
||||
"HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True),
|
||||
"HyperCLOVAXForCausalLM": _HfExamplesInfo(
|
||||
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
|
||||
min_transformers_version="5.9.0",
|
||||
@@ -1723,6 +1725,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"tencent/Hy3-preview",
|
||||
speculative_model="tencent/Hy3-preview",
|
||||
),
|
||||
# Internal checkpoint, not publicly available yet.
|
||||
"HYV4MTPModel": _HfExamplesInfo(
|
||||
"tencent/Hy4-preview",
|
||||
speculative_model="tencent/Hy4-preview",
|
||||
),
|
||||
"InklingMTPModel": _HfExamplesInfo(
|
||||
"thinkingmachines/Inkling-NVFP4",
|
||||
speculative_model="thinkingmachines/Inkling-NVFP4",
|
||||
|
||||
@@ -59,6 +59,11 @@ def test_registry_imports(model_arch):
|
||||
):
|
||||
pytest.skip("Dots3 NOTE is only supported on CUDA")
|
||||
|
||||
if model_arch in ("HYV4ForCausalLM", "HYV4MTPModel") and not (
|
||||
current_platform.is_cuda()
|
||||
):
|
||||
pytest.skip("HY V4 is only supported on CUDA")
|
||||
|
||||
# Ensure all model classes can be imported successfully
|
||||
model_cls = ModelRegistry._try_load_model_cls(model_arch)
|
||||
assert model_cls is not None
|
||||
|
||||
@@ -552,6 +552,11 @@ def dummy_hf_overrides(
|
||||
|
||||
text_config.update(update_dict)
|
||||
|
||||
# Keep per-layer metadata consistent with the reduced layer count.
|
||||
# HY V4 uses this to decide which sparse-attention indexers are shared.
|
||||
if indexer_types := getattr(text_config, "indexer_types", None):
|
||||
text_config.update({"indexer_types": indexer_types[:num_hidden_layers]})
|
||||
|
||||
# Update n_layers and moe configs for Moondream3 model
|
||||
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
|
||||
text_config.update(
|
||||
|
||||
@@ -57,6 +57,7 @@ MTPModelTypes = Literal[
|
||||
"pangu_ultra_moe_mtp",
|
||||
"step3p5_mtp",
|
||||
"hy_v3_mtp",
|
||||
"hy_v4_mtp",
|
||||
"gemma4_mtp",
|
||||
"inkling_mtp",
|
||||
]
|
||||
@@ -917,6 +918,13 @@ class SpeculativeConfig:
|
||||
{"n_predict": n_predict, "architectures": ["HYV3MTPModel"]}
|
||||
)
|
||||
|
||||
if hf_config.model_type == "hy_v4":
|
||||
hf_config.model_type = "hy_v4_mtp"
|
||||
n_predict = getattr(hf_config, "num_nextn_predict_layers", None)
|
||||
hf_config.update(
|
||||
{"n_predict": n_predict, "architectures": ["HYV4MTPModel"]}
|
||||
)
|
||||
|
||||
if hf_config.model_type in ("inkling_mm_model", "inkling_model"):
|
||||
mtp_config = getattr(hf_config, "mtp_config", None) or {}
|
||||
hf_config = getattr(hf_config, "text_config", hf_config)
|
||||
|
||||
@@ -79,6 +79,8 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset(
|
||||
"DeepseekV4ForCausalLM",
|
||||
"DeepSeekV4MTPModel",
|
||||
"GlmMoeDsaForCausalLM",
|
||||
"HYV4ForCausalLM",
|
||||
"HYV4MTPModel",
|
||||
"InklingForCausalLM",
|
||||
"InklingForConditionalGeneration",
|
||||
"KimiK3ForConditionalGeneration",
|
||||
|
||||
@@ -321,6 +321,7 @@ if TYPE_CHECKING:
|
||||
VLLM_GPU_NIC_PCIE_MAPPING: str = ""
|
||||
VLLM_NIC_SELECTION_VARS: str = ""
|
||||
VLLM_PREFIX_CACHE_RETENTION_INTERVAL: int | None = None
|
||||
VLLM_ENABLE_HPC_OPS: bool = False
|
||||
|
||||
|
||||
def get_default_cache_root():
|
||||
@@ -2155,6 +2156,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Each entry is VAR_NAME or VAR_NAME:<suffix> (suffix appended to
|
||||
# RDMA device name). Must be set together with VLLM_GPU_NIC_PCIE_MAPPING.
|
||||
"VLLM_NIC_SELECTION_VARS": lambda: os.getenv("VLLM_NIC_SELECTION_VARS", ""),
|
||||
# If set to 1, enable the HPC fused kernels (requires the hpc package
|
||||
# (.so) and an sm100/sm103 device). Covers:
|
||||
# * the HY V4 iHC ops -- each of the eager HYV4HCPreLayer /
|
||||
# HYV4HCPostLayer / HYV4HCHeadLayer bodies becomes one kernel launch;
|
||||
# * the gated-MLA output gating (attn_out * sigmoid(gate projection)),
|
||||
# fused into gated_mla_gemm; elementwise gating only.
|
||||
# Each op additionally checks its own shape / dtype constraints and falls
|
||||
# back to the eager path when they do not hold.
|
||||
"VLLM_ENABLE_HPC_OPS": lambda: bool(int(os.getenv("VLLM_ENABLE_HPC_OPS", "0"))),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -79,12 +79,27 @@ class TrtLlmFp8ExpertsBase:
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
if not supported or moe_config.num_experts <= 2048:
|
||||
if not supported:
|
||||
return supported, reason
|
||||
return False, (
|
||||
"FlashInfer TRTLLM routing supports at most 2048 experts, "
|
||||
f"but got {moe_config.num_experts}"
|
||||
)
|
||||
# FlashInfer accepts gemm1_alpha/beta/clamp_limit only for MXFP8; the
|
||||
# DeepSeek-FP8 block-scale kernel rejects them and the per-tensor kernel
|
||||
# has no clamp at all, so a model-requested SwiGLU clamp cannot be
|
||||
# honored on those paths.
|
||||
if (
|
||||
moe_config.swiglu_limit is not None
|
||||
or moe_config.swiglu_alpha is not None
|
||||
or moe_config.swiglu_beta is not None
|
||||
) and (weight_key, activation_key) != (kMxfp8Static, kMxfp8Dynamic):
|
||||
return False, (
|
||||
"the TRTLLM FP8 kernels apply the SwiGLU alpha/beta/clamp "
|
||||
"parameters only for MXFP8 weights"
|
||||
)
|
||||
if moe_config.num_experts > 2048:
|
||||
return False, (
|
||||
"FlashInfer TRTLLM routing supports at most 2048 experts, "
|
||||
f"but got {moe_config.num_experts}"
|
||||
)
|
||||
return True, None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -1,11 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.layers.hpc.gated_mla import (
|
||||
hpc_gated_mla_gemm,
|
||||
hpc_gated_mla_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.hpc.hpc_ihc import (
|
||||
HpcIHCHead,
|
||||
HpcIHCPost,
|
||||
HpcIHCPre,
|
||||
)
|
||||
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
|
||||
from vllm.model_executor.layers.hpc.rope_norm import HpcRopeNorm, QkNormPolicy
|
||||
|
||||
__all__ = [
|
||||
"HpcIHCHead",
|
||||
"HpcIHCPost",
|
||||
"HpcIHCPre",
|
||||
"HpcModule",
|
||||
"HpcRopeNorm",
|
||||
"QkNormPolicy",
|
||||
"hpc_gated_mla_gemm",
|
||||
"hpc_gated_mla_supported",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Enablement gate for the HPC fused gated-MLA GEMM.
|
||||
|
||||
``hpc.gated_mla_gemm`` fuses the MLA output gating -- the gate projection, the
|
||||
sigmoid and the elementwise product -- into a single kernel launch, replacing
|
||||
the eager ``attn_out * sigmoid(hidden_states @ weight.T)`` sequence.
|
||||
|
||||
Kernel constraints:
|
||||
- Requires VLLM_ENABLE_HPC_OPS=1
|
||||
- Requires the hpc package (.so) built for the current arch
|
||||
- Only sm100 / sm103 (compute capability 100, 103)
|
||||
- All three operands must be bfloat16 and contiguous
|
||||
- The gate width must be a multiple of 256
|
||||
- Only elementwise gating (headwise has no fused form)
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.hpc import has_hpc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# hpc.gated_mla_gemm fuses `attn_out * sigmoid(x @ weight.T)` into one launch.
|
||||
# It is built for sm100/sm103 and needs all three operands bf16 + contiguous,
|
||||
# with the gate width a multiple of 256.
|
||||
_HPC_GATED_GEMM_N_ALIGN = 256
|
||||
_HPC_GATED_GEMM_CAPABILITIES: frozenset[int] = frozenset({100, 103})
|
||||
|
||||
|
||||
def hpc_gated_mla_supported(
|
||||
gating_type: str | None, attn_output_gate: torch.nn.Module | None
|
||||
) -> bool:
|
||||
"""Gate for running the MLA output gating through hpc.gated_mla_gemm."""
|
||||
if not envs.VLLM_ENABLE_HPC_OPS:
|
||||
return False
|
||||
|
||||
if attn_output_gate is None:
|
||||
return False
|
||||
|
||||
if not has_hpc():
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: 'hpc' package is not installed. "
|
||||
"Please install the HPC library (.so) to enable the fused kernel."
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
logger.warning_once("HPC gated MLA disabled: only CUDA is supported.")
|
||||
return False
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.to_int() not in _HPC_GATED_GEMM_CAPABILITIES:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: compute capability %s not in %s.",
|
||||
capability,
|
||||
_HPC_GATED_GEMM_CAPABILITIES,
|
||||
)
|
||||
return False
|
||||
|
||||
# Headwise gating broadcasts one scalar per head over v_head_dim; the
|
||||
# kernel only implements the elementwise product.
|
||||
if gating_type != "elementwise":
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gating_type '%s' has no fused form.",
|
||||
gating_type,
|
||||
)
|
||||
return False
|
||||
|
||||
weight = getattr(attn_output_gate, "weight", None)
|
||||
if weight is None or weight.dtype != torch.bfloat16:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gate weight dtype is %s, the kernel needs "
|
||||
"bfloat16 (keep the layer out of quantization).",
|
||||
None if weight is None else weight.dtype,
|
||||
)
|
||||
return False
|
||||
|
||||
if weight.shape[0] % _HPC_GATED_GEMM_N_ALIGN != 0:
|
||||
logger.warning_once(
|
||||
"HPC gated MLA disabled: gate width %d is not a multiple of %d.",
|
||||
weight.shape[0],
|
||||
_HPC_GATED_GEMM_N_ALIGN,
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info_once("HPC gated MLA enabled by set VLLM_ENABLE_HPC_OPS.")
|
||||
return True
|
||||
|
||||
|
||||
def hpc_gated_mla_gemm(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
atten_output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Fused ``atten_output * sigmoid(x @ weight.T)``.
|
||||
|
||||
Args:
|
||||
x: ``[m, k]`` bfloat16, contiguous.
|
||||
weight: ``[n, k]`` bfloat16, contiguous and row-major (the kernel
|
||||
transposes it). ``n`` must be a multiple of 256.
|
||||
atten_output: ``[m, n]`` bfloat16, contiguous.
|
||||
|
||||
Returns:
|
||||
``[m, n]`` bfloat16.
|
||||
"""
|
||||
from hpc.gemm import gated_mla_gemm
|
||||
|
||||
return gated_mla_gemm(x, weight, atten_output)
|
||||
@@ -0,0 +1,261 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HPC fused iHC (independent Hyper-Connections) kernels for HY V4.
|
||||
|
||||
Replaces the eager HYV4HCPreLayer / HYV4HCPostLayer / HYV4HCHeadLayer bodies
|
||||
with single-kernel HPC implementations:
|
||||
|
||||
pre : rms square sum + w projection + sigmoid gates + hc-dim weighted sum
|
||||
post : y[n, i, :] = H_post[n, i] * x[n, :] + residual[n, i, :]
|
||||
head : same as pre but hc_mult projection rows and no H_post output
|
||||
|
||||
The eager path issues 20 / 5 / 15 kernels for pre / post / head; each HPC op is
|
||||
a single kernel.
|
||||
|
||||
Constraints:
|
||||
- Requires VLLM_ENABLE_HPC_OPS=1
|
||||
- Requires the hpc package (.so) built for the current arch
|
||||
- Only sm100 / sm103 (compute capability 100, 103)
|
||||
- Only hc_mult == 4 and hidden_size in {4096, 6144}
|
||||
- x / residual must be bfloat16; hc_fn weights stay float32 (the checkpoint
|
||||
keeps them out of fp8 quantization via modules_to_not_convert)
|
||||
|
||||
The fused modules own no parameters: they hold references to the weights of the
|
||||
eager layer they replace, bound in ``process_weights_after_loading`` (invoked
|
||||
generically for every ``HpcModule`` by the model loader).
|
||||
|
||||
NOTE: The reference implementation also offers a cross-layer post+pre fusion
|
||||
(``HpcIHCPostPre``) that folds one segment's post into the next segment's pre.
|
||||
It requires reworking the decoder-layer forward scheduling and is not ported
|
||||
here. TODO: add it once the decoder dataflow is restructured for it.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.hpc.hpc_module import HpcModule
|
||||
from vllm.utils.hpc import has_hpc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Shapes the HPC kernels are instantiated for. Other shapes launch nothing,
|
||||
# so gate on this rather than relying on a silent no-op.
|
||||
_SUPPORTED_HIDDEN_SIZES: frozenset[int] = frozenset({4096, 6144})
|
||||
_SUPPORTED_HC_MULTS: frozenset[int] = frozenset({4})
|
||||
_SUPPORTED_CAPABILITIES: frozenset[int] = frozenset({100, 103})
|
||||
|
||||
|
||||
def _ihc_supported(hc_mult: int, hidden_size: int) -> bool:
|
||||
"""Shared gate for all three iHC ops."""
|
||||
if not envs.VLLM_ENABLE_HPC_OPS:
|
||||
return False
|
||||
|
||||
if not has_hpc():
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: 'hpc' package is not installed. "
|
||||
"Please install the HPC library (.so) to enable fused kernels."
|
||||
)
|
||||
return False
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
logger.warning_once("HPC iHC disabled: only CUDA is supported.")
|
||||
return False
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.to_int() not in _SUPPORTED_CAPABILITIES:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: compute capability %s not in %s.",
|
||||
capability,
|
||||
_SUPPORTED_CAPABILITIES,
|
||||
)
|
||||
return False
|
||||
|
||||
if hc_mult not in _SUPPORTED_HC_MULTS:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: hc_mult=%d not in %s.", hc_mult, _SUPPORTED_HC_MULTS
|
||||
)
|
||||
return False
|
||||
|
||||
if hidden_size not in _SUPPORTED_HIDDEN_SIZES:
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: hidden_size=%d not in %s.",
|
||||
hidden_size,
|
||||
_SUPPORTED_HIDDEN_SIZES,
|
||||
)
|
||||
return False
|
||||
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
# Batch-invariant mode relies on torch's deterministic reductions; the
|
||||
# HPC kernel reduces in a different order, so keep the two regimes
|
||||
# apart.
|
||||
logger.warning_once(
|
||||
"HPC iHC disabled: not supported under VLLM_BATCH_INVARIANT."
|
||||
)
|
||||
return False
|
||||
|
||||
logger.info_once("HPC iHC enabled by set VLLM_ENABLE_HPC_OPS.")
|
||||
return True
|
||||
|
||||
|
||||
class HpcIHCPre(HpcModule):
|
||||
"""Fused iHC pre block.
|
||||
|
||||
Computes, in one kernel:
|
||||
x_flat = x.flatten(1)
|
||||
r = rsqrt(x_flat.square().mean(-1) + rms_norm_eps)
|
||||
mixes = (x_flat @ w.T) * r
|
||||
H_pre = sigmoid(mixes[:, :hc] * hc_scale[0] + hc_base[:hc]) + hc_eps
|
||||
H_post = magnitude * sigmoid(
|
||||
mixes[:, hc:] * hc_scale[1] + hc_base[hc:]) + hc_eps
|
||||
y = sum_i H_pre[:, i] * x[:, i, :]
|
||||
|
||||
Args:
|
||||
hc_mult: HC expand ratio.
|
||||
hidden_size: Model hidden dimension.
|
||||
magnitude: H_post multiplier (config.hc_magnitude).
|
||||
hc_eps: Epsilon added to both gates (config.hc_eps).
|
||||
norm_eps: Epsilon inside the rsqrt (config.rms_norm_eps).
|
||||
fallback_op: The eager HYV4HCPreLayer to source weights from.
|
||||
norm_owner: Optional RMSNorm that immediately follows this pre block.
|
||||
When given, its weight/eps are folded into the kernel so a single
|
||||
launch covers pre + RMSNorm (the caller then skips the separate
|
||||
layernorm). None keeps the original pre-only behaviour.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
magnitude: float,
|
||||
hc_eps: float,
|
||||
norm_eps: float,
|
||||
fallback_op: torch.nn.Module,
|
||||
norm_owner: torch.nn.Module | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
self.magnitude = magnitude
|
||||
self.hc_eps = hc_eps
|
||||
self.norm_eps = norm_eps
|
||||
# Stash the owner outside nn.Module's attribute machinery: callers pass
|
||||
# their own `self` here, and registering that as a submodule would make
|
||||
# the module tree cyclic (state_dict() then hits RecursionError).
|
||||
object.__setattr__(self, "_fallback_op", fallback_op)
|
||||
# Same anti-cycle idiom for the fused RMSNorm owner (a sibling module).
|
||||
object.__setattr__(self, "_norm_owner", norm_owner)
|
||||
self.weight = None
|
||||
self.hc_scale = None
|
||||
self.hc_base = None
|
||||
self.rms_weight = None
|
||||
self.rms_eps = 0.0
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def process_weights_after_loading(self, model=None):
|
||||
# hc_fn is a ReplicatedLinear whose weight is [2 * hc_mult, hc_mult * d].
|
||||
# It stays float32 -- the HPC kernel consumes fp32 weights directly, so
|
||||
# there is no conversion (and no precision loss) here.
|
||||
owner = self._fallback_op
|
||||
self.weight = owner.hc_fn.weight
|
||||
self.hc_scale = owner.hc_scale
|
||||
self.hc_base = owner.hc_base
|
||||
if self._norm_owner is not None:
|
||||
self.rms_weight = self._norm_owner.weight
|
||||
self.rms_eps = self._norm_owner.variance_epsilon
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
import hpc
|
||||
|
||||
return hpc.fuse_ihc_pre(
|
||||
x,
|
||||
self.weight,
|
||||
self.hc_scale,
|
||||
self.hc_base,
|
||||
self.norm_eps,
|
||||
self.hc_eps,
|
||||
self.magnitude,
|
||||
self.rms_weight,
|
||||
self.rms_eps,
|
||||
)
|
||||
|
||||
|
||||
class HpcIHCPost(HpcModule):
|
||||
"""Fused iHC post block: H_post gating plus multi-channel residual add.
|
||||
|
||||
Unlike mHC there is no comb matrix, so each output channel only needs its
|
||||
own residual channel and the whole thing is one fused multiply-add per
|
||||
element.
|
||||
"""
|
||||
|
||||
def __init__(self, hc_mult: int, hidden_size: int) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual: torch.Tensor, H_post: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
import hpc
|
||||
|
||||
return hpc.fuse_ihc_post(x, residual, H_post)
|
||||
|
||||
|
||||
class HpcIHCHead(HpcModule):
|
||||
"""Fused iHC head block: merge the hc channels back into one hidden state.
|
||||
|
||||
Same structure as HpcIHCPre but the projection emits only hc_mult gate
|
||||
logits and there is no H_post output. Called once per forward, after the
|
||||
last decoder layer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hc_mult: int,
|
||||
hidden_size: int,
|
||||
hc_eps: float,
|
||||
norm_eps: float,
|
||||
fallback_op: torch.nn.Module,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hc_mult = hc_mult
|
||||
self.hidden_size = hidden_size
|
||||
self.hc_eps = hc_eps
|
||||
self.norm_eps = norm_eps
|
||||
# See HpcIHCPre: keep the owner out of the module tree.
|
||||
object.__setattr__(self, "_fallback_op", fallback_op)
|
||||
self.weight = None
|
||||
self.hc_scale = None
|
||||
self.hc_base = None
|
||||
|
||||
@classmethod
|
||||
def support(cls, hc_mult: int, hidden_size: int) -> bool:
|
||||
return _ihc_supported(hc_mult, hidden_size)
|
||||
|
||||
def process_weights_after_loading(self, model=None):
|
||||
# hc_head_fn weight is [hc_mult, hc_mult * d], float32.
|
||||
owner = self._fallback_op
|
||||
self.weight = owner.hc_head_fn.weight
|
||||
self.hc_scale = owner.hc_head_scale
|
||||
self.hc_base = owner.hc_head_base
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
import hpc
|
||||
|
||||
return hpc.fuse_ihc_head(
|
||||
x,
|
||||
self.weight,
|
||||
self.hc_scale,
|
||||
self.hc_base,
|
||||
self.norm_eps,
|
||||
self.hc_eps,
|
||||
)
|
||||
@@ -15,6 +15,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
@@ -144,7 +145,12 @@ class LogitsProcessor(PluggableLayer):
|
||||
lm_head, hidden_states, bias=embedding_bias
|
||||
)
|
||||
|
||||
if not isinstance(lm_head.quant_method, UnquantizedEmbeddingMethod):
|
||||
# A quant config that excludes lm_head hands out UnquantizedLinearMethod
|
||||
# rather than UnquantizedEmbeddingMethod, so accept both: either way the
|
||||
# weight is plain and `lm_head.weight` can be cast directly.
|
||||
if not isinstance(
|
||||
lm_head.quant_method, (UnquantizedEmbeddingMethod, UnquantizedLinearMethod)
|
||||
):
|
||||
raise ValueError(
|
||||
"A head_dtype different from the model dtype is only "
|
||||
"supported for an unquantized lm_head."
|
||||
|
||||
@@ -874,12 +874,20 @@ def _try_load_fp8_indexer_wk(
|
||||
# We have both weight and scale: dequantize FP8 to BF16.
|
||||
weight_fp8, scale_inv = entry["weight"], entry["scale"]
|
||||
del buf[layer_prefix]
|
||||
if scale_inv.dtype in (torch.uint8, torch.float8_e8m0fnu):
|
||||
# MXFP8 checkpoints store power-of-two E8M0 scales, decode them to float.
|
||||
scale_inv = scale_inv.view(torch.float8_e8m0fnu).to(torch.float32)
|
||||
if scale_inv.ndim == 1:
|
||||
# Per-channel scale: one scale per row of [out, in]
|
||||
group_shape = GroupShape(1, weight_fp8.shape[1])
|
||||
else:
|
||||
block_size = weight_fp8.shape[1] // scale_inv.shape[1]
|
||||
group_shape = GroupShape(block_size, block_size)
|
||||
# Derive the block size independently per dim so that per-channel
|
||||
# ([out, 1]), MX ([out, in / 32]) and 2D block ([out / 128, in / 128])
|
||||
# scale layouts are all handled.
|
||||
group_shape = GroupShape(
|
||||
weight_fp8.shape[0] // scale_inv.shape[0],
|
||||
weight_fp8.shape[1] // scale_inv.shape[1],
|
||||
)
|
||||
weight_bf16 = scaled_dequantize(
|
||||
weight_fp8,
|
||||
scale_inv,
|
||||
|
||||
@@ -128,6 +128,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"GraniteSWAForCausalLM": ("granite", "GraniteForCausalLM"),
|
||||
"HrmTextForCausalLM": ("hrm_text", "HrmTextForCausalLM"),
|
||||
"HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"),
|
||||
"HYV4ForCausalLM": ("vllm.models.hy_v4", "HYV4ForCausalLM"),
|
||||
"HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"),
|
||||
"HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"),
|
||||
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
|
||||
@@ -663,6 +664,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
"InternS2MobiusMTP": ("interns2_mobius", "InternS2MobiusMTP"),
|
||||
"HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"),
|
||||
"HYV4MTPModel": ("vllm.models.hy_v4", "HYV4MTP"),
|
||||
"KimiK3MTPModel": ("vllm.models.kimi_k3", "KimiK3MTP"),
|
||||
# Temporarily disabled.
|
||||
# # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HY V4 (``hy_v4``) model — hardware-isolated entry point.
|
||||
|
||||
HY V4 combines three architectural pieces:
|
||||
|
||||
- **iHC** (independent Hyper-Connections): the single residual stream is
|
||||
replaced by ``hc_mult`` parallel residual channels, gated per sub-block.
|
||||
- **MLA + lightning indexer**: multi-head latent attention with an optional
|
||||
DSA-style sparse top-k selection, plus an output gate and a learnable sink.
|
||||
- **MoE**: sigmoid-routed experts with a clamped SwiGLU and shared experts.
|
||||
|
||||
The package is organized like `vllm.models.deepseek_v32`: this module is the
|
||||
only public entry point and dispatches on the current platform, so registry
|
||||
entries never reach into a platform subpackage.
|
||||
|
||||
Only NVIDIA is supported for now. The port also drops the reference
|
||||
implementation's HPC/TPCP fusion paths, which depend on infrastructure that
|
||||
does not exist in this tree.
|
||||
"""
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
raise NotImplementedError("hy_v4 does not yet support ROCm.")
|
||||
elif current_platform.is_xpu():
|
||||
raise NotImplementedError("hy_v4 does not yet support XPU.")
|
||||
else:
|
||||
# Covers Blackwell (sm100) and all other CUDA devices.
|
||||
from .nvidia.model import HYV4ForCausalLM
|
||||
from .nvidia.mtp import HYV4MTP
|
||||
|
||||
__all__ = [
|
||||
"HYV4ForCausalLM",
|
||||
"HYV4MTP",
|
||||
]
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,765 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MLA attention and lightning indexer for HY V4 (NVIDIA).
|
||||
|
||||
NOTE: The reference implementation also offers a fused MLA preprocessing
|
||||
wrapper, which is not available here, so this port keeps the eager MLA path
|
||||
with an FP8 indexer cache. TODO: restore the fused preprocessing once the
|
||||
kernel lands. The HPC gated-MLA GEMM *is* wired up: it replaces the eager
|
||||
output gating when the hpc package is installed, ``VLLM_ENABLE_HPC_OPS=1``
|
||||
and the elementwise/dtype/alignment constraints hold.
|
||||
|
||||
The per-head learnable sink is supported through `.flashmla_sparse`, which
|
||||
subclasses the platform's sparse MLA backend to forward ``attn_sink``.
|
||||
"""
|
||||
|
||||
from typing import cast
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import DeepseekV2Config, DeepseekV3Config, PretrainedConfig
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.hpc import hpc_gated_mla_gemm, hpc_gated_mla_supported
|
||||
from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionType
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SPARSE_LAYER_TYPES = ("sparse_attention", "sparse", "deepseek_sparse_attention")
|
||||
_WEIGHT_LAYER_INDEX_RE = re.compile(r"(?:^|\.)layers\.(\d+)(?:\.|$)")
|
||||
|
||||
|
||||
def compute_skip_topk_layers(config: PretrainedConfig) -> set[int]:
|
||||
"""Return the backbone layers that reuse a previous layer's top-k indices.
|
||||
|
||||
A "shared" indexer layer performs sparse attention with the indices computed
|
||||
by the closest preceding "full" indexer layer, so it does not build its own
|
||||
indexer and its checkpoint indexer weights must be skipped.
|
||||
|
||||
Args:
|
||||
config: The model config.
|
||||
|
||||
Returns:
|
||||
The set of layer indices that share another layer's top-k indices.
|
||||
|
||||
Raises:
|
||||
ValueError: If ``indexer_types`` has the wrong length or an unknown
|
||||
entry, or if ``index_topk_freq`` is not a positive integer.
|
||||
"""
|
||||
if not hasattr(config, "index_topk"):
|
||||
return set()
|
||||
|
||||
num_hidden_layers = config.num_hidden_layers
|
||||
indexer_types = getattr(config, "indexer_types", None)
|
||||
if indexer_types is not None:
|
||||
if len(indexer_types) != num_hidden_layers:
|
||||
raise ValueError(
|
||||
"indexer_types must contain one entry per hidden layer: "
|
||||
f"expected {num_hidden_layers}, got {len(indexer_types)}."
|
||||
)
|
||||
invalid_types = sorted(set(indexer_types) - {"full", "shared"})
|
||||
if invalid_types:
|
||||
raise ValueError(
|
||||
f"indexer_types only supports 'full' and 'shared', got {invalid_types}."
|
||||
)
|
||||
return {
|
||||
layer_idx
|
||||
for layer_idx, indexer_type in enumerate(indexer_types)
|
||||
if indexer_type == "shared"
|
||||
}
|
||||
|
||||
freq = getattr(config, "index_topk_freq", 1)
|
||||
if not isinstance(freq, int) or freq <= 0:
|
||||
raise ValueError(f"index_topk_freq must be a positive integer, got {freq!r}.")
|
||||
pattern = getattr(config, "index_topk_pattern", None)
|
||||
offset = getattr(config, "index_skip_topk_offset", 2)
|
||||
skip_layers: set[int] = set()
|
||||
for layer_idx in range(num_hidden_layers):
|
||||
if pattern is None:
|
||||
if max(layer_idx - offset + 1, 0) % freq != 0:
|
||||
skip_layers.add(layer_idx)
|
||||
elif 0 <= layer_idx < len(pattern) and pattern[layer_idx] == "S":
|
||||
skip_layers.add(layer_idx)
|
||||
return skip_layers
|
||||
|
||||
|
||||
def is_skip_topk_indexer_weight(weight_name: str, skip_topk_layers: set[int]) -> bool:
|
||||
"""Return whether an indexer weight belongs to a top-k sharing layer.
|
||||
|
||||
Args:
|
||||
weight_name: Checkpoint weight name.
|
||||
skip_topk_layers: Result of `compute_skip_topk_layers`.
|
||||
|
||||
Returns:
|
||||
True when the weight is an indexer weight of a layer that has no
|
||||
indexer module and therefore must be dropped.
|
||||
"""
|
||||
if ".indexer." not in weight_name or not skip_topk_layers:
|
||||
return False
|
||||
match = _WEIGHT_LAYER_INDEX_RE.search(weight_name)
|
||||
return match is not None and int(match.group(1)) in skip_topk_layers
|
||||
|
||||
|
||||
class Indexer(nn.Module):
|
||||
"""Lightning indexer selecting the top-k tokens for sparse MLA."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: DeepseekV2Config | DeepseekV3Config,
|
||||
hidden_size: int,
|
||||
q_lora_rank: int,
|
||||
quant_config: QuantizationConfig | None,
|
||||
cache_config: CacheConfig | None,
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.topk_tokens = config.index_topk
|
||||
self.n_head = config.index_n_heads
|
||||
self.head_dim = config.index_head_dim
|
||||
self.rope_dim = config.qk_rope_head_dim
|
||||
self.q_lora_rank = q_lora_rank
|
||||
|
||||
# No tensor parallelism, just replicated.
|
||||
self.wq_b = ReplicatedLinear(
|
||||
self.q_lora_rank,
|
||||
self.head_dim * self.n_head,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.wq_b",
|
||||
)
|
||||
# Fused wk + weights_proj: single GEMM producing [head_dim + n_head].
|
||||
# FP8 wk weights are upcast to BF16 while loading to keep the fusion.
|
||||
self.wk_weights_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[self.head_dim, self.n_head],
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
disable_tp=True,
|
||||
prefix=f"{prefix}.wk_weights_proj",
|
||||
)
|
||||
self.k_norm = LayerNorm(self.head_dim, eps=1e-6)
|
||||
self.softmax_scale = self.head_dim**-0.5
|
||||
|
||||
self.scale_fmt = "ue8m0"
|
||||
self.quant_block_size = 128
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
|
||||
# FP8 naive cache: values in fp8 plus one fp32 scale per
|
||||
# ``quant_block_size`` elements.
|
||||
assert cache_config is not None, "HYV4 indexer requires cache_config"
|
||||
self.k_cache = DeepseekV32IndexerCache(
|
||||
head_dim=self.head_dim + self.head_dim // self.quant_block_size * 4,
|
||||
dtype=torch.uint8,
|
||||
prefix=f"{prefix}.k_cache",
|
||||
cache_config=cache_config,
|
||||
)
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
self.prefix = prefix
|
||||
|
||||
from vllm.v1.attention.backends.mla.indexer import get_max_prefill_buffer_size
|
||||
|
||||
self.max_total_seq_len = get_max_prefill_buffer_size(vllm_config)
|
||||
self.indexer_op = SparseAttnIndexer(
|
||||
self.k_cache,
|
||||
self.quant_block_size,
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rotary_emb: nn.Module,
|
||||
) -> torch.Tensor:
|
||||
hidden_states, q_quant, k, weights = self.prepare_inputs(
|
||||
hidden_states, qr, positions, rotary_emb
|
||||
)
|
||||
return self.indexer_op(hidden_states, q_quant, k, weights)
|
||||
|
||||
def prepare_inputs(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rotary_emb: nn.Module,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build the quantized query, key and per-head weights of the indexer."""
|
||||
q, _ = self.wq_b(qr)
|
||||
q = q.view(-1, self.n_head, self.head_dim)
|
||||
# Checkpoint (PTM) layout: pe occupies the LAST rope_dim dims.
|
||||
q_nope, q_pe = torch.split(
|
||||
q, [self.head_dim - self.rope_dim, self.rope_dim], dim=-1
|
||||
)
|
||||
|
||||
kw, _ = self.wk_weights_proj(hidden_states)
|
||||
k = kw[:, : self.head_dim]
|
||||
weights = kw[:, self.head_dim :]
|
||||
|
||||
k = self.k_norm(k)
|
||||
k_nope, k_pe = torch.split(
|
||||
k, [self.head_dim - self.rope_dim, self.rope_dim], dim=-1
|
||||
)
|
||||
|
||||
q_pe, k_pe = rotary_emb(positions, q_pe, k_pe.unsqueeze(1))
|
||||
# RoPE (NeoX) can introduce extra leading dims, so flatten back to the
|
||||
# token-major shapes.
|
||||
q_pe = q_pe.reshape(-1, self.n_head, self.rope_dim)
|
||||
k_pe = k_pe.reshape(-1, 1, self.rope_dim)
|
||||
|
||||
# Reassemble with the original physical layout: no_pe first, pe last.
|
||||
q = torch.cat([q_nope, q_pe], dim=-1)
|
||||
# ``k_pe`` is [num_tokens, 1, rope_dim] (MQA).
|
||||
k = torch.cat([k_nope, k_pe.squeeze(-2)], dim=-1)
|
||||
|
||||
# Only q is quantized here; k quantization is fused with cache insertion.
|
||||
q = q.view(-1, self.head_dim)
|
||||
q_fp8, q_scale = per_token_group_quant_fp8(
|
||||
q,
|
||||
self.quant_block_size,
|
||||
column_major_scales=False,
|
||||
use_ue8m0=self.scale_fmt is not None,
|
||||
)
|
||||
q_fp8 = q_fp8.view(-1, self.n_head, self.head_dim)
|
||||
q_scale = q_scale.view(-1, self.n_head, 1)
|
||||
|
||||
weights = (
|
||||
weights.unsqueeze(-1) * q_scale * self.softmax_scale * self.n_head**-0.5
|
||||
)
|
||||
weights = weights.squeeze(-1)
|
||||
|
||||
return hidden_states, q_fp8, k, weights
|
||||
|
||||
|
||||
class HYV4MLAAttention(nn.Module):
|
||||
"""Multi-head latent attention with optional sparse lightning indexer.
|
||||
|
||||
Main reference: the DeepSeek-V2 paper and the FlashInfer implementation
|
||||
(https://arxiv.org/abs/2405.04434). HY V4 additionally supports an output
|
||||
gate (``gated_mla``) and a per-head learnable attention sink.
|
||||
|
||||
The sink is applied by binding the sink-capable backend from
|
||||
`.flashmla_sparse`; if no backend on this platform can consume sinks, the
|
||||
weight is still loaded but the bias is disabled with a warning.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: PretrainedConfig,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
max_position_embeddings: int = 8192,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
layer_idx: int = 0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_size = hidden_size
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.layer_idx = layer_idx
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
|
||||
self.num_heads = num_heads
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
assert num_heads % tp_size == 0
|
||||
self.num_local_heads = num_heads // tp_size
|
||||
|
||||
self.layer_id = int(prefix.split(".")[-2])
|
||||
layer_types = getattr(config, "layer_types", None)
|
||||
requested_sparse = (
|
||||
hasattr(config, "index_topk")
|
||||
and layer_types is not None
|
||||
and self.layer_id < len(layer_types)
|
||||
and layer_types[self.layer_id] in _SPARSE_LAYER_TYPES
|
||||
)
|
||||
# Only actual sparse layers may share another layer's top-k indices.
|
||||
self.skip_topk = requested_sparse and self.layer_id in compute_skip_topk_layers(
|
||||
config
|
||||
)
|
||||
# The skip pattern only governs backbone layers. MTP/nextn layers
|
||||
# (layer_id >= num_hidden_layers) always build a full indexer: they
|
||||
# compute indices at draft step 0 and toggle at runtime.
|
||||
num_hidden_layers = getattr(config, "num_hidden_layers", None)
|
||||
is_mtp_layer = (
|
||||
num_hidden_layers is not None and self.layer_id >= num_hidden_layers
|
||||
)
|
||||
self.create_indexer = requested_sparse and (not self.skip_topk or is_mtp_layer)
|
||||
self.is_sparse = requested_sparse
|
||||
|
||||
# Do not silently degrade sparse layers into dense attention. Probe the
|
||||
# sparse MLA backend directly and fail fast with the real error.
|
||||
kv_cache_dtype = cache_config.cache_dtype if cache_config else "auto"
|
||||
if self.is_sparse:
|
||||
try:
|
||||
get_attn_backend(
|
||||
head_size=self.kv_lora_rank + self.qk_rope_head_dim,
|
||||
dtype=torch.get_default_dtype(),
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=True,
|
||||
num_heads=self.num_local_heads,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
"HYV4 sparse attention was requested, but no valid sparse MLA "
|
||||
"backend is available for current runtime/config. "
|
||||
"Refusing to fall back to dense attention."
|
||||
) from exc
|
||||
|
||||
self.scaling = self.qk_head_dim**-0.5
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.fused_qkv_a_proj = None
|
||||
self.kv_a_proj_with_mqa = None
|
||||
if self.q_lora_rank is not None:
|
||||
# ``q_a_proj`` and ``kv_a_proj_with_mqa`` read the same
|
||||
# ``hidden_states`` and are both TP-replicated, so they run as one
|
||||
# GEMM. The checkpoint keeps them separate; `load_weights` merges
|
||||
# them through ``stacked_params_mapping``.
|
||||
self.fused_qkv_a_proj = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fused_qkv_a_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
else:
|
||||
self.kv_a_proj_with_mqa = ReplicatedLinear(
|
||||
self.hidden_size,
|
||||
self.kv_lora_rank + self.qk_rope_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_a_proj_with_mqa",
|
||||
)
|
||||
|
||||
self.q_a_layernorm = None
|
||||
self.q_b_proj = None
|
||||
self.q_proj = None
|
||||
if self.q_lora_rank is not None:
|
||||
self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
|
||||
self.q_b_proj = ColumnParallelLinear(
|
||||
self.q_lora_rank,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.q_b_proj",
|
||||
)
|
||||
else:
|
||||
self.q_proj = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.q_proj",
|
||||
)
|
||||
self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps)
|
||||
self.kv_b_proj = ColumnParallelLinear(
|
||||
self.kv_lora_rank,
|
||||
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.kv_b_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.num_heads * self.v_head_dim,
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
is_neox_style=False,
|
||||
)
|
||||
self.indexer_rope_emb: nn.Module | None
|
||||
self.indexer: Indexer | None
|
||||
if self.create_indexer:
|
||||
# The checkpoint stores indexer q_pe/k_pe in interleaved
|
||||
# (Megatron/PTM) layout, so the indexer must use interleaved RoPE
|
||||
# (is_neox_style=False) like the main attention path. Using NeoX
|
||||
# here loses the relative-position dependence and corrupts the DSA
|
||||
# top-k selection.
|
||||
self.indexer_rope_emb = get_rope(
|
||||
qk_rope_head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
is_neox_style=False,
|
||||
)
|
||||
# The indexer projects its queries from the MLA q_lora activations,
|
||||
# so a sparse layer requires a query down-projection.
|
||||
assert q_lora_rank is not None, (
|
||||
"HYV4 sparse attention requires q_lora_rank to be set"
|
||||
)
|
||||
self.indexer = Indexer(
|
||||
vllm_config,
|
||||
config,
|
||||
hidden_size,
|
||||
q_lora_rank,
|
||||
quant_config,
|
||||
cache_config,
|
||||
topk_indices_buffer,
|
||||
f"{prefix}.indexer",
|
||||
)
|
||||
else:
|
||||
self.indexer_rope_emb = None
|
||||
self.indexer = None
|
||||
|
||||
self.gated_mla = bool(getattr(config, "gated_mla", False))
|
||||
self.linear_gate: ColumnParallelLinear | None
|
||||
if self.gated_mla:
|
||||
if config.gating_type == "headwise":
|
||||
self.gate_projection_size_per_head = 1
|
||||
elif config.gating_type == "elementwise":
|
||||
self.gate_projection_size_per_head = self.v_head_dim
|
||||
else:
|
||||
raise ValueError(f"Unknown gating type: {config.gating_type}")
|
||||
self.linear_gate = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
self.num_heads * self.gate_projection_size_per_head,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_gate",
|
||||
)
|
||||
self.use_hpc_gated_mla = hpc_gated_mla_supported(
|
||||
config.gating_type, self.linear_gate
|
||||
)
|
||||
else:
|
||||
self.linear_gate = None
|
||||
self.use_hpc_gated_mla = False
|
||||
self.prefix = prefix
|
||||
|
||||
# Per-head learnable attention sink. Created BEFORE ``MLAAttention`` so
|
||||
# it can be forwarded as the ``sinks`` impl kwarg. The parameter always
|
||||
# holds the local TP shard.
|
||||
self.learnable_sink = bool(getattr(config, "learnable_sink", False))
|
||||
sinks = None
|
||||
sink_backend: type[AttentionBackend] | None = None
|
||||
if self.learnable_sink:
|
||||
sink_backend = self._resolve_sink_backend(kv_cache_dtype)
|
||||
enable_sink = sink_backend is not None
|
||||
self.learnable_sink_param = nn.Parameter(
|
||||
torch.empty(
|
||||
self.num_local_heads,
|
||||
# The kernels require fp32 sinks; the disabled path keeps
|
||||
# the checkpoint dtype since the value is never consumed.
|
||||
dtype=torch.float32 if enable_sink else torch.bfloat16,
|
||||
)
|
||||
)
|
||||
if enable_sink:
|
||||
sinks = self.learnable_sink_param
|
||||
self._force_sparse_mqa()
|
||||
|
||||
extra_impl_args = {} if sinks is None else {"sinks": sinks}
|
||||
self.mla_attn = MLAAttention(
|
||||
num_heads=self.num_local_heads,
|
||||
scale=self.scaling,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
q_lora_rank=self.q_lora_rank,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
use_sparse=self.is_sparse,
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
attn_backend=sink_backend,
|
||||
**extra_impl_args,
|
||||
)
|
||||
|
||||
@property
|
||||
def topk_indices_buffer(self) -> torch.Tensor | None:
|
||||
"""The sparse top-k index buffer this layer reads from.
|
||||
|
||||
The real consumer is ``mla_attn.impl``, an ``AttentionImpl`` rather than
|
||||
an ``nn.Module``, so a plain ``named_modules()`` walk cannot reach it.
|
||||
The MTP proposer rebinds the draft layers onto the target model's buffer
|
||||
through such a walk, so expose it here: without this the draft attention
|
||||
would keep reading its own buffer while the indices it needs are written
|
||||
to the target's.
|
||||
"""
|
||||
return getattr(self.mla_attn.impl, "topk_indices_buffer", None)
|
||||
|
||||
@topk_indices_buffer.setter
|
||||
def topk_indices_buffer(self, buffer: torch.Tensor) -> None:
|
||||
self.mla_attn.impl.topk_indices_buffer = buffer # type: ignore[attr-defined]
|
||||
|
||||
def _resolve_sink_backend(
|
||||
self, kv_cache_dtype: str
|
||||
) -> type[AttentionBackend] | None:
|
||||
"""Return an MLA backend that can apply this layer's learnable sink.
|
||||
|
||||
The sink is part of the architecture, so a backend that cannot apply it
|
||||
changes the model's output. Resolution order:
|
||||
|
||||
1. If the backend the selector would pick already advertises
|
||||
`supports_sink`, keep it — this also honours an explicit
|
||||
``--attention-backend`` choice.
|
||||
2. Otherwise fall back to the sink-capable ``FLASHMLA_SPARSE`` subclass
|
||||
in `.flashmla_sparse`, whose kernels accept ``attn_sink``, provided
|
||||
it validates against the current runtime configuration.
|
||||
3. Otherwise give up on the bias rather than failing the load.
|
||||
|
||||
Args:
|
||||
kv_cache_dtype: The layer's KV cache dtype string.
|
||||
|
||||
Returns:
|
||||
The backend class to bind, or None when no sink-capable backend is
|
||||
available; the caller then loads the sink weight but disables the
|
||||
bias.
|
||||
"""
|
||||
head_size = self.kv_lora_rank + self.qk_rope_head_dim
|
||||
dtype = torch.get_default_dtype()
|
||||
try:
|
||||
selected_cls = get_attn_backend(
|
||||
head_size=head_size,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
use_mla=True,
|
||||
use_sparse=self.is_sparse,
|
||||
num_heads=self.num_local_heads,
|
||||
)
|
||||
except Exception as exc:
|
||||
# Stringify before logging: warning_once dedupes on the arguments,
|
||||
# and a fresh exception object per layer would defeat it.
|
||||
logger.warning_once(
|
||||
"HYV4 failed to select an MLA backend for the learnable sink "
|
||||
"(%s); the sink parameter is loaded but the sink bias is "
|
||||
"disabled.",
|
||||
str(exc),
|
||||
)
|
||||
return None
|
||||
|
||||
if selected_cls.supports_sink():
|
||||
return selected_cls
|
||||
|
||||
from .flashmla_sparse import HYV4FlashMLASparseBackend
|
||||
|
||||
# Mirror how the selector derives the configuration-dependent inputs so
|
||||
# this check accepts exactly what the backend would accept at runtime.
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None:
|
||||
logger.warning_once(
|
||||
"HYV4 learnable sink is unavailable: the device compute "
|
||||
"capability is unknown. The sink parameter is loaded but the "
|
||||
"sink bias is disabled."
|
||||
)
|
||||
return None
|
||||
cache_config = get_current_vllm_config().cache_config
|
||||
block_size = (
|
||||
cache_config.block_size
|
||||
if cache_config is not None and cache_config.user_specified_block_size
|
||||
else None
|
||||
)
|
||||
invalid_reasons = HYV4FlashMLASparseBackend.validate_configuration(
|
||||
head_size=head_size,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=cast(CacheDType, kv_cache_dtype),
|
||||
block_size=block_size,
|
||||
use_mla=True,
|
||||
has_sink=True,
|
||||
use_sparse=self.is_sparse,
|
||||
use_mm_prefix=False,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=capability,
|
||||
attn_type=AttentionType.DECODER,
|
||||
)
|
||||
if invalid_reasons:
|
||||
logger.warning_once(
|
||||
"HYV4 learnable sink is unavailable: the selected backend %s "
|
||||
"cannot apply sinks and the sink-capable FLASHMLA_SPARSE path "
|
||||
"is invalid here (%s). The sink parameter is loaded but the "
|
||||
"sink bias is disabled.",
|
||||
selected_cls.get_name(),
|
||||
", ".join(invalid_reasons),
|
||||
)
|
||||
return None
|
||||
|
||||
logger.info_once(
|
||||
"HYV4 learnable sink enabled: using the sink-capable "
|
||||
"FLASHMLA_SPARSE impl instead of %s, which cannot apply sinks.",
|
||||
selected_cls.get_name(),
|
||||
)
|
||||
return HYV4FlashMLASparseBackend
|
||||
|
||||
def _force_sparse_mqa(self) -> None:
|
||||
"""Keep every token on the sink-capable sparse MQA path.
|
||||
|
||||
``_resolve_sink_backend`` only binds the backend that serves decode.
|
||||
``MLAAttention`` additionally routes short prefills to a separate dense
|
||||
MLA prefill backend, and none of those accept ``attn_sink``, so prefill
|
||||
would silently drop the sink while decode applies it. Such a partially
|
||||
applied sink is not the trained architecture and corrupts the output, so
|
||||
opt out of the dense split instead.
|
||||
|
||||
Prefills up to ``index_topk`` keep every token inside the sparse top-k,
|
||||
making the sparse path numerically equivalent to the dense one apart from
|
||||
also applying the sink.
|
||||
"""
|
||||
attention_config = get_current_vllm_config().attention_config
|
||||
if attention_config.sparse_mla_force_mqa:
|
||||
return
|
||||
attention_config.sparse_mla_force_mqa = True
|
||||
logger.info_once(
|
||||
"HYV4 learnable sink enabled: forcing sparse MQA for prefill too, "
|
||||
"as the dense MLA prefill backends cannot apply sinks."
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
llama_4_scaling: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
q_c = None
|
||||
if self.q_lora_rank is not None:
|
||||
assert self.fused_qkv_a_proj is not None
|
||||
assert self.q_a_layernorm is not None
|
||||
assert self.q_b_proj is not None
|
||||
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
|
||||
q_c, kv_lora = qkv_lora.split(
|
||||
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
|
||||
dim=-1,
|
||||
)
|
||||
q_c = self.q_a_layernorm(q_c)
|
||||
q = self.q_b_proj(q_c)[0]
|
||||
else:
|
||||
assert self.q_proj is not None
|
||||
assert self.kv_a_proj_with_mqa is not None
|
||||
q = self.q_proj(hidden_states)[0]
|
||||
kv_lora = self.kv_a_proj_with_mqa(hidden_states)[0]
|
||||
|
||||
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
|
||||
kv_c_normed = self.kv_a_layernorm(kv_c)
|
||||
|
||||
q = q.view(-1, self.num_local_heads, self.qk_head_dim)
|
||||
# Add a head dim of 1 to k_pe.
|
||||
k_pe = k_pe.unsqueeze(1)
|
||||
q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb(
|
||||
positions, q[..., self.qk_nope_head_dim :], k_pe
|
||||
)
|
||||
|
||||
if llama_4_scaling is not None:
|
||||
q *= llama_4_scaling
|
||||
|
||||
output_shape = (
|
||||
hidden_states.shape[0],
|
||||
self.num_local_heads * self.v_head_dim,
|
||||
)
|
||||
# Single coarse eager break covering the indexer and MLA attention, as
|
||||
# the breakable cudagraph contract requires: everything that reads
|
||||
# per-batch metadata runs in one eager segment, so no tensor has to stay
|
||||
# alive across a capture-segment boundary.
|
||||
attn_out = torch.empty(
|
||||
output_shape, dtype=hidden_states.dtype, device=hidden_states.device
|
||||
)
|
||||
self._indexer_and_attn(
|
||||
hidden_states, q_c, positions, q, kv_c_normed, k_pe, attn_out
|
||||
)
|
||||
|
||||
if self.gated_mla and self.linear_gate is not None:
|
||||
if self.use_hpc_gated_mla:
|
||||
# Projection, sigmoid and the product in one launch. The gate
|
||||
# is column-parallel and unbiased, so its local weight shard
|
||||
# maps straight onto the local attn_out columns.
|
||||
assert hidden_states.is_contiguous() and attn_out.is_contiguous()
|
||||
attn_out = hpc_gated_mla_gemm(
|
||||
hidden_states,
|
||||
self.linear_gate.weight,
|
||||
attn_out,
|
||||
)
|
||||
else:
|
||||
gate_score = self.linear_gate(hidden_states)[0]
|
||||
if self.config.gating_type == "headwise":
|
||||
gate_score = gate_score.unsqueeze(-1)
|
||||
attn_out = attn_out.reshape(
|
||||
*attn_out.shape[:-1], -1, self.v_head_dim
|
||||
)
|
||||
attn_out = attn_out * torch.sigmoid(gate_score)
|
||||
attn_out = attn_out.reshape(*attn_out.shape[:-2], -1)
|
||||
else:
|
||||
attn_out = attn_out * torch.sigmoid(gate_score)
|
||||
|
||||
out, _ = self.o_proj(attn_out)
|
||||
return out
|
||||
|
||||
@eager_break_during_capture
|
||||
def _indexer_and_attn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_c: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
out: torch.Tensor, # [num_tokens, heads * v_head_dim], written in place
|
||||
) -> None:
|
||||
"""Run the lightning indexer and MLA attention in one eager segment.
|
||||
|
||||
Both read per-batch attention metadata, so under the breakable cudagraph
|
||||
they must not be captured. Keeping them in a single break (instead of one
|
||||
break each) also means the attention inputs never have to survive a
|
||||
capture-segment boundary. The nested ``sparse_attn_indexer`` and
|
||||
``unified_mla_attention_with_output`` breaks short-circuit here, since
|
||||
the capture is no longer active inside an eager segment.
|
||||
"""
|
||||
if self.indexer is not None and self.is_sparse and not self.skip_topk:
|
||||
self.indexer(hidden_states, q_c, positions, self.indexer_rope_emb)
|
||||
out.copy_(
|
||||
self.mla_attn(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_shape=out.shape,
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,283 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sink-capable FlashMLA sparse backend for HY V4 (NVIDIA).
|
||||
|
||||
HY V4 adds a per-head learnable attention sink on top of sparse MLA. The
|
||||
vendored FlashMLA kernels already accept an ``attn_sink`` argument, but vLLM's
|
||||
shared ``FLASHMLA_SPARSE`` backend neither advertises sink support nor forwards
|
||||
the tensor, so the bias would be silently dropped.
|
||||
|
||||
This module supplies the missing wiring inside the model package, mirroring how
|
||||
`vllm.models.deepseek_v4.nvidia.flashinfer_sparse` hands ``sinks`` to the
|
||||
FlashInfer sparse MLA kernels: subclass the platform backend, declare
|
||||
`supports_sink`, and thread the sink into every kernel call.
|
||||
|
||||
The subclass intentionally keeps the inherited ``get_name()``
|
||||
(``"FLASHMLA_SPARSE"``). Several shared code paths key off that exact string —
|
||||
`_canonicalize_sparse_mla_kv_cache_dtype` promotes a quantized KV cache to
|
||||
``fp8_ds_mla`` for it, and `FlashMLASparseImpl` asserts that layout — so a new
|
||||
name would silently change KV cache behaviour. Only``supports_sink`` and the
|
||||
two kernel wrappers differ from the parent.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
FlashMLASparseBackend,
|
||||
FlashMLASparseImpl,
|
||||
FlashMLASparseMetadata,
|
||||
)
|
||||
from vllm.v1.attention.ops.flashmla import (
|
||||
flash_mla_sparse_fwd,
|
||||
flash_mla_with_kvcache,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.models.deepseek_v2 import Indexer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class HYV4FlashMLASparseImpl(FlashMLASparseImpl):
|
||||
"""FlashMLA sparse impl that applies HY V4's per-head learnable sink.
|
||||
|
||||
The sink enters as the ``sinks`` impl kwarg of
|
||||
`vllm.model_executor.layers.attention.MLAAttention` and is consumed by the
|
||||
FlashMLA kernels, which fold it into the softmax denominator:
|
||||
``out *= exp(lse) / (exp(lse) + exp(sink))``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int,
|
||||
alibi_slopes: list[float] | None,
|
||||
sliding_window: int | None,
|
||||
kv_cache_dtype: str,
|
||||
logits_soft_cap: float | None,
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
# MLA Specific Arguments
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
indexer: "Indexer | None" = None,
|
||||
**mla_args,
|
||||
) -> None:
|
||||
# ``SparseMLACommonImpl`` takes explicit keyword arguments only, so the
|
||||
# sink has to be removed before the base classes see``mla_args``.
|
||||
sinks: torch.Tensor | None = mla_args.pop("sinks", None)
|
||||
super().__init__(
|
||||
num_heads,
|
||||
head_size,
|
||||
scale,
|
||||
num_kv_heads,
|
||||
alibi_slopes,
|
||||
sliding_window,
|
||||
kv_cache_dtype,
|
||||
logits_soft_cap,
|
||||
attn_type,
|
||||
kv_sharing_target_layer_name,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
indexer=indexer,
|
||||
**mla_args,
|
||||
)
|
||||
self._validate_sinks(sinks, num_heads)
|
||||
self.sinks = sinks
|
||||
|
||||
@staticmethod
|
||||
def _validate_sinks(sinks: torch.Tensor | None, num_heads: int) -> None:
|
||||
"""Reject sink tensors the FlashMLA kernels cannot consume.
|
||||
|
||||
Args:
|
||||
sinks: Candidate sink tensor, or None when the layer has no sink.
|
||||
num_heads: Local (TP-sharded) query head count of this layer.
|
||||
|
||||
Raises:
|
||||
ValueError: If the dtype is not float32 or the shape is not
|
||||
``(num_heads,)``.
|
||||
"""
|
||||
if sinks is None:
|
||||
return
|
||||
if sinks.dtype != torch.float32:
|
||||
raise ValueError(
|
||||
"HYV4 FlashMLA sparse attention sinks must have dtype "
|
||||
f"torch.float32, but got {sinks.dtype}."
|
||||
)
|
||||
if sinks.ndim != 1 or sinks.shape[0] != num_heads:
|
||||
raise ValueError(
|
||||
"HYV4 FlashMLA sparse attention sinks must have shape "
|
||||
f"({num_heads},), but got {tuple(sinks.shape)}."
|
||||
)
|
||||
|
||||
def _sinks_for_query(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
head_dim: int,
|
||||
kernel_heads: int,
|
||||
) -> torch.Tensor | None:
|
||||
"""Return the sink laid out for the kernel's query head count.
|
||||
|
||||
Args:
|
||||
q: Query tensor, before any head padding.
|
||||
head_dim: Axis of ``q`` holding the query heads.
|
||||
kernel_heads: Head count the kernel is invoked with, which may
|
||||
exceed the query head count because of padding.
|
||||
|
||||
Returns:
|
||||
The sink tensor padded to ``kernel_heads`` with ``-inf`` (a no-op
|
||||
sink) for the padded lanes, or None when the layer has no sink.
|
||||
|
||||
Raises:
|
||||
ValueError: If the sink and query head layouts disagree, or if they
|
||||
live on different devices.
|
||||
"""
|
||||
sinks = self.sinks
|
||||
if sinks is None:
|
||||
return None
|
||||
|
||||
query_heads = q.shape[head_dim]
|
||||
if sinks.shape[0] != query_heads:
|
||||
raise ValueError(
|
||||
"HYV4 FlashMLA sparse attention sink head count must match the "
|
||||
f"runtime query layout: sinks={sinks.shape[0]}, "
|
||||
f"query_heads={query_heads}. The sink must use the same "
|
||||
"unpadded head layout as the query."
|
||||
)
|
||||
if sinks.device != q.device:
|
||||
raise ValueError(
|
||||
"HYV4 FlashMLA sparse attention sinks and query must be on the "
|
||||
f"same device, but got sinks={sinks.device}, query={q.device}."
|
||||
)
|
||||
if kernel_heads < query_heads:
|
||||
raise ValueError(
|
||||
"HYV4 FlashMLA sparse kernel head count cannot be smaller than "
|
||||
f"the runtime query layout: query_heads={query_heads}, "
|
||||
f"kernel_heads={kernel_heads}."
|
||||
)
|
||||
if kernel_heads == query_heads:
|
||||
return sinks
|
||||
|
||||
# Mirror the query padding the kernels require. Reading ``sinks`` here
|
||||
# (rather than caching a padded copy at construction time) keeps the
|
||||
# values correct for weights loaded after the module is built, and the
|
||||
# allocation plus copy are captured in the CUDA graph.
|
||||
padded_sinks = sinks.new_full((kernel_heads,), float("-inf"))
|
||||
padded_sinks[:query_heads] = sinks
|
||||
return padded_sinks
|
||||
|
||||
def _fp8_flash_mla_kernel(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
kernel_metadata: FlashMLASparseMetadata.FP8KernelMetadata,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# q shape: (batch, seq_len, num_heads, head_dim)
|
||||
actual_num_heads = q.size(2)
|
||||
padded_num_heads = self.fp8_decode_padded_heads
|
||||
attn_sink = self._sinks_for_query(q, head_dim=2, kernel_heads=padded_num_heads)
|
||||
|
||||
# Pad query if needed (kernel only supports h_q = 64 or 128)
|
||||
if actual_num_heads < padded_num_heads:
|
||||
logger.warning_once(
|
||||
f"Padding num_heads from {actual_num_heads} to "
|
||||
f"{padded_num_heads} for FP8 sparse decode kernel"
|
||||
)
|
||||
q_padded = q.new_zeros((q.size(0), q.size(1), padded_num_heads, q.size(3)))
|
||||
q_padded[:, :, :actual_num_heads, :] = q
|
||||
q = q_padded
|
||||
|
||||
out, lse = flash_mla_with_kvcache(
|
||||
q=q,
|
||||
k_cache=kv_c_and_k_pe_cache.view(torch.uint8).unsqueeze(-2),
|
||||
block_table=kernel_metadata.dummy_block_table,
|
||||
head_dim_v=512,
|
||||
cache_seqlens=kernel_metadata.cache_lens,
|
||||
tile_scheduler_metadata=kernel_metadata.scheduler_metadata,
|
||||
is_fp8_kvcache=True,
|
||||
indices=topk_indices,
|
||||
softmax_scale=self.softmax_scale,
|
||||
attn_sink=attn_sink,
|
||||
)
|
||||
|
||||
# Slice output and lse back to actual head count if we padded
|
||||
if actual_num_heads < padded_num_heads:
|
||||
out = out[:, :, :actual_num_heads, :]
|
||||
lse = lse[:, :actual_num_heads, :]
|
||||
|
||||
return out, lse
|
||||
|
||||
def _bf16_flash_mla_kernel(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_length: torch.Tensor | None = None,
|
||||
actual_num_heads: int | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
num_tokens = q.shape[0]
|
||||
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(
|
||||
-1, 1, kv_c_and_k_pe_cache.shape[-1]
|
||||
)
|
||||
|
||||
# NOTE(Chen): kernel requires num_local_head to be a multiple of
|
||||
# 64 on hopper and 128 on blackwell. Pad from q's head count, not
|
||||
# self.num_heads: under DCP the heads are all-gathered before this.
|
||||
if actual_num_heads is None:
|
||||
actual_num_heads = q.shape[1]
|
||||
padded_num_heads = (
|
||||
(actual_num_heads + self.prefill_padding - 1)
|
||||
// self.prefill_padding
|
||||
* self.prefill_padding
|
||||
)
|
||||
# q_concat_buffer may already include the kernel-required padding.
|
||||
# The sink remains defined only for the real query heads.
|
||||
attn_sink = self._sinks_for_query(
|
||||
q[:, :actual_num_heads], head_dim=1, kernel_heads=padded_num_heads
|
||||
)
|
||||
|
||||
if q.shape[1] < padded_num_heads:
|
||||
logger.warning_once(
|
||||
f"Padding num_heads from {actual_num_heads} to "
|
||||
f"{padded_num_heads} for BF16 sparse prefill kernel"
|
||||
)
|
||||
# Zero (not new_empty) the padded lanes: topk_indices is shared by
|
||||
# all heads, so the kernel reduces across the head group and NaNs
|
||||
# from uninitialized memory would leak into the real heads.
|
||||
q_padded = q.new_zeros((q.shape[0], padded_num_heads, q.shape[2]))
|
||||
q_padded[:, :actual_num_heads, :] = q
|
||||
q = q_padded
|
||||
|
||||
topk_indices = topk_indices.view(num_tokens, 1, -1)
|
||||
output, _, lse = flash_mla_sparse_fwd(
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
topk_indices,
|
||||
self.softmax_scale,
|
||||
attn_sink=attn_sink,
|
||||
topk_length=topk_length,
|
||||
)
|
||||
|
||||
output = output[:, :actual_num_heads, :]
|
||||
lse = lse[:, :actual_num_heads]
|
||||
return output, lse
|
||||
|
||||
|
||||
class HYV4FlashMLASparseBackend(FlashMLASparseBackend):
|
||||
"""``FLASHMLA_SPARSE`` with attention-sink support for HY V4.
|
||||
|
||||
Keeps the parent's name, metadata and builder; only the impl class and the
|
||||
sink capability differ. See the module docstring for why the name is reused.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type[HYV4FlashMLASparseImpl]:
|
||||
return HYV4FlashMLASparseImpl
|
||||
|
||||
@classmethod
|
||||
def supports_sink(cls) -> bool:
|
||||
return True
|
||||
@@ -0,0 +1,370 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""iHC (independent Hyper-Connections) layers for HY V4 (NVIDIA).
|
||||
|
||||
iHC replaces the single residual stream of a standard transformer with
|
||||
``hc_mult`` parallel residual channels. Each decoder sub-block reduces the
|
||||
channels to one hidden state (``HYV4HCPreLayer``), runs the sub-block, then
|
||||
scatters the result back over the channels (``HYV4HCPostLayer``). The final
|
||||
``HYV4HCHeadLayer`` merges the channels before the model's output norm.
|
||||
|
||||
NOTE: Each of the three steps has an optional single-kernel HPC replacement
|
||||
(``HpcIHCPre`` / ``HpcIHCPost`` / ``HpcIHCHead``). They are only constructed
|
||||
when the hpc package is installed, ``VLLM_ENABLE_HPC_OPS=1`` and the shape /
|
||||
device constraints hold; otherwise the eager path below runs unchanged.
|
||||
TODO: port the cross-layer post+pre fusion (``HpcIHCPostPre``) as well; it
|
||||
requires restructuring the decoder-layer forward scheduling.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.model_executor.layers.hpc import HpcIHCHead, HpcIHCPost, HpcIHCPre
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
|
||||
|
||||
class HYV4HCPreLayer(nn.Module):
|
||||
"""iHC pre-processing layer (2D-activation adaptation).
|
||||
|
||||
Steps:
|
||||
1. RMS-normalize the flattened ``[num_tokens, hc * d]`` input.
|
||||
2. Project to the pre/post gating logits.
|
||||
3. Turn the logits into sigmoid gates.
|
||||
4. Reduce over the channel dim with the pre gates.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
hidden_dim: int,
|
||||
hc_mult: int = 4,
|
||||
magnitude: float = 2.0,
|
||||
init_std: float = 6e-3,
|
||||
base_noise_std: float = 0.0,
|
||||
hc_eps: float = 1e-6,
|
||||
layernorm_epsilon: float = 1e-5,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_dim = hidden_dim
|
||||
self.hc_mult = hc_mult
|
||||
self.magnitude = magnitude
|
||||
self.hc_eps = hc_eps
|
||||
self.layernorm_epsilon = layernorm_epsilon
|
||||
hc_in_dim = hc_mult * hidden_dim
|
||||
mix_hc = 2 * hc_mult # pre + post only, no comb
|
||||
|
||||
self.hc_fn = ReplicatedLinear(
|
||||
input_size=hc_in_dim,
|
||||
output_size=mix_hc,
|
||||
params_dtype=torch.float32,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.hc_fn",
|
||||
)
|
||||
self.hc_scale = nn.Parameter(torch.empty(2, dtype=torch.float32))
|
||||
self.hc_base = nn.Parameter(torch.empty(mix_hc, dtype=torch.float32))
|
||||
|
||||
self.reset_parameters(init_std, base_noise_std)
|
||||
|
||||
# Optional single-kernel HPC replacement for the whole forward below.
|
||||
# ``norm_owner`` is left unset: the RMSNorm that follows the pre block
|
||||
# lives in the decoder layer and is not folded in yet.
|
||||
self.hpc_op: HpcIHCPre | None = None
|
||||
if HpcIHCPre.support(hc_mult, hidden_dim):
|
||||
self.hpc_op = HpcIHCPre(
|
||||
hc_mult=hc_mult,
|
||||
hidden_size=hidden_dim,
|
||||
magnitude=magnitude,
|
||||
hc_eps=hc_eps,
|
||||
norm_eps=layernorm_epsilon,
|
||||
fallback_op=self,
|
||||
)
|
||||
|
||||
def reset_parameters(self, init_std: float, base_noise_std: float = 0.0) -> None:
|
||||
"""Initialize the gate scale and per-channel gate bias."""
|
||||
del init_std # hc_fn is initialized by ReplicatedLinear
|
||||
nn.init.constant_(self.hc_scale, 0.01)
|
||||
with torch.no_grad():
|
||||
self.hc_base[: self.hc_mult].fill_(
|
||||
-torch.log(torch.tensor(self.hc_mult - 1.0, dtype=self.hc_base.dtype))
|
||||
)
|
||||
self.hc_base[self.hc_mult : 2 * self.hc_mult].fill_(0.0)
|
||||
if base_noise_std > 0.0:
|
||||
self.hc_base.add_(torch.randn_like(self.hc_base) * base_noise_std)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Reduce the iHC channels and emit the post gates.
|
||||
|
||||
Args:
|
||||
x: Input of shape ``[num_tokens, hc, d]``.
|
||||
|
||||
Returns:
|
||||
A tuple of the pre-gated reduction ``[num_tokens, d]`` and the post
|
||||
gates ``[num_tokens, hc]`` consumed by `HYV4HCPostLayer`.
|
||||
"""
|
||||
if self.hpc_op is not None:
|
||||
return self.hpc_op(x)
|
||||
|
||||
shape = x.size() # [num_tokens, hc, d]
|
||||
hc = self.hc_mult
|
||||
hc_eps = self.hc_eps
|
||||
|
||||
x_flat = x.flatten(1).float() # [num_tokens, hc*d]
|
||||
|
||||
rsqrt = torch.rsqrt(
|
||||
x_flat.square().mean(-1, keepdim=True) + self.layernorm_epsilon
|
||||
)
|
||||
mixes = self.hc_fn(x_flat)[0] * rsqrt # [num_tokens, 2*hc]
|
||||
|
||||
pre_raw = mixes[..., :hc]
|
||||
post_raw = mixes[..., hc : 2 * hc]
|
||||
|
||||
pre = (
|
||||
torch.sigmoid(
|
||||
pre_raw * self.hc_scale[0].float() + self.hc_base[:hc].float()
|
||||
)
|
||||
+ hc_eps
|
||||
)
|
||||
post = (
|
||||
self.magnitude
|
||||
* torch.sigmoid(
|
||||
post_raw * self.hc_scale[1].float() + self.hc_base[hc : 2 * hc].float()
|
||||
)
|
||||
+ hc_eps
|
||||
)
|
||||
|
||||
y = torch.sum(pre.unsqueeze(-1) * x.reshape(shape), dim=1) # [num_tokens, d]
|
||||
return y.to(x.dtype), post
|
||||
|
||||
|
||||
class HYV4HCPostLayer(nn.Module):
|
||||
"""iHC post-processing layer (2D-activation adaptation).
|
||||
|
||||
Applies post-gating to the sub-block output and adds the multi-channel
|
||||
residual (no comb mixing)::
|
||||
|
||||
y[n, i, d] = post[n, i] * x[n, d] + residual[n, i, d]
|
||||
"""
|
||||
|
||||
def __init__(self, config: PretrainedConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
|
||||
# Optional single-kernel HPC replacement for the whole forward below.
|
||||
# Only constructed under enable_ihc (see HYV4HCLayer), so hc_mult is
|
||||
# present; getattr keeps this robust if that ever changes.
|
||||
self.hpc_op: HpcIHCPost | None = None
|
||||
hc_mult = getattr(config, "hc_mult", 0)
|
||||
if HpcIHCPost.support(hc_mult, config.hidden_size):
|
||||
self.hpc_op = HpcIHCPost(hc_mult=hc_mult, hidden_size=config.hidden_size)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Scatter the sub-block output back onto the iHC channels.
|
||||
|
||||
Args:
|
||||
x: Attention/MLP output of shape ``[num_tokens, d]``.
|
||||
residual: Multi-channel residual ``[num_tokens, hc, d]``.
|
||||
post: Post gates ``[num_tokens, hc]`` from `HYV4HCPreLayer`.
|
||||
|
||||
Returns:
|
||||
The updated residual channels ``[num_tokens, hc, d]``.
|
||||
"""
|
||||
if self.hpc_op is not None:
|
||||
return self.hpc_op(x, residual, post)
|
||||
|
||||
dtype = x.dtype
|
||||
x = x.float()
|
||||
residual = residual.float()
|
||||
post = post.float()
|
||||
|
||||
post_gated = post.unsqueeze(-1) * x.unsqueeze(-2) # [num_tokens, hc, d]
|
||||
y = post_gated + residual
|
||||
return y.to(dtype)
|
||||
|
||||
|
||||
class HYV4HCHeadLayer(nn.Module):
|
||||
"""iHC head layer (2D-activation adaptation).
|
||||
|
||||
Merges the iHC channels back into a single hidden state before the final
|
||||
layer norm, using an RMS-normed projection plus sigmoid-gated reduction.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
hidden_size: int,
|
||||
hc_mult: int = 4,
|
||||
hc_eps: float = 1e-6,
|
||||
init_std: float = 6e-3,
|
||||
base_noise_std: float = 0.0,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_size = hidden_size
|
||||
self.hc_mult = hc_mult
|
||||
self.hc_eps = hc_eps
|
||||
self.hc_head_fn = ReplicatedLinear(
|
||||
input_size=hc_mult * hidden_size,
|
||||
output_size=hc_mult,
|
||||
params_dtype=torch.float32,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.hc_head_fn",
|
||||
)
|
||||
self.hc_head_base = nn.Parameter(torch.empty(hc_mult, dtype=torch.float32))
|
||||
self.hc_head_scale = nn.Parameter(torch.empty(1, dtype=torch.float32))
|
||||
|
||||
self.reset_parameters(init_std, base_noise_std)
|
||||
|
||||
# Optional single-kernel HPC replacement for the whole forward below.
|
||||
self.hpc_op: HpcIHCHead | None = None
|
||||
if HpcIHCHead.support(hc_mult, hidden_size):
|
||||
self.hpc_op = HpcIHCHead(
|
||||
hc_mult=hc_mult,
|
||||
hidden_size=hidden_size,
|
||||
hc_eps=hc_eps,
|
||||
norm_eps=config.rms_norm_eps,
|
||||
fallback_op=self,
|
||||
)
|
||||
|
||||
def reset_parameters(
|
||||
self, init_std: float = 6e-3, base_noise_std: float = 0.0
|
||||
) -> None:
|
||||
"""Initialize the head gate scale and per-channel gate bias."""
|
||||
del init_std # hc_head_fn is initialized by ReplicatedLinear
|
||||
nn.init.constant_(self.hc_head_scale, 0.01)
|
||||
with torch.no_grad():
|
||||
self.hc_head_base.fill_(
|
||||
-torch.log(
|
||||
torch.tensor(self.hc_mult - 1.0, dtype=self.hc_head_base.dtype)
|
||||
)
|
||||
)
|
||||
if base_noise_std > 0.0:
|
||||
self.hc_head_base.add_(
|
||||
torch.randn_like(self.hc_head_base) * base_noise_std
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Merge the iHC channels into a single hidden state.
|
||||
|
||||
Args:
|
||||
x: Input of shape ``[num_tokens, hc, d]``.
|
||||
|
||||
Returns:
|
||||
The merged hidden state ``[num_tokens, d]``.
|
||||
"""
|
||||
if self.hpc_op is not None:
|
||||
return self.hpc_op(x)
|
||||
|
||||
shape, x_dtype = x.size(), x.dtype
|
||||
|
||||
x = x.flatten(1).float() # [num_tokens, hc*d]
|
||||
rsqrt = torch.rsqrt(
|
||||
x.square().mean(-1, keepdim=True) + self.config.rms_norm_eps
|
||||
)
|
||||
mixes = self.hc_head_fn(x)[0] * rsqrt # [num_tokens, hc]
|
||||
pre = (
|
||||
torch.sigmoid(
|
||||
mixes * self.hc_head_scale.float() + self.hc_head_base.float()
|
||||
)
|
||||
+ self.hc_eps
|
||||
)
|
||||
y = torch.sum(pre.unsqueeze(-1) * x.reshape(shape), dim=1) # [num_tokens, d]
|
||||
return y.to(x_dtype)
|
||||
|
||||
|
||||
class HYV4HCLayer(nn.Module):
|
||||
"""Wrapper owning one iHC boundary (pre + post) of a decoder sub-block."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
layer_idx: int,
|
||||
init_std: float = 6e-3,
|
||||
base_noise_std: float = 0.0,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.layer_idx = layer_idx
|
||||
self.enable_ihc = getattr(config, "enable_ihc", False)
|
||||
if self.enable_ihc:
|
||||
self.hc_pre = HYV4HCPreLayer(
|
||||
config,
|
||||
config.hidden_size,
|
||||
config.hc_mult,
|
||||
config.hc_magnitude,
|
||||
init_std,
|
||||
base_noise_std,
|
||||
config.hc_eps,
|
||||
config.rms_norm_eps,
|
||||
prefix=f"{prefix}.hc_pre",
|
||||
)
|
||||
self.hc_post = HYV4HCPostLayer(config)
|
||||
|
||||
def prepare_input(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Normalize the sub-block input to 3D when iHC is enabled."""
|
||||
if not self.enable_ihc:
|
||||
return hidden_states
|
||||
return self._prepare_input_to_3d(hidden_states)
|
||||
|
||||
def _prepare_input_to_3d(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Reshape the iHC input to ``[num_tokens, hc, h]``.
|
||||
|
||||
Accepted inputs are ``[num_tokens, hc, h]`` (no-op), ``[num_tokens, h]``
|
||||
(broadcast over the channels) and ``[num_tokens, hc * h]`` (reshape).
|
||||
"""
|
||||
if hidden_states.dim() == 3:
|
||||
return hidden_states
|
||||
if hidden_states.dim() != 2:
|
||||
raise RuntimeError(
|
||||
f"HC expects a 2D/3D tensor, got shape={tuple(hidden_states.shape)}"
|
||||
)
|
||||
|
||||
d0, d1 = hidden_states.shape
|
||||
h = self.config.hidden_size
|
||||
hc = self.config.hc_mult
|
||||
|
||||
if d1 == h:
|
||||
return hidden_states.unsqueeze(1).repeat(1, hc, 1)
|
||||
|
||||
expected = hc * h
|
||||
if d1 == expected:
|
||||
return hidden_states.reshape(d0, hc, h)
|
||||
|
||||
raise RuntimeError(
|
||||
f"HC expects last dim to be hidden_size ({h})"
|
||||
f"or hc_mult*hidden_size ({expected}), got {d1}. "
|
||||
f"hc_mult={hc}, hidden_size={h}."
|
||||
)
|
||||
|
||||
def pre(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
|
||||
"""Reduce the iHC channels and produce the post gates.
|
||||
|
||||
Returns:
|
||||
A tuple of the reduced hidden states ``[num_tokens, d]``, the post
|
||||
gates ``[num_tokens, hc]`` (``None`` when iHC is disabled) and the
|
||||
residual (the untouched input).
|
||||
"""
|
||||
if not self.enable_ihc:
|
||||
return hidden_states, None, hidden_states
|
||||
reduced, post_gates = self.hc_pre(hidden_states)
|
||||
return reduced, post_gates, hidden_states
|
||||
|
||||
def post(
|
||||
self,
|
||||
output_with_bias: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_gates: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply post-gating and add the residual."""
|
||||
if not self.enable_ihc:
|
||||
return output_with_bias + residual
|
||||
assert post_gates is not None
|
||||
return self.hc_post(output_with_bias, residual, post_gates)
|
||||
@@ -0,0 +1,716 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
||||
# and OPT implementations in this library. It has been modified from its
|
||||
# original forms to accommodate minor architectural differences compared
|
||||
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only HY V4 model compatible with HuggingFace weights (NVIDIA)."""
|
||||
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable, MutableSequence, Sequence
|
||||
from itertools import islice
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.deepseek_v2 import _try_load_fp8_indexer_wk
|
||||
from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
get_pp_missing_layer_names,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .attention import (
|
||||
HYV4MLAAttention,
|
||||
compute_skip_topk_layers,
|
||||
is_skip_topk_indexer_weight,
|
||||
)
|
||||
from .hc import HYV4HCHeadLayer, HYV4HCLayer
|
||||
from .moe import HYV4FeedForward, HYV4MoEFused
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _normalize_hyv4_config(config: PretrainedConfig) -> PretrainedConfig:
|
||||
"""Populate the aliases consumed by the shared MoE implementation."""
|
||||
config.router_scaling_factor = config.routed_scaling_factor
|
||||
config.num_experts = config.n_routed_experts
|
||||
config.expert_hidden_dim = config.moe_intermediate_size
|
||||
config.num_shared_experts = config.n_shared_experts
|
||||
config.route_norm = config.norm_topk_prob
|
||||
return config
|
||||
|
||||
|
||||
class HYV4DecoderLayer(nn.Module):
|
||||
"""One HY V4 decoder layer: MLA attention plus a dense or MoE MLP.
|
||||
|
||||
When``config.enable_ihc`` is set the layer runs on ``hc_mult`` residual
|
||||
channels and each sub-block is wrapped by an `HYV4HCLayer` boundary;
|
||||
otherwise it uses the standard single-stream residual.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
vllm_config: VllmConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.config = config
|
||||
self.enable_ihc = getattr(config, "enable_ihc", False)
|
||||
layer_idx = int(prefix.split(".")[-1])
|
||||
self.layer_idx = layer_idx
|
||||
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
|
||||
self.self_attn = HYV4MLAAttention(
|
||||
vllm_config=vllm_config,
|
||||
config=config,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
qk_nope_head_dim=config.qk_nope_head_dim,
|
||||
qk_rope_head_dim=config.qk_rope_head_dim,
|
||||
v_head_dim=config.v_head_dim,
|
||||
q_lora_rank=config.q_lora_rank,
|
||||
kv_lora_rank=config.kv_lora_rank,
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
layer_idx=layer_idx,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps)
|
||||
if config.mlp_layer_types[layer_idx] == "dense":
|
||||
self.mlp = HYV4FeedForward(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.block_type = "feedforward"
|
||||
else:
|
||||
self.mlp = HYV4MoEFused(
|
||||
config=config, quant_config=quant_config, prefix=f"{prefix}.mlp"
|
||||
)
|
||||
self.block_type = "moe"
|
||||
self.post_attention_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps)
|
||||
self.hc_attn_layer = HYV4HCLayer(
|
||||
config, layer_idx, prefix=f"{prefix}.hc_attn_layer"
|
||||
)
|
||||
self.hc_mlp_layer = HYV4HCLayer(
|
||||
config, layer_idx, prefix=f"{prefix}.hc_mlp_layer"
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if self.enable_ihc:
|
||||
return self._forward_ihc(positions, hidden_states)
|
||||
return self._forward_normal(positions, hidden_states, residual)
|
||||
|
||||
def _forward_normal(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Standard single-residual-stream forward (iHC disabled)."""
|
||||
if residual is not None:
|
||||
hidden_states = hidden_states + residual
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
|
||||
hidden_states = hidden_states + residual
|
||||
residual = hidden_states
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
def _forward_ihc(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, None]:
|
||||
"""iHC forward: each sub-block reduces and re-scatters the channels."""
|
||||
hidden_states = self.hc_attn_layer.prepare_input(hidden_states)
|
||||
hidden_states, post_gates, residual = self.hc_attn_layer.pre(hidden_states)
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
hidden_states = self.hc_attn_layer.post(hidden_states, residual, post_gates)
|
||||
|
||||
hidden_states = self.hc_mlp_layer.prepare_input(hidden_states)
|
||||
hidden_states, post_gates, residual = self.hc_mlp_layer.pre(hidden_states)
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = self.hc_mlp_layer.post(hidden_states, residual, post_gates)
|
||||
|
||||
# Under iHC the residual is carried inside hidden_states.
|
||||
return hidden_states, None
|
||||
|
||||
|
||||
class HYV4Model(nn.Module):
|
||||
"""HY V4 backbone."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = _normalize_hyv4_config(vllm_config.model_config.hf_config)
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
eplb_config = parallel_config.eplb_config
|
||||
self.num_redundant_experts = eplb_config.num_redundant_experts
|
||||
self.device = current_platform.device_type
|
||||
self.vocab_size = config.vocab_size
|
||||
self.is_sparse = hasattr(config, "index_topk")
|
||||
if self.is_sparse:
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
else:
|
||||
self.topk_indices_buffer = None
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.enable_ihc = getattr(config, "enable_ihc", False)
|
||||
|
||||
if get_pp_group().is_first_rank or (
|
||||
self.config.tie_word_embeddings and get_pp_group().is_last_rank
|
||||
):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: HYV4DecoderLayer(
|
||||
config=config,
|
||||
vllm_config=vllm_config,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
if self.enable_ihc:
|
||||
# iHC head layer: merge the residual channels into one stream.
|
||||
if get_pp_group().is_last_rank:
|
||||
self.hc_head = HYV4HCHeadLayer(
|
||||
config,
|
||||
hidden_size=config.hidden_size,
|
||||
hc_mult=config.hc_mult,
|
||||
hc_eps=config.hc_eps,
|
||||
prefix=f"{prefix}.hc_head",
|
||||
)
|
||||
else:
|
||||
self.hc_head = PPMissingLayer()
|
||||
self.make_empty_intermediate_tensors = (
|
||||
make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], config.hc_mult * config.hidden_size
|
||||
)
|
||||
)
|
||||
else:
|
||||
self.make_empty_intermediate_tensors = (
|
||||
make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
)
|
||||
|
||||
# MoE hyperparameters (consumed by EPLB).
|
||||
self.expert_weights: MutableSequence[Sequence[torch.Tensor]] = []
|
||||
self.num_expert_groups = 1
|
||||
self.moe_layers: list[nn.Module] = []
|
||||
example_layer: HYV4MoEFused | None = None
|
||||
for layer in self.layers:
|
||||
if isinstance(layer, PPMissingLayer):
|
||||
continue
|
||||
|
||||
assert isinstance(layer, HYV4DecoderLayer)
|
||||
if layer.block_type == "moe":
|
||||
assert isinstance(layer.mlp, HYV4MoEFused)
|
||||
example_layer = layer.mlp
|
||||
self.moe_layers.append(layer.mlp.experts)
|
||||
|
||||
if example_layer is None:
|
||||
self.num_moe_layers = 0
|
||||
self.num_logical_experts = 0
|
||||
self.num_physical_experts = 0
|
||||
self.num_local_physical_experts = 0
|
||||
self.num_routed_experts = 0
|
||||
self.num_redundant_experts = 0
|
||||
return
|
||||
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
self.num_logical_experts = example_layer.n_logical_experts
|
||||
self.num_physical_experts = example_layer.n_physical_experts
|
||||
self.num_local_physical_experts = example_layer.n_local_physical_experts
|
||||
self.num_routed_experts = example_layer.n_routed_experts
|
||||
self.num_redundant_experts = example_layer.n_redundant_experts
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def update_physical_experts_metadata(
|
||||
self,
|
||||
num_physical_experts: int,
|
||||
num_local_physical_experts: int,
|
||||
) -> None:
|
||||
assert self.num_local_physical_experts == num_local_physical_experts
|
||||
self.num_physical_experts = num_physical_experts
|
||||
self.num_local_physical_experts = num_local_physical_experts
|
||||
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
|
||||
for layer in self.layers:
|
||||
if isinstance(layer, PPMissingLayer):
|
||||
continue
|
||||
if isinstance(layer.mlp, HYV4MoEFused):
|
||||
moe = layer.mlp
|
||||
moe.n_local_physical_experts = num_local_physical_experts
|
||||
moe.n_physical_experts = num_physical_experts
|
||||
moe.n_redundant_experts = self.num_redundant_experts
|
||||
moe.experts.update_expert_map()
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
# (param_name, weight_name, expert_id, shard_id) for weights, fp8
|
||||
# weight scales and fp8 activation scales.
|
||||
if not hasattr(self, "_cached_expert_params_mapping"):
|
||||
self._cached_expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
return self._cached_expert_params_mapping
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
# In iHC mode the flattened [num_tokens, hc*h] tensor from the
|
||||
# previous PP stage is reshaped back to 3D by the first layer's
|
||||
# prepare_input, and residual is unused.
|
||||
residual = None if self.enable_ihc else intermediate_tensors["residual"]
|
||||
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
if self.enable_ihc:
|
||||
# hidden_states is [num_tokens, hc, h]; flatten the channel dim
|
||||
# for PP transfer (matches the 2D receive buffer).
|
||||
return IntermediateTensors({"hidden_states": hidden_states.flatten(1)})
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
if self.enable_ihc:
|
||||
hidden_states = self.hc_head(hidden_states)
|
||||
else:
|
||||
hidden_states = hidden_states + residual
|
||||
|
||||
return self.norm(hidden_states)
|
||||
|
||||
def load_fused_expert_weights(
|
||||
self,
|
||||
name: str,
|
||||
params_dict: dict,
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_id: str,
|
||||
num_experts: int,
|
||||
) -> bool:
|
||||
param = params_dict[name]
|
||||
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
|
||||
loaded_local_expert = False
|
||||
for expert_id in range(num_experts):
|
||||
curr_expert_weight = loaded_weight[expert_id]
|
||||
success = weight_loader(
|
||||
param,
|
||||
curr_expert_weight,
|
||||
name,
|
||||
shard_id,
|
||||
expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
loaded_local_expert = True
|
||||
|
||||
return loaded_local_expert
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# FP8 indexer wk dequant buffer (weight and scale arrive separately).
|
||||
pending_wk_fp8: dict[str, dict[str, torch.Tensor]] = {}
|
||||
pp_missing_layer_names = get_pp_missing_layer_names(self)
|
||||
skip_topk_layers = compute_skip_topk_layers(self.config)
|
||||
|
||||
# Must not be cached on `self`: `process_weights_after_loading` swaps in
|
||||
# kernel-specific expert weights via `replace_parameter`, and a cache
|
||||
# outliving this call would pin the pre-shuffle storage (OOM on large
|
||||
# MoE) and make a later reload target orphaned tensors.
|
||||
params_dict = dict(self.named_parameters())
|
||||
# Split per-expert mapping (V3 style): experts.0.gate_proj.weight
|
||||
split_expert_params_mapping = self.get_expert_mapping()
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
# Sink weights are sharded like the q/k/v linears.
|
||||
sink_tp_size = get_tensor_model_parallel_world_size()
|
||||
sink_tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // sink_tp_size
|
||||
head_rank_start = n_local_head * sink_tp_rank
|
||||
head_rank_end = n_local_head * (sink_tp_rank + 1)
|
||||
base_layer = (
|
||||
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
|
||||
)
|
||||
# Fused expert mapping: experts.gate_up_proj (all experts in one tensor).
|
||||
# The packed weights are owned by the RoutedExperts submodule, so the
|
||||
# targets are experts.routed_experts.[base_layer.]w{13,2}_weight.
|
||||
fused_expert_prefix = f".experts.routed_experts.{base_layer}"
|
||||
fused_expert_params_mapping = [
|
||||
(f"{fused_expert_prefix}w13_weight", ".experts.gate_up_proj", 0, "w1"),
|
||||
(f"{fused_expert_prefix}w2_weight", ".experts.down_proj", 0, "w2"),
|
||||
]
|
||||
num_experts = getattr(self.config, "num_experts", 0)
|
||||
|
||||
def _should_skip_missing_param(param_name: str) -> bool:
|
||||
# Sparse checkpoints may contain indexer weights for layers that
|
||||
# fall back to dense MLA on unsupported GPUs.
|
||||
return ".indexer." in param_name
|
||||
|
||||
def _is_split_expert_weight(weight_name: str) -> bool:
|
||||
"""Whether this weight is in split (per-expert) format."""
|
||||
# Split format: mlp.experts.<id>.gate_proj.weight
|
||||
return bool(re.search(r"\.experts\.\d+\.", weight_name))
|
||||
|
||||
def _is_fused_expert_weight(weight_name: str) -> bool:
|
||||
"""Whether this weight is in fused (all-experts-packed) format."""
|
||||
return ".experts.gate_up_proj" in weight_name or (
|
||||
".experts.down_proj" in weight_name
|
||||
and not _is_split_expert_weight(weight_name)
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if is_skip_topk_indexer_weight(name, skip_topk_layers):
|
||||
continue
|
||||
if _try_load_fp8_indexer_wk(
|
||||
name,
|
||||
loaded_weight,
|
||||
pending_wk_fp8,
|
||||
params_dict,
|
||||
loaded_params,
|
||||
pp_missing_layer_names,
|
||||
):
|
||||
continue
|
||||
if self.config.tie_word_embeddings and "lm_head.weight" in name:
|
||||
continue
|
||||
# Names and stacked shard ids are normalized upstream by
|
||||
# AutoWeightsLoader, through quant_config.get_cache_scale_mapper()
|
||||
# and hf_to_vllm_mapper.
|
||||
stacked_shard_id = getattr(loaded_weight, "shard_id", None)
|
||||
if ".indexer.wk." in name:
|
||||
name = name.replace(".indexer.wk.", ".indexer.wk_weights_proj.")
|
||||
stacked_shard_id = 0
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
# Determine per-weight whether this is fused or split format.
|
||||
is_fused_expert = _is_fused_expert_weight(name)
|
||||
expert_params_mapping = (
|
||||
fused_expert_params_mapping
|
||||
if is_fused_expert
|
||||
else split_expert_params_mapping
|
||||
)
|
||||
|
||||
is_expert_weight = False
|
||||
loaded_expert_param_names: set[str] = set()
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
# This is an expert weight and must not be attempted as any
|
||||
# other kind of weight later on.
|
||||
is_expert_weight = True
|
||||
|
||||
# Do not modify `name`: the loop may continue past this point.
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
if is_fused_expert:
|
||||
if "experts.gate_up_proj" in name:
|
||||
chunks = loaded_weight.chunk(2, dim=-2)
|
||||
success_w1 = self.load_fused_expert_weights(
|
||||
name_mapped, params_dict, chunks[0], "w1", num_experts
|
||||
)
|
||||
success_w3 = self.load_fused_expert_weights(
|
||||
name_mapped, params_dict, chunks[1], "w3", num_experts
|
||||
)
|
||||
success = success_w1 and success_w3
|
||||
else:
|
||||
success = self.load_fused_expert_weights(
|
||||
name_mapped,
|
||||
params_dict,
|
||||
loaded_weight,
|
||||
shard_id,
|
||||
num_experts,
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
# Split per-expert format (V3 style).
|
||||
if name_mapped not in params_dict:
|
||||
if _should_skip_missing_param(name_mapped):
|
||||
continue
|
||||
logger.warning_once(
|
||||
"Skipping unknown checkpoint weight: %s",
|
||||
name_mapped,
|
||||
)
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
# Ask the weight loader whether it succeeded, otherwise we
|
||||
# may skip experts that have other available replicas.
|
||||
weight_loader: Callable[..., typing.Any] = param.weight_loader
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
if not is_fused_expert:
|
||||
loaded_expert_param_names.add(name_mapped)
|
||||
continue
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if loaded_expert_param_names:
|
||||
loaded_params.update(loaded_expert_param_names)
|
||||
continue
|
||||
if "learnable_sink_param" in name:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
with torch.no_grad():
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
else:
|
||||
if is_expert_weight:
|
||||
# An expert weight that is not mapped to this rank.
|
||||
continue
|
||||
if "gate.e_score_correction_bias" in name:
|
||||
name = name.replace(
|
||||
"gate.e_score_correction_bias", "expert_bias"
|
||||
)
|
||||
if "router.gate." in name:
|
||||
name = name.replace("router.", "")
|
||||
if "hc_fn" in name:
|
||||
name = name.replace("hc_fn", "hc_fn.weight")
|
||||
if "hc_head_fn" in name:
|
||||
name = name.replace("hc_head_fn", "hc_head_fn.weight")
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
if name not in params_dict:
|
||||
if _should_skip_missing_param(name):
|
||||
continue
|
||||
logger.warning_once(
|
||||
"Skipping unknown checkpoint weight: %s",
|
||||
name,
|
||||
)
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
if stacked_shard_id is None:
|
||||
weight_loader(param, loaded_weight)
|
||||
else:
|
||||
weight_loader(param, loaded_weight, stacked_shard_id)
|
||||
loaded_params.add(name)
|
||||
|
||||
return loaded_params
|
||||
|
||||
|
||||
class HYV4ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_a_proj": (".fused_qkv_a_proj", 0),
|
||||
".kv_a_proj_with_mqa": (".fused_qkv_a_proj", 1),
|
||||
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
|
||||
".mlp.up_proj": (".mlp.gate_up_proj", 1),
|
||||
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
|
||||
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
|
||||
".indexer.weights_proj.": (".indexer.wk_weights_proj.", 1),
|
||||
}
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
# MLA runs both latent down-projections as one GEMM.
|
||||
"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"],
|
||||
# The indexer fuses wk and weights_proj into one GEMM.
|
||||
"wk_weights_proj": ["wk", "weights_proj"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
if quant_config is not None:
|
||||
quant_config.packed_modules_mapping = self.packed_modules_mapping
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
eplb_config = parallel_config.eplb_config
|
||||
self.num_redundant_experts = eplb_config.num_redundant_experts
|
||||
|
||||
self.model = HYV4Model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
# The head stays in the model dtype; ``enable_lm_head_fp32`` is
|
||||
# surfaced as ``head_dtype`` on the config (see HYV4Config), which
|
||||
# makes LogitsProcessor accumulate the projection straight into
|
||||
# fp32 instead of materializing an fp32 copy of the weight.
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
# With tie_word_embeddings, embed_tokens is kept on the last rank
|
||||
# (see HYV4Model.__init__) so the weight can be shared here.
|
||||
if self.config.tie_word_embeddings:
|
||||
self.lm_head.weight = self.model.embed_tokens.weight
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
if getattr(self.config, "soft_logits_capping", False):
|
||||
soft_cap = self.config.soft_logits_capping_logits
|
||||
logits = soft_cap * torch.nn.functional.tanh(logits / soft_cap)
|
||||
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# MTP layers are loaded by the MTP head, not by the target model. They
|
||||
# appear as model.mtp_layers.<i>.* and as the layer ids past the backbone.
|
||||
mtp_start = self.config.num_hidden_layers
|
||||
drop_prefixes: dict[str, str | None] = {"model.mtp_layers.": None}
|
||||
for i in range(getattr(self.config, "num_nextn_predict_layers", 0)):
|
||||
drop_prefixes[f"model.layers.{mtp_start + i}."] = None
|
||||
if self.config.tie_word_embeddings:
|
||||
drop_prefixes["lm_head."] = None
|
||||
drop_mapper = WeightsMapper(orig_to_new_prefix=drop_prefixes)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper | drop_mapper)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Dense FFN and MoE blocks for HY V4 (NVIDIA)."""
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, GateLinear
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class HYV4FeedForward(nn.Module):
|
||||
"""Dense SwiGLU feed-forward block.
|
||||
|
||||
Used both for the dense decoder layers and for the MoE shared experts.
|
||||
The routed-expert SwiGLU clamp does not apply here, matching the reference
|
||||
implementation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
out = self.act_fn(gate_up)
|
||||
out, _ = self.down_proj(out)
|
||||
return out
|
||||
|
||||
|
||||
class HYV4MoEFused(nn.Module):
|
||||
"""HY V4 MoE layer with optional SwiGLU clamp support.
|
||||
|
||||
When ``config.swiglu_limit > 0`` the routed experts use a clamped SwiGLU::
|
||||
|
||||
gate = clamp(gate, max=limit)
|
||||
up = clamp(up, -limit, limit)
|
||||
output = silu(gate) * up
|
||||
|
||||
Dense layers and shared experts are NOT clamped. The clamp is forwarded to
|
||||
``FusedMoEFactory`` as ``swiglu_limit`` so it travels through the quant
|
||||
config to every expert backend instead of mutating global state.
|
||||
|
||||
NOTE: The reference implementation additionally supports HPC gate kernels,
|
||||
TPCP/MLPSP sharded shared experts and a side-stream overlap for the shared
|
||||
experts. Those depend on infrastructure that is absent here, so this port
|
||||
keeps only the standard path. TODO: restore once available.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
enable_eplb: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.ep_group = get_ep_group().device_group
|
||||
self.ep_rank = get_ep_group().rank_in_group
|
||||
self.ep_size = self.ep_group.size()
|
||||
self.n_routed_experts = config.num_experts
|
||||
if self.tp_size > config.num_experts:
|
||||
raise ValueError(
|
||||
f"Tensor parallel size {self.tp_size} is greater than "
|
||||
f"the number of experts {config.num_experts}."
|
||||
)
|
||||
top_k = config.num_experts_per_tok
|
||||
intermediate_size = config.expert_hidden_dim
|
||||
router_scaling_factor = getattr(config, "router_scaling_factor", 1.0)
|
||||
vllm_config = get_current_vllm_config()
|
||||
eplb_config = vllm_config.parallel_config.eplb_config
|
||||
self.enable_eplb = enable_eplb
|
||||
|
||||
self.n_logical_experts = self.n_routed_experts
|
||||
self.n_redundant_experts = eplb_config.num_redundant_experts
|
||||
self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts
|
||||
self.n_local_physical_experts = self.n_physical_experts // self.ep_size
|
||||
self.physical_expert_start = self.ep_rank * self.n_local_physical_experts
|
||||
self.physical_expert_end = (
|
||||
self.physical_expert_start + self.n_local_physical_experts
|
||||
)
|
||||
|
||||
self.gate = GateLinear(
|
||||
config.hidden_size,
|
||||
config.num_experts,
|
||||
bias=False,
|
||||
out_dtype=torch.float32,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
self.shared_experts: HYV4FeedForward | None
|
||||
if config.num_shared_experts > 0:
|
||||
self.shared_experts = HYV4FeedForward(
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.expert_hidden_dim * config.num_shared_experts,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
reduce_results=False,
|
||||
)
|
||||
else:
|
||||
self.shared_experts = None
|
||||
|
||||
self.expert_bias = nn.Parameter(
|
||||
torch.empty(config.num_experts, dtype=torch.float32)
|
||||
)
|
||||
|
||||
raw_swiglu_limit = getattr(config, "swiglu_limit", 0)
|
||||
use_swiglu_clamp = bool(raw_swiglu_limit) and float(raw_swiglu_limit) > 0
|
||||
moe_swiglu_limit = float(raw_swiglu_limit) if use_swiglu_clamp else None
|
||||
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=self.n_routed_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
renormalize=config.route_norm,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
enable_eplb=self.enable_eplb,
|
||||
num_redundant_experts=self.n_redundant_experts,
|
||||
scoring_func="sigmoid",
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=1,
|
||||
topk_group=1,
|
||||
routed_scaling_factor=router_scaling_factor,
|
||||
e_score_correction_bias=self.expert_bias,
|
||||
shared_experts=self.shared_experts,
|
||||
swiglu_limit=moe_swiglu_limit,
|
||||
)
|
||||
self.prefix = prefix
|
||||
if use_swiglu_clamp:
|
||||
logger.debug_once(
|
||||
"HYV4MoEFused: swiglu_limit=%.1f enabled for routed experts",
|
||||
moe_swiglu_limit,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_dim = hidden_states.shape[-1]
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
|
||||
# router_logits: (num_tokens, n_experts); the gate stores fp32 weights
|
||||
# and emits fp32 logits, so no explicit upcast is needed here.
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states, router_logits=router_logits
|
||||
)
|
||||
return final_hidden_states.view(orig_shape)
|
||||
@@ -0,0 +1,937 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Multi-token prediction (MTP) head for HY V4 (NVIDIA)."""
|
||||
|
||||
import copy
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_mapping
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_v2 import _try_load_fp8_indexer_wk
|
||||
from vllm.model_executor.models.utils import (
|
||||
get_pp_missing_layer_names,
|
||||
is_pp_missing_parameter,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from .model import HYV4DecoderLayer, _normalize_hyv4_config
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MTP_SCALE_ONLY_PARAM_SUBSTRINGS = (
|
||||
"_scale",
|
||||
"_weight_scale",
|
||||
"_weight_scale_inv",
|
||||
"_input_scale",
|
||||
)
|
||||
|
||||
# Names of the quant-config attribute that stores the unquantized-module
|
||||
# prefix list, in the order we look them up. Different QuantizationConfig
|
||||
# subclasses expose the list under different names:
|
||||
# * ``ignored_layers`` -- Fp8Config, GPTQ / AWQ configs.
|
||||
# * ``exclude_modules`` -- ModelOpt* configs (Fp8, NvFp4, MxFp8, Mixed).
|
||||
_MTP_QUANT_EXCLUSION_ATTRS = ("ignored_layers", "exclude_modules")
|
||||
|
||||
|
||||
def _get_spec_layer_idx_from_weight_name(
|
||||
config: PretrainedConfig, weight_name: str
|
||||
) -> int | None:
|
||||
"""Return the MTP layer index a checkpoint weight belongs to, or None.
|
||||
|
||||
Compatible with``num_nextn_predict_layers`` of 1 and 2 for a single MTP
|
||||
head, matching how the HY V4 checkpoints are exported.
|
||||
"""
|
||||
if getattr(config, "num_nextn_predict_layers", 0) < 1:
|
||||
return None
|
||||
layer_idx = config.num_hidden_layers
|
||||
num_mtp_layers = max(config.num_nextn_predict_layers - 1, 1)
|
||||
for i in range(num_mtp_layers):
|
||||
if weight_name.startswith(f"model.layers.{layer_idx + i}."):
|
||||
return layer_idx + i
|
||||
return None
|
||||
|
||||
|
||||
def _should_skip_missing_mtp_scale_param(
|
||||
quant_config: QuantizationConfig | None,
|
||||
name: str,
|
||||
) -> bool:
|
||||
"""Whether an unmatched scale parameter can be silently ignored."""
|
||||
return quant_config is None and any(
|
||||
scale_name in name for scale_name in _MTP_SCALE_ONLY_PARAM_SUBSTRINGS
|
||||
)
|
||||
|
||||
|
||||
def _resolve_fused_expert_param(
|
||||
param_base: str,
|
||||
ckpt_suffix: str,
|
||||
params_dict: dict,
|
||||
) -> str | None:
|
||||
"""Map a fused expert checkpoint suffix onto a draft parameter name.
|
||||
|
||||
Fused checkpoints pack every expert into one tensor and name the companion
|
||||
scales by suffixing the projection (``experts.gate_up_proj_scale_inv``),
|
||||
while the draft model owns ``experts.routed_experts.w13_weight`` plus a
|
||||
separate ``..._scale_inv`` parameter. Dropping the suffix would push the
|
||||
scale into the weight parameter and leave the scale at its sentinel init.
|
||||
|
||||
Args:
|
||||
param_base: Draft parameter holding the packed weight.
|
||||
ckpt_suffix: Checkpoint text following the projection name.
|
||||
params_dict: The draft model's named parameters.
|
||||
|
||||
Returns:
|
||||
The matching draft parameter name, or None when there is none.
|
||||
"""
|
||||
if not ckpt_suffix:
|
||||
return param_base if param_base in params_dict else None
|
||||
tail = ckpt_suffix.lstrip(".").removeprefix("weight_")
|
||||
for candidate in (
|
||||
f"{param_base}_{tail}",
|
||||
f"{param_base}_scale_inv",
|
||||
f"{param_base}_scale",
|
||||
):
|
||||
if candidate in params_dict:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _prepare_mtp_fp8_expert_scale(
|
||||
quant_config: QuantizationConfig | None,
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
) -> tuple[str, torch.Tensor]:
|
||||
"""Normalize block-wise FP8 expert scale names and dtypes."""
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
|
||||
if (
|
||||
isinstance(quant_config, Fp8Config)
|
||||
and quant_config.weight_block_size is not None
|
||||
and ".mlp.experts." in name
|
||||
and name.endswith(".scale")
|
||||
):
|
||||
name = name[: -len(".scale")] + ".weight_scale_inv"
|
||||
if (
|
||||
getattr(quant_config, "is_scale_e8m0", False)
|
||||
and loaded_weight.dtype == torch.uint8
|
||||
):
|
||||
# UE8M0 scale exponents are stored as raw uint8 bytes: reinterpret
|
||||
# the bits instead of numerically converting them.
|
||||
loaded_weight = loaded_weight.view(torch.float8_e8m0fnu)
|
||||
return name, loaded_weight
|
||||
|
||||
|
||||
def _create_mtp_quant_config(
|
||||
hf_config: PretrainedConfig,
|
||||
backbone_quant_config: QuantizationConfig | None = None,
|
||||
) -> QuantizationConfig | None:
|
||||
"""Create the quantization config for the MTP layers.
|
||||
|
||||
The MTP quantization algorithm is given by the ``mtp_quant_algo`` field in
|
||||
``config.json``, independently of the backbone's quantization. Supported
|
||||
values are ``"FP8"``, ``"NONE"`` (inherit the backbone) and ``"BF16"`` /
|
||||
``"FP16"`` (unquantized). A missing or ``"NONE"`` value falls back to the
|
||||
backbone config.
|
||||
|
||||
Args:
|
||||
hf_config: The draft model's HF config.
|
||||
backbone_quant_config: The target model's quantization config.
|
||||
|
||||
Returns:
|
||||
The quantization config to use for the MTP layers, or None when the MTP
|
||||
layers are unquantized.
|
||||
"""
|
||||
mtp_quant_algo = getattr(hf_config, "mtp_quant_algo", None)
|
||||
|
||||
if mtp_quant_algo is None or mtp_quant_algo.upper() == "NONE":
|
||||
logger.info_once(
|
||||
"No explicit MTP quant algo (mtp_quant_algo=%s). "
|
||||
"Using the backbone's quant config.",
|
||||
mtp_quant_algo,
|
||||
)
|
||||
return backbone_quant_config
|
||||
|
||||
mtp_quant_algo = mtp_quant_algo.upper()
|
||||
|
||||
if mtp_quant_algo in ("BF16", "FP16"):
|
||||
logger.info_once(
|
||||
"MTP layers are unquantized (mtp_quant_algo=%s).", mtp_quant_algo
|
||||
)
|
||||
return None
|
||||
|
||||
if mtp_quant_algo == "FP8":
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
|
||||
logger.info_once("MTP uses FP8 quantization (mtp_quant_algo=FP8).")
|
||||
hf_quant_config = getattr(hf_config, "quantization_config", None) or {}
|
||||
weight_block_size = hf_quant_config.get("weight_block_size")
|
||||
activation_scheme = hf_quant_config.get("activation_scheme", "dynamic")
|
||||
if weight_block_size is not None:
|
||||
activation_scheme = "dynamic"
|
||||
ignored_layers = hf_quant_config.get("ignored_layers") or hf_quant_config.get(
|
||||
"modules_to_not_convert"
|
||||
)
|
||||
|
||||
fp8_config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
activation_scheme=activation_scheme,
|
||||
ignored_layers=ignored_layers or [],
|
||||
weight_block_size=weight_block_size,
|
||||
)
|
||||
if hf_quant_config.get("scale_fmt") == "ue8m0":
|
||||
# Fp8LinearMethod reads this through getattr, so it is not a
|
||||
# declared Fp8Config field.
|
||||
setattr(fp8_config, "is_scale_e8m0", True) # noqa: B010
|
||||
return fp8_config
|
||||
|
||||
logger.warning(
|
||||
"Unknown mtp_quant_algo=%s, falling back to the backbone config.",
|
||||
mtp_quant_algo,
|
||||
)
|
||||
return backbone_quant_config
|
||||
|
||||
|
||||
def _remap_mtp_quant_exclusions(
|
||||
quant_config: QuantizationConfig | None,
|
||||
mtp_start_layer_idx: int,
|
||||
num_mtp_layers: int,
|
||||
) -> QuantizationConfig | None:
|
||||
"""Translate checkpoint-named MTP quant exclusions to draft prefixes.
|
||||
|
||||
``modules_to_not_convert`` / ``exclude_modules`` name MTP modules the way
|
||||
the checkpoint does (``model.mtp_layers.0.self_attn.linear_gate``), while
|
||||
the draft model builds them under ``model.layers.<num_hidden_layers + i>``.
|
||||
``is_layer_skipped`` compares prefixes for exact equality, so without this
|
||||
translation an excluded MTP module gets a quant method even though its
|
||||
checkpoint weight is BF16 with no ``weight_scale`` / ``weight_scale_inv``
|
||||
companion: the scale then keeps its ``finfo(float32).min`` sentinel and the
|
||||
layer silently produces garbage.
|
||||
|
||||
The list lives under a different attribute per config class, hence the
|
||||
lookup over ``_MTP_QUANT_EXCLUSION_ATTRS``.
|
||||
|
||||
Args:
|
||||
quant_config: The MTP quantization config to widen.
|
||||
mtp_start_layer_idx: Index of the first MTP layer in the draft model.
|
||||
num_mtp_layers: Number of MTP layers.
|
||||
|
||||
Returns:
|
||||
A shallow copy with the translated exclusions, or the input unchanged.
|
||||
"""
|
||||
if quant_config is None:
|
||||
return None
|
||||
|
||||
# Different QuantizationConfig subclasses expose the exclusion list under
|
||||
# different attribute names; only the one that is actually populated on
|
||||
# the backbone config governs which draft modules stay unquantized. For
|
||||
# ModelOpt checkpoints (``quant_method: modelopt`` and friends) this is
|
||||
# ``exclude_modules``; for the legacy fp8 path it is ``ignored_layers``.
|
||||
exclusion_attr: str | None = None
|
||||
patterns: list[str] | None = None
|
||||
for attr in _MTP_QUANT_EXCLUSION_ATTRS:
|
||||
value = getattr(quant_config, attr, None)
|
||||
if value:
|
||||
exclusion_attr = attr
|
||||
patterns = list(value)
|
||||
break
|
||||
if not patterns or exclusion_attr is None:
|
||||
return quant_config
|
||||
|
||||
extra: list[str] = []
|
||||
for offset in range(num_mtp_layers):
|
||||
source_prefix = f"model.mtp_layers.{offset}."
|
||||
draft_prefix = f"model.layers.{mtp_start_layer_idx + offset}."
|
||||
for pattern in patterns:
|
||||
if pattern.startswith(source_prefix):
|
||||
extra.append(draft_prefix + pattern[len(source_prefix) :])
|
||||
# ModelOpt exclusion patterns may end with a ``*`` wildcard; the
|
||||
# remap must preserve that suffix so ``is_layer_skipped`` still
|
||||
# matches the draft module prefixes.
|
||||
elif pattern.rstrip("*").startswith(source_prefix):
|
||||
stripped = pattern.rstrip("*")
|
||||
trailing_stars = pattern[len(stripped) :]
|
||||
extra.append(
|
||||
draft_prefix + stripped[len(source_prefix) :] + trailing_stars
|
||||
)
|
||||
if not extra:
|
||||
return quant_config
|
||||
|
||||
# The backbone shares this object with the target model; copy before
|
||||
# widening the exclusion list. The attribute is only present on the
|
||||
# concrete configs that support exclusions, hence the getattr/setattr pair.
|
||||
quant_config = copy.copy(quant_config)
|
||||
setattr(quant_config, exclusion_attr, patterns + extra) # noqa: B010
|
||||
logger.info_once(
|
||||
"HYV4 MTP: mapped %d checkpoint-named quant exclusions (%s) onto the "
|
||||
"draft module prefixes: %s",
|
||||
len(extra),
|
||||
exclusion_attr,
|
||||
", ".join(extra),
|
||||
)
|
||||
return quant_config
|
||||
|
||||
|
||||
class HYV4SharedHead(nn.Module):
|
||||
"""Holds the draft LM head shared with the target model."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Use the same ``lm_head`` prefix as the backbone so that quant-config
|
||||
# exclusion lists (e.g. ModelOpt's ``exclude_modules: ["lm_head", ...]``)
|
||||
# also apply here: an unquantized checkpoint lm_head must not gain a
|
||||
# spurious ``weight_scale`` parameter on the draft side, which would
|
||||
# otherwise stay at its FP-min sentinel until the proposer swaps the
|
||||
# whole ParallelLMHead for the target model's ``lm_head``.
|
||||
self.head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix="lm_head",
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
return hidden_states
|
||||
|
||||
|
||||
class HYV4MultiTokenPredictorLayer(nn.Module):
|
||||
"""A single MTP draft block."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
prefix: str,
|
||||
vllm_config: VllmConfig,
|
||||
model_config: ModelConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
topk_indices_buffer: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
del model_config # kept for signature parity with the other MTP heads
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
|
||||
self.shared_head = HYV4SharedHead(config=config, quant_config=quant_config)
|
||||
|
||||
# HYV4DecoderLayer indexes layer_types/mlp_layer_types by the numeric
|
||||
# layer id parsed from the prefix. MTP layers live after the backbone
|
||||
# layers, so extend the per-layer config lists to cover those ids.
|
||||
layer_idx = int(prefix.rsplit(".", 1)[-1])
|
||||
mtp_config = copy.deepcopy(config)
|
||||
# MTP checkpoints do not carry iHC pre/post weights for mtp_layers, so
|
||||
# the draft blocks are built without iHC to match their structure.
|
||||
mtp_config.enable_ihc = False
|
||||
mtp_config.layer_types = _extend_layer_types(
|
||||
getattr(mtp_config, "layer_types", None), layer_idx, "full_attention"
|
||||
)
|
||||
mtp_config.mlp_layer_types = _extend_layer_types(
|
||||
getattr(mtp_config, "mlp_layer_types", None), layer_idx, "sparse"
|
||||
)
|
||||
|
||||
self.mtp_block = HYV4DecoderLayer(
|
||||
config=mtp_config,
|
||||
vllm_config=vllm_config,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
)
|
||||
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
del input_ids # embeddings are resolved by the caller
|
||||
assert inputs_embeds is not None
|
||||
inputs_embeds = self.enorm(inputs_embeds)
|
||||
previous_hidden_states = self.hnorm(previous_hidden_states)
|
||||
hidden_states = self.eh_proj(
|
||||
torch.cat([inputs_embeds, previous_hidden_states], dim=-1)
|
||||
)
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
hidden_states, _ = self.final_layernorm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
|
||||
def _extend_layer_types(
|
||||
layer_types: list[str] | None, layer_idx: int, fallback: str
|
||||
) -> list[str] | None:
|
||||
"""Pad a per-layer config list so``layer_idx`` is addressable."""
|
||||
if layer_types is None or len(layer_types) > layer_idx:
|
||||
return layer_types
|
||||
fill = layer_types[-1] if layer_types else fallback
|
||||
return list(layer_types) + [fill] * (layer_idx + 1 - len(layer_types))
|
||||
|
||||
|
||||
class HYV4MultiTokenPredictor(nn.Module):
|
||||
"""Owns the MTP draft blocks and their shared embedding / logits path."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
target_config = vllm_config.model_config.hf_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None, (
|
||||
"HYV4MTP can only be built as a speculative draft model"
|
||||
)
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
assert draft_model_config is not None
|
||||
config = _normalize_hyv4_config(draft_model_config.hf_config)
|
||||
self.mtp_start_layer_idx = target_config.num_hidden_layers
|
||||
self.num_mtp_layers = max(
|
||||
getattr(target_config, "num_nextn_predict_layers", 1), 1
|
||||
)
|
||||
self.quant_config = _remap_mtp_quant_exclusions(
|
||||
_create_mtp_quant_config(config, vllm_config.quant_config),
|
||||
self.mtp_start_layer_idx,
|
||||
self.num_mtp_layers,
|
||||
)
|
||||
if self.quant_config is not None:
|
||||
self.quant_config.packed_modules_mapping = HYV4MTP.packed_modules_mapping
|
||||
|
||||
# Sparse DSA draft blocks must share the target model's top-k buffer.
|
||||
# The proposer injects it through HYV4MTP.set_topk_indices_buffer()
|
||||
# before any draft forward.
|
||||
self.topk_indices_buffer: torch.Tensor | None = None
|
||||
if hasattr(config, "index_topk"):
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
config.index_topk,
|
||||
dtype=torch.int32,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
|
||||
self.layers = nn.ModuleDict(
|
||||
{
|
||||
str(idx): HYV4MultiTokenPredictorLayer(
|
||||
config,
|
||||
f"{prefix}.layers.{idx}",
|
||||
vllm_config=vllm_config,
|
||||
model_config=draft_model_config,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
)
|
||||
for idx in range(
|
||||
self.mtp_start_layer_idx,
|
||||
self.mtp_start_layer_idx + self.num_mtp_layers,
|
||||
)
|
||||
}
|
||||
)
|
||||
self.requires_topk_indices_buffer = any(
|
||||
layer.mtp_block.self_attn.is_sparse for layer in self.layers.values()
|
||||
)
|
||||
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.spec_step_idx: int = 0
|
||||
|
||||
def set_skip_topk(self, skip: bool) -> None:
|
||||
"""Toggle the draft indexer for ``index_share_for_mtp_iteration``.
|
||||
|
||||
The proposer clears the flag for draft step 0 so the MTP layer builds
|
||||
its own top-k indices, then sets it for steps 1+ so they reuse what
|
||||
step 0 wrote into the shared buffer instead of re-running the indexer.
|
||||
"""
|
||||
for layer in self.layers.values():
|
||||
layer.mtp_block.self_attn.skip_topk = skip
|
||||
|
||||
def compact_topk_indices(self, slot_ids: torch.Tensor) -> None:
|
||||
"""Move the top-k rows at ``slot_ids`` to the front of the buffer.
|
||||
|
||||
Step 0 writes one row per query token of the multi-token batch, while
|
||||
steps 1+ decode a single token per request and index the buffer from 0.
|
||||
Without this gather they would read another token's rows.
|
||||
"""
|
||||
num_slots = slot_ids.numel()
|
||||
if self.topk_indices_buffer is None or num_slots == 0:
|
||||
return
|
||||
self.topk_indices_buffer[:num_slots] = self.topk_indices_buffer[slot_ids]
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
inputs_embeds = torch.where((positions == 0).unsqueeze(-1), 0, inputs_embeds)
|
||||
|
||||
current_step_idx = self.spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
current_step_idx = self.spec_step_idx % self.num_mtp_layers
|
||||
mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)]
|
||||
lm_head = mtp_layer.shared_head.head
|
||||
proj_input = mtp_layer.shared_head(hidden_states)
|
||||
return self.logits_processor(lm_head, proj_input)
|
||||
|
||||
|
||||
class HYV4MTP(nn.Module):
|
||||
"""HY V4 MTP draft head.
|
||||
|
||||
Not a pipeline-parallel stage: the draft head always runs on a single rank,
|
||||
matching `DeepseekV32MTP` / `KimiK3MTP` / `HYV3MTP`.
|
||||
"""
|
||||
|
||||
packed_modules_mapping = {
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
# MLA runs both latent down-projections as one GEMM.
|
||||
"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"],
|
||||
# The indexer fuses wk and weights_proj into one GEMM.
|
||||
"wk_weights_proj": ["wk", "weights_proj"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.model = HYV4MultiTokenPredictor(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.quant_config = self.model.quant_config
|
||||
self.sampler = Sampler()
|
||||
|
||||
def set_topk_indices_buffer(self, topk_indices_buffer: torch.Tensor) -> None:
|
||||
"""Share the target sparse-index buffer with every draft consumer.
|
||||
|
||||
Proposers that walk ``named_modules()`` instead of calling this reach
|
||||
the same consumers via ``HYV4MLAAttention.topk_indices_buffer``.
|
||||
"""
|
||||
self.model.topk_indices_buffer = topk_indices_buffer
|
||||
for layer in self.model.layers.values():
|
||||
self_attn = layer.mtp_block.self_attn
|
||||
if not self_attn.is_sparse:
|
||||
continue
|
||||
|
||||
indexer = self_attn.indexer
|
||||
assert indexer is not None, "Sparse HYV4 MTP attention requires an indexer"
|
||||
indexer.topk_indices_buffer = topk_indices_buffer
|
||||
indexer.indexer_op.topk_indices_buffer = topk_indices_buffer
|
||||
|
||||
attn_impl = self_attn.mla_attn.impl
|
||||
assert hasattr(attn_impl, "topk_indices_buffer"), (
|
||||
"Sparse HYV4 MTP attention backend requires a top-k indices buffer"
|
||||
)
|
||||
attn_impl.topk_indices_buffer = topk_indices_buffer
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
del intermediate_tensors # the MTP head is single-stage
|
||||
if (
|
||||
self.model.requires_topk_indices_buffer
|
||||
and self.model.topk_indices_buffer is None
|
||||
):
|
||||
raise RuntimeError(
|
||||
"HYV4 sparse MTP requires the target model's top-k indices buffer. "
|
||||
"The proposer must call HYV4MTP.set_topk_indices_buffer() before "
|
||||
"the first draft forward."
|
||||
)
|
||||
self.model.spec_step_idx = spec_step_idx
|
||||
return self.model(input_ids, positions, hidden_states, inputs_embeds)
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor | None:
|
||||
self.model.spec_step_idx = spec_step_idx
|
||||
return self.model.compute_logits(hidden_states)
|
||||
|
||||
def sample(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> SamplerOutput | None:
|
||||
return self.sampler(logits, sampling_metadata)
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
if f"model.layers.{spec_layer}.embed_tokens" in name:
|
||||
return "__skip__"
|
||||
if f"model.layers.{spec_layer}.shared_head" in name:
|
||||
return "__skip__"
|
||||
|
||||
spec_layer_weight_names = ["enorm", "hnorm", "eh_proj", "final_layernorm"]
|
||||
spec_layer_weight = any(
|
||||
weight_name in name for weight_name in spec_layer_weight_names
|
||||
)
|
||||
if not spec_layer_weight:
|
||||
name = name.replace(
|
||||
f"model.layers.{spec_layer}.",
|
||||
f"model.layers.{spec_layer}.mtp_block.",
|
||||
)
|
||||
return name
|
||||
|
||||
def _load_fused_expert_weights(
|
||||
self,
|
||||
name: str,
|
||||
params_dict: dict,
|
||||
loaded_weight: torch.Tensor,
|
||||
shard_id: str,
|
||||
num_experts: int,
|
||||
) -> bool:
|
||||
if name not in params_dict:
|
||||
return False
|
||||
param = params_dict[name]
|
||||
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
|
||||
loaded_local_expert = False
|
||||
for expert_id in range(num_experts):
|
||||
curr_expert_weight = loaded_weight[expert_id]
|
||||
success = weight_loader(
|
||||
param,
|
||||
curr_expert_weight,
|
||||
name,
|
||||
shard_id,
|
||||
expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
loaded_local_expert = True
|
||||
return loaded_local_expert
|
||||
|
||||
def _load_expert_weight(
|
||||
self,
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
params_dict: dict,
|
||||
loaded_params: set[str],
|
||||
split_expert_params_mapping: list[tuple[str, str, int, str]],
|
||||
fused_expert_param_names: dict[tuple[str, str], str],
|
||||
num_experts: int,
|
||||
) -> bool:
|
||||
"""Load one routed-expert weight in either checkpoint layout.
|
||||
|
||||
Args:
|
||||
name: Weight name already rewritten to draft-module naming.
|
||||
loaded_weight: The checkpoint tensor.
|
||||
params_dict: The draft model's named parameters.
|
||||
loaded_params: Set updated with the parameters that received a value.
|
||||
split_expert_params_mapping: Mapping for the per-expert layout.
|
||||
fused_expert_param_names: ``(mlp_prefix, tag) -> param name`` for the
|
||||
all-experts-packed layout.
|
||||
num_experts: Total number of routed experts.
|
||||
|
||||
Returns:
|
||||
True when the weight was consumed (even if this rank holds none of
|
||||
the addressed experts).
|
||||
"""
|
||||
base = name.split(".experts.")[0]
|
||||
for ckpt_proj, tag in (
|
||||
(".experts.gate_up_proj", "w13_weight"),
|
||||
(".experts.down_proj", "w2_weight"),
|
||||
):
|
||||
if ckpt_proj not in name:
|
||||
continue
|
||||
param_base = fused_expert_param_names.get((base, tag))
|
||||
if param_base is None:
|
||||
return False
|
||||
# Keep the checkpoint suffix (e.g. `_scale_inv`) so block-scale
|
||||
# tensors land in the scale parameter, not in the weight.
|
||||
target = _resolve_fused_expert_param(
|
||||
param_base, name.split(ckpt_proj, 1)[1], params_dict
|
||||
)
|
||||
if target is None:
|
||||
return False
|
||||
if tag == "w13_weight":
|
||||
chunks = loaded_weight.chunk(2, dim=-2)
|
||||
loaded_w1 = self._load_fused_expert_weights(
|
||||
target, params_dict, chunks[0], "w1", num_experts
|
||||
)
|
||||
loaded_w3 = self._load_fused_expert_weights(
|
||||
target, params_dict, chunks[1], "w3", num_experts
|
||||
)
|
||||
loaded = loaded_w1 and loaded_w3
|
||||
else:
|
||||
loaded = self._load_fused_expert_weights(
|
||||
target, params_dict, loaded_weight, "w2", num_experts
|
||||
)
|
||||
if loaded:
|
||||
loaded_params.add(target)
|
||||
# The weight belongs to the experts either way; never fall through.
|
||||
return True
|
||||
|
||||
consumed = False
|
||||
for param_name, weight_name, expert_id, shard_id in split_expert_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
consumed = True
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if name_mapped not in params_dict:
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
|
||||
if weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
):
|
||||
loaded_params.add(name_mapped)
|
||||
return consumed
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
pp_missing_layer_names = get_pp_missing_layer_names(self)
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
mtp_start = self.config.num_hidden_layers
|
||||
shared_weights = {
|
||||
"model.embed_tokens.weight": "model.embed_tokens.weight",
|
||||
"lm_head.weight": f"model.layers.{mtp_start}.shared_head.head.weight",
|
||||
}
|
||||
|
||||
num_experts = getattr(self.config, "n_routed_experts", 0)
|
||||
sink_tp_size = get_tensor_model_parallel_world_size()
|
||||
sink_tp_rank = get_tensor_model_parallel_rank()
|
||||
n_local_head = self.config.num_attention_heads // sink_tp_size
|
||||
head_rank_start = n_local_head * sink_tp_rank
|
||||
head_rank_end = n_local_head * (sink_tp_rank + 1)
|
||||
|
||||
# Routed-expert weights come in two checkpoint layouts:
|
||||
# split: mlp.experts.<id>.gate_proj.weight
|
||||
# fused: mlp.experts.gate_up_proj (all experts in one tensor)
|
||||
# The split layout is resolved through the shared EPLB helper so the
|
||||
# target param names stay in sync with the RoutedExperts module layout;
|
||||
# the fused layout is resolved from the live parameter names.
|
||||
split_expert_params_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=num_experts,
|
||||
)
|
||||
fused_expert_param_names: dict[tuple[str, str], str] = {}
|
||||
for param_name in params_dict:
|
||||
for tag in ("w13_weight", "w2_weight"):
|
||||
if param_name.endswith(tag) and ".experts." in param_name:
|
||||
base = param_name.split(".experts.")[0]
|
||||
fused_expert_param_names[base, tag] = param_name
|
||||
stacked_mapping = [
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
# MLA runs both latent down-projections as one GEMM.
|
||||
(".fused_qkv_a_proj", ".q_a_proj", 0),
|
||||
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
|
||||
]
|
||||
# Sparse DSA draft blocks build an Indexer whose wk / weights_proj are
|
||||
# fused into a single MergedColumnParallelLinear (wk_weights_proj).
|
||||
indexer_stacked_mapping = [
|
||||
(".wk_weights_proj", ".wk", 0),
|
||||
(".wk_weights_proj", ".weights_proj", 1),
|
||||
]
|
||||
# FP8 indexer wk dequant buffer (weight and scale arrive separately).
|
||||
pending_wk_fp8: dict[str, dict[str, torch.Tensor]] = {}
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if name in shared_weights:
|
||||
target_name = shared_weights[name]
|
||||
if target_name in params_dict:
|
||||
param = params_dict[target_name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(target_name)
|
||||
continue
|
||||
|
||||
spec_layer = None
|
||||
if name.startswith("model.mtp_layers."):
|
||||
parts = name.split(".")
|
||||
if len(parts) > 3 and parts[2].isdigit():
|
||||
spec_layer = mtp_start + int(parts[2])
|
||||
name = name.replace(
|
||||
f"model.mtp_layers.{parts[2]}.",
|
||||
f"model.layers.{spec_layer}.",
|
||||
)
|
||||
else:
|
||||
spec_layer = _get_spec_layer_idx_from_weight_name(self.config, name)
|
||||
|
||||
if spec_layer is None:
|
||||
continue
|
||||
|
||||
name = self._rewrite_spec_layer_name(spec_layer, name)
|
||||
if name == "__skip__":
|
||||
continue
|
||||
name, loaded_weight = _prepare_mtp_fp8_expert_scale(
|
||||
self.quant_config, name, loaded_weight
|
||||
)
|
||||
|
||||
if "mlp.gate.e_score_correction_bias" in name:
|
||||
name = name.replace("gate.e_score_correction_bias", "expert_bias")
|
||||
|
||||
is_loaded = False
|
||||
for param_name, weight_name, shard_id in stacked_mapping:
|
||||
if weight_name not in name or ".experts." in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if name_mapped not in params_dict:
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
is_loaded = True
|
||||
break
|
||||
param = params_dict[name_mapped]
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(name_mapped)
|
||||
is_loaded = True
|
||||
break
|
||||
if is_loaded:
|
||||
continue
|
||||
|
||||
# FP8 indexer wk: dequantize to BF16 and load into the fused
|
||||
# wk_weights_proj. PP-aware (skips layers not held by this rank).
|
||||
if _try_load_fp8_indexer_wk(
|
||||
name,
|
||||
loaded_weight,
|
||||
pending_wk_fp8,
|
||||
params_dict,
|
||||
loaded_params,
|
||||
pp_missing_layer_names,
|
||||
):
|
||||
continue
|
||||
|
||||
# BF16 indexer wk / weights_proj: merge into the fused param.
|
||||
is_loaded = False
|
||||
for param_name, weight_name, shard_id in indexer_stacked_mapping:
|
||||
if weight_name not in name or "wk_weights" in name:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if name_mapped not in params_dict:
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
is_loaded = True
|
||||
break
|
||||
param = params_dict[name_mapped]
|
||||
param.weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(name_mapped)
|
||||
is_loaded = True
|
||||
break
|
||||
if is_loaded:
|
||||
continue
|
||||
|
||||
is_loaded = False
|
||||
if ".experts." in name:
|
||||
is_loaded = self._load_expert_weight(
|
||||
name,
|
||||
loaded_weight,
|
||||
params_dict,
|
||||
loaded_params,
|
||||
split_expert_params_mapping,
|
||||
fused_expert_param_names,
|
||||
num_experts,
|
||||
)
|
||||
if is_loaded:
|
||||
continue
|
||||
|
||||
if "learnable_sink_param" in name:
|
||||
if name in params_dict:
|
||||
narrow_weight = loaded_weight[head_rank_start:head_rank_end]
|
||||
n = narrow_weight.shape[0]
|
||||
with torch.no_grad():
|
||||
params_dict[name][:n].copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
if name not in params_dict:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
if _should_skip_missing_mtp_scale_param(self.quant_config, name):
|
||||
continue
|
||||
logger.warning_once("Skipping unknown MTP weight: %s", name)
|
||||
continue
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
|
||||
logger.info_once("HYV4 MTP draft model loaded: %d params", len(loaded_params))
|
||||
unassigned_all = sorted(set(params_dict) - loaded_params)
|
||||
# KVCacheScaleParameter (``k_scale`` / ``v_scale`` / ``q_scale`` /
|
||||
# ``prob_scale``) is created by ``BaseKVCacheMethod.create_weights`` for
|
||||
# every Attention layer that runs on an fp8-family quant method, and
|
||||
# its -1.0 sentinel is replaced by the runtime default (1.0) in
|
||||
# ``process_weights_after_loading`` -- which runs *after* this method.
|
||||
# HY V4 checkpoints intentionally omit these (``kv_cache_quant_algo:
|
||||
# null``); silence them only when the parameter is actually the
|
||||
# sentinel type so a genuine mismatch on a same-named tensor still
|
||||
# reaches the warning.
|
||||
from vllm.model_executor.layers.quantization.kv_cache import (
|
||||
KVCacheScaleParameter,
|
||||
)
|
||||
|
||||
unassigned = [
|
||||
name
|
||||
for name in unassigned_all
|
||||
if not isinstance(params_dict.get(name), KVCacheScaleParameter)
|
||||
]
|
||||
if unassigned:
|
||||
# A draft parameter with no checkpoint source keeps its sentinel
|
||||
# init value (FP8 scales start at finfo(float32).min), which
|
||||
# silently destroys draft quality instead of failing the load.
|
||||
logger.warning(
|
||||
"HYV4 MTP draft model: %d parameters received no checkpoint value: %s",
|
||||
len(unassigned),
|
||||
", ".join(unassigned),
|
||||
)
|
||||
return loaded_params
|
||||
@@ -451,9 +451,10 @@ def get_harmony_structural_tag(
|
||||
builtin_tools: list[BuiltinToolParam],
|
||||
tool_choice: SimplifiedToolChoice,
|
||||
reasoning: bool,
|
||||
token_suffix: str = "",
|
||||
) -> StructuralTag:
|
||||
# reasoning always enabled for Harmony
|
||||
del reasoning
|
||||
del reasoning, token_suffix
|
||||
|
||||
if builtin_tools:
|
||||
# Fallback for built-in tools
|
||||
|
||||
@@ -84,6 +84,10 @@ _REASONING_PARSERS_TO_REGISTER = {
|
||||
"hy_v3_reasoning_parser",
|
||||
"HYV3ReasoningParser",
|
||||
),
|
||||
"hy_v4": (
|
||||
"hy_v4_reasoning_parser",
|
||||
"HYV4ReasoningParser",
|
||||
),
|
||||
"kimi_k2": (
|
||||
"kimi_k2_reasoning_parser",
|
||||
"KimiK2ReasoningParser",
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import TypedDict
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Must match the chat template's own default for ``reasoning_effort``.
|
||||
DEFAULT_REASONING_EFFORT = "high"
|
||||
|
||||
|
||||
class ReasoningDelta(TypedDict):
|
||||
"""One streaming reasoning delta. Either/both fields may be None."""
|
||||
|
||||
reasoning: str | None
|
||||
content: str | None
|
||||
|
||||
|
||||
# NOTE: mirrored in ``vllm.tool_parsers.hy_v4_tool_parser`` (same pattern as
|
||||
# ``gemma4_utils``) so neither package depends on the other.
|
||||
def detect_token_suffix(tokenizer: TokenizerLike) -> str:
|
||||
"""Detect the per-checkpoint suffix used by Hunyuan structural tokens.
|
||||
|
||||
Args:
|
||||
tokenizer: Tokenizer of the served checkpoint.
|
||||
|
||||
Returns:
|
||||
The suffix including its leading colon (e.g. ``":6124c78e"``), or ``""``
|
||||
when the checkpoint uses unsuffixed tokens.
|
||||
|
||||
Raises:
|
||||
RuntimeError: The tokenizer declares the structural tokens through
|
||||
``model_specific_special_tokens``, which transformers 5 no longer
|
||||
round-trips.
|
||||
"""
|
||||
|
||||
import transformers
|
||||
|
||||
if int(transformers.__version__.split(".")[0]) >= 5:
|
||||
init_kwargs = getattr(tokenizer, "init_kwargs", None) or {}
|
||||
think_begin_as_special = init_kwargs.get(
|
||||
"model_specific_special_tokens", {}
|
||||
).get("think_begin_token", "")
|
||||
if think_begin_as_special:
|
||||
raise RuntimeError(
|
||||
"This checkpoint declares HYV4 structural tokens (think_begin_token"
|
||||
"/toolcalls_begin_token/argkey_begin_token) in "
|
||||
"tokenizer_config.json, which transformers 5 no longer supports. "
|
||||
"Remove those fields and keep the tokens in the tokenizer's own "
|
||||
"token definitions so the suffix can be read from the vocab."
|
||||
)
|
||||
|
||||
structural_token_re = re.compile(
|
||||
r"<(?:think|tool_calls|tool_call|arg_key|arg_value)(:[^\s>]+)?>"
|
||||
)
|
||||
for token in tokenizer.get_vocab():
|
||||
match = structural_token_re.fullmatch(token)
|
||||
if match:
|
||||
return match.group(1) or ""
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
class HYV4ReasoningExtractor:
|
||||
"""Reasoning extraction for HYV4, on plain data.
|
||||
|
||||
``thinking`` selects the mode:
|
||||
- ``True``: parse ``<think>...</think>``; the start token is injected at the
|
||||
end of the prompt and is therefore usually absent from the output.
|
||||
- ``False``: ``no_think`` -- the whole output is content, no reasoning.
|
||||
"""
|
||||
|
||||
def __init__(self, vocab: dict[str, int], token_suffix: str, thinking: bool = True):
|
||||
self.thinking = thinking
|
||||
self.start_token: str = f"<think{token_suffix}>"
|
||||
self.end_token: str = f"</think{token_suffix}>"
|
||||
|
||||
start_token_id = vocab.get(self.start_token)
|
||||
end_token_id = vocab.get(self.end_token)
|
||||
if thinking and (start_token_id is None or end_token_id is None):
|
||||
raise RuntimeError(
|
||||
"HYV4 reasoning extractor could not locate think "
|
||||
"start/end tokens in the tokenizer!"
|
||||
)
|
||||
# In no_think mode the ids are unused (kept for symmetry / debugging).
|
||||
self.start_token_id: int | None = start_token_id
|
||||
self.end_token_id: int | None = end_token_id
|
||||
|
||||
def has_reasoning_ended(self, token_ids: Sequence[int]) -> bool:
|
||||
"""Idempotent STATE query: has ``</think>`` been emitted yet?
|
||||
|
||||
Monotonic -- once the end token is present this stays True. Use for
|
||||
"has reasoning finished, given everything so far". (no_think is always
|
||||
ended.) Contrast with ``reasoning_ended_in_delta`` (edge detector).
|
||||
"""
|
||||
if not self.thinking:
|
||||
return True
|
||||
for i in range(len(token_ids) - 1, -1, -1):
|
||||
if token_ids[i] == self.start_token_id:
|
||||
return False
|
||||
if token_ids[i] == self.end_token_id:
|
||||
return True
|
||||
return False
|
||||
|
||||
def reasoning_ended_in_delta(
|
||||
self, token_ids: Sequence[int], delta_ids: Iterable[int]
|
||||
) -> bool:
|
||||
"""Edge detector (NOT idempotent): did reasoning end *in this* step?
|
||||
|
||||
True only on the step whose ``delta_ids`` contains ``</think>``. Callers
|
||||
use it once behind their own "already ended" latch, so it need not be
|
||||
monotonic; it only checks the new delta (cheaper than scanning the whole
|
||||
sequence). Contrast with ``has_reasoning_ended`` (cumulative state).
|
||||
"""
|
||||
if not self.thinking:
|
||||
return True
|
||||
end_token_id = self.end_token_id
|
||||
if end_token_id is None:
|
||||
return False
|
||||
return end_token_id in delta_ids
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
"""Return the token ids after ``</think>`` (all of them in no_think)."""
|
||||
if not self.thinking:
|
||||
return input_ids
|
||||
head = input_ids[: max(0, len(input_ids) - 1)]
|
||||
if self.end_token_id not in head:
|
||||
return []
|
||||
return input_ids[input_ids.index(self.end_token_id) + 1 :]
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
"""Count reasoning tokens (0 in no_think).
|
||||
|
||||
The ``<think>`` start token is injected at the END of the prompt, so it
|
||||
is normally absent from ``token_ids``. Only skip a leading start token
|
||||
when it actually appears first (legacy / no prompt-injection); otherwise
|
||||
count every token up to ``</think>``.
|
||||
"""
|
||||
if not self.thinking:
|
||||
return 0
|
||||
start = 1 if (token_ids and token_ids[0] == self.start_token_id) else 0
|
||||
try:
|
||||
return token_ids.index(self.end_token_id, start) - start
|
||||
except ValueError:
|
||||
# No end token -- reasoning was truncated; everything counts.
|
||||
return len(token_ids) - start
|
||||
|
||||
def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]:
|
||||
"""Split a complete output into ``(reasoning, content)``.
|
||||
|
||||
In no_think mode reasoning is None and the whole output is content. The
|
||||
start token may be absent (injected in the prompt), so reasoning is
|
||||
assumed to begin at the start of the output.
|
||||
"""
|
||||
if not self.thinking:
|
||||
return None, model_output
|
||||
|
||||
parts = model_output.partition(self.start_token)
|
||||
model_output = parts[2] if parts[1] else parts[0]
|
||||
|
||||
if self.end_token not in model_output:
|
||||
return model_output, None
|
||||
reasoning, _, content = model_output.partition(self.end_token)
|
||||
return reasoning, content or None
|
||||
|
||||
def extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> ReasoningDelta | None:
|
||||
"""Classify a streaming delta as reasoning and/or content.
|
||||
|
||||
In no_think mode every delta is content. Otherwise: same logic as vLLM's
|
||||
generic thinking parser, except the final branch (no start token seen
|
||||
anywhere) treats output as reasoning-then-content, because HYV4 injects
|
||||
``<think>`` in the prompt (absent from the output).
|
||||
"""
|
||||
if not self.thinking:
|
||||
if not delta_text:
|
||||
return None
|
||||
return ReasoningDelta(reasoning=None, content=delta_text)
|
||||
|
||||
# Skip a delta that is a single lone start/end special token.
|
||||
if len(delta_token_ids) == 1 and delta_token_ids[0] in (
|
||||
self.start_token_id,
|
||||
self.end_token_id,
|
||||
):
|
||||
return None
|
||||
|
||||
if self.start_token_id in previous_token_ids:
|
||||
if self.end_token_id in delta_token_ids:
|
||||
end_index = delta_text.find(self.end_token)
|
||||
reasoning = delta_text[:end_index]
|
||||
content = delta_text[end_index + len(self.end_token) :]
|
||||
return ReasoningDelta(reasoning=reasoning, content=content or None)
|
||||
elif self.end_token_id in previous_token_ids:
|
||||
return ReasoningDelta(reasoning=None, content=delta_text)
|
||||
else:
|
||||
return ReasoningDelta(reasoning=delta_text, content=None)
|
||||
elif self.start_token_id in delta_token_ids:
|
||||
if self.end_token_id in delta_token_ids:
|
||||
start_index = delta_text.find(self.start_token)
|
||||
end_index = delta_text.find(self.end_token)
|
||||
reasoning = delta_text[start_index + len(self.start_token) : end_index]
|
||||
content = delta_text[end_index + len(self.end_token) :]
|
||||
return ReasoningDelta(reasoning=reasoning, content=content or None)
|
||||
else:
|
||||
return ReasoningDelta(reasoning=delta_text, content=None)
|
||||
else:
|
||||
# HYV4: <think> was injected at the end of the prompt, so it never
|
||||
# appears in the output. Treat the stream as reasoning until
|
||||
# </think>, then content.
|
||||
if self.end_token_id in delta_token_ids:
|
||||
end_index = delta_text.find(self.end_token)
|
||||
reasoning = delta_text[:end_index]
|
||||
content = delta_text[end_index + len(self.end_token) :]
|
||||
return ReasoningDelta(reasoning=reasoning, content=content or None)
|
||||
elif self.end_token_id in previous_token_ids:
|
||||
return ReasoningDelta(reasoning=None, content=delta_text)
|
||||
else:
|
||||
return ReasoningDelta(reasoning=delta_text, content=None)
|
||||
|
||||
|
||||
def build_reasoning_extractor(
|
||||
tokenizer: TokenizerLike, *, thinking: bool = True
|
||||
) -> HYV4ReasoningExtractor:
|
||||
return HYV4ReasoningExtractor(
|
||||
tokenizer.get_vocab(), detect_token_suffix(tokenizer), thinking
|
||||
)
|
||||
|
||||
|
||||
class HYV4ReasoningParser(ReasoningParser):
|
||||
"""vLLM adapter around :class:`HYV4ReasoningExtractor`.
|
||||
|
||||
Decides ``thinking`` from the request's ``reasoning_effort`` and converts
|
||||
the extractor's ``ReasoningDelta`` to ``DeltaMessage``.
|
||||
"""
|
||||
|
||||
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
|
||||
super().__init__(tokenizer, *args, **kwargs)
|
||||
|
||||
# Decide thinking vs no_think from the request; the extractor handles
|
||||
# both modes internally. Prefer chat_template_kwargs.reasoning_effort,
|
||||
# then the top-level reasoning_effort. The default must match the chat
|
||||
# template, which sets reasoning_effort='high' when the request omits
|
||||
# it -- defaulting to no_think here would return the whole
|
||||
# chain-of-thought (and a stray ``</think:SUF>``) as content.
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
reasoning_effort = chat_kwargs.get("reasoning_effort", None)
|
||||
if not reasoning_effort:
|
||||
reasoning_effort = (
|
||||
kwargs.get("reasoning_effort") or DEFAULT_REASONING_EFFORT
|
||||
)
|
||||
logger.debug("reasoning_effort for choosing parser: %s", reasoning_effort)
|
||||
thinking = reasoning_effort != "no_think"
|
||||
|
||||
self._extractor = build_reasoning_extractor(tokenizer, thinking=thinking)
|
||||
|
||||
# Reported regardless of the per-request thinking mode: ``ReasoningConfig``
|
||||
# is engine-wide and needs the checkpoint's delimiters to derive the token
|
||||
# ids used for thinking budget enforcement.
|
||||
@property
|
||||
def reasoning_start_str(self) -> str:
|
||||
return self._extractor.start_token
|
||||
|
||||
@property
|
||||
def reasoning_end_str(self) -> str:
|
||||
return self._extractor.end_token
|
||||
|
||||
def is_reasoning_end(self, input_ids) -> bool:
|
||||
return self._extractor.has_reasoning_ended(input_ids)
|
||||
|
||||
def is_reasoning_end_streaming(self, input_ids, delta_ids) -> bool:
|
||||
return self._extractor.reasoning_ended_in_delta(input_ids, delta_ids)
|
||||
|
||||
def extract_content_ids(self, input_ids) -> list[int]:
|
||||
return self._extractor.extract_content_ids(input_ids)
|
||||
|
||||
def count_reasoning_tokens(self, token_ids) -> int:
|
||||
return self._extractor.count_reasoning_tokens(token_ids)
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
return self._extractor.extract_reasoning(model_output)
|
||||
|
||||
def extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
) -> DeltaMessage | None:
|
||||
delta = self._extractor.extract_reasoning_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
)
|
||||
if delta is None:
|
||||
return None
|
||||
return DeltaMessage(reasoning=delta["reasoning"], content=delta["content"])
|
||||
@@ -94,6 +94,10 @@ _TOOL_PARSERS_TO_REGISTER = {
|
||||
"hy_v3_tool_parser",
|
||||
"HYV3ToolParser",
|
||||
),
|
||||
"hy_v4": (
|
||||
"hy_v4_tool_parser",
|
||||
"HYV4ToolParser",
|
||||
),
|
||||
"internlm": (
|
||||
"internlm2_tool_parser",
|
||||
"Internlm2ToolParser",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -49,6 +49,7 @@ StructuralTagBuilder: TypeAlias = Callable[
|
||||
list[BuiltinToolParam],
|
||||
SimplifiedToolChoice,
|
||||
bool,
|
||||
str,
|
||||
],
|
||||
StructuralTag,
|
||||
]
|
||||
@@ -70,7 +71,7 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
|
||||
"deepseek_v4",
|
||||
}
|
||||
)
|
||||
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "kimi_k3"})
|
||||
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"})
|
||||
SUPPORTED_STRUCTURAL_TAG_MODELS = (
|
||||
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
|
||||
)
|
||||
@@ -78,7 +79,9 @@ SUPPORTED_STRUCTURAL_TAG_MODELS = (
|
||||
_VLLM_STRUCTURAL_TAG_REGISTRY: dict[str, StructuralTagBuilder] = {}
|
||||
|
||||
|
||||
def register_vllm_structural_tag(model: str):
|
||||
def register_vllm_structural_tag(
|
||||
model: str,
|
||||
) -> Callable[[StructuralTagBuilder], StructuralTagBuilder]:
|
||||
"""Register a vLLM-owned structural tag builder."""
|
||||
|
||||
def decorator(func: StructuralTagBuilder) -> StructuralTagBuilder:
|
||||
@@ -104,6 +107,7 @@ def get_model_structural_tag(
|
||||
tools: Sequence[ChatCompletionToolsParam | ResponsesTool] | None,
|
||||
tool_choice: ToolChoice,
|
||||
reasoning: bool,
|
||||
token_suffix: str = "",
|
||||
) -> StructuralTag | None:
|
||||
"""Build a structural tag with xgrammar's builtin model templates."""
|
||||
|
||||
@@ -126,12 +130,19 @@ def get_model_structural_tag(
|
||||
builtin_tools,
|
||||
simplified_tool_choice,
|
||||
reasoning,
|
||||
token_suffix,
|
||||
)
|
||||
|
||||
if model not in XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS:
|
||||
supported = sorted(SUPPORTED_STRUCTURAL_TAG_MODELS)
|
||||
raise ValueError(f"Unknown format type: {model}, supported types: {supported}")
|
||||
|
||||
if token_suffix:
|
||||
raise ValueError(
|
||||
f"Structural tag model {model!r} is an xgrammar builtin with fixed "
|
||||
f"tokens and cannot apply token_suffix={token_suffix!r}"
|
||||
)
|
||||
|
||||
return get_xgrammar_model_structural_tag(
|
||||
model=model,
|
||||
tools=dumped_tools,
|
||||
@@ -244,8 +255,9 @@ def get_hermes_structural_tag(
|
||||
builtin_tools: list[BuiltinToolParam],
|
||||
tool_choice: SimplifiedToolChoice,
|
||||
reasoning: bool,
|
||||
token_suffix: str = "",
|
||||
) -> StructuralTag:
|
||||
del builtin_tools, reasoning
|
||||
del builtin_tools, reasoning, token_suffix
|
||||
|
||||
tool_call_trigger = "<tool_call>"
|
||||
|
||||
@@ -293,8 +305,9 @@ def get_minimax_structural_tag(
|
||||
builtin_tools: list[BuiltinToolParam],
|
||||
tool_choice: SimplifiedToolChoice,
|
||||
reasoning: bool,
|
||||
token_suffix: str = "",
|
||||
) -> StructuralTag:
|
||||
del builtin_tools, reasoning
|
||||
del builtin_tools, reasoning, token_suffix
|
||||
|
||||
tool_call_begin = "<minimax:tool_call>\n"
|
||||
tool_call_end = "</minimax:tool_call>"
|
||||
@@ -589,8 +602,9 @@ def get_kimi_k3_structural_tag(
|
||||
builtin_tools: list[BuiltinToolParam],
|
||||
tool_choice: SimplifiedToolChoice,
|
||||
reasoning: bool,
|
||||
token_suffix: str = "",
|
||||
) -> StructuralTag:
|
||||
del builtin_tools, reasoning
|
||||
del builtin_tools, reasoning, token_suffix
|
||||
|
||||
trailer = OptionalFormat(content=ConstStringFormat(value=_K3_MESSAGE_CLOSE))
|
||||
|
||||
@@ -611,3 +625,142 @@ def get_kimi_k3_structural_tag(
|
||||
return StructuralTag(
|
||||
format=SequenceFormat(elements=[*_k3_response_prefix(), tools_part, trailer])
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HYV4 (<tool_calls>/<tool_call>/<arg_key>/<arg_value> structural tokens)
|
||||
# ---------------------------------------------------------------------------
|
||||
# HYV4 assistant output after the reasoning gate (``</think:SUF>``):
|
||||
# <tool_calls:SUF>
|
||||
# <tool_call:SUF>NAME
|
||||
# <arg_key:SUF>K</arg_key:SUF><arg_value:SUF>V</arg_value:SUF>
|
||||
# </tool_call:SUF>
|
||||
# </tool_calls:SUF>
|
||||
# Adjacent <tool_call> blocks have no separator, and argument values are emitted
|
||||
# verbatim (no surrounding quotes), so the value body cannot be described by
|
||||
# JSONSchemaFormat. Only the skeleton (tool names, argument keys, tag order) is
|
||||
# constrained; each <arg_value> body stays free text.
|
||||
|
||||
|
||||
def _hy_v4_argument_keys(parameters: dict[str, Any]) -> tuple[list[str], list[str]]:
|
||||
"""Split a tool's argument keys into ``(required, optional)``.
|
||||
|
||||
Both lists keep declaration order. Required keys missing from
|
||||
``properties`` are still treated as required.
|
||||
"""
|
||||
properties = parameters.get("properties") or {}
|
||||
required = parameters.get("required") or []
|
||||
required_set = set(required)
|
||||
required_keys = [k for k in properties if k in required_set]
|
||||
required_keys += [k for k in required if k not in properties]
|
||||
optional_keys = [k for k in properties if k not in required_set]
|
||||
return required_keys, optional_keys
|
||||
|
||||
|
||||
@register_vllm_structural_tag("hy_v4")
|
||||
def get_hy_v4_structural_tag(
|
||||
tools: list[FunctionToolParam],
|
||||
builtin_tools: list[BuiltinToolParam],
|
||||
tool_choice: SimplifiedToolChoice,
|
||||
reasoning: bool,
|
||||
token_suffix: str = "",
|
||||
) -> StructuralTag:
|
||||
"""Build HYV4 <tool_calls>/<tool_call>/<arg_key>/<arg_value> structural tags.
|
||||
|
||||
Args:
|
||||
tools: Normalized function tools the model may call.
|
||||
builtin_tools: Unused; HYV4 has no builtin tools.
|
||||
tool_choice: Simplified tool choice.
|
||||
reasoning: Whether the grammar also covers the reasoning phase.
|
||||
token_suffix: Per-checkpoint structural-token suffix including the
|
||||
leading colon (e.g. ``":6124c78e"``), or ``""`` when the checkpoint
|
||||
uses unsuffixed tokens. The HYV4 tool parser reads it off the
|
||||
tokenizer vocab and passes it to ``get_model_structural_tag``.
|
||||
"""
|
||||
del builtin_tools
|
||||
|
||||
think_end = f"</think{token_suffix}>"
|
||||
tool_calls_begin = f"<tool_calls{token_suffix}>"
|
||||
tool_calls_end = f"</tool_calls{token_suffix}>"
|
||||
tool_call_begin = f"<tool_call{token_suffix}>"
|
||||
tool_call_end = f"</tool_call{token_suffix}>"
|
||||
arg_key_begin = f"<arg_key{token_suffix}>"
|
||||
arg_key_end = f"</arg_key{token_suffix}>"
|
||||
arg_value_begin = f"<arg_value{token_suffix}>"
|
||||
arg_value_end = f"</arg_value{token_suffix}>"
|
||||
|
||||
def arg_pair(key: str) -> SequenceFormat:
|
||||
return SequenceFormat(
|
||||
elements=[
|
||||
ConstStringFormat(value=arg_key_begin),
|
||||
ConstStringFormat(value=key),
|
||||
ConstStringFormat(value=arg_key_end),
|
||||
ConstStringFormat(value=arg_value_begin),
|
||||
AnyTextFormat(),
|
||||
ConstStringFormat(value=arg_value_end),
|
||||
]
|
||||
)
|
||||
|
||||
def single_tool_call(tool: FunctionToolParam) -> TagFormat:
|
||||
function = tool.function
|
||||
parameters = (
|
||||
function.parameters if isinstance(function.parameters, dict) else {}
|
||||
)
|
||||
required_keys, optional_keys = _hy_v4_argument_keys(parameters)
|
||||
elements: list[Any] = [arg_pair(k) for k in required_keys]
|
||||
if optional_keys:
|
||||
# Optional keys may appear in any order, or not at all.
|
||||
optional_pairs = [arg_pair(k) for k in optional_keys]
|
||||
elements.append(
|
||||
StarFormat(
|
||||
content=optional_pairs[0]
|
||||
if len(optional_pairs) == 1
|
||||
else OrFormat(elements=optional_pairs)
|
||||
)
|
||||
)
|
||||
content: Any = (
|
||||
SequenceFormat(elements=elements) if elements else AnyTextFormat()
|
||||
)
|
||||
return TagFormat(
|
||||
begin=tool_call_begin + function.name,
|
||||
content=content,
|
||||
end=tool_call_end,
|
||||
)
|
||||
|
||||
tags = [single_tool_call(tool) for tool in tools]
|
||||
|
||||
if tool_choice == "auto":
|
||||
block = TagFormat(
|
||||
begin=tool_calls_begin,
|
||||
content=TagsWithSeparatorFormat(
|
||||
tags=tags,
|
||||
separator="",
|
||||
at_least_one=True,
|
||||
),
|
||||
end=tool_calls_end,
|
||||
)
|
||||
suffix_tag: Any = TriggeredTagsFormat(triggers=[tool_calls_begin], tags=[block])
|
||||
else:
|
||||
# HYV4 places tool_call blocks back-to-back with no separator. "forced"
|
||||
# has already been filtered down to the single named tool and must emit
|
||||
# exactly one block.
|
||||
suffix_tag = SequenceFormat(
|
||||
elements=[
|
||||
ConstStringFormat(value=tool_calls_begin),
|
||||
TagsWithSeparatorFormat(
|
||||
tags=tags,
|
||||
separator="",
|
||||
at_least_one=True,
|
||||
stop_after_first=tool_choice == "forced",
|
||||
),
|
||||
ConstStringFormat(value=tool_calls_end),
|
||||
]
|
||||
)
|
||||
|
||||
if not reasoning:
|
||||
return StructuralTag(format=suffix_tag)
|
||||
|
||||
# The reasoning phase is constrained too, so the tag must explicitly skip
|
||||
# the ``<think>...</think:SUF>`` prefix.
|
||||
prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=think_end)
|
||||
return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag]))
|
||||
|
||||
@@ -91,6 +91,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
granite4_vision="Granite4VisionConfig",
|
||||
hyperclovax="HyperCLOVAXConfig",
|
||||
hy_v3="HYV3Config",
|
||||
hy_v4="HYV4Config",
|
||||
isaac="IsaacConfig",
|
||||
kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3
|
||||
kimi_linear="KimiLinearConfig",
|
||||
|
||||
@@ -41,6 +41,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat",
|
||||
"Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision",
|
||||
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
|
||||
"HYV4Config": "vllm.transformers_utils.configs.hy_v4",
|
||||
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
|
||||
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
|
||||
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
|
||||
@@ -130,6 +131,7 @@ __all__ = [
|
||||
"FunAudioChatAudioEncoderConfig",
|
||||
"Granite4VisionConfig",
|
||||
"HYV3Config",
|
||||
"HYV4Config",
|
||||
"HyperCLOVAXConfig",
|
||||
"IsaacConfig",
|
||||
"RWConfig",
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from huggingface_hub.dataclasses import strict
|
||||
from transformers.configuration_utils import PreTrainedConfig
|
||||
from transformers.modeling_rope_utils import RopeParameters
|
||||
|
||||
|
||||
@strict
|
||||
class HYV4Config(PreTrainedConfig):
|
||||
r"""Configuration class for the HY V4 (`HYV4ForCausalLM`) model.
|
||||
|
||||
Kept field-for-field in sync with
|
||||
`transformers.models.hy_v4.configuration_hy_v4.HYV4Config` so that a
|
||||
checkpoint resolves to the same values through either class. Registering it
|
||||
in vLLM's `_CONFIG_REGISTRY` lets the architecture load without
|
||||
`trust_remote_code` and without an `auto_map` module next to the weights.
|
||||
|
||||
HYV4 is a mixture-of-experts causal language model using Multi-head Latent
|
||||
Attention (MLA), DeepSeek-style sparse attention (DSA), gated MLA,
|
||||
learnable attention sinks, and independent Hyper-Connections (iHC).
|
||||
|
||||
The `layer_types`, `num_nextn_predict_layers` and `mtp_loss_factor` fields
|
||||
below are vLLM-only additions: upstream leaves them to `**kwargs`, but
|
||||
vLLM reads them directly (sparse-attention selection and MTP speculative
|
||||
decoding), so they are declared to give checkpoints that omit them a
|
||||
working default instead of a `None`.
|
||||
|
||||
Args:
|
||||
mlp_layer_types: Per-layer MLP kind, `"dense"` or `"sparse"`. Defaults
|
||||
to one dense layer followed by sparse MoE layers.
|
||||
layer_types: Per-layer attention kind, e.g. `"full_attention"` or
|
||||
`"deepseek_sparse_attention"`.
|
||||
indexer_types: Per-layer DSA indexer kind, `"full"` or `"shared"`. A
|
||||
shared layer reuses the most recent full indexer in the same
|
||||
forward request.
|
||||
index_topk: Maximum number of key positions selected by each DSA query.
|
||||
index_head_dim: Hidden dimension of each DSA indexer head.
|
||||
index_n_heads: Number of DSA indexer heads.
|
||||
enable_lm_head_fp32: Whether the language-model head emits float32
|
||||
logits. Surfaced as ``head_dtype`` so `LogitsProcessor` accumulates
|
||||
the projection into fp32 while the weight stays in the model dtype.
|
||||
enable_ihc: Whether independent Hyper-Connections are enabled.
|
||||
hc_mult: Number of hidden-state channels maintained by iHC.
|
||||
hc_magnitude: Scale applied to the iHC post-gating branch.
|
||||
hc_eps: Numerical epsilon added to iHC sigmoid gates.
|
||||
gated_mla: Whether to gate the MLA output.
|
||||
gating_type: MLA gate granularity, `"elementwise"` or `"headwise"`.
|
||||
learnable_sink: Whether to add a learned per-head attention sink.
|
||||
learnable_sink_init: Initial value of each learned attention-sink
|
||||
logit.
|
||||
swiglu_limit: Magnitude of the routed-expert SwiGLU clamp. Values at or
|
||||
below zero disable the clamp.
|
||||
num_nextn_predict_layers: Number of MTP layers in the checkpoint.
|
||||
mtp_loss_factor: Training-time MTP loss weight (unused at inference).
|
||||
"""
|
||||
|
||||
model_type = "hy_v4"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
attribute_map = {
|
||||
"num_local_experts": "n_routed_experts",
|
||||
}
|
||||
base_model_tp_plan = {
|
||||
"layers.*.self_attn.q_b_proj": "colwise",
|
||||
"layers.*.self_attn.kv_a_proj_with_mqa": "mla_kv_a_proj",
|
||||
"layers.*.self_attn.kv_b_proj": "colwise",
|
||||
"layers.*.self_attn.o_proj": "rowwise",
|
||||
"layers.*.self_attn.linear_gate": "colwise",
|
||||
"layers.*.mlp.experts.gate_up_proj": "packed_colwise",
|
||||
"layers.*.mlp.experts.down_proj": "rowwise",
|
||||
"layers.*.mlp.experts": "moe_tp_experts",
|
||||
"layers.*.mlp.shared_experts.gate_proj": "colwise",
|
||||
"layers.*.mlp.shared_experts.up_proj": "colwise",
|
||||
"layers.*.mlp.shared_experts.down_proj": "rowwise",
|
||||
"layers.*.mlp.gate_proj": "colwise",
|
||||
"layers.*.mlp.up_proj": "colwise",
|
||||
"layers.*.mlp.down_proj": "rowwise",
|
||||
}
|
||||
base_model_pp_plan = {
|
||||
"embed_tokens": (["input_ids"], ["inputs_embeds"]),
|
||||
"layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
|
||||
"norm": (["hidden_states"], ["hidden_states"]),
|
||||
}
|
||||
|
||||
vocab_size: int = 120832
|
||||
hidden_size: int = 2816
|
||||
intermediate_size: int = 6912
|
||||
moe_intermediate_size: int = 768
|
||||
num_hidden_layers: int = 34
|
||||
num_attention_heads: int = 32
|
||||
num_key_value_heads: int = 32
|
||||
head_dim: int = 256
|
||||
hidden_act: str = "silu"
|
||||
max_position_embeddings: int = 262144
|
||||
initializer_range: float = 0.006
|
||||
rms_norm_eps: float = 1e-5
|
||||
use_cache: bool = True
|
||||
pad_token_id: int | None = 120002
|
||||
bos_token_id: int | None = 120000
|
||||
eos_token_id: int | list[int] | None = 120025
|
||||
tie_word_embeddings: bool = False
|
||||
attention_bias: bool = False
|
||||
attention_dropout: float = 0.0
|
||||
n_routed_experts: int = 256
|
||||
n_shared_experts: int = 1
|
||||
num_experts_per_tok: int = 8
|
||||
routed_scaling_factor: float = 2.827
|
||||
norm_topk_prob: bool = True
|
||||
q_lora_rank: int = 1536
|
||||
kv_lora_rank: int = 512
|
||||
qk_nope_head_dim: int = 192
|
||||
qk_rope_head_dim: int = 64
|
||||
v_head_dim: int = 256
|
||||
mlp_layer_types: list[str] | None = None
|
||||
layer_types: list[str] | None = None
|
||||
index_topk: int = 2048
|
||||
index_head_dim: int = 128
|
||||
index_n_heads: int = 16
|
||||
indexer_types: list[str] | None = None
|
||||
enable_lm_head_fp32: bool = True
|
||||
enable_ihc: bool = True
|
||||
hc_mult: int = 4
|
||||
hc_magnitude: float = 2.0
|
||||
hc_eps: float = 1e-6
|
||||
gated_mla: bool = True
|
||||
gating_type: str = "elementwise"
|
||||
learnable_sink: bool = True
|
||||
learnable_sink_init: float = 0.0
|
||||
swiglu_limit: float = 10.0
|
||||
rope_parameters: RopeParameters | dict | None = None
|
||||
num_nextn_predict_layers: int = 1
|
||||
mtp_loss_factor: float = 0.1
|
||||
|
||||
def __post_init__(self, **kwargs):
|
||||
self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
|
||||
# RoPE applies only to the rope slice, so point `head_dim` at it: the
|
||||
# inherited rotary embedding reads `config.head_dim` and then computes
|
||||
# the right frequencies with no override.
|
||||
self.head_dim = self.qk_rope_head_dim
|
||||
|
||||
if self.mlp_layer_types is None:
|
||||
self.mlp_layer_types = ["dense"] * min(1, self.num_hidden_layers) + [
|
||||
"sparse"
|
||||
] * max(self.num_hidden_layers - 1, 0)
|
||||
if self.indexer_types is None:
|
||||
self.indexer_types = [
|
||||
"full" if layer_idx == 0 or (layer_idx - 1) % 4 == 0 else "shared"
|
||||
for layer_idx in range(self.num_hidden_layers)
|
||||
]
|
||||
# `PreTrainedConfig` validates `layer_types` against a fixed allow-list
|
||||
# in which "deepseek_sparse_attention" is the only accepted spelling of
|
||||
# a DSA layer. The model treats "sparse_attention" and "sparse" as
|
||||
# equivalent, so fold them in before the base class validates.
|
||||
if self.layer_types is not None:
|
||||
self.layer_types = [
|
||||
"deepseek_sparse_attention"
|
||||
if lt in ("sparse_attention", "sparse")
|
||||
else lt
|
||||
for lt in self.layer_types
|
||||
]
|
||||
|
||||
# `ModelConfig.head_dtype` reads this off the HF config, so an fp32 head
|
||||
# needs no model-side dtype juggling: LogitsProcessor picks the
|
||||
# `torch.mm(out_dtype=float32)` path and leaves the weight in the model
|
||||
# dtype. Only fill it in when the checkpoint (or --hf-overrides) has not
|
||||
# already pinned a value.
|
||||
if self.enable_lm_head_fp32 and getattr(self, "head_dtype", None) is None:
|
||||
self.head_dtype = "float32"
|
||||
|
||||
super().__post_init__(**kwargs)
|
||||
|
||||
|
||||
__all__ = ["HYV4Config"]
|
||||
@@ -332,6 +332,8 @@ class ModelArchConfigConvertorBase:
|
||||
"bailing_hybrid",
|
||||
"bailing_hybrid_mtp",
|
||||
"bailing_hybrid_v3_mtp",
|
||||
"hy_v4",
|
||||
"hy_v4_mtp",
|
||||
):
|
||||
# check is deepseek_v4 model
|
||||
if hasattr(self.hf_text_config, "compress_ratios"):
|
||||
|
||||
@@ -30,6 +30,19 @@ from vllm.v1.worker.utils import AttentionGroup
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _target_feeds_hc_residual(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether the target replaces the drafter's input with its HC residual.
|
||||
|
||||
Keyed on the same hook the model runner calls to perform that swap. It is
|
||||
resolved from the target model class because speculators are built before
|
||||
the target model is instantiated.
|
||||
"""
|
||||
from vllm.model_executor.model_loader.utils import get_model_cls
|
||||
|
||||
target_cls = get_model_cls(vllm_config.model_config)
|
||||
return hasattr(target_cls, "get_mtp_target_hidden_states")
|
||||
|
||||
|
||||
class BaseSpeculator(ABC):
|
||||
@abstractmethod
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
@@ -90,11 +103,16 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
# the draft model's hidden size can be different from the target model's
|
||||
# hidden size (e.g., Llama 3.3 70B).
|
||||
self.hidden_size = self.draft_model_config.get_hidden_size()
|
||||
# Widen for HC-multiplexed residuals (e.g. DeepSeek V4 feeds the MTP
|
||||
# draft the target's pre-hc_head (T, hc_mult * hidden_size) residual).
|
||||
# Non-HC models default to hc_mult=1 and are unaffected.
|
||||
hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1)
|
||||
self.hidden_size = self.hidden_size * hc_mult
|
||||
# Widen for HC-multiplexed residuals: a target that implements
|
||||
# get_mtp_target_hidden_states() (e.g. DeepSeek V4) hands the drafter
|
||||
# its pre-hc_head (T, hc_mult * hidden_size) residual in place of the
|
||||
# collapsed hidden states, so the drafter's buffers must match. Key off
|
||||
# that hook rather than hc_mult alone -- HY V4 runs iHC in its backbone
|
||||
# (hc_mult=4) but its MTP head consumes the collapsed states, so
|
||||
# widening it feeds propose() a 4x-too-wide buffer.
|
||||
if _target_feeds_hc_residual(vllm_config):
|
||||
hc_mult = getattr(self.draft_model_config.hf_config, "hc_mult", 1)
|
||||
self.hidden_size = self.hidden_size * hc_mult
|
||||
self.vocab_size = self.draft_model_config.get_vocab_size()
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
self.use_fp64_gumbel = vllm_config.model_config.use_fp64_gumbel
|
||||
|
||||
Reference in New Issue
Block a user