[Model][Spec Decode] Nemotron-H MTP and Mamba Speculative Decoding Support (#33726)

Signed-off-by: Shahar Mor <smor@nvidia.com>
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Shahar Mor <smor@nvidia.com>
Co-authored-by: Roi Koren <roik@nvidia.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Benjamin Chislett
2026-02-24 09:49:56 -08:00
committed by GitHub
co-authored by Shahar Mor Roi Koren Lucas Wilkinson
parent a9e15e040d
commit f5972a872f
19 changed files with 799 additions and 157 deletions
@@ -51,6 +51,8 @@ class NemotronHConfig(PretrainedConfig):
The pattern of the hybrid model. The pattern is a string of
characters where each character represents
M: Mamba2, *: Attention, -: MLP
mtp_hybrid_override_pattern (`str`, *optional*, defaults to `"*E"`):
The pattern of the MTP layers.
num_attention_heads (`int`, *optional*, defaults to 32):
Number of attention heads for each attention layer in the
Transformer encoder.
@@ -150,6 +152,7 @@ class NemotronHConfig(PretrainedConfig):
intermediate_size=21504,
num_hidden_layers=52,
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
mtp_hybrid_override_pattern="*E",
num_attention_heads=32,
head_dim=128,
num_key_value_heads=8, # nemo: num_query_groups
@@ -203,6 +206,7 @@ class NemotronHConfig(PretrainedConfig):
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.hybrid_override_pattern = hybrid_override_pattern
self.mtp_hybrid_override_pattern = mtp_hybrid_override_pattern
self.num_attention_heads = num_attention_heads
self.head_dim = head_dim
self.sliding_window = sliding_window
@@ -215,10 +219,9 @@ class NemotronHConfig(PretrainedConfig):
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, (
"hybrid_override_pattern must have same length as num_hidden_layers"
)
assert re.match(r"^[*-M]+$", self.hybrid_override_pattern), (
"hybrid_override_pattern must only contain characters 'M', '*', or '-'"
assert re.match(r"^[*-ME]+$", self.hybrid_override_pattern), (
"hybrid_override_pattern must only contain characters 'M', '*', '-', or 'E'"
)
# for backward compatibility
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads