[V1] Enable Mamba2 layers other than MambaMixer2 in the v1 engine (#20660)

Signed-off-by: nopperl <54780682+nopperl@users.noreply.github.com>
This commit is contained in:
nopperl
2025-07-11 05:53:31 +00:00
committed by GitHub
parent 31d5c1797f
commit 5d09152ff1
11 changed files with 68 additions and 45 deletions
+1 -2
View File
@@ -99,8 +99,7 @@ class BambaMixerDecoderLayer(nn.Module):
rms_norm_eps=config.rms_norm_eps,
activation=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mixer",
chunk_size=config.mamba_chunk_size)
prefix=f"{prefix}.mixer")
self.feed_forward = BambaMLP(config, quant_config=quant_config)
self.input_layernorm = RMSNorm(config.hidden_size,
-1
View File
@@ -109,7 +109,6 @@ class FalconH1SSMDecoderLayer(nn.Module):
quant_config=quant_config,
use_rms_norm=config.mamba_rms_norm,
prefix=f"{prefix}.mixer",
chunk_size=config.mamba_chunk_size,
)
# n_groups is overridden later by `MambaMixer2`
self.groups_time_state_size = self.mamba.n_groups * config.mamba_d_state
@@ -69,8 +69,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
rms_norm_eps=config.rms_norm_eps,
activation=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mixer",
chunk_size=config.mamba_chunk_size)
prefix=f"{prefix}.mixer")
self.block_sparse_moe = None
if getattr(config, "num_local_experts", 0) > 0:
+1 -2
View File
@@ -62,8 +62,7 @@ class Mamba2DecoderLayer(nn.Module):
rms_norm_eps=config.layer_norm_epsilon,
activation=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mixer",
chunk_size=config.chunk_size)
prefix=f"{prefix}.mixer")
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
-1
View File
@@ -154,7 +154,6 @@ class NemotronHMambaDecoderLayer(nn.Module):
activation=config.mamba_hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mixer",
chunk_size=config.chunk_size,
)
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+1 -2
View File
@@ -501,8 +501,7 @@ class Zamba2MambaDecoderLayer(nn.Module):
rms_norm_eps=config.rms_norm_eps,
activation="silu",
quant_config=quant_config,
prefix=f"{prefix}.mixer",
chunk_size=config.chunk_size)
prefix=f"{prefix}.mixer")
# Input normalization
self.input_layernorm = RMSNorm(config.hidden_size,