[V1] Enable Mamba2 layers other than MambaMixer2 in the v1 engine (#20660)
Signed-off-by: nopperl <54780682+nopperl@users.noreply.github.com>
This commit is contained in:
@@ -99,8 +99,7 @@ class BambaMixerDecoderLayer(nn.Module):
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.mamba_chunk_size)
|
||||
prefix=f"{prefix}.mixer")
|
||||
|
||||
self.feed_forward = BambaMLP(config, quant_config=quant_config)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size,
|
||||
|
||||
@@ -109,7 +109,6 @@ class FalconH1SSMDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
use_rms_norm=config.mamba_rms_norm,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.mamba_chunk_size,
|
||||
)
|
||||
# n_groups is overridden later by `MambaMixer2`
|
||||
self.groups_time_state_size = self.mamba.n_groups * config.mamba_d_state
|
||||
|
||||
@@ -69,8 +69,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.mamba_chunk_size)
|
||||
prefix=f"{prefix}.mixer")
|
||||
|
||||
self.block_sparse_moe = None
|
||||
if getattr(config, "num_local_experts", 0) > 0:
|
||||
|
||||
@@ -62,8 +62,7 @@ class Mamba2DecoderLayer(nn.Module):
|
||||
rms_norm_eps=config.layer_norm_epsilon,
|
||||
activation=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.chunk_size)
|
||||
prefix=f"{prefix}.mixer")
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
|
||||
|
||||
@@ -154,7 +154,6 @@ class NemotronHMambaDecoderLayer(nn.Module):
|
||||
activation=config.mamba_hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.chunk_size,
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -501,8 +501,7 @@ class Zamba2MambaDecoderLayer(nn.Module):
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation="silu",
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.chunk_size)
|
||||
prefix=f"{prefix}.mixer")
|
||||
|
||||
# Input normalization
|
||||
self.input_layernorm = RMSNorm(config.hidden_size,
|
||||
|
||||
Reference in New Issue
Block a user