[Bugfix] Load stacked expert weights for non-gated MoE (#59031)

Signed-off-by: Roi Koren <roik@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
roikoren755
2026-10-05 15:26:55 +03:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4ff028d77e
commit 51eeb0c58f
6 changed files with 88 additions and 18 deletions
+2 -2
View File
@@ -235,7 +235,7 @@ class NemotronHMoE(nn.Module):
intermediate_size=config.moe_intermediate_size,
renormalize=config.norm_topk_prob,
quant_config=quant_config,
ckpt_names=("up_proj", "down_proj", ""),
ckpt_names=("up_proj", "down_proj", None),
use_grouped_topk=True,
num_expert_group=config.n_group,
topk_group=config.topk_group,
@@ -707,7 +707,7 @@ class NemotronHModel(nn.Module, EagleModelMixin):
self,
ckpt_gate_proj_name="up_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="",
ckpt_up_proj_name=None,
num_experts=self._get_max_n_routed_experts(),
num_redundant_experts=getattr(self, "num_redundant_experts", 0),
)
+1 -1
View File
@@ -444,7 +444,7 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
self,
ckpt_gate_proj_name="up_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="", # Empty - non-gated MoE
ckpt_up_proj_name=None, # non-gated MoE
num_experts=num_experts,
num_redundant_experts=self.num_redundant_experts,
)