[Bugfix] Load stacked expert weights for non-gated MoE (#59031)
Signed-off-by: Roi Koren <roik@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4ff028d77e
commit
51eeb0c58f
@@ -235,7 +235,7 @@ class NemotronHMoE(nn.Module):
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config,
|
||||
ckpt_names=("up_proj", "down_proj", ""),
|
||||
ckpt_names=("up_proj", "down_proj", None),
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=config.n_group,
|
||||
topk_group=config.topk_group,
|
||||
@@ -707,7 +707,7 @@ class NemotronHModel(nn.Module, EagleModelMixin):
|
||||
self,
|
||||
ckpt_gate_proj_name="up_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="",
|
||||
ckpt_up_proj_name=None,
|
||||
num_experts=self._get_max_n_routed_experts(),
|
||||
num_redundant_experts=getattr(self, "num_redundant_experts", 0),
|
||||
)
|
||||
|
||||
@@ -444,7 +444,7 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
|
||||
self,
|
||||
ckpt_gate_proj_name="up_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="", # Empty - non-gated MoE
|
||||
ckpt_up_proj_name=None, # non-gated MoE
|
||||
num_experts=num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user