[Bugfix] Load stacked expert weights for non-gated MoE (#59031)

Signed-off-by: Roi Koren <roik@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
roikoren755
2026-10-05 15:26:55 +03:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4ff028d77e
commit 51eeb0c58f
6 changed files with 88 additions and 18 deletions
@@ -118,7 +118,7 @@ def FusedMoEFactory(
has_bias: bool = False,
is_sequence_parallel: bool = False,
reduce_results: bool = True,
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
ckpt_names: tuple[str, str, str | None] = ("gate_proj", "down_proj", "up_proj"),
is_fused_checkpoint_transposed: bool = False,
n_shared_experts: int | None = None,
fuse_shared_experts: bool = False,
@@ -442,7 +442,7 @@ def fused_moe_make_expert_params_mapping(
model: torch.nn.Module,
ckpt_gate_proj_name: str,
ckpt_down_proj_name: str,
ckpt_up_proj_name: str,
ckpt_up_proj_name: str | None,
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
@@ -87,7 +87,7 @@ class RoutedExperts(PluggableLayer):
expert_map_manager: ExpertMapManager,
ckpt_gate_proj_name: str = "gate_proj",
ckpt_down_proj_name: str = "down_proj",
ckpt_up_proj_name: str = "up_proj",
ckpt_up_proj_name: str | None = "up_proj",
is_fused_checkpoint_transposed: bool = False,
#
# Extra params that are needed by quant_methods, pass along for now
@@ -114,6 +114,13 @@ class RoutedExperts(PluggableLayer):
self.ckpt_gate_proj_name = ckpt_gate_proj_name
self.ckpt_down_proj_name = ckpt_down_proj_name
self.ckpt_up_proj_name = ckpt_up_proj_name
if moe_config.is_act_and_mul == (ckpt_up_proj_name is None):
raise ValueError(
f"{layer_name}: ckpt_up_proj_name={ckpt_up_proj_name!r} is "
f"inconsistent with activation {moe_config.activation.value!r}. "
"Gated MoE needs an up projection name; non-gated MoE must "
"pass None."
)
self.is_fused_checkpoint_transposed = is_fused_checkpoint_transposed
self.expert_map_manager = expert_map_manager
self.hidden_size = moe_config.hidden_dim
@@ -971,7 +978,7 @@ class RoutedExperts(PluggableLayer):
loaded_weight,
self.is_fused_checkpoint_transposed and uses_weight_layout,
)
if shard_id in {"w1", "w3"}:
if shard_id in {"w1", "w3"} and self.moe_config.is_act_and_mul:
# Repurpose expert_id for deconcatenating w1 and w3
experts_shard = fused_weight.chunk(2, dim=1)[expert_id]
else:
@@ -1029,6 +1036,7 @@ class RoutedExperts(PluggableLayer):
routed_experts_prefix="",
lora_base_layer_prefix=self.lora_base_layer_prefix,
include_fused=include_fused,
is_gated=moe_config.is_act_and_mul,
)
@staticmethod
@@ -1036,7 +1044,7 @@ class RoutedExperts(PluggableLayer):
model: torch.nn.Module,
ckpt_gate_proj_name: str,
ckpt_down_proj_name: str,
ckpt_up_proj_name: str,
ckpt_up_proj_name: str | None,
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
@@ -1067,13 +1075,14 @@ class RoutedExperts(PluggableLayer):
def build_expert_params_mapping(
ckpt_gate_proj_name: str,
ckpt_down_proj_name: str,
ckpt_up_proj_name: str,
ckpt_up_proj_name: str | None,
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
lora_base_layer_prefix: str = "",
lora_base_layer_prefix_on_param_name: str = "",
include_fused: bool = False,
is_gated: bool = True,
) -> list[tuple[str, str, int, str]]:
"""Create expert parameter mapping for weight loading with redundant experts.
@@ -1095,6 +1104,8 @@ class RoutedExperts(PluggableLayer):
``make_expert_params_mapping`` indexes the model-wide
``params_dict`` (prefix included).
include_fused: Prepend the fused pre-fused-checkpoint entries
is_gated: Whether w13 holds gate and up (gated activation) or a
single up projection. Selects the fused checkpoint layout.
routed_experts_prefix: Prefix of the routed experts submodule
Returns:
@@ -1129,7 +1140,13 @@ class RoutedExperts(PluggableLayer):
fused_mapping = []
if include_fused:
gate_up = None
if ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
w13_shards: tuple[str, ...] = ("w1", "w3")
if not is_gated:
# Non-gated: the stacked checkpoint tensor is the up projection
# itself, nothing to split into gate and up.
gate_up = ckpt_gate_proj_name
w13_shards = ("w1",)
elif ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
gate_up = "gate_up_proj"
elif ckpt_gate_proj_name == "w1" and ckpt_up_proj_name == "w3":
gate_up = "w13"
@@ -1143,10 +1160,16 @@ class RoutedExperts(PluggableLayer):
if gate_up is not None:
fused_mapping = [
# (param_name, weight_name, expert_id, shard_id)
(f"{w13}weight", f"experts.{gate_up}", 0, "w1"),
(f"{w13}weight", f"experts.{gate_up}", 1, "w3"),
# expert_id doubles as the chunk index when splitting w13.
*(
(f"{w13}weight", f"experts.{gate_up}", chunk, shard_id)
for chunk, shard_id in enumerate(w13_shards)
),
(f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"),
]
if gate_up is not None and is_gated:
# Per-expert gate_up_proj tensors; the non-gated equivalent is
# already covered by the per-expert mapping below.
fused_mapping.extend(
(
w13,
@@ -1175,6 +1198,7 @@ class RoutedExperts(PluggableLayer):
("w2", ckpt_down_proj_name),
("w3", ckpt_up_proj_name),
]
if weight_name is not None
]
return fused_mapping + per_expert_mapping