[Bugfix] Load stacked expert weights for non-gated MoE (#59031)
Signed-off-by: Roi Koren <roik@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4ff028d77e
commit
51eeb0c58f
@@ -9,6 +9,8 @@ have the original unpadded size. These tests verify that weight loading
|
||||
correctly handles this mismatch.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -564,9 +566,11 @@ class TestLoadWeightsExpertMapping:
|
||||
transposed=False,
|
||||
quant_method="tensor",
|
||||
layer_name="model.layers.0.mlp.experts",
|
||||
is_gated=True,
|
||||
):
|
||||
experts = torch.nn.Module()
|
||||
experts.layer_name = layer_name
|
||||
experts.moe_config = SimpleNamespace(is_act_and_mul=is_gated)
|
||||
experts.get_expert_mapping = lambda **_: mapping
|
||||
experts.is_fused_checkpoint_transposed = transposed
|
||||
experts._orient_fused_weight = RoutedExperts._orient_fused_weight
|
||||
@@ -593,7 +597,7 @@ class TestLoadWeightsExpertMapping:
|
||||
(("gate_proj", "down_proj", "up_proj"), ""),
|
||||
(("w1", "w2", "w3"), ""),
|
||||
(("up_proj", "down_proj", "up_proj"), ""),
|
||||
(("up_proj", "down_proj", ""), ""),
|
||||
(("up_proj", "down_proj", None), ""),
|
||||
(("gate_proj", "down_proj", "up_proj"), "base_layer."),
|
||||
],
|
||||
)
|
||||
@@ -609,7 +613,10 @@ class TestLoadWeightsExpertMapping:
|
||||
self, projs, lora_prefix, suffix, shape, dtype
|
||||
):
|
||||
mapping = self._mapping(
|
||||
projs, num_redundant_experts=4, lora_base_layer_prefix=lora_prefix
|
||||
projs,
|
||||
num_redundant_experts=4,
|
||||
lora_base_layer_prefix=lora_prefix,
|
||||
is_gated=projs[2] is not None,
|
||||
)
|
||||
for logical_id, physical_ids in [(1, (1, 17)), (10, (10,))]:
|
||||
for proj in dict.fromkeys(projs):
|
||||
@@ -651,6 +658,47 @@ class TestLoadWeightsExpertMapping:
|
||||
for call, expected in zip(calls, weight.chunk(2) * 2):
|
||||
torch.testing.assert_close(call[3], expected, rtol=0, atol=0)
|
||||
|
||||
def test_non_gated_mapping_uses_stacked_up_proj_without_warning(self, caplog_vllm):
|
||||
mapping = self._mapping(("up_proj", "down_proj", None), is_gated=False)
|
||||
assert "Unexpected gate/up projection names" not in caplog_vllm.text
|
||||
assert mapping[:2] == [
|
||||
("experts.w13_weight", "experts.up_proj", 0, "w1"),
|
||||
("experts.w2_weight", "experts.down_proj", 0, "w2"),
|
||||
]
|
||||
assert mapping[2:] == [
|
||||
(param, f"experts.{expert}.{proj}.", expert, shard)
|
||||
for expert in range(16)
|
||||
for param, proj, shard in (
|
||||
("experts.w13_", "up_proj", "w1"),
|
||||
("experts.w2_", "down_proj", "w2"),
|
||||
)
|
||||
]
|
||||
|
||||
def test_gated_mapping_with_unknown_names_warns_and_skips_fused(self, caplog_vllm):
|
||||
mapping = self._mapping(("fc1", "fc2", "fc3"))
|
||||
assert "Unexpected gate/up projection names: fc1, fc3" in caplog_vllm.text
|
||||
assert [name for _, name, _, _ in mapping[:3]] == [
|
||||
"experts.0.fc1.",
|
||||
"experts.0.fc2.",
|
||||
"experts.0.fc3.",
|
||||
]
|
||||
|
||||
def test_stacked_non_gated_tensors_load_unsplit(self):
|
||||
mapping = self._mapping(("up_proj", "down_proj", None), is_gated=False)
|
||||
up_proj = torch.arange(16 * 4 * 3).reshape(16, 4, 3)
|
||||
down_proj = torch.arange(16 * 3 * 4).reshape(16, 3, 4)
|
||||
calls = self._load(
|
||||
mapping,
|
||||
[("up_proj", up_proj), ("down_proj", down_proj)],
|
||||
is_gated=False,
|
||||
)
|
||||
assert [call[:3] for call in calls] == [
|
||||
*(("w13_weight", "w1", expert) for expert in range(16)),
|
||||
*(("w2_weight", "w2", expert) for expert in range(16)),
|
||||
]
|
||||
for call, expected in zip(calls, [*up_proj, *down_proj]):
|
||||
torch.testing.assert_close(call[3], expected, rtol=0, atol=0)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"suffix,quant_method,transposed",
|
||||
[
|
||||
|
||||
+2
-4
@@ -409,13 +409,11 @@ def process_packed_modules_mapping(
|
||||
"experts.w3",
|
||||
]
|
||||
elif (not model.is_3d_moe_weight) or force_2d_moe:
|
||||
# Filter out malformed entries: non-gated MoE has empty
|
||||
# ckpt_up_proj_name which results in weight_name containing ".."
|
||||
# (e.g., "experts.0.." instead of "experts.0.layer_name.")
|
||||
# Non-gated MoE yields two entries per expert (no w3); the
|
||||
# manager pads them to triplets before packing.
|
||||
packed_modules_mapping["experts"] = [
|
||||
weight_name.rstrip(".")
|
||||
for _, weight_name, _, _ in get_moe_expert_mapping(model)
|
||||
if ".." not in weight_name
|
||||
]
|
||||
|
||||
return packed_modules_mapping
|
||||
|
||||
@@ -118,7 +118,7 @@ def FusedMoEFactory(
|
||||
has_bias: bool = False,
|
||||
is_sequence_parallel: bool = False,
|
||||
reduce_results: bool = True,
|
||||
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
|
||||
ckpt_names: tuple[str, str, str | None] = ("gate_proj", "down_proj", "up_proj"),
|
||||
is_fused_checkpoint_transposed: bool = False,
|
||||
n_shared_experts: int | None = None,
|
||||
fuse_shared_experts: bool = False,
|
||||
@@ -442,7 +442,7 @@ def fused_moe_make_expert_params_mapping(
|
||||
model: torch.nn.Module,
|
||||
ckpt_gate_proj_name: str,
|
||||
ckpt_down_proj_name: str,
|
||||
ckpt_up_proj_name: str,
|
||||
ckpt_up_proj_name: str | None,
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
|
||||
@@ -87,7 +87,7 @@ class RoutedExperts(PluggableLayer):
|
||||
expert_map_manager: ExpertMapManager,
|
||||
ckpt_gate_proj_name: str = "gate_proj",
|
||||
ckpt_down_proj_name: str = "down_proj",
|
||||
ckpt_up_proj_name: str = "up_proj",
|
||||
ckpt_up_proj_name: str | None = "up_proj",
|
||||
is_fused_checkpoint_transposed: bool = False,
|
||||
#
|
||||
# Extra params that are needed by quant_methods, pass along for now
|
||||
@@ -114,6 +114,13 @@ class RoutedExperts(PluggableLayer):
|
||||
self.ckpt_gate_proj_name = ckpt_gate_proj_name
|
||||
self.ckpt_down_proj_name = ckpt_down_proj_name
|
||||
self.ckpt_up_proj_name = ckpt_up_proj_name
|
||||
if moe_config.is_act_and_mul == (ckpt_up_proj_name is None):
|
||||
raise ValueError(
|
||||
f"{layer_name}: ckpt_up_proj_name={ckpt_up_proj_name!r} is "
|
||||
f"inconsistent with activation {moe_config.activation.value!r}. "
|
||||
"Gated MoE needs an up projection name; non-gated MoE must "
|
||||
"pass None."
|
||||
)
|
||||
self.is_fused_checkpoint_transposed = is_fused_checkpoint_transposed
|
||||
self.expert_map_manager = expert_map_manager
|
||||
self.hidden_size = moe_config.hidden_dim
|
||||
@@ -971,7 +978,7 @@ class RoutedExperts(PluggableLayer):
|
||||
loaded_weight,
|
||||
self.is_fused_checkpoint_transposed and uses_weight_layout,
|
||||
)
|
||||
if shard_id in {"w1", "w3"}:
|
||||
if shard_id in {"w1", "w3"} and self.moe_config.is_act_and_mul:
|
||||
# Repurpose expert_id for deconcatenating w1 and w3
|
||||
experts_shard = fused_weight.chunk(2, dim=1)[expert_id]
|
||||
else:
|
||||
@@ -1029,6 +1036,7 @@ class RoutedExperts(PluggableLayer):
|
||||
routed_experts_prefix="",
|
||||
lora_base_layer_prefix=self.lora_base_layer_prefix,
|
||||
include_fused=include_fused,
|
||||
is_gated=moe_config.is_act_and_mul,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1036,7 +1044,7 @@ class RoutedExperts(PluggableLayer):
|
||||
model: torch.nn.Module,
|
||||
ckpt_gate_proj_name: str,
|
||||
ckpt_down_proj_name: str,
|
||||
ckpt_up_proj_name: str,
|
||||
ckpt_up_proj_name: str | None,
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
@@ -1067,13 +1075,14 @@ class RoutedExperts(PluggableLayer):
|
||||
def build_expert_params_mapping(
|
||||
ckpt_gate_proj_name: str,
|
||||
ckpt_down_proj_name: str,
|
||||
ckpt_up_proj_name: str,
|
||||
ckpt_up_proj_name: str | None,
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
lora_base_layer_prefix: str = "",
|
||||
lora_base_layer_prefix_on_param_name: str = "",
|
||||
include_fused: bool = False,
|
||||
is_gated: bool = True,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
"""Create expert parameter mapping for weight loading with redundant experts.
|
||||
|
||||
@@ -1095,6 +1104,8 @@ class RoutedExperts(PluggableLayer):
|
||||
``make_expert_params_mapping`` indexes the model-wide
|
||||
``params_dict`` (prefix included).
|
||||
include_fused: Prepend the fused pre-fused-checkpoint entries
|
||||
is_gated: Whether w13 holds gate and up (gated activation) or a
|
||||
single up projection. Selects the fused checkpoint layout.
|
||||
routed_experts_prefix: Prefix of the routed experts submodule
|
||||
|
||||
Returns:
|
||||
@@ -1129,7 +1140,13 @@ class RoutedExperts(PluggableLayer):
|
||||
fused_mapping = []
|
||||
if include_fused:
|
||||
gate_up = None
|
||||
if ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
|
||||
w13_shards: tuple[str, ...] = ("w1", "w3")
|
||||
if not is_gated:
|
||||
# Non-gated: the stacked checkpoint tensor is the up projection
|
||||
# itself, nothing to split into gate and up.
|
||||
gate_up = ckpt_gate_proj_name
|
||||
w13_shards = ("w1",)
|
||||
elif ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
|
||||
gate_up = "gate_up_proj"
|
||||
elif ckpt_gate_proj_name == "w1" and ckpt_up_proj_name == "w3":
|
||||
gate_up = "w13"
|
||||
@@ -1143,10 +1160,16 @@ class RoutedExperts(PluggableLayer):
|
||||
if gate_up is not None:
|
||||
fused_mapping = [
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
(f"{w13}weight", f"experts.{gate_up}", 0, "w1"),
|
||||
(f"{w13}weight", f"experts.{gate_up}", 1, "w3"),
|
||||
# expert_id doubles as the chunk index when splitting w13.
|
||||
*(
|
||||
(f"{w13}weight", f"experts.{gate_up}", chunk, shard_id)
|
||||
for chunk, shard_id in enumerate(w13_shards)
|
||||
),
|
||||
(f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"),
|
||||
]
|
||||
if gate_up is not None and is_gated:
|
||||
# Per-expert gate_up_proj tensors; the non-gated equivalent is
|
||||
# already covered by the per-expert mapping below.
|
||||
fused_mapping.extend(
|
||||
(
|
||||
w13,
|
||||
@@ -1175,6 +1198,7 @@ class RoutedExperts(PluggableLayer):
|
||||
("w2", ckpt_down_proj_name),
|
||||
("w3", ckpt_up_proj_name),
|
||||
]
|
||||
if weight_name is not None
|
||||
]
|
||||
|
||||
return fused_mapping + per_expert_mapping
|
||||
|
||||
@@ -235,7 +235,7 @@ class NemotronHMoE(nn.Module):
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
renormalize=config.norm_topk_prob,
|
||||
quant_config=quant_config,
|
||||
ckpt_names=("up_proj", "down_proj", ""),
|
||||
ckpt_names=("up_proj", "down_proj", None),
|
||||
use_grouped_topk=True,
|
||||
num_expert_group=config.n_group,
|
||||
topk_group=config.topk_group,
|
||||
@@ -707,7 +707,7 @@ class NemotronHModel(nn.Module, EagleModelMixin):
|
||||
self,
|
||||
ckpt_gate_proj_name="up_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="",
|
||||
ckpt_up_proj_name=None,
|
||||
num_experts=self._get_max_n_routed_experts(),
|
||||
num_redundant_experts=getattr(self, "num_redundant_experts", 0),
|
||||
)
|
||||
|
||||
@@ -444,7 +444,7 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant):
|
||||
self,
|
||||
ckpt_gate_proj_name="up_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="", # Empty - non-gated MoE
|
||||
ckpt_up_proj_name=None, # non-gated MoE
|
||||
num_experts=num_experts,
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user