diff --git a/tests/kernels/moe/test_moe_weight_loading_padded.py b/tests/kernels/moe/test_moe_weight_loading_padded.py index f7da37dc92..2ec1aaded7 100644 --- a/tests/kernels/moe/test_moe_weight_loading_padded.py +++ b/tests/kernels/moe/test_moe_weight_loading_padded.py @@ -9,6 +9,8 @@ have the original unpadded size. These tests verify that weight loading correctly handles this mismatch. """ +from types import SimpleNamespace + import pytest import torch @@ -564,9 +566,11 @@ class TestLoadWeightsExpertMapping: transposed=False, quant_method="tensor", layer_name="model.layers.0.mlp.experts", + is_gated=True, ): experts = torch.nn.Module() experts.layer_name = layer_name + experts.moe_config = SimpleNamespace(is_act_and_mul=is_gated) experts.get_expert_mapping = lambda **_: mapping experts.is_fused_checkpoint_transposed = transposed experts._orient_fused_weight = RoutedExperts._orient_fused_weight @@ -593,7 +597,7 @@ class TestLoadWeightsExpertMapping: (("gate_proj", "down_proj", "up_proj"), ""), (("w1", "w2", "w3"), ""), (("up_proj", "down_proj", "up_proj"), ""), - (("up_proj", "down_proj", ""), ""), + (("up_proj", "down_proj", None), ""), (("gate_proj", "down_proj", "up_proj"), "base_layer."), ], ) @@ -609,7 +613,10 @@ class TestLoadWeightsExpertMapping: self, projs, lora_prefix, suffix, shape, dtype ): mapping = self._mapping( - projs, num_redundant_experts=4, lora_base_layer_prefix=lora_prefix + projs, + num_redundant_experts=4, + lora_base_layer_prefix=lora_prefix, + is_gated=projs[2] is not None, ) for logical_id, physical_ids in [(1, (1, 17)), (10, (10,))]: for proj in dict.fromkeys(projs): @@ -651,6 +658,47 @@ class TestLoadWeightsExpertMapping: for call, expected in zip(calls, weight.chunk(2) * 2): torch.testing.assert_close(call[3], expected, rtol=0, atol=0) + def test_non_gated_mapping_uses_stacked_up_proj_without_warning(self, caplog_vllm): + mapping = self._mapping(("up_proj", "down_proj", None), is_gated=False) + assert "Unexpected gate/up projection names" not in caplog_vllm.text + assert mapping[:2] == [ + ("experts.w13_weight", "experts.up_proj", 0, "w1"), + ("experts.w2_weight", "experts.down_proj", 0, "w2"), + ] + assert mapping[2:] == [ + (param, f"experts.{expert}.{proj}.", expert, shard) + for expert in range(16) + for param, proj, shard in ( + ("experts.w13_", "up_proj", "w1"), + ("experts.w2_", "down_proj", "w2"), + ) + ] + + def test_gated_mapping_with_unknown_names_warns_and_skips_fused(self, caplog_vllm): + mapping = self._mapping(("fc1", "fc2", "fc3")) + assert "Unexpected gate/up projection names: fc1, fc3" in caplog_vllm.text + assert [name for _, name, _, _ in mapping[:3]] == [ + "experts.0.fc1.", + "experts.0.fc2.", + "experts.0.fc3.", + ] + + def test_stacked_non_gated_tensors_load_unsplit(self): + mapping = self._mapping(("up_proj", "down_proj", None), is_gated=False) + up_proj = torch.arange(16 * 4 * 3).reshape(16, 4, 3) + down_proj = torch.arange(16 * 3 * 4).reshape(16, 3, 4) + calls = self._load( + mapping, + [("up_proj", up_proj), ("down_proj", down_proj)], + is_gated=False, + ) + assert [call[:3] for call in calls] == [ + *(("w13_weight", "w1", expert) for expert in range(16)), + *(("w2_weight", "w2", expert) for expert in range(16)), + ] + for call, expected in zip(calls, [*up_proj, *down_proj]): + torch.testing.assert_close(call[3], expected, rtol=0, atol=0) + @pytest.mark.parametrize( "suffix,quant_method,transposed", [ diff --git a/vllm/lora/utils.py b/vllm/lora/utils.py index 71a2e60dc7..3a4009d28b 100644 --- a/vllm/lora/utils.py +++ b/vllm/lora/utils.py @@ -409,13 +409,11 @@ def process_packed_modules_mapping( "experts.w3", ] elif (not model.is_3d_moe_weight) or force_2d_moe: - # Filter out malformed entries: non-gated MoE has empty - # ckpt_up_proj_name which results in weight_name containing ".." - # (e.g., "experts.0.." instead of "experts.0.layer_name.") + # Non-gated MoE yields two entries per expert (no w3); the + # manager pads them to triplets before packing. packed_modules_mapping["experts"] = [ weight_name.rstrip(".") for _, weight_name, _, _ in get_moe_expert_mapping(model) - if ".." not in weight_name ] return packed_modules_mapping diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index ff620622c1..294c538c7b 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -118,7 +118,7 @@ def FusedMoEFactory( has_bias: bool = False, is_sequence_parallel: bool = False, reduce_results: bool = True, - ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"), + ckpt_names: tuple[str, str, str | None] = ("gate_proj", "down_proj", "up_proj"), is_fused_checkpoint_transposed: bool = False, n_shared_experts: int | None = None, fuse_shared_experts: bool = False, @@ -442,7 +442,7 @@ def fused_moe_make_expert_params_mapping( model: torch.nn.Module, ckpt_gate_proj_name: str, ckpt_down_proj_name: str, - ckpt_up_proj_name: str, + ckpt_up_proj_name: str | None, num_experts: int, num_redundant_experts: int = 0, routed_experts_prefix: str = "routed_experts", diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index f3c9522df1..adc8a27010 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -87,7 +87,7 @@ class RoutedExperts(PluggableLayer): expert_map_manager: ExpertMapManager, ckpt_gate_proj_name: str = "gate_proj", ckpt_down_proj_name: str = "down_proj", - ckpt_up_proj_name: str = "up_proj", + ckpt_up_proj_name: str | None = "up_proj", is_fused_checkpoint_transposed: bool = False, # # Extra params that are needed by quant_methods, pass along for now @@ -114,6 +114,13 @@ class RoutedExperts(PluggableLayer): self.ckpt_gate_proj_name = ckpt_gate_proj_name self.ckpt_down_proj_name = ckpt_down_proj_name self.ckpt_up_proj_name = ckpt_up_proj_name + if moe_config.is_act_and_mul == (ckpt_up_proj_name is None): + raise ValueError( + f"{layer_name}: ckpt_up_proj_name={ckpt_up_proj_name!r} is " + f"inconsistent with activation {moe_config.activation.value!r}. " + "Gated MoE needs an up projection name; non-gated MoE must " + "pass None." + ) self.is_fused_checkpoint_transposed = is_fused_checkpoint_transposed self.expert_map_manager = expert_map_manager self.hidden_size = moe_config.hidden_dim @@ -971,7 +978,7 @@ class RoutedExperts(PluggableLayer): loaded_weight, self.is_fused_checkpoint_transposed and uses_weight_layout, ) - if shard_id in {"w1", "w3"}: + if shard_id in {"w1", "w3"} and self.moe_config.is_act_and_mul: # Repurpose expert_id for deconcatenating w1 and w3 experts_shard = fused_weight.chunk(2, dim=1)[expert_id] else: @@ -1029,6 +1036,7 @@ class RoutedExperts(PluggableLayer): routed_experts_prefix="", lora_base_layer_prefix=self.lora_base_layer_prefix, include_fused=include_fused, + is_gated=moe_config.is_act_and_mul, ) @staticmethod @@ -1036,7 +1044,7 @@ class RoutedExperts(PluggableLayer): model: torch.nn.Module, ckpt_gate_proj_name: str, ckpt_down_proj_name: str, - ckpt_up_proj_name: str, + ckpt_up_proj_name: str | None, num_experts: int, num_redundant_experts: int = 0, routed_experts_prefix: str = "routed_experts", @@ -1067,13 +1075,14 @@ class RoutedExperts(PluggableLayer): def build_expert_params_mapping( ckpt_gate_proj_name: str, ckpt_down_proj_name: str, - ckpt_up_proj_name: str, + ckpt_up_proj_name: str | None, num_experts: int, num_redundant_experts: int = 0, routed_experts_prefix: str = "routed_experts", lora_base_layer_prefix: str = "", lora_base_layer_prefix_on_param_name: str = "", include_fused: bool = False, + is_gated: bool = True, ) -> list[tuple[str, str, int, str]]: """Create expert parameter mapping for weight loading with redundant experts. @@ -1095,6 +1104,8 @@ class RoutedExperts(PluggableLayer): ``make_expert_params_mapping`` indexes the model-wide ``params_dict`` (prefix included). include_fused: Prepend the fused pre-fused-checkpoint entries + is_gated: Whether w13 holds gate and up (gated activation) or a + single up projection. Selects the fused checkpoint layout. routed_experts_prefix: Prefix of the routed experts submodule Returns: @@ -1129,7 +1140,13 @@ class RoutedExperts(PluggableLayer): fused_mapping = [] if include_fused: gate_up = None - if ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj": + w13_shards: tuple[str, ...] = ("w1", "w3") + if not is_gated: + # Non-gated: the stacked checkpoint tensor is the up projection + # itself, nothing to split into gate and up. + gate_up = ckpt_gate_proj_name + w13_shards = ("w1",) + elif ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj": gate_up = "gate_up_proj" elif ckpt_gate_proj_name == "w1" and ckpt_up_proj_name == "w3": gate_up = "w13" @@ -1143,10 +1160,16 @@ class RoutedExperts(PluggableLayer): if gate_up is not None: fused_mapping = [ # (param_name, weight_name, expert_id, shard_id) - (f"{w13}weight", f"experts.{gate_up}", 0, "w1"), - (f"{w13}weight", f"experts.{gate_up}", 1, "w3"), + # expert_id doubles as the chunk index when splitting w13. + *( + (f"{w13}weight", f"experts.{gate_up}", chunk, shard_id) + for chunk, shard_id in enumerate(w13_shards) + ), (f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"), ] + if gate_up is not None and is_gated: + # Per-expert gate_up_proj tensors; the non-gated equivalent is + # already covered by the per-expert mapping below. fused_mapping.extend( ( w13, @@ -1175,6 +1198,7 @@ class RoutedExperts(PluggableLayer): ("w2", ckpt_down_proj_name), ("w3", ckpt_up_proj_name), ] + if weight_name is not None ] return fused_mapping + per_expert_mapping diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index bef3da5cc5..e477c65302 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -235,7 +235,7 @@ class NemotronHMoE(nn.Module): intermediate_size=config.moe_intermediate_size, renormalize=config.norm_topk_prob, quant_config=quant_config, - ckpt_names=("up_proj", "down_proj", ""), + ckpt_names=("up_proj", "down_proj", None), use_grouped_topk=True, num_expert_group=config.n_group, topk_group=config.topk_group, @@ -707,7 +707,7 @@ class NemotronHModel(nn.Module, EagleModelMixin): self, ckpt_gate_proj_name="up_proj", ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="", + ckpt_up_proj_name=None, num_experts=self._get_max_n_routed_experts(), num_redundant_experts=getattr(self, "num_redundant_experts", 0), ) diff --git a/vllm/model_executor/models/nemotron_h_mtp.py b/vllm/model_executor/models/nemotron_h_mtp.py index 6b9de63009..519f85a631 100644 --- a/vllm/model_executor/models/nemotron_h_mtp.py +++ b/vllm/model_executor/models/nemotron_h_mtp.py @@ -444,7 +444,7 @@ class NemotronHMTP(nn.Module, SupportsPP, SupportsQuant): self, ckpt_gate_proj_name="up_proj", ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="", # Empty - non-gated MoE + ckpt_up_proj_name=None, # non-gated MoE num_experts=num_experts, num_redundant_experts=self.num_redundant_experts, )