diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index fa50456a61..718ab9126c 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -896,6 +896,14 @@ class RoutedExperts(PluggableLayer): break continue matched = True + is_per_expert_fused_w13 = ( + not is_fused + and shard_id in {"w1", "w3"} + and any( + f".{fused_name}." in qual_name + for fused_name in ("gate_up_proj", "w13") + ) + ) weight_name = qual_name.replace(weight_name, param_name) param_name = weight_name.removeprefix(f"{self.layer_name}.") param = getattr(self, param_name, None) @@ -919,6 +927,12 @@ class RoutedExperts(PluggableLayer): else: experts_shard = fused_weight start = 0 + elif is_per_expert_fused_w13: + shard_index = 0 if shard_id == "w1" else 1 + experts_shard = loaded_weight.chunk(2, dim=0)[ + shard_index + ].unsqueeze(0) + start = expert_id else: # loaded_weight is a single expert weight, so we add a dummy expert # dimension to unify the loading logic with the fused case @@ -1069,6 +1083,19 @@ class RoutedExperts(PluggableLayer): (f"{w13}weight", f"experts.{gate_up}", 1, "w3"), (f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"), ] + fused_mapping.extend( + ( + w13, + ( + f"experts.{physical_to_logical_map[expert_id]}." + f"{gate_up}.{lora_base_layer_prefix}" + ), + expert_id, + shard_id, + ) + for expert_id in range(num_physical_experts) + for shard_id in ("w1", "w3") + ) per_expert_mapping = [ # (param_name, weight_name, expert_id, shard_id) diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 26a396f23a..de659e4f50 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -569,6 +569,38 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): activation=self.moe.activation, ) + @staticmethod + def _encode_mxfp4_weight_scale(loaded_weight: torch.Tensor) -> torch.Tensor: + if loaded_weight.dtype == torch.uint8: + return loaded_weight + if loaded_weight.dtype == torch.float8_e8m0fnu: + return loaded_weight.view(torch.uint8) + if loaded_weight.is_floating_point(): + return loaded_weight.to(torch.float8_e8m0fnu).view(torch.uint8) + return loaded_weight + + @staticmethod + def get_scale_weight_loader(weight_loader): + def mxfp4_weight_loader( + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + return_success: bool = False, + ) -> bool | None: + loaded_weight = Mxfp4MoEMethod._encode_mxfp4_weight_scale(loaded_weight) + return weight_loader( + param, + loaded_weight, + weight_name, + shard_id, + expert_id, + return_success=return_success, + ) + + return mxfp4_weight_loader + def create_weights( self, layer: RoutedExperts, @@ -587,6 +619,8 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): layer.num_experts = num_experts self.intermediate_size = intermediate_size_per_partition self.hidden_size = hidden_size + weight_loader = extra_weight_attrs.pop("weight_loader") + scale_weight_loader = Mxfp4MoEMethod.get_scale_weight_loader(weight_loader) # Fused gate_up_proj (column parallel) w13_weight = torch.nn.Parameter( @@ -600,6 +634,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): ) layer.register_parameter("w13_weight", w13_weight) set_weight_attrs(w13_weight, extra_weight_attrs) + set_weight_attrs(w13_weight, {"weight_loader": weight_loader}) w13_weight_scale = torch.nn.Parameter( torch.zeros( @@ -612,6 +647,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): ) layer.register_parameter("w13_weight_scale", w13_weight_scale) set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w13_weight_scale, {"weight_loader": scale_weight_loader}) w13_weight_scale.quant_method = "block" # down_proj (row parallel) @@ -626,6 +662,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): ) layer.register_parameter("w2_weight", w2_weight) set_weight_attrs(w2_weight, extra_weight_attrs) + set_weight_attrs(w2_weight, {"weight_loader": weight_loader}) w2_weight_scale = torch.nn.Parameter( torch.zeros( @@ -638,6 +675,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): ) layer.register_parameter("w2_weight_scale", w2_weight_scale) set_weight_attrs(w2_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, {"weight_loader": scale_weight_loader}) w2_weight_scale.quant_method = "block" if self.moe.has_bias: @@ -651,6 +689,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): ) layer.register_parameter("w13_bias", w13_bias) set_weight_attrs(w13_bias, extra_weight_attrs) + set_weight_attrs(w13_bias, {"weight_loader": weight_loader}) w2_bias = torch.nn.Parameter( torch.zeros( @@ -662,6 +701,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): ) layer.register_parameter("w2_bias", w2_bias) set_weight_attrs(w2_bias, extra_weight_attrs) + set_weight_attrs(w2_bias, {"weight_loader": weight_loader}) def _setup_kernel( self,