diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index ce87d0b1b3..80e4064ea6 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -886,17 +886,21 @@ class RoutedExperts(PluggableLayer): param_name = weight_name.removeprefix(f"{self.layer_name}.") param = getattr(self, param_name) if is_fused: + # w1 and w3 share one fused tensor; use a local copy so the + # transpose below doesn't mutate loaded_weight across + # iterations (else w3 is transposed twice and wrongly chunked) + fused_weight = loaded_weight if shard_id in {"w1", "w3"}: - if loaded_weight.shape[-1] != unpadded_hidden: + if fused_weight.shape[-1] != unpadded_hidden: # [..., hidden, intermediate] -> [..., intermediate, hidden] - loaded_weight = loaded_weight.transpose(-1, -2) + fused_weight = fused_weight.transpose(-1, -2) # Repurpose expert_id for deconcatenating w1 and w3 - experts_shard = loaded_weight.chunk(2, dim=1)[expert_id] + experts_shard = fused_weight.chunk(2, dim=1)[expert_id] else: - if loaded_weight.shape[-2] != unpadded_hidden: + if fused_weight.shape[-2] != unpadded_hidden: # [..., intermediate, hidden] -> [..., hidden, intermediate] - loaded_weight = loaded_weight.transpose(-1, -2) - experts_shard = loaded_weight + fused_weight = fused_weight.transpose(-1, -2) + experts_shard = fused_weight start = 0 else: # loaded_weight is a single expert weight, so we add a dummy expert