From 94e9ef0768d0297299069502bf46813dc7479ce5 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:46:52 +0100 Subject: [PATCH] [Bugfix] Don't transpose fused MoE quantization scales in `RoutedExperts.load_weights` (#50137) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../moe/test_moe_weight_loading_padded.py | 49 +++++++++++++++++++ .../layers/fused_moe/routed_experts.py | 36 +++++++++++--- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/tests/kernels/moe/test_moe_weight_loading_padded.py b/tests/kernels/moe/test_moe_weight_loading_padded.py index b7dad4da5a..bb377049d9 100644 --- a/tests/kernels/moe/test_moe_weight_loading_padded.py +++ b/tests/kernels/moe/test_moe_weight_loading_padded.py @@ -61,6 +61,55 @@ class TestGetHiddenDim: RoutedExperts._get_hidden_dim(shard_dim=0, ndim=3) +class TestOrientFusedWeight: + """Unit tests for _orient_fused_weight. + + E=8 experts, hidden=3072, intermediate=1024. + """ + + HIDDEN = 3072 + + def test_w13_standard_orientation_is_untouched(self): + weight = torch.randn(8, 2048, self.HIDDEN) + result = RoutedExperts._orient_fused_weight(weight, "w1", self.HIDDEN) + assert result.shape == (8, 2048, self.HIDDEN) + + def test_w13_transposed_checkpoint_is_normalised(self): + # e.g. Qwen3 VL MoE stores [experts, hidden, 2 * intermediate] + weight = torch.randn(8, self.HIDDEN, 2048) + result = RoutedExperts._orient_fused_weight(weight, "w3", self.HIDDEN) + assert result.shape == (8, 2048, self.HIDDEN) + + def test_w2_standard_orientation_is_untouched(self): + weight = torch.randn(8, self.HIDDEN, 1024) + result = RoutedExperts._orient_fused_weight(weight, "w2", self.HIDDEN) + assert result.shape == (8, self.HIDDEN, 1024) + + def test_w2_transposed_checkpoint_is_normalised(self): + weight = torch.randn(8, 1024, self.HIDDEN) + result = RoutedExperts._orient_fused_weight(weight, "w2", self.HIDDEN) + assert result.shape == (8, self.HIDDEN, 1024) + + def test_w13_per_channel_scale_is_untouched(self): + # A fused per-channel scale has no hidden dim, so transposing it would + # leave chunk()/TP sharding operating on the wrong axis. + scale = torch.randn(8, 2048, 1) + result = RoutedExperts._orient_fused_weight(scale, "w1", self.HIDDEN) + assert result.shape == (8, 2048, 1) + assert result.chunk(2, dim=1)[0].shape == (8, 1024, 1) + + def test_w2_per_channel_scale_is_untouched(self): + scale = torch.randn(8, self.HIDDEN, 1) + result = RoutedExperts._orient_fused_weight(scale, "w2", self.HIDDEN) + assert result.shape == (8, self.HIDDEN, 1) + + def test_block_scale_is_untouched(self): + # Block scales are [experts, 2 * intermediate / block, hidden / block] + scale = torch.randn(8, 16, 24) + result = RoutedExperts._orient_fused_weight(scale, "w1", self.HIDDEN) + assert result.shape == (8, 16, 24) + + class TestNarrowExpertDataForPadding: """Unit tests for _narrow_expert_data_for_padding.""" diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 6f46283829..f9fbe80b2c 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -422,6 +422,28 @@ class RoutedExperts(PluggableLayer): f"for a {ndim}D tensor (expected {dim_a} or {dim_b})" ) + @staticmethod + def _orient_fused_weight( + fused_weight: torch.Tensor, shard_id: str, unpadded_hidden: int + ) -> torch.Tensor: + """Normalise a fused expert tensor from either checkpoint orientation + to (intermediate, hidden) for w1/w3 and (hidden, intermediate) for w2. + + Only transposes when the hidden dim is definitively on the wrong axis, + leaving tensors that have no hidden dim alone (e.g. a fused per-channel + scale, (2 * intermediate, 1)). + """ + if shard_id == "w2": + hidden_axis, intermediate_axis = -2, -1 + else: + hidden_axis, intermediate_axis = -1, -2 + if ( + fused_weight.shape[hidden_axis] != unpadded_hidden + and fused_weight.shape[intermediate_axis] == unpadded_hidden + ): + return fused_weight.transpose(-1, -2) + return fused_weight + @staticmethod def _narrow_expert_data_for_padding( expert_data: torch.Tensor, @@ -900,7 +922,9 @@ class RoutedExperts(PluggableLayer): self, weights: Iterable[tuple[str, torch.Tensor]] ) -> Iterable[str]: expert_mapping = self.get_expert_mapping(include_fused=True) - unpadded_hidden = self.moe_config.hidden_dim_unpadded + unpadded_hidden = ( + self.moe_config.hidden_dim_unpadded or self.moe_config.hidden_dim + ) for expert_name, loaded_weight in weights: qual_name = f"{self.layer_name}.{expert_name}" # Fused expert weights can be identified by their 3D tensors @@ -919,17 +943,13 @@ class RoutedExperts(PluggableLayer): # w1 and w3 share one fused tensor; use a local copy so the # transpose below doesn't mutate loaded_weight across # iterations (else w3 is transposed twice and wrongly chunked) - fused_weight = loaded_weight + fused_weight = self._orient_fused_weight( + loaded_weight, shard_id, unpadded_hidden + ) if shard_id in {"w1", "w3"}: - if fused_weight.shape[-1] != unpadded_hidden: - # [..., hidden, intermediate] -> [..., intermediate, hidden] - fused_weight = fused_weight.transpose(-1, -2) # Repurpose expert_id for deconcatenating w1 and w3 experts_shard = fused_weight.chunk(2, dim=1)[expert_id] else: - if fused_weight.shape[-2] != unpadded_hidden: - # [..., intermediate, hidden] -> [..., hidden, intermediate] - fused_weight = fused_weight.transpose(-1, -2) experts_shard = fused_weight start = 0 else: