[Bugfix] Don't transpose fused MoE quantization scales in RoutedExperts.load_weights (#50137)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-31 08:46:52 -07:00
committed by GitHub
co-authored by Claude Opus 5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent df71917cf1
commit 94e9ef0768
2 changed files with 77 additions and 8 deletions
@@ -61,6 +61,55 @@ class TestGetHiddenDim:
RoutedExperts._get_hidden_dim(shard_dim=0, ndim=3)
class TestOrientFusedWeight:
"""Unit tests for _orient_fused_weight.
E=8 experts, hidden=3072, intermediate=1024.
"""
HIDDEN = 3072
def test_w13_standard_orientation_is_untouched(self):
weight = torch.randn(8, 2048, self.HIDDEN)
result = RoutedExperts._orient_fused_weight(weight, "w1", self.HIDDEN)
assert result.shape == (8, 2048, self.HIDDEN)
def test_w13_transposed_checkpoint_is_normalised(self):
# e.g. Qwen3 VL MoE stores [experts, hidden, 2 * intermediate]
weight = torch.randn(8, self.HIDDEN, 2048)
result = RoutedExperts._orient_fused_weight(weight, "w3", self.HIDDEN)
assert result.shape == (8, 2048, self.HIDDEN)
def test_w2_standard_orientation_is_untouched(self):
weight = torch.randn(8, self.HIDDEN, 1024)
result = RoutedExperts._orient_fused_weight(weight, "w2", self.HIDDEN)
assert result.shape == (8, self.HIDDEN, 1024)
def test_w2_transposed_checkpoint_is_normalised(self):
weight = torch.randn(8, 1024, self.HIDDEN)
result = RoutedExperts._orient_fused_weight(weight, "w2", self.HIDDEN)
assert result.shape == (8, self.HIDDEN, 1024)
def test_w13_per_channel_scale_is_untouched(self):
# A fused per-channel scale has no hidden dim, so transposing it would
# leave chunk()/TP sharding operating on the wrong axis.
scale = torch.randn(8, 2048, 1)
result = RoutedExperts._orient_fused_weight(scale, "w1", self.HIDDEN)
assert result.shape == (8, 2048, 1)
assert result.chunk(2, dim=1)[0].shape == (8, 1024, 1)
def test_w2_per_channel_scale_is_untouched(self):
scale = torch.randn(8, self.HIDDEN, 1)
result = RoutedExperts._orient_fused_weight(scale, "w2", self.HIDDEN)
assert result.shape == (8, self.HIDDEN, 1)
def test_block_scale_is_untouched(self):
# Block scales are [experts, 2 * intermediate / block, hidden / block]
scale = torch.randn(8, 16, 24)
result = RoutedExperts._orient_fused_weight(scale, "w1", self.HIDDEN)
assert result.shape == (8, 16, 24)
class TestNarrowExpertDataForPadding:
"""Unit tests for _narrow_expert_data_for_padding."""