[Bugfix] Don't transpose fused MoE quantization scales in RoutedExperts.load_weights (#50137)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
df71917cf1
commit
94e9ef0768
@@ -61,6 +61,55 @@ class TestGetHiddenDim:
|
||||
RoutedExperts._get_hidden_dim(shard_dim=0, ndim=3)
|
||||
|
||||
|
||||
class TestOrientFusedWeight:
|
||||
"""Unit tests for _orient_fused_weight.
|
||||
|
||||
E=8 experts, hidden=3072, intermediate=1024.
|
||||
"""
|
||||
|
||||
HIDDEN = 3072
|
||||
|
||||
def test_w13_standard_orientation_is_untouched(self):
|
||||
weight = torch.randn(8, 2048, self.HIDDEN)
|
||||
result = RoutedExperts._orient_fused_weight(weight, "w1", self.HIDDEN)
|
||||
assert result.shape == (8, 2048, self.HIDDEN)
|
||||
|
||||
def test_w13_transposed_checkpoint_is_normalised(self):
|
||||
# e.g. Qwen3 VL MoE stores [experts, hidden, 2 * intermediate]
|
||||
weight = torch.randn(8, self.HIDDEN, 2048)
|
||||
result = RoutedExperts._orient_fused_weight(weight, "w3", self.HIDDEN)
|
||||
assert result.shape == (8, 2048, self.HIDDEN)
|
||||
|
||||
def test_w2_standard_orientation_is_untouched(self):
|
||||
weight = torch.randn(8, self.HIDDEN, 1024)
|
||||
result = RoutedExperts._orient_fused_weight(weight, "w2", self.HIDDEN)
|
||||
assert result.shape == (8, self.HIDDEN, 1024)
|
||||
|
||||
def test_w2_transposed_checkpoint_is_normalised(self):
|
||||
weight = torch.randn(8, 1024, self.HIDDEN)
|
||||
result = RoutedExperts._orient_fused_weight(weight, "w2", self.HIDDEN)
|
||||
assert result.shape == (8, self.HIDDEN, 1024)
|
||||
|
||||
def test_w13_per_channel_scale_is_untouched(self):
|
||||
# A fused per-channel scale has no hidden dim, so transposing it would
|
||||
# leave chunk()/TP sharding operating on the wrong axis.
|
||||
scale = torch.randn(8, 2048, 1)
|
||||
result = RoutedExperts._orient_fused_weight(scale, "w1", self.HIDDEN)
|
||||
assert result.shape == (8, 2048, 1)
|
||||
assert result.chunk(2, dim=1)[0].shape == (8, 1024, 1)
|
||||
|
||||
def test_w2_per_channel_scale_is_untouched(self):
|
||||
scale = torch.randn(8, self.HIDDEN, 1)
|
||||
result = RoutedExperts._orient_fused_weight(scale, "w2", self.HIDDEN)
|
||||
assert result.shape == (8, self.HIDDEN, 1)
|
||||
|
||||
def test_block_scale_is_untouched(self):
|
||||
# Block scales are [experts, 2 * intermediate / block, hidden / block]
|
||||
scale = torch.randn(8, 16, 24)
|
||||
result = RoutedExperts._orient_fused_weight(scale, "w1", self.HIDDEN)
|
||||
assert result.shape == (8, 16, 24)
|
||||
|
||||
|
||||
class TestNarrowExpertDataForPadding:
|
||||
"""Unit tests for _narrow_expert_data_for_padding."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user