[Bugfix] Don't transpose fused MoE quantization scales in RoutedExperts.load_weights (#50137)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-31 08:46:52 -07:00
committed by GitHub
co-authored by Claude Opus 5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent df71917cf1
commit 94e9ef0768
2 changed files with 77 additions and 8 deletions
@@ -61,6 +61,55 @@ class TestGetHiddenDim:
RoutedExperts._get_hidden_dim(shard_dim=0, ndim=3)
class TestOrientFusedWeight:
"""Unit tests for _orient_fused_weight.
E=8 experts, hidden=3072, intermediate=1024.
"""
HIDDEN = 3072
def test_w13_standard_orientation_is_untouched(self):
weight = torch.randn(8, 2048, self.HIDDEN)
result = RoutedExperts._orient_fused_weight(weight, "w1", self.HIDDEN)
assert result.shape == (8, 2048, self.HIDDEN)
def test_w13_transposed_checkpoint_is_normalised(self):
# e.g. Qwen3 VL MoE stores [experts, hidden, 2 * intermediate]
weight = torch.randn(8, self.HIDDEN, 2048)
result = RoutedExperts._orient_fused_weight(weight, "w3", self.HIDDEN)
assert result.shape == (8, 2048, self.HIDDEN)
def test_w2_standard_orientation_is_untouched(self):
weight = torch.randn(8, self.HIDDEN, 1024)
result = RoutedExperts._orient_fused_weight(weight, "w2", self.HIDDEN)
assert result.shape == (8, self.HIDDEN, 1024)
def test_w2_transposed_checkpoint_is_normalised(self):
weight = torch.randn(8, 1024, self.HIDDEN)
result = RoutedExperts._orient_fused_weight(weight, "w2", self.HIDDEN)
assert result.shape == (8, self.HIDDEN, 1024)
def test_w13_per_channel_scale_is_untouched(self):
# A fused per-channel scale has no hidden dim, so transposing it would
# leave chunk()/TP sharding operating on the wrong axis.
scale = torch.randn(8, 2048, 1)
result = RoutedExperts._orient_fused_weight(scale, "w1", self.HIDDEN)
assert result.shape == (8, 2048, 1)
assert result.chunk(2, dim=1)[0].shape == (8, 1024, 1)
def test_w2_per_channel_scale_is_untouched(self):
scale = torch.randn(8, self.HIDDEN, 1)
result = RoutedExperts._orient_fused_weight(scale, "w2", self.HIDDEN)
assert result.shape == (8, self.HIDDEN, 1)
def test_block_scale_is_untouched(self):
# Block scales are [experts, 2 * intermediate / block, hidden / block]
scale = torch.randn(8, 16, 24)
result = RoutedExperts._orient_fused_weight(scale, "w1", self.HIDDEN)
assert result.shape == (8, 16, 24)
class TestNarrowExpertDataForPadding:
"""Unit tests for _narrow_expert_data_for_padding."""
@@ -422,6 +422,28 @@ class RoutedExperts(PluggableLayer):
f"for a {ndim}D tensor (expected {dim_a} or {dim_b})"
)
@staticmethod
def _orient_fused_weight(
fused_weight: torch.Tensor, shard_id: str, unpadded_hidden: int
) -> torch.Tensor:
"""Normalise a fused expert tensor from either checkpoint orientation
to (intermediate, hidden) for w1/w3 and (hidden, intermediate) for w2.
Only transposes when the hidden dim is definitively on the wrong axis,
leaving tensors that have no hidden dim alone (e.g. a fused per-channel
scale, (2 * intermediate, 1)).
"""
if shard_id == "w2":
hidden_axis, intermediate_axis = -2, -1
else:
hidden_axis, intermediate_axis = -1, -2
if (
fused_weight.shape[hidden_axis] != unpadded_hidden
and fused_weight.shape[intermediate_axis] == unpadded_hidden
):
return fused_weight.transpose(-1, -2)
return fused_weight
@staticmethod
def _narrow_expert_data_for_padding(
expert_data: torch.Tensor,
@@ -900,7 +922,9 @@ class RoutedExperts(PluggableLayer):
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
expert_mapping = self.get_expert_mapping(include_fused=True)
unpadded_hidden = self.moe_config.hidden_dim_unpadded
unpadded_hidden = (
self.moe_config.hidden_dim_unpadded or self.moe_config.hidden_dim
)
for expert_name, loaded_weight in weights:
qual_name = f"{self.layer_name}.{expert_name}"
# Fused expert weights can be identified by their 3D tensors
@@ -919,17 +943,13 @@ class RoutedExperts(PluggableLayer):
# w1 and w3 share one fused tensor; use a local copy so the
# transpose below doesn't mutate loaded_weight across
# iterations (else w3 is transposed twice and wrongly chunked)
fused_weight = loaded_weight
fused_weight = self._orient_fused_weight(
loaded_weight, shard_id, unpadded_hidden
)
if shard_id in {"w1", "w3"}:
if fused_weight.shape[-1] != unpadded_hidden:
# [..., hidden, intermediate] -> [..., intermediate, hidden]
fused_weight = fused_weight.transpose(-1, -2)
# Repurpose expert_id for deconcatenating w1 and w3
experts_shard = fused_weight.chunk(2, dim=1)[expert_id]
else:
if fused_weight.shape[-2] != unpadded_hidden:
# [..., intermediate, hidden] -> [..., hidden, intermediate]
fused_weight = fused_weight.transpose(-1, -2)
experts_shard = fused_weight
start = 0
else: