[Bugfix] Don't transpose fused MoE quantization scales in RoutedExperts.load_weights (#50137)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
df71917cf1
commit
94e9ef0768
@@ -61,6 +61,55 @@ class TestGetHiddenDim:
|
||||
RoutedExperts._get_hidden_dim(shard_dim=0, ndim=3)
|
||||
|
||||
|
||||
class TestOrientFusedWeight:
|
||||
"""Unit tests for _orient_fused_weight.
|
||||
|
||||
E=8 experts, hidden=3072, intermediate=1024.
|
||||
"""
|
||||
|
||||
HIDDEN = 3072
|
||||
|
||||
def test_w13_standard_orientation_is_untouched(self):
|
||||
weight = torch.randn(8, 2048, self.HIDDEN)
|
||||
result = RoutedExperts._orient_fused_weight(weight, "w1", self.HIDDEN)
|
||||
assert result.shape == (8, 2048, self.HIDDEN)
|
||||
|
||||
def test_w13_transposed_checkpoint_is_normalised(self):
|
||||
# e.g. Qwen3 VL MoE stores [experts, hidden, 2 * intermediate]
|
||||
weight = torch.randn(8, self.HIDDEN, 2048)
|
||||
result = RoutedExperts._orient_fused_weight(weight, "w3", self.HIDDEN)
|
||||
assert result.shape == (8, 2048, self.HIDDEN)
|
||||
|
||||
def test_w2_standard_orientation_is_untouched(self):
|
||||
weight = torch.randn(8, self.HIDDEN, 1024)
|
||||
result = RoutedExperts._orient_fused_weight(weight, "w2", self.HIDDEN)
|
||||
assert result.shape == (8, self.HIDDEN, 1024)
|
||||
|
||||
def test_w2_transposed_checkpoint_is_normalised(self):
|
||||
weight = torch.randn(8, 1024, self.HIDDEN)
|
||||
result = RoutedExperts._orient_fused_weight(weight, "w2", self.HIDDEN)
|
||||
assert result.shape == (8, self.HIDDEN, 1024)
|
||||
|
||||
def test_w13_per_channel_scale_is_untouched(self):
|
||||
# A fused per-channel scale has no hidden dim, so transposing it would
|
||||
# leave chunk()/TP sharding operating on the wrong axis.
|
||||
scale = torch.randn(8, 2048, 1)
|
||||
result = RoutedExperts._orient_fused_weight(scale, "w1", self.HIDDEN)
|
||||
assert result.shape == (8, 2048, 1)
|
||||
assert result.chunk(2, dim=1)[0].shape == (8, 1024, 1)
|
||||
|
||||
def test_w2_per_channel_scale_is_untouched(self):
|
||||
scale = torch.randn(8, self.HIDDEN, 1)
|
||||
result = RoutedExperts._orient_fused_weight(scale, "w2", self.HIDDEN)
|
||||
assert result.shape == (8, self.HIDDEN, 1)
|
||||
|
||||
def test_block_scale_is_untouched(self):
|
||||
# Block scales are [experts, 2 * intermediate / block, hidden / block]
|
||||
scale = torch.randn(8, 16, 24)
|
||||
result = RoutedExperts._orient_fused_weight(scale, "w1", self.HIDDEN)
|
||||
assert result.shape == (8, 16, 24)
|
||||
|
||||
|
||||
class TestNarrowExpertDataForPadding:
|
||||
"""Unit tests for _narrow_expert_data_for_padding."""
|
||||
|
||||
|
||||
@@ -422,6 +422,28 @@ class RoutedExperts(PluggableLayer):
|
||||
f"for a {ndim}D tensor (expected {dim_a} or {dim_b})"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _orient_fused_weight(
|
||||
fused_weight: torch.Tensor, shard_id: str, unpadded_hidden: int
|
||||
) -> torch.Tensor:
|
||||
"""Normalise a fused expert tensor from either checkpoint orientation
|
||||
to (intermediate, hidden) for w1/w3 and (hidden, intermediate) for w2.
|
||||
|
||||
Only transposes when the hidden dim is definitively on the wrong axis,
|
||||
leaving tensors that have no hidden dim alone (e.g. a fused per-channel
|
||||
scale, (2 * intermediate, 1)).
|
||||
"""
|
||||
if shard_id == "w2":
|
||||
hidden_axis, intermediate_axis = -2, -1
|
||||
else:
|
||||
hidden_axis, intermediate_axis = -1, -2
|
||||
if (
|
||||
fused_weight.shape[hidden_axis] != unpadded_hidden
|
||||
and fused_weight.shape[intermediate_axis] == unpadded_hidden
|
||||
):
|
||||
return fused_weight.transpose(-1, -2)
|
||||
return fused_weight
|
||||
|
||||
@staticmethod
|
||||
def _narrow_expert_data_for_padding(
|
||||
expert_data: torch.Tensor,
|
||||
@@ -900,7 +922,9 @@ class RoutedExperts(PluggableLayer):
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
expert_mapping = self.get_expert_mapping(include_fused=True)
|
||||
unpadded_hidden = self.moe_config.hidden_dim_unpadded
|
||||
unpadded_hidden = (
|
||||
self.moe_config.hidden_dim_unpadded or self.moe_config.hidden_dim
|
||||
)
|
||||
for expert_name, loaded_weight in weights:
|
||||
qual_name = f"{self.layer_name}.{expert_name}"
|
||||
# Fused expert weights can be identified by their 3D tensors
|
||||
@@ -919,17 +943,13 @@ class RoutedExperts(PluggableLayer):
|
||||
# w1 and w3 share one fused tensor; use a local copy so the
|
||||
# transpose below doesn't mutate loaded_weight across
|
||||
# iterations (else w3 is transposed twice and wrongly chunked)
|
||||
fused_weight = loaded_weight
|
||||
fused_weight = self._orient_fused_weight(
|
||||
loaded_weight, shard_id, unpadded_hidden
|
||||
)
|
||||
if shard_id in {"w1", "w3"}:
|
||||
if fused_weight.shape[-1] != unpadded_hidden:
|
||||
# [..., hidden, intermediate] -> [..., intermediate, hidden]
|
||||
fused_weight = fused_weight.transpose(-1, -2)
|
||||
# Repurpose expert_id for deconcatenating w1 and w3
|
||||
experts_shard = fused_weight.chunk(2, dim=1)[expert_id]
|
||||
else:
|
||||
if fused_weight.shape[-2] != unpadded_hidden:
|
||||
# [..., intermediate, hidden] -> [..., hidden, intermediate]
|
||||
fused_weight = fused_weight.transpose(-1, -2)
|
||||
experts_shard = fused_weight
|
||||
start = 0
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user