[ROCm][CI][MoE] Fix double-transpose of fused w3 expert weights (#47874)

Signed-off-by: stefankoncarevic <stefan.koncarevic@amd.com>
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
stefankoncarevic
2026-07-08 16:59:28 -07:00
committed by GitHub
co-authored by Andreas Karatzas mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7802c20c4e
commit bc44f9feb7
@@ -886,17 +886,21 @@ class RoutedExperts(PluggableLayer):
param_name = weight_name.removeprefix(f"{self.layer_name}.")
param = getattr(self, param_name)
if is_fused:
# w1 and w3 share one fused tensor; use a local copy so the
# transpose below doesn't mutate loaded_weight across
# iterations (else w3 is transposed twice and wrongly chunked)
fused_weight = loaded_weight
if shard_id in {"w1", "w3"}:
if loaded_weight.shape[-1] != unpadded_hidden:
if fused_weight.shape[-1] != unpadded_hidden:
# [..., hidden, intermediate] -> [..., intermediate, hidden]
loaded_weight = loaded_weight.transpose(-1, -2)
fused_weight = fused_weight.transpose(-1, -2)
# Repurpose expert_id for deconcatenating w1 and w3
experts_shard = loaded_weight.chunk(2, dim=1)[expert_id]
experts_shard = fused_weight.chunk(2, dim=1)[expert_id]
else:
if loaded_weight.shape[-2] != unpadded_hidden:
if fused_weight.shape[-2] != unpadded_hidden:
# [..., intermediate, hidden] -> [..., hidden, intermediate]
loaded_weight = loaded_weight.transpose(-1, -2)
experts_shard = loaded_weight
fused_weight = fused_weight.transpose(-1, -2)
experts_shard = fused_weight
start = 0
else:
# loaded_weight is a single expert weight, so we add a dummy expert