[ROCm][CI][MoE] Fix double-transpose of fused w3 expert weights (#47874)
Signed-off-by: stefankoncarevic <stefan.koncarevic@amd.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Andreas Karatzas
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7802c20c4e
commit
bc44f9feb7
@@ -886,17 +886,21 @@ class RoutedExperts(PluggableLayer):
|
||||
param_name = weight_name.removeprefix(f"{self.layer_name}.")
|
||||
param = getattr(self, param_name)
|
||||
if is_fused:
|
||||
# w1 and w3 share one fused tensor; use a local copy so the
|
||||
# transpose below doesn't mutate loaded_weight across
|
||||
# iterations (else w3 is transposed twice and wrongly chunked)
|
||||
fused_weight = loaded_weight
|
||||
if shard_id in {"w1", "w3"}:
|
||||
if loaded_weight.shape[-1] != unpadded_hidden:
|
||||
if fused_weight.shape[-1] != unpadded_hidden:
|
||||
# [..., hidden, intermediate] -> [..., intermediate, hidden]
|
||||
loaded_weight = loaded_weight.transpose(-1, -2)
|
||||
fused_weight = fused_weight.transpose(-1, -2)
|
||||
# Repurpose expert_id for deconcatenating w1 and w3
|
||||
experts_shard = loaded_weight.chunk(2, dim=1)[expert_id]
|
||||
experts_shard = fused_weight.chunk(2, dim=1)[expert_id]
|
||||
else:
|
||||
if loaded_weight.shape[-2] != unpadded_hidden:
|
||||
if fused_weight.shape[-2] != unpadded_hidden:
|
||||
# [..., intermediate, hidden] -> [..., hidden, intermediate]
|
||||
loaded_weight = loaded_weight.transpose(-1, -2)
|
||||
experts_shard = loaded_weight
|
||||
fused_weight = fused_weight.transpose(-1, -2)
|
||||
experts_shard = fused_weight
|
||||
start = 0
|
||||
else:
|
||||
# loaded_weight is a single expert weight, so we add a dummy expert
|
||||
|
||||
Reference in New Issue
Block a user