[Bugfix][ROCm] Keep MLA derived weights in place on weight reload (#60389)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt
2026-10-07 20:38:06 -05:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 8733a21d55
commit 9dba339d07
2 changed files with 68 additions and 13 deletions
@@ -1272,23 +1272,34 @@ class MLAAttention(nn.Module, AttentionLayerBase):
quark_quantize_weight_to_mxfp4,
)
self.W_K, self.W_K_scale = quark_quantize_weight_to_mxfp4(W_UK)
# Convert from (L, N, P) to (N, L, P)
self.W_K = self.W_K.transpose(0, 1)
self.W_K_scale = self.W_K_scale.transpose(0, 1)
self.W_V, self.W_V_scale = quark_quantize_weight_to_mxfp4(
W_UV.permute(1, 2, 0)
)
W_K, W_K_scale = quark_quantize_weight_to_mxfp4(W_UK)
W_V, W_V_scale = quark_quantize_weight_to_mxfp4(W_UV.permute(1, 2, 0))
# Copy in place on reload so captured CUDA graphs stay valid.
# Convert W_K from (L, N, P) to (N, L, P).
for name, tensor in (
("W_K", W_K.transpose(0, 1)),
("W_K_scale", W_K_scale.transpose(0, 1)),
("W_V", W_V),
("W_V_scale", W_V_scale),
):
replace_parameter(self, name, tensor, prefer_copy=True)
elif self.is_aiter_triton_fp8_bmm_enabled:
W_K = W_UK.transpose(0, 1) # 16 512 128
W_V = W_UV.permute(1, 2, 0) # 16 128 512
self.W_K, self.W_K_scale = dynamic_per_batched_tensor_quant(
W_K, W_K_scale = dynamic_per_batched_tensor_quant(
W_K, dtype=current_platform.fp8_dtype()
)
self.W_V, self.W_V_scale = dynamic_per_batched_tensor_quant(
W_V, W_V_scale = dynamic_per_batched_tensor_quant(
W_V, dtype=current_platform.fp8_dtype()
)
# Copy in place on reload so captured CUDA graphs stay valid.
for name, tensor in (
("W_K", W_K),
("W_K_scale", W_K_scale),
("W_V", W_V),
("W_V_scale", W_V_scale),
):
replace_parameter(self, name, tensor, prefer_copy=True)
# The kernel operates on non-padded inputs. Hence, pre-compiling
# triton kernel to avoid runtime compilation for unseen batch sizes
@@ -1327,8 +1338,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
# Convert from (L, N, P) to (N, P, L)
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
if self.dcp_q_replicate:
self.W_UK_T_dcp_qrep = get_dcp_group().all_gather(
self.W_UK_T.contiguous(), dim=0
replace_parameter(
self,
"W_UK_T_dcp_qrep",
get_dcp_group().all_gather(self.W_UK_T.contiguous(), dim=0),
prefer_copy=True,
)
# If we should not load quant weights, we initialize the scales to 1.0