[Bugfix][ROCm] Keep MLA derived weights in place on weight reload (#60389)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1272,23 +1272,34 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
quark_quantize_weight_to_mxfp4,
|
||||
)
|
||||
|
||||
self.W_K, self.W_K_scale = quark_quantize_weight_to_mxfp4(W_UK)
|
||||
# Convert from (L, N, P) to (N, L, P)
|
||||
self.W_K = self.W_K.transpose(0, 1)
|
||||
self.W_K_scale = self.W_K_scale.transpose(0, 1)
|
||||
|
||||
self.W_V, self.W_V_scale = quark_quantize_weight_to_mxfp4(
|
||||
W_UV.permute(1, 2, 0)
|
||||
)
|
||||
W_K, W_K_scale = quark_quantize_weight_to_mxfp4(W_UK)
|
||||
W_V, W_V_scale = quark_quantize_weight_to_mxfp4(W_UV.permute(1, 2, 0))
|
||||
# Copy in place on reload so captured CUDA graphs stay valid.
|
||||
# Convert W_K from (L, N, P) to (N, L, P).
|
||||
for name, tensor in (
|
||||
("W_K", W_K.transpose(0, 1)),
|
||||
("W_K_scale", W_K_scale.transpose(0, 1)),
|
||||
("W_V", W_V),
|
||||
("W_V_scale", W_V_scale),
|
||||
):
|
||||
replace_parameter(self, name, tensor, prefer_copy=True)
|
||||
elif self.is_aiter_triton_fp8_bmm_enabled:
|
||||
W_K = W_UK.transpose(0, 1) # 16 512 128
|
||||
W_V = W_UV.permute(1, 2, 0) # 16 128 512
|
||||
self.W_K, self.W_K_scale = dynamic_per_batched_tensor_quant(
|
||||
W_K, W_K_scale = dynamic_per_batched_tensor_quant(
|
||||
W_K, dtype=current_platform.fp8_dtype()
|
||||
)
|
||||
self.W_V, self.W_V_scale = dynamic_per_batched_tensor_quant(
|
||||
W_V, W_V_scale = dynamic_per_batched_tensor_quant(
|
||||
W_V, dtype=current_platform.fp8_dtype()
|
||||
)
|
||||
# Copy in place on reload so captured CUDA graphs stay valid.
|
||||
for name, tensor in (
|
||||
("W_K", W_K),
|
||||
("W_K_scale", W_K_scale),
|
||||
("W_V", W_V),
|
||||
("W_V_scale", W_V_scale),
|
||||
):
|
||||
replace_parameter(self, name, tensor, prefer_copy=True)
|
||||
|
||||
# The kernel operates on non-padded inputs. Hence, pre-compiling
|
||||
# triton kernel to avoid runtime compilation for unseen batch sizes
|
||||
@@ -1327,8 +1338,11 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# Convert from (L, N, P) to (N, P, L)
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
if self.dcp_q_replicate:
|
||||
self.W_UK_T_dcp_qrep = get_dcp_group().all_gather(
|
||||
self.W_UK_T.contiguous(), dim=0
|
||||
replace_parameter(
|
||||
self,
|
||||
"W_UK_T_dcp_qrep",
|
||||
get_dcp_group().all_gather(self.W_UK_T.contiguous(), dim=0),
|
||||
prefer_copy=True,
|
||||
)
|
||||
|
||||
# If we should not load quant weights, we initialize the scales to 1.0
|
||||
|
||||
Reference in New Issue
Block a user