[ROCm][Perf] DSv3.2: fuse MLA Q concat+fp8-quant in forward_mqa (#43673)
Signed-off-by: Frida Andersson <fanderss@amd.com>
This commit is contained in:
@@ -713,11 +713,16 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata])
|
||||
# NOTE(lucas): for the sparse FlashMLA kernels the kernels want to use
|
||||
# MQA 576/512 approach for both prefill and decode
|
||||
|
||||
# Concatenate q if it's a tuple (ql_nope, q_pe)
|
||||
fp8_attention = self.kv_cache_dtype.startswith("fp8")
|
||||
if isinstance(q, tuple):
|
||||
ql_nope, q_pe = q
|
||||
q = self.q_concat_buffer[: ql_nope.shape[0]]
|
||||
ops.concat_mla_q(ql_nope, q_pe, q)
|
||||
if fp8_attention:
|
||||
q = layer._decode_concat_quant_fp8_op( # type: ignore[attr-defined]
|
||||
ql_nope, q_pe, layer._q_scale
|
||||
)
|
||||
else:
|
||||
q = self.q_concat_buffer[: ql_nope.shape[0]]
|
||||
ops.concat_mla_q(ql_nope, q_pe, q)
|
||||
|
||||
num_actual_toks = attn_metadata.num_actual_tokens
|
||||
|
||||
@@ -736,12 +741,12 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata])
|
||||
)
|
||||
|
||||
# write the latent and rope to kv cache
|
||||
fp8_attention = self.kv_cache_dtype.startswith("fp8")
|
||||
if fp8_attention:
|
||||
original_q_shape = q.shape
|
||||
kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(current_platform.fp8_dtype())
|
||||
q, _ = ops.scaled_fp8_quant(q.view(q.shape[0], -1), layer._q_scale)
|
||||
q = q.view(original_q_shape)
|
||||
if q.dtype != current_platform.fp8_dtype():
|
||||
original_q_shape = q.shape
|
||||
q, _ = ops.scaled_fp8_quant(q.view(q.shape[0], -1), layer._q_scale)
|
||||
q = q.view(original_q_shape)
|
||||
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self.num_heads, q)
|
||||
attn_out = self._forward_mla(
|
||||
layer, mla_padded_q, kv_c_and_k_pe_cache, attn_metadata
|
||||
|
||||
Reference in New Issue
Block a user