From 20b5af55c1c9ad0d1a00dc45771997dd490c8461 Mon Sep 17 00:00:00 2001 From: frida-andersson Date: Tue, 23 Jun 2026 12:12:04 +0200 Subject: [PATCH] [ROCm][Perf] DSv3.2: fuse MLA Q concat+fp8-quant in forward_mqa (#43673) Signed-off-by: Frida Andersson --- .../backends/mla/rocm_aiter_mla_sparse.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index 1225352ace..9982b7aacb 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -713,11 +713,16 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata]) # NOTE(lucas): for the sparse FlashMLA kernels the kernels want to use # MQA 576/512 approach for both prefill and decode - # Concatenate q if it's a tuple (ql_nope, q_pe) + fp8_attention = self.kv_cache_dtype.startswith("fp8") if isinstance(q, tuple): ql_nope, q_pe = q - q = self.q_concat_buffer[: ql_nope.shape[0]] - ops.concat_mla_q(ql_nope, q_pe, q) + if fp8_attention: + q = layer._decode_concat_quant_fp8_op( # type: ignore[attr-defined] + ql_nope, q_pe, layer._q_scale + ) + else: + q = self.q_concat_buffer[: ql_nope.shape[0]] + ops.concat_mla_q(ql_nope, q_pe, q) num_actual_toks = attn_metadata.num_actual_tokens @@ -736,12 +741,12 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata]) ) # write the latent and rope to kv cache - fp8_attention = self.kv_cache_dtype.startswith("fp8") if fp8_attention: - original_q_shape = q.shape kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view(current_platform.fp8_dtype()) - q, _ = ops.scaled_fp8_quant(q.view(q.shape[0], -1), layer._q_scale) - q = q.view(original_q_shape) + if q.dtype != current_platform.fp8_dtype(): + original_q_shape = q.shape + q, _ = ops.scaled_fp8_quant(q.view(q.shape[0], -1), layer._q_scale) + q = q.view(original_q_shape) mla_padded_q = AiterMLAHelper.get_mla_padded_q(self.num_heads, q) attn_out = self._forward_mla( layer, mla_padded_q, kv_c_and_k_pe_cache, attn_metadata