From baf068d8be09b22db636ae8c6a2082033e2b6911 Mon Sep 17 00:00:00 2001 From: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Date: Thu, 7 May 2026 22:10:50 -0500 Subject: [PATCH] enable persistent mla for sparse mla backend (#41990) Signed-off-by: ganyi Signed-off-by: Douglas Lehr Co-authored-by: ganyi --- .../attention/backends/mla/rocm_aiter_mla.py | 3 - .../backends/mla/rocm_aiter_mla_sparse.py | 122 +++++++++++++++++- 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index 121194b144..f16fd5dc14 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -420,9 +420,6 @@ class AiterMLAHelper: @staticmethod def get_mla_padded_q(num_heads: int, q: torch.Tensor) -> torch.Tensor: - assert num_heads not in AiterMLAHelper._AITER_UNSUPPORTED_HEADS, ( - f"unsupported head_num: {num_heads}" - ) return ( q if num_heads >= AiterMLAHelper._AITER_MIN_MLA_HEADS diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index dc343b639f..dd25e721d3 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -332,6 +332,15 @@ class ROCMAiterMLASparseMetadata(AttentionMetadata): block_size: int = 1 topk_tokens: int = 2048 + # Persistent MLA metadata (only populated when persistent mode is enabled, + # i.e. when the aiter sparse decode kernel supports work-stealing splits). + work_meta_data: torch.Tensor | None = None + work_indptr: torch.Tensor | None = None + work_info_set: torch.Tensor | None = None + reduce_indptr: torch.Tensor | None = None + reduce_final_map: torch.Tensor | None = None + reduce_partial_map: torch.Tensor | None = None + @dataclass class ROCMAiterMLASparseMetadataBuilder( @@ -389,6 +398,66 @@ class ROCMAiterMLASparseMetadataBuilder( [max_num_batched_tokens + 1], dtype=torch.int32, device=device ) + # ----- Persistent MLA metadata buffers ----- + # The aiter sparse decode kernel supports a "persistent" path that + # uses precomputed work-splitting metadata for better load balancing + # across CUs. Mirrors the approach used in rocm_aiter_mla.py. + # + # In the sparse case each query token is its own "batch" entry in the + # qo_indptr (qo_indptr = [0, 1, 2, ..., num_tokens]) and max_qo_len=1. + # We pad get_mla_metadata_info_v1's batch_size to max_num_batched_tokens + # so the buffers are large enough for any decode shape we might see. + from aiter import dtypes, get_mla_metadata_info_v1 + + # Aiter sparse MLA also requires num_heads >= 16 (will be padded by + # AiterMLAHelper.get_mla_padded_q in forward). + self._num_attention_heads = max(16, self.num_heads) + + q_dtype = self.model_dtype + kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto") + if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"): + kv_cache_dtype_str = "fp8" + else: + kv_cache_dtype_str = "bf16" + kv_dtype = dtypes.d_dtypes.get(kv_cache_dtype_str, dtypes.bf16) + + ( + (work_meta_data_size, work_meta_data_type), + (work_indptr_size, work_indptr_type), + (work_info_set_size, work_info_set_type), + (reduce_indptr_size, reduce_indptr_type), + (reduce_final_map_size, reduce_final_map_type), + (reduce_partial_map_size, reduce_partial_map_type), + ) = get_mla_metadata_info_v1( + max_num_batched_tokens, + 1, + self._num_attention_heads, + q_dtype, + kv_dtype, + is_sparse=True, + fast_mode=True, + ) + self._mla_work_meta_data = torch.empty( + work_meta_data_size, dtype=work_meta_data_type, device=device + ) + self._mla_work_indptr = torch.empty( + work_indptr_size, dtype=work_indptr_type, device=device + ) + self._mla_work_info_set = torch.empty( + work_info_set_size, dtype=work_info_set_type, device=device + ) + self._mla_reduce_indptr = torch.empty( + reduce_indptr_size, dtype=reduce_indptr_type, device=device + ) + self._mla_reduce_final_map = torch.empty( + reduce_final_map_size, dtype=reduce_final_map_type, device=device + ) + self._mla_reduce_partial_map = torch.empty( + reduce_partial_map_size, + dtype=reduce_partial_map_type, + device=device, + ) + def build( self, common_prefix_len: int, @@ -431,6 +500,33 @@ class ROCMAiterMLASparseMetadataBuilder( paged_kv_indptr = self.paged_kv_indptr[: num_tokens + 1] paged_kv_indices = self.paged_kv_indices[: num_tokens * self.topk_tokens] + # ----- Compute persistent MLA metadata ----- + # The aiter sparse decode kernel uses qseqlen=1 (each query token is + # treated as its own batch entry), so persistent metadata can always + # be precomputed here. The kernel switches to the persistent + # work-stealing path automatically when work_meta_data is non-None. + from aiter import get_mla_metadata_v1 + + get_mla_metadata_v1( + qo_indptr, + paged_kv_indptr, + paged_kv_last_page_len, + self._num_attention_heads, + 1, + True, + self._mla_work_meta_data, + self._mla_work_info_set, + self._mla_work_indptr, + self._mla_reduce_indptr, + self._mla_reduce_final_map, + self._mla_reduce_partial_map, + page_size=1, + kv_granularity=16, + max_seqlen_qo=1, + uni_seqlen_qo=1, + fast_mode=True, + ) + metadata = ROCMAiterMLASparseMetadata( num_reqs=common_attn_metadata.num_reqs, max_query_len=common_attn_metadata.max_query_len, @@ -447,6 +543,12 @@ class ROCMAiterMLASparseMetadataBuilder( paged_kv_last_page_len=paged_kv_last_page_len, paged_kv_indices=paged_kv_indices, paged_kv_indptr=paged_kv_indptr, + work_meta_data=self._mla_work_meta_data, + work_indptr=self._mla_work_indptr, + work_info_set=self._mla_work_info_set, + reduce_indptr=self._mla_reduce_indptr, + reduce_final_map=self._mla_reduce_final_map, + reduce_partial_map=self._mla_reduce_partial_map, ) return metadata @@ -525,6 +627,23 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata]) device=q.device, ) + # Build kwargs and forward the persistent MLA metadata when it has + # been computed. The aiter mla_decode_fwd switches to its + # work-stealing persistent kernel path when work_meta_data is given. + mla_kwargs: dict = dict( + q_scale=layer._q_scale, + kv_scale=layer._k_scale, + ) + if attn_metadata.work_meta_data is not None: + mla_kwargs.update( + work_meta_data=attn_metadata.work_meta_data, + work_indptr=attn_metadata.work_indptr, + work_info_set=attn_metadata.work_info_set, + reduce_indptr=attn_metadata.reduce_indptr, + reduce_final_map=attn_metadata.reduce_final_map, + reduce_partial_map=attn_metadata.reduce_partial_map, + ) + rocm_aiter_ops.mla_decode_fwd( q, kv_c_and_k_pe_cache, @@ -535,8 +654,7 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata]) attn_metadata.paged_kv_indptr, attn_metadata.paged_kv_indices, attn_metadata.paged_kv_last_page_len, - q_scale=layer._q_scale, - kv_scale=layer._k_scale, + **mla_kwargs, ) return AiterMLAHelper.get_mla_unpadded_o(self.num_heads, output)