enable persistent mla for sparse mla backend (#41990)
Signed-off-by: ganyi <ygan@amd.com> Signed-off-by: Douglas Lehr <Doug.Lehr@amd.com> Co-authored-by: ganyi <ygan@amd.com>
This commit is contained in:
@@ -420,9 +420,6 @@ class AiterMLAHelper:
|
||||
|
||||
@staticmethod
|
||||
def get_mla_padded_q(num_heads: int, q: torch.Tensor) -> torch.Tensor:
|
||||
assert num_heads not in AiterMLAHelper._AITER_UNSUPPORTED_HEADS, (
|
||||
f"unsupported head_num: {num_heads}"
|
||||
)
|
||||
return (
|
||||
q
|
||||
if num_heads >= AiterMLAHelper._AITER_MIN_MLA_HEADS
|
||||
|
||||
@@ -332,6 +332,15 @@ class ROCMAiterMLASparseMetadata(AttentionMetadata):
|
||||
block_size: int = 1
|
||||
topk_tokens: int = 2048
|
||||
|
||||
# Persistent MLA metadata (only populated when persistent mode is enabled,
|
||||
# i.e. when the aiter sparse decode kernel supports work-stealing splits).
|
||||
work_meta_data: torch.Tensor | None = None
|
||||
work_indptr: torch.Tensor | None = None
|
||||
work_info_set: torch.Tensor | None = None
|
||||
reduce_indptr: torch.Tensor | None = None
|
||||
reduce_final_map: torch.Tensor | None = None
|
||||
reduce_partial_map: torch.Tensor | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ROCMAiterMLASparseMetadataBuilder(
|
||||
@@ -389,6 +398,66 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
[max_num_batched_tokens + 1], dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
# ----- Persistent MLA metadata buffers -----
|
||||
# The aiter sparse decode kernel supports a "persistent" path that
|
||||
# uses precomputed work-splitting metadata for better load balancing
|
||||
# across CUs. Mirrors the approach used in rocm_aiter_mla.py.
|
||||
#
|
||||
# In the sparse case each query token is its own "batch" entry in the
|
||||
# qo_indptr (qo_indptr = [0, 1, 2, ..., num_tokens]) and max_qo_len=1.
|
||||
# We pad get_mla_metadata_info_v1's batch_size to max_num_batched_tokens
|
||||
# so the buffers are large enough for any decode shape we might see.
|
||||
from aiter import dtypes, get_mla_metadata_info_v1
|
||||
|
||||
# Aiter sparse MLA also requires num_heads >= 16 (will be padded by
|
||||
# AiterMLAHelper.get_mla_padded_q in forward).
|
||||
self._num_attention_heads = max(16, self.num_heads)
|
||||
|
||||
q_dtype = self.model_dtype
|
||||
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
|
||||
if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
|
||||
kv_cache_dtype_str = "fp8"
|
||||
else:
|
||||
kv_cache_dtype_str = "bf16"
|
||||
kv_dtype = dtypes.d_dtypes.get(kv_cache_dtype_str, dtypes.bf16)
|
||||
|
||||
(
|
||||
(work_meta_data_size, work_meta_data_type),
|
||||
(work_indptr_size, work_indptr_type),
|
||||
(work_info_set_size, work_info_set_type),
|
||||
(reduce_indptr_size, reduce_indptr_type),
|
||||
(reduce_final_map_size, reduce_final_map_type),
|
||||
(reduce_partial_map_size, reduce_partial_map_type),
|
||||
) = get_mla_metadata_info_v1(
|
||||
max_num_batched_tokens,
|
||||
1,
|
||||
self._num_attention_heads,
|
||||
q_dtype,
|
||||
kv_dtype,
|
||||
is_sparse=True,
|
||||
fast_mode=True,
|
||||
)
|
||||
self._mla_work_meta_data = torch.empty(
|
||||
work_meta_data_size, dtype=work_meta_data_type, device=device
|
||||
)
|
||||
self._mla_work_indptr = torch.empty(
|
||||
work_indptr_size, dtype=work_indptr_type, device=device
|
||||
)
|
||||
self._mla_work_info_set = torch.empty(
|
||||
work_info_set_size, dtype=work_info_set_type, device=device
|
||||
)
|
||||
self._mla_reduce_indptr = torch.empty(
|
||||
reduce_indptr_size, dtype=reduce_indptr_type, device=device
|
||||
)
|
||||
self._mla_reduce_final_map = torch.empty(
|
||||
reduce_final_map_size, dtype=reduce_final_map_type, device=device
|
||||
)
|
||||
self._mla_reduce_partial_map = torch.empty(
|
||||
reduce_partial_map_size,
|
||||
dtype=reduce_partial_map_type,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -431,6 +500,33 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
paged_kv_indptr = self.paged_kv_indptr[: num_tokens + 1]
|
||||
paged_kv_indices = self.paged_kv_indices[: num_tokens * self.topk_tokens]
|
||||
|
||||
# ----- Compute persistent MLA metadata -----
|
||||
# The aiter sparse decode kernel uses qseqlen=1 (each query token is
|
||||
# treated as its own batch entry), so persistent metadata can always
|
||||
# be precomputed here. The kernel switches to the persistent
|
||||
# work-stealing path automatically when work_meta_data is non-None.
|
||||
from aiter import get_mla_metadata_v1
|
||||
|
||||
get_mla_metadata_v1(
|
||||
qo_indptr,
|
||||
paged_kv_indptr,
|
||||
paged_kv_last_page_len,
|
||||
self._num_attention_heads,
|
||||
1,
|
||||
True,
|
||||
self._mla_work_meta_data,
|
||||
self._mla_work_info_set,
|
||||
self._mla_work_indptr,
|
||||
self._mla_reduce_indptr,
|
||||
self._mla_reduce_final_map,
|
||||
self._mla_reduce_partial_map,
|
||||
page_size=1,
|
||||
kv_granularity=16,
|
||||
max_seqlen_qo=1,
|
||||
uni_seqlen_qo=1,
|
||||
fast_mode=True,
|
||||
)
|
||||
|
||||
metadata = ROCMAiterMLASparseMetadata(
|
||||
num_reqs=common_attn_metadata.num_reqs,
|
||||
max_query_len=common_attn_metadata.max_query_len,
|
||||
@@ -447,6 +543,12 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
paged_kv_last_page_len=paged_kv_last_page_len,
|
||||
paged_kv_indices=paged_kv_indices,
|
||||
paged_kv_indptr=paged_kv_indptr,
|
||||
work_meta_data=self._mla_work_meta_data,
|
||||
work_indptr=self._mla_work_indptr,
|
||||
work_info_set=self._mla_work_info_set,
|
||||
reduce_indptr=self._mla_reduce_indptr,
|
||||
reduce_final_map=self._mla_reduce_final_map,
|
||||
reduce_partial_map=self._mla_reduce_partial_map,
|
||||
)
|
||||
return metadata
|
||||
|
||||
@@ -525,6 +627,23 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata])
|
||||
device=q.device,
|
||||
)
|
||||
|
||||
# Build kwargs and forward the persistent MLA metadata when it has
|
||||
# been computed. The aiter mla_decode_fwd switches to its
|
||||
# work-stealing persistent kernel path when work_meta_data is given.
|
||||
mla_kwargs: dict = dict(
|
||||
q_scale=layer._q_scale,
|
||||
kv_scale=layer._k_scale,
|
||||
)
|
||||
if attn_metadata.work_meta_data is not None:
|
||||
mla_kwargs.update(
|
||||
work_meta_data=attn_metadata.work_meta_data,
|
||||
work_indptr=attn_metadata.work_indptr,
|
||||
work_info_set=attn_metadata.work_info_set,
|
||||
reduce_indptr=attn_metadata.reduce_indptr,
|
||||
reduce_final_map=attn_metadata.reduce_final_map,
|
||||
reduce_partial_map=attn_metadata.reduce_partial_map,
|
||||
)
|
||||
|
||||
rocm_aiter_ops.mla_decode_fwd(
|
||||
q,
|
||||
kv_c_and_k_pe_cache,
|
||||
@@ -535,8 +654,7 @@ class ROCMAiterMLASparseImpl(SparseMLAAttentionImpl[ROCMAiterMLASparseMetadata])
|
||||
attn_metadata.paged_kv_indptr,
|
||||
attn_metadata.paged_kv_indices,
|
||||
attn_metadata.paged_kv_last_page_len,
|
||||
q_scale=layer._q_scale,
|
||||
kv_scale=layer._k_scale,
|
||||
**mla_kwargs,
|
||||
)
|
||||
|
||||
return AiterMLAHelper.get_mla_unpadded_o(self.num_heads, output)
|
||||
|
||||
Reference in New Issue
Block a user