[ROCm][Perf] Skip redundant sparse index remap on non-indexer layers (#51309)

Signed-off-by: Sriram Kumar <sriramkumar.kishorekumar@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
This commit is contained in:
Sriram Kumar
2026-10-13 05:25:56 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Douglas Lehr Andreas Karatzas Simon Danielsson
parent 2ce8012fa0
commit 0d2689e0df
5 changed files with 349 additions and 16 deletions
@@ -401,6 +401,18 @@ class ROCMAiterMLASparseBackend(AttentionBackend):
return on_mi3xx()
# Indexer writes per shared top-k buffer, in layer construction order.
_INDEX_EPOCHS: dict[int, int] = {}
def _index_epoch(buf: object, owns_indexer: bool) -> int:
"""Number of indexer writes up to and including this layer."""
# id(): tensors are unhashable, and the buffer outlives every reader.
epoch = _INDEX_EPOCHS.get(id(buf), 0) + (1 if owns_indexer else 0)
_INDEX_EPOCHS[id(buf)] = epoch
return epoch
@dataclass(kw_only=True)
class ROCMAiterMLASparseMetadata(SparseMLACommonMetadata):
qo_indptr: torch.Tensor | None = None
@@ -409,6 +421,10 @@ class ROCMAiterMLASparseMetadata(SparseMLACommonMetadata):
paged_kv_indptr: torch.Tensor | None = None
attn_out_dtype: torch.dtype | None = None
# Source buffer and index_epoch of paged_kv_indices; -1 before any remap.
remapped_buf: object = None
remapped_epoch: int = -1
# Persistent MLA metadata (only populated when persistent mode is enabled,
# i.e. when the aiter sparse decode kernel supports work-stealing splits).
work_meta_data: torch.Tensor | None = None
@@ -818,6 +834,10 @@ class ROCMAiterMLASparseImpl(SparseMLACommonImpl[ROCMAiterMLASparseMetadata]):
self.masked_mha_available = False
vllm_config = get_current_vllm_config()
self.owns_indexer: bool = indexer is not None
self.index_epoch: int = _index_epoch(
self.topk_indices_buffer, self.owns_indexer
)
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
q_concat_shape = (max_tokens, num_heads, head_size)
(self.q_concat_buffer,) = current_workspace_manager().get_simultaneous(
@@ -1162,25 +1182,35 @@ class ROCMAiterMLASparseImpl(SparseMLACommonImpl[ROCMAiterMLASparseMetadata]):
num_actual_toks = q.shape[0]
# Get topk indices
assert self.topk_indices_buffer is not None
topk_indices = fit_kpool_indices_to_aiter(
self.topk_indices_buffer[:num_actual_toks], attn_metadata.topk_tokens
)
kv_c_and_k_pe_cache, block_stride_rows = flat_kv_row_view(
kv_c_and_k_pe_cache, attn_metadata.block_size
)
triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token[:num_actual_toks],
attn_metadata.block_table,
topk_indices,
attn_metadata.paged_kv_indptr,
attn_metadata.paged_kv_indices,
BLOCK_SIZE=attn_metadata.block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=attn_metadata.topk_tokens,
)
# Indexer layers may rewrite their buffer, so always remap. Non-indexer
# layers have their own KV cache group: remap once per indexer write.
if (
self.owns_indexer
or attn_metadata.remapped_epoch != self.index_epoch
or attn_metadata.remapped_buf is not self.topk_indices_buffer
):
# Get topk indices
assert self.topk_indices_buffer is not None
topk_indices = fit_kpool_indices_to_aiter(
self.topk_indices_buffer[:num_actual_toks], attn_metadata.topk_tokens
)
triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token[:num_actual_toks],
attn_metadata.block_table,
topk_indices,
attn_metadata.paged_kv_indptr,
attn_metadata.paged_kv_indices,
BLOCK_SIZE=attn_metadata.block_size,
BLOCK_STRIDE_ROWS=block_stride_rows,
NUM_TOPK_TOKENS=attn_metadata.topk_tokens,
)
attn_metadata.remapped_buf = self.topk_indices_buffer
attn_metadata.remapped_epoch = self.index_epoch
# write the latent and rope to kv cache
if fp8_attention: