[ROCm][Perf] Skip redundant sparse index remap on non-indexer layers (#51309)
Signed-off-by: Sriram Kumar <sriramkumar.kishorekumar@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Douglas Lehr
Andreas Karatzas
Simon Danielsson
parent
2ce8012fa0
commit
0d2689e0df
@@ -401,6 +401,18 @@ class ROCMAiterMLASparseBackend(AttentionBackend):
|
||||
return on_mi3xx()
|
||||
|
||||
|
||||
# Indexer writes per shared top-k buffer, in layer construction order.
|
||||
_INDEX_EPOCHS: dict[int, int] = {}
|
||||
|
||||
|
||||
def _index_epoch(buf: object, owns_indexer: bool) -> int:
|
||||
"""Number of indexer writes up to and including this layer."""
|
||||
# id(): tensors are unhashable, and the buffer outlives every reader.
|
||||
epoch = _INDEX_EPOCHS.get(id(buf), 0) + (1 if owns_indexer else 0)
|
||||
_INDEX_EPOCHS[id(buf)] = epoch
|
||||
return epoch
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class ROCMAiterMLASparseMetadata(SparseMLACommonMetadata):
|
||||
qo_indptr: torch.Tensor | None = None
|
||||
@@ -409,6 +421,10 @@ class ROCMAiterMLASparseMetadata(SparseMLACommonMetadata):
|
||||
paged_kv_indptr: torch.Tensor | None = None
|
||||
attn_out_dtype: torch.dtype | None = None
|
||||
|
||||
# Source buffer and index_epoch of paged_kv_indices; -1 before any remap.
|
||||
remapped_buf: object = None
|
||||
remapped_epoch: int = -1
|
||||
|
||||
# Persistent MLA metadata (only populated when persistent mode is enabled,
|
||||
# i.e. when the aiter sparse decode kernel supports work-stealing splits).
|
||||
work_meta_data: torch.Tensor | None = None
|
||||
@@ -818,6 +834,10 @@ class ROCMAiterMLASparseImpl(SparseMLACommonImpl[ROCMAiterMLASparseMetadata]):
|
||||
self.masked_mha_available = False
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.owns_indexer: bool = indexer is not None
|
||||
self.index_epoch: int = _index_epoch(
|
||||
self.topk_indices_buffer, self.owns_indexer
|
||||
)
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
q_concat_shape = (max_tokens, num_heads, head_size)
|
||||
(self.q_concat_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
@@ -1162,25 +1182,35 @@ class ROCMAiterMLASparseImpl(SparseMLACommonImpl[ROCMAiterMLASparseMetadata]):
|
||||
|
||||
num_actual_toks = q.shape[0]
|
||||
|
||||
# Get topk indices
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = fit_kpool_indices_to_aiter(
|
||||
self.topk_indices_buffer[:num_actual_toks], attn_metadata.topk_tokens
|
||||
)
|
||||
|
||||
kv_c_and_k_pe_cache, block_stride_rows = flat_kv_row_view(
|
||||
kv_c_and_k_pe_cache, attn_metadata.block_size
|
||||
)
|
||||
triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token[:num_actual_toks],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
attn_metadata.paged_kv_indptr,
|
||||
attn_metadata.paged_kv_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=attn_metadata.topk_tokens,
|
||||
)
|
||||
|
||||
# Indexer layers may rewrite their buffer, so always remap. Non-indexer
|
||||
# layers have their own KV cache group: remap once per indexer write.
|
||||
if (
|
||||
self.owns_indexer
|
||||
or attn_metadata.remapped_epoch != self.index_epoch
|
||||
or attn_metadata.remapped_buf is not self.topk_indices_buffer
|
||||
):
|
||||
# Get topk indices
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = fit_kpool_indices_to_aiter(
|
||||
self.topk_indices_buffer[:num_actual_toks], attn_metadata.topk_tokens
|
||||
)
|
||||
|
||||
triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token[:num_actual_toks],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
attn_metadata.paged_kv_indptr,
|
||||
attn_metadata.paged_kv_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
BLOCK_STRIDE_ROWS=block_stride_rows,
|
||||
NUM_TOPK_TOKENS=attn_metadata.topk_tokens,
|
||||
)
|
||||
attn_metadata.remapped_buf = self.topk_indices_buffer
|
||||
attn_metadata.remapped_epoch = self.index_epoch
|
||||
|
||||
# write the latent and rope to kv cache
|
||||
if fp8_attention:
|
||||
|
||||
Reference in New Issue
Block a user