[ROCm][DSv4.1][Perf] Reuse the decode topk ragged metadata across layers (#57434)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
Fangzhou Ai
2026-09-20 11:46:33 +08:00
committed by GitHub
co-authored by Shanshan Shen
parent 03f8301387
commit 1596fa5f88
+66 -13
View File
@@ -13,7 +13,10 @@ from vllm.distributed import (
)
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.attention import (
DeepseekV4Attention,
_replace_layer_index,
)
from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4FlashMLAMetadata,
@@ -391,6 +394,10 @@ def _copy_ragged_to_graph_buffers(
return ragged_out, indptr_out
# (ragged_indices, ragged_indptr, lens) as consumed by rocm_sparse_attn_decode.
_TopkRagged = tuple[torch.Tensor, torch.Tensor, torch.Tensor]
@dataclass
class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
decode_swa_ragged_indices: torch.Tensor | None = None
@@ -500,6 +507,23 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
self._fused_compressor_weight: torch.Tensor | None
self.register_buffer("_fused_compressor_weight", None, persistent=False)
self._fused_compressor_split_sizes: tuple[int, int] | None = None
# Decode ragged topk metadata is a pure function of the indices its
# index source published, so every consumer of a source rebuilds the
# same thing. The source owns one cache per compress ratio, refreshed
# when it runs; consumers below it read through.
self._topk_ragged_cache: dict[int, _TopkRagged] = {}
self._index_source_prefix: str | None = None
if self.compress_ratio > 0:
assert self.index_source_layer_id is not None
self._index_source_prefix = _replace_layer_index(
self.prefix, self.index_source_layer_id
)
if self._index_source_prefix not in self._static_forward_context:
raise NotImplementedError(
f"Index source {self._index_source_prefix} not found on "
"this rank; PP splits inside a v4.1 index-sharing group "
"are not supported."
)
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
@@ -733,6 +757,42 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
output=output[:num_decode_tokens],
)
def _decode_topk_ragged(
self,
swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata,
attn_metadata: DeepseekV4FlashMLAMetadata | None,
num_decodes: int,
num_decode_tokens: int,
) -> _TopkRagged:
"""Ragged form of the topk indices this layer's index source published.
The packing depends only on the shared ``topk_indices_buffer`` and
per-step metadata, apart from the layer's own compress ratio, so the
source memoizes one result per ratio for all the layers below it.
"""
assert attn_metadata is not None
assert swa_metadata.is_valid_token is not None
assert self.topk_indices_buffer is not None
assert self._index_source_prefix is not None
source = self._static_forward_context[self._index_source_prefix]
if source is self:
# Fresh indices as of this layer; drop what the last step cached.
source._topk_ragged_cache = {}
cached = source._topk_ragged_cache.get(self.compress_ratio)
if cached is not None:
return cached
built = compute_global_topk_ragged_indices_and_indptr(
self.topk_indices_buffer[:num_decode_tokens],
swa_metadata.token_to_req_indices,
attn_metadata.block_table[:num_decodes],
attn_metadata.block_size // self.compress_ratio,
swa_metadata.is_valid_token[:num_decode_tokens],
)
source._topk_ragged_cache[self.compress_ratio] = built
return built
def _forward_decode(
self,
q: torch.Tensor,
@@ -749,22 +809,15 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
topk_ragged_indices = None
topk_ragged_indptr = None
if not swa_only:
# Local indices filled by the index-source layer's indexer.
assert attn_metadata is not None
assert swa_metadata.is_valid_token is not None
assert self.topk_indices_buffer is not None
block_size = attn_metadata.block_size // self.compress_ratio
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
(
topk_ragged_indices,
topk_ragged_indptr,
topk_lens,
) = compute_global_topk_ragged_indices_and_indptr(
self.topk_indices_buffer[:num_decode_tokens],
swa_metadata.token_to_req_indices,
attn_metadata.block_table[:num_decodes],
block_size,
is_valid,
) = self._decode_topk_ragged(
swa_metadata=swa_metadata,
attn_metadata=attn_metadata,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
)
rocm_sparse_attn_decode(