[ROCm][DSv4.1][Perf] Reuse the decode topk ragged metadata across layers (#57434)
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Shanshan Shen
parent
03f8301387
commit
1596fa5f88
@@ -13,7 +13,10 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v41.attention import (
|
||||
DeepseekV4Attention,
|
||||
_replace_layer_index,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache
|
||||
from vllm.models.deepseek_v41.sparse_mla import (
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
@@ -391,6 +394,10 @@ def _copy_ragged_to_graph_buffers(
|
||||
return ragged_out, indptr_out
|
||||
|
||||
|
||||
# (ragged_indices, ragged_indptr, lens) as consumed by rocm_sparse_attn_decode.
|
||||
_TopkRagged = tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
|
||||
decode_swa_ragged_indices: torch.Tensor | None = None
|
||||
@@ -500,6 +507,23 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
self._fused_compressor_weight: torch.Tensor | None
|
||||
self.register_buffer("_fused_compressor_weight", None, persistent=False)
|
||||
self._fused_compressor_split_sizes: tuple[int, int] | None = None
|
||||
# Decode ragged topk metadata is a pure function of the indices its
|
||||
# index source published, so every consumer of a source rebuilds the
|
||||
# same thing. The source owns one cache per compress ratio, refreshed
|
||||
# when it runs; consumers below it read through.
|
||||
self._topk_ragged_cache: dict[int, _TopkRagged] = {}
|
||||
self._index_source_prefix: str | None = None
|
||||
if self.compress_ratio > 0:
|
||||
assert self.index_source_layer_id is not None
|
||||
self._index_source_prefix = _replace_layer_index(
|
||||
self.prefix, self.index_source_layer_id
|
||||
)
|
||||
if self._index_source_prefix not in self._static_forward_context:
|
||||
raise NotImplementedError(
|
||||
f"Index source {self._index_source_prefix} not found on "
|
||||
"this rank; PP splits inside a v4.1 index-sharing group "
|
||||
"are not supported."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
@@ -733,6 +757,42 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
output=output[:num_decode_tokens],
|
||||
)
|
||||
|
||||
def _decode_topk_ragged(
|
||||
self,
|
||||
swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata,
|
||||
attn_metadata: DeepseekV4FlashMLAMetadata | None,
|
||||
num_decodes: int,
|
||||
num_decode_tokens: int,
|
||||
) -> _TopkRagged:
|
||||
"""Ragged form of the topk indices this layer's index source published.
|
||||
|
||||
The packing depends only on the shared ``topk_indices_buffer`` and
|
||||
per-step metadata, apart from the layer's own compress ratio, so the
|
||||
source memoizes one result per ratio for all the layers below it.
|
||||
"""
|
||||
assert attn_metadata is not None
|
||||
assert swa_metadata.is_valid_token is not None
|
||||
assert self.topk_indices_buffer is not None
|
||||
assert self._index_source_prefix is not None
|
||||
|
||||
source = self._static_forward_context[self._index_source_prefix]
|
||||
if source is self:
|
||||
# Fresh indices as of this layer; drop what the last step cached.
|
||||
source._topk_ragged_cache = {}
|
||||
cached = source._topk_ragged_cache.get(self.compress_ratio)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
built = compute_global_topk_ragged_indices_and_indptr(
|
||||
self.topk_indices_buffer[:num_decode_tokens],
|
||||
swa_metadata.token_to_req_indices,
|
||||
attn_metadata.block_table[:num_decodes],
|
||||
attn_metadata.block_size // self.compress_ratio,
|
||||
swa_metadata.is_valid_token[:num_decode_tokens],
|
||||
)
|
||||
source._topk_ragged_cache[self.compress_ratio] = built
|
||||
return built
|
||||
|
||||
def _forward_decode(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -749,22 +809,15 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
topk_ragged_indices = None
|
||||
topk_ragged_indptr = None
|
||||
if not swa_only:
|
||||
# Local indices filled by the index-source layer's indexer.
|
||||
assert attn_metadata is not None
|
||||
assert swa_metadata.is_valid_token is not None
|
||||
assert self.topk_indices_buffer is not None
|
||||
block_size = attn_metadata.block_size // self.compress_ratio
|
||||
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
|
||||
(
|
||||
topk_ragged_indices,
|
||||
topk_ragged_indptr,
|
||||
topk_lens,
|
||||
) = compute_global_topk_ragged_indices_and_indptr(
|
||||
self.topk_indices_buffer[:num_decode_tokens],
|
||||
swa_metadata.token_to_req_indices,
|
||||
attn_metadata.block_table[:num_decodes],
|
||||
block_size,
|
||||
is_valid,
|
||||
) = self._decode_topk_ragged(
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=attn_metadata,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
)
|
||||
|
||||
rocm_sparse_attn_decode(
|
||||
|
||||
Reference in New Issue
Block a user