diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index 4d838e121e..b7a67e85bd 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -13,7 +13,10 @@ from vllm.distributed import ( ) from vllm.forward_context import get_forward_context from vllm.logger import init_logger -from vllm.models.deepseek_v41.attention import DeepseekV4Attention +from vllm.models.deepseek_v41.attention import ( + DeepseekV4Attention, + _replace_layer_index, +) from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4FlashMLAMetadata, @@ -391,6 +394,10 @@ def _copy_ragged_to_graph_buffers( return ragged_out, indptr_out +# (ragged_indices, ragged_indptr, lens) as consumed by rocm_sparse_attn_decode. +_TopkRagged = tuple[torch.Tensor, torch.Tensor, torch.Tensor] + + @dataclass class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata): decode_swa_ragged_indices: torch.Tensor | None = None @@ -500,6 +507,23 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): self._fused_compressor_weight: torch.Tensor | None self.register_buffer("_fused_compressor_weight", None, persistent=False) self._fused_compressor_split_sizes: tuple[int, int] | None = None + # Decode ragged topk metadata is a pure function of the indices its + # index source published, so every consumer of a source rebuilds the + # same thing. The source owns one cache per compress ratio, refreshed + # when it runs; consumers below it read through. + self._topk_ragged_cache: dict[int, _TopkRagged] = {} + self._index_source_prefix: str | None = None + if self.compress_ratio > 0: + assert self.index_source_layer_id is not None + self._index_source_prefix = _replace_layer_index( + self.prefix, self.index_source_layer_id + ) + if self._index_source_prefix not in self._static_forward_context: + raise NotImplementedError( + f"Index source {self._index_source_prefix} not found on " + "this rank; PP splits inside a v4.1 index-sharing group " + "are not supported." + ) @classmethod def get_padded_num_q_heads(cls, num_heads: int) -> int: @@ -733,6 +757,42 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): output=output[:num_decode_tokens], ) + def _decode_topk_ragged( + self, + swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, + attn_metadata: DeepseekV4FlashMLAMetadata | None, + num_decodes: int, + num_decode_tokens: int, + ) -> _TopkRagged: + """Ragged form of the topk indices this layer's index source published. + + The packing depends only on the shared ``topk_indices_buffer`` and + per-step metadata, apart from the layer's own compress ratio, so the + source memoizes one result per ratio for all the layers below it. + """ + assert attn_metadata is not None + assert swa_metadata.is_valid_token is not None + assert self.topk_indices_buffer is not None + assert self._index_source_prefix is not None + + source = self._static_forward_context[self._index_source_prefix] + if source is self: + # Fresh indices as of this layer; drop what the last step cached. + source._topk_ragged_cache = {} + cached = source._topk_ragged_cache.get(self.compress_ratio) + if cached is not None: + return cached + + built = compute_global_topk_ragged_indices_and_indptr( + self.topk_indices_buffer[:num_decode_tokens], + swa_metadata.token_to_req_indices, + attn_metadata.block_table[:num_decodes], + attn_metadata.block_size // self.compress_ratio, + swa_metadata.is_valid_token[:num_decode_tokens], + ) + source._topk_ragged_cache[self.compress_ratio] = built + return built + def _forward_decode( self, q: torch.Tensor, @@ -749,22 +809,15 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): topk_ragged_indices = None topk_ragged_indptr = None if not swa_only: - # Local indices filled by the index-source layer's indexer. - assert attn_metadata is not None - assert swa_metadata.is_valid_token is not None - assert self.topk_indices_buffer is not None - block_size = attn_metadata.block_size // self.compress_ratio - is_valid = swa_metadata.is_valid_token[:num_decode_tokens] ( topk_ragged_indices, topk_ragged_indptr, topk_lens, - ) = compute_global_topk_ragged_indices_and_indptr( - self.topk_indices_buffer[:num_decode_tokens], - swa_metadata.token_to_req_indices, - attn_metadata.block_table[:num_decodes], - block_size, - is_valid, + ) = self._decode_topk_ragged( + swa_metadata=swa_metadata, + attn_metadata=attn_metadata, + num_decodes=num_decodes, + num_decode_tokens=num_decode_tokens, ) rocm_sparse_attn_decode(