diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py new file mode 100644 index 0000000000..aefcfeee4d --- /dev/null +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -0,0 +1,377 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm.platforms import current_platform + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), reason="Only used by ROCm" +) + +NOPE_HEAD_DIM = 448 +ROPE_HEAD_DIM = 64 +HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM + + +def _ref_global_topk_ragged( + topk_indices: torch.Tensor, + token_to_req_indices: torch.Tensor, + block_table: torch.Tensor, + block_size: int, + is_valid_token: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + topk = topk_indices.reshape(topk_indices.shape[0], -1) + valid = (topk >= 0) & is_valid_token[:, None] + lens = valid.sum(dim=1, dtype=torch.int32) + indptr = torch.zeros(lens.shape[0] + 1, dtype=torch.int32, device=topk.device) + torch.cumsum(lens, dim=0, out=indptr[1:]) + + safe_topk = torch.clamp(topk, min=0) + block_indices = safe_topk // block_size + block_offsets = safe_topk % block_size + req_indices = token_to_req_indices[:, None].expand_as(topk) + slot_ids = block_table[req_indices, block_indices] * block_size + block_offsets + + offsets = torch.arange(topk.shape[1], dtype=torch.int32, device=topk.device) + positions = indptr[:-1, None] + offsets[None, :] + return slot_ids[valid], positions[valid].to(torch.long), indptr, lens + + +def _ref_sparse_prefill_ragged( + q: torch.Tensor, + kv: torch.Tensor, + rows: list[list[int]], + scale: float, + attn_sink: torch.Tensor | None, +) -> torch.Tensor: + q_f32 = q.float() + kv_f32 = kv.float() + out = torch.empty_like(q_f32) + + for query_idx in range(q.shape[0]): + row_indices = rows[query_idx] + for head_idx in range(q.shape[1]): + if row_indices: + selected_kv = kv_f32[row_indices] + scores = torch.mv(selected_kv, q_f32[query_idx, head_idx]) * scale + if attn_sink is not None: + scores_with_sink = torch.cat( + [scores, attn_sink[head_idx].float().reshape(1)] + ) + probs = torch.softmax(scores_with_sink, dim=0)[:-1] + else: + probs = torch.softmax(scores, dim=0) + out[query_idx, head_idx] = torch.sum( + probs[:, None] * selected_kv, dim=0 + ) + else: + out[query_idx, head_idx] = 0 + return out.to(torch.bfloat16) + + +def _pack_fp8_ds_mla_cache(kv: torch.Tensor, block_size: int) -> torch.Tensor: + assert kv.shape[-1] == HEAD_DIM + num_tokens = kv.shape[0] + num_blocks = (num_tokens + block_size - 1) // block_size + cache = torch.zeros( + (num_blocks, block_size, 584), + dtype=torch.uint8, + device=kv.device, + ) + cache_flat = cache.view(torch.uint8).flatten() + kv_nope_fp8 = ( + kv[:, :NOPE_HEAD_DIM].to(current_platform.fp8_dtype()).view(torch.uint8) + ) + kv_rope_u8 = kv[:, NOPE_HEAD_DIM:].contiguous().view(torch.uint8) + + for slot in range(num_tokens): + block_idx = slot // block_size + pos = slot % block_size + block_base = block_idx * cache.stride(0) + token_base = block_base + pos * 576 + scale_base = block_base + block_size * 576 + pos * 8 + cache_flat[token_base : token_base + NOPE_HEAD_DIM].copy_(kv_nope_fp8[slot]) + cache_flat[ + token_base + NOPE_HEAD_DIM : token_base + NOPE_HEAD_DIM + ROPE_HEAD_DIM * 2 + ].copy_(kv_rope_u8[slot]) + cache_flat[scale_base : scale_base + 7].fill_(127) + return cache + + +def _read_fp8_ds_mla_cache( + cache: torch.Tensor, slot: int, block_size: int +) -> torch.Tensor: + cache_flat = cache.view(torch.uint8).flatten() + block_idx = slot // block_size + pos = slot % block_size + block_base = block_idx * cache.stride(0) + token_base = block_base + pos * 576 + + nope_u8 = cache_flat[token_base : token_base + NOPE_HEAD_DIM] + nope = nope_u8.view(current_platform.fp8_dtype()).to(torch.float32) + rope_u8 = cache_flat[ + token_base + NOPE_HEAD_DIM : token_base + NOPE_HEAD_DIM + ROPE_HEAD_DIM * 2 + ] + rope = rope_u8.view(torch.bfloat16).to(torch.float32) + return torch.cat([nope, rope]) + + +def _ref_sparse_decode_ragged( + q: torch.Tensor, + main_cache: torch.Tensor, + main_rows: list[list[int]], + scale: float, + attn_sink: torch.Tensor | None, + block_size: int, + extra_cache: torch.Tensor | None = None, + extra_rows: list[list[int]] | None = None, +) -> torch.Tensor: + q_f32 = q.float() + out = torch.empty_like(q_f32) + + for query_idx in range(q.shape[0]): + row_kv = [ + _read_fp8_ds_mla_cache(main_cache, int(slot), block_size) + for slot in main_rows[query_idx] + ] + if extra_cache is not None and extra_rows is not None: + row_kv.extend( + _read_fp8_ds_mla_cache(extra_cache, int(slot), block_size) + for slot in extra_rows[query_idx] + ) + + kv = torch.stack(row_kv).to(q.device) + for head_idx in range(q.shape[1]): + scores = torch.mv(kv, q_f32[query_idx, head_idx]) * scale + if attn_sink is not None: + scores_with_sink = torch.cat( + [scores, attn_sink[head_idx].float().reshape(1)] + ) + probs = torch.softmax(scores_with_sink, dim=0)[:-1] + else: + probs = torch.softmax(scores, dim=0) + out[query_idx, head_idx] = torch.sum(probs[:, None] * kv, dim=0) + return out.to(torch.bfloat16) + + +def _ref_combine_topk_swa_ragged( + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + expected_ragged = torch.tensor( + [ + 100, + 101, + 7, + 8, + 9, + 110, + 111, + 8, + 9, + 10, + 120, + 121, + 122, + 9, + 10, + 11, + 150, + 27, + 28, + 29, + 160, + 161, + 28, + 29, + 30, + ], + dtype=torch.int32, + device=device, + ) + expected_lens = torch.tensor([5, 5, 6, 4, 5], dtype=torch.int32, device=device) + expected_indptr = torch.zeros(6, dtype=torch.int32, device=device) + torch.cumsum(expected_lens, dim=0, out=expected_indptr[1:]) + return expected_ragged, expected_indptr, expected_lens + + +@torch.inference_mode() +def test_compute_global_topk_ragged_indices_and_indptr() -> None: + from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import ( + compute_global_topk_ragged_indices_and_indptr, + ) + + device = torch.device("cuda") + block_size = 4 + topk_indices = torch.tensor( + [ + [0, 3, 4, -1], + [5, 8, -1, -1], + [2, 7, 9, -1], + ], + dtype=torch.int32, + device=device, + ) + token_to_req_indices = torch.tensor([0, 1, 1], dtype=torch.int32, device=device) + block_table = torch.tensor( + [ + [10, 11, 12], + [20, 21, 22], + ], + dtype=torch.int32, + device=device, + ) + is_valid_token = torch.tensor([True, False, True], dtype=torch.bool, device=device) + + actual_ragged, actual_indptr, actual_lens = ( + compute_global_topk_ragged_indices_and_indptr( + topk_indices, + token_to_req_indices, + block_table, + block_size, + is_valid_token, + ) + ) + expected_values, expected_positions, expected_indptr, expected_lens = ( + _ref_global_topk_ragged( + topk_indices, + token_to_req_indices, + block_table, + block_size, + is_valid_token, + ) + ) + + torch.testing.assert_close(actual_ragged[expected_positions], expected_values) + torch.testing.assert_close(actual_indptr, expected_indptr) + torch.testing.assert_close(actual_lens, expected_lens) + + +@torch.inference_mode() +def test_sparse_attn_prefill_ragged_kernel() -> None: + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _rocm_sparse_attn_prefill_ragged_triton, + ) + + device = torch.device("cuda") + torch.manual_seed(0) + q = torch.randn(3, 3, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125 + kv = torch.randn(5, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125 + indices = torch.tensor([0, 2, 1, 3, 4], dtype=torch.int32, device=device) + indptr = torch.tensor([0, 2, 5, 5], dtype=torch.int32, device=device) + attn_sink = torch.tensor([-0.25, 0.0, 0.25], dtype=torch.float32, device=device) + scale = HEAD_DIM**-0.5 + + actual = _rocm_sparse_attn_prefill_ragged_triton( + q=q, + kv=kv, + indices=indices, + indptr=indptr, + scale=scale, + attn_sink=attn_sink, + nope_head_dim=NOPE_HEAD_DIM, + rope_head_dim=ROPE_HEAD_DIM, + ) + expected = _ref_sparse_prefill_ragged( + q, kv, [[0, 2], [1, 3, 4], []], scale, attn_sink + ) + + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) + + +@torch.inference_mode() +def test_sparse_attn_decode_ragged_kernel() -> None: + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _rocm_sparse_attn_decode_ragged_triton, + ) + + device = torch.device("cuda") + torch.manual_seed(1) + block_size = 4 + q = torch.randn(2, 3, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125 + main_kv = torch.randn(6, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125 + extra_kv = torch.randn(5, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125 + main_cache = _pack_fp8_ds_mla_cache(main_kv, block_size) + extra_cache = _pack_fp8_ds_mla_cache(extra_kv, block_size) + main_indices = torch.tensor([0, 2, 4, 1], dtype=torch.int32, device=device) + main_indptr = torch.tensor([0, 2, 4], dtype=torch.int32, device=device) + extra_indices = torch.tensor([1, 3, 0], dtype=torch.int32, device=device) + extra_indptr = torch.tensor([0, 1, 3], dtype=torch.int32, device=device) + attn_sink = torch.tensor([-0.1, 0.0, 0.1], dtype=torch.float32, device=device) + scale = HEAD_DIM**-0.5 + + actual = _rocm_sparse_attn_decode_ragged_triton( + q=q, + main_cache=main_cache, + main_indices=main_indices, + main_indptr=main_indptr, + scale=scale, + attn_sink=attn_sink, + nope_head_dim=NOPE_HEAD_DIM, + rope_head_dim=ROPE_HEAD_DIM, + extra_cache=extra_cache, + extra_indices=extra_indices, + extra_indptr=extra_indptr, + ) + expected = _ref_sparse_decode_ragged( + q=q, + main_cache=main_cache, + main_rows=[[0, 2], [4, 1]], + scale=scale, + attn_sink=attn_sink, + block_size=block_size, + extra_cache=extra_cache, + extra_rows=[[1], [3, 0]], + ) + + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) + + +@torch.inference_mode() +def test_combine_topk_swa_indices_ragged() -> None: + from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import ( + combine_topk_swa_indices_ragged, + ) + + device = torch.device("cuda") + topk_indices = torch.tensor( + [ + [100, 101, 102, 103], + [110, 111, 112, 113], + [120, 121, 122, 123], + [130, 131, 132, 133], + [140, 141, 142, 143], + ], + dtype=torch.int32, + device=device, + ) + query_start_loc = torch.tensor([0, 3, 5], dtype=torch.int32, device=device) + seq_lens = torch.tensor([6, 4], dtype=torch.int32, device=device) + gather_lens = torch.tensor([4, 3], dtype=torch.int32, device=device) + window_size = 3 + compress_ratio = 2 + topk = 4 + M = 20 + N = 8 + + actual_ragged, actual_indptr, actual_lens = combine_topk_swa_indices_ragged( + topk_indices, + query_start_loc, + seq_lens, + gather_lens, + window_size, + compress_ratio, + topk, + M, + N, + ) + expected_ragged, expected_indptr, expected_lens = _ref_combine_topk_swa_ragged( + device + ) + + torch.testing.assert_close( + actual_ragged[: expected_ragged.numel()], expected_ragged + ) + torch.testing.assert_close(actual_indptr, expected_indptr) + torch.testing.assert_close(actual_lens, expected_lens) diff --git a/vllm/model_executor/layers/deepseek_v4_attention.py b/vllm/model_executor/layers/deepseek_v4_attention.py index 494d613380..e9a6ec9a58 100644 --- a/vllm/model_executor/layers/deepseek_v4_attention.py +++ b/vllm/model_executor/layers/deepseek_v4_attention.py @@ -28,11 +28,7 @@ from vllm.v1.attention.ops.deepseek_v4_ops import ( fused_inv_rope_fp8_quant, fused_q_kv_rmsnorm, ) -from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( - rocm_forward_decode_fallback, - rocm_inv_rope_einsum, - rocm_sparse_attn_prefill, -) +from vllm.v1.attention.ops.rocm_aiter_mla_sparse import rocm_inv_rope_einsum if TYPE_CHECKING: from vllm.v1.attention.backends.mla.sparse_swa import ( @@ -725,6 +721,12 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase): self.kv_cache = torch.tensor([]) def get_attn_backend(self) -> type[AttentionBackend]: + if current_platform.is_rocm(): + from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import ( + DeepseekV4ROCMAiterMLASparseBackend, + ) + + return DeepseekV4ROCMAiterMLASparseBackend return DeepseekV4FlashMLASparseBackend def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None: @@ -757,6 +759,14 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase): f"output buffer dtype {output.dtype} must match q dtype {q.dtype}" ) + if current_platform.is_rocm(): + from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import ( + DeepseekV4ROCMAiterMLASparseImpl, + ) + + DeepseekV4ROCMAiterMLASparseImpl.forward(self, q, kv, positions, output) + return + # Get SWA and indexer metadata from forward context forward_context = get_forward_context() attn_metadata = forward_context.attn_metadata @@ -839,25 +849,6 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase): swa_indices = swa_metadata.decode_swa_indices swa_lens = swa_metadata.decode_swa_lens - if current_platform.is_rocm(): - rocm_forward_decode_fallback( - q=q, - kv_cache=kv_cache, - swa_k_cache=self.swa_cache_layer.kv_cache, - swa_only=swa_only, - topk_indices=topk_indices, - topk_lens=topk_lens, - swa_indices=swa_indices, - swa_lens=swa_lens, - attn_sink=self.attn_sink, - scale=self.scale, - head_dim=self.head_dim, - nope_head_dim=self.nope_head_dim, - rope_head_dim=self.rope_head_dim, - output=output, - ) - return - # We treat queries in the same seq as different queries # and later we only attend by generated indices. # q arrives pre-padded to self.padded_heads by the outer wrapper. @@ -1021,28 +1012,15 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase): M, N, ) - - if current_platform.is_rocm(): - rocm_sparse_attn_prefill( - q=q[query_start:query_end], - kv=kv.view(-1, 1, q.shape[-1]), - indices=combined_indices.unsqueeze(1), - topk_length=combined_lens, - scale=self.scale, - head_dim=self.head_dim, - attn_sink=self.attn_sink, - output=output[query_start:query_end], - ) - else: - output_chunk, _, _ = flash_mla_sparse_fwd( - q=q[query_start:query_end], - kv=kv.view(-1, 1, q.shape[-1]), - indices=combined_indices.unsqueeze(1), - sm_scale=self.scale, - attn_sink=self.attn_sink, - topk_length=combined_lens, - out=output[query_start:query_end], - ) + flash_mla_sparse_fwd( + q=q[query_start:query_end], + kv=kv.view(-1, 1, q.shape[-1]), + indices=combined_indices.unsqueeze(1), + sm_scale=self.scale, + attn_sink=self.attn_sink, + topk_length=combined_lens, + out=output[query_start:query_end], + ) class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase): diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 474a5b2d42..7971790769 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar import numpy as np import torch @@ -112,7 +112,7 @@ class FlashMLASparseBackend(AttentionBackend): return FlashMLASparseMetadataBuilder @staticmethod - def get_impl_cls() -> type["FlashMLASparseImpl"]: + def get_impl_cls() -> type[SparseMLAAttentionImpl[Any]]: return FlashMLASparseImpl @classmethod diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse_dsv4.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse_dsv4.py new file mode 100644 index 0000000000..a647b7d2c6 --- /dev/null +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse_dsv4.py @@ -0,0 +1,682 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from dataclasses import dataclass +from typing import TYPE_CHECKING, cast + +import torch + +from vllm.forward_context import get_forward_context +from vllm.triton_utils import tl, triton +from vllm.v1.attention.backend import ( + AttentionLayer, + CommonAttentionMetadata, + SparseMLAAttentionImpl, +) +from vllm.v1.attention.backends.mla.flashmla_sparse import ( + DeepseekV4FlashMLASparseBackend, + FlashMLASparseMetadata, + FlashMLASparseMetadataBuilder, +) +from vllm.v1.attention.backends.mla.sparse_swa import ( + DeepseekSparseSWAMetadata, + DeepseekSparseSWAMetadataBuilder, +) +from vllm.v1.attention.ops.deepseek_v4_ops import dequantize_and_gather_k_cache +from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + build_ragged_indices_from_dense, + rocm_sparse_attn_decode, + rocm_sparse_attn_prefill, +) +from vllm.v1.worker.workspace import current_workspace_manager + +if TYPE_CHECKING: + from vllm.model_executor.layers.deepseek_v4_attention import ( + DeepseekV4MLAAttention, + ) + + +def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor: + lengths = lengths.to(dtype=torch.int32).contiguous() + indptr = torch.zeros(lengths.shape[0] + 1, dtype=torch.int32, device=lengths.device) + torch.cumsum(lengths, dim=0, out=indptr[1:]) + return indptr + + +@triton.jit +def _compute_topk_lens_kernel( + topk_lens_ptr, + topk_indices_ptr, + topk_indices_stride, + topk, + is_valid_token_ptr, + TRITON_BLOCK_SIZE: tl.constexpr, +): + token_idx = tl.program_id(0) + is_valid_token = tl.load(is_valid_token_ptr + token_idx) + + count = tl.zeros((), dtype=tl.int32) + for i in range(0, topk, TRITON_BLOCK_SIZE): + offset = i + tl.arange(0, TRITON_BLOCK_SIZE) + mask = offset < topk + local_idx = tl.load( + topk_indices_ptr + token_idx * topk_indices_stride + offset, + mask=mask, + other=-1, + ) + count += tl.sum((local_idx >= 0).to(tl.int32), axis=0) + + tl.store(topk_lens_ptr + token_idx, tl.where(is_valid_token, count, 0)) + + +@triton.jit +def _pack_global_topk_ragged_kernel( + global_topk_ragged_ptr, + topk_indptr_ptr, + topk_indices_ptr, + topk_indices_stride, + token_to_req_indices_ptr, + block_table_ptr, + block_table_stride, + block_size, + topk, + BLOCK_SIZE: tl.constexpr, +): + token_idx = tl.program_id(0) + block_idx = tl.program_id(1) + offset = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + + out_start = tl.load(topk_indptr_ptr + token_idx) + out_end = tl.load(topk_indptr_ptr + token_idx + 1) + out_len = out_end - out_start + if block_idx * BLOCK_SIZE >= out_len: + return + + req_idx = tl.load(token_to_req_indices_ptr + token_idx) + mask = (offset < out_len) & (offset < topk) + local_idx = tl.load( + topk_indices_ptr + token_idx * topk_indices_stride + offset, + mask=mask, + other=-1, + ) + valid = mask & (local_idx >= 0) + block_indices = local_idx // block_size + block_numbers = tl.load( + block_table_ptr + req_idx * block_table_stride + block_indices, + mask=valid, + other=0, + ) + block_offsets = local_idx % block_size + slot_ids = tl.where(valid, block_numbers * block_size + block_offsets, -1) + tl.store(global_topk_ragged_ptr + out_start + offset, slot_ids, mask=mask) + + +def compute_global_topk_ragged_indices_and_indptr( + topk_indices: torch.Tensor, + token_to_req_indices: torch.Tensor, + block_table: torch.Tensor, + block_size: int, + is_valid_token: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous() + num_tokens = topk_indices.shape[0] + topk = topk_indices.shape[1] + + topk_lens = torch.empty(num_tokens, dtype=torch.int32, device=topk_indices.device) + _compute_topk_lens_kernel[(num_tokens,)]( + topk_lens, + topk_indices, + topk_indices.stride(0), + topk, + is_valid_token, + TRITON_BLOCK_SIZE=1024, + ) + + topk_indptr = _build_indptr_from_lengths(topk_lens) + global_topk_ragged = torch.empty( + num_tokens * topk, + dtype=torch.int32, + device=topk_indices.device, + ) + if global_topk_ragged.numel() > 0: + block = 128 + _pack_global_topk_ragged_kernel[(num_tokens, triton.cdiv(topk, block))]( + global_topk_ragged, + topk_indptr, + topk_indices, + topk_indices.stride(0), + token_to_req_indices, + block_table, + block_table.stride(0), + block_size, + topk, + BLOCK_SIZE=block, + ) + return global_topk_ragged, topk_indptr, topk_lens + + +@triton.jit +def _compute_combined_lens_kernel( + combined_lens_ptr, + query_start_loc_ptr, + seq_lens_ptr, + TOP_K: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + WINDOW_SIZE: tl.constexpr, +): + batch_idx = tl.program_id(0) + worker_id = tl.program_id(1) + num_workers = tl.num_programs(1) + + base = tl.load(query_start_loc_ptr) + query_start = tl.load(query_start_loc_ptr + batch_idx) - base + query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base + query_len = query_end - query_start + seq_len = tl.load(seq_lens_ptr + batch_idx) + start_pos = seq_len - query_len + + for token_idx in range(query_start + worker_id, query_end, num_workers): + token_idx_in_query = token_idx - query_start + pos = start_pos + token_idx_in_query + topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K) + swa_len = tl.minimum(pos + 1, WINDOW_SIZE) + tl.store(combined_lens_ptr + token_idx, topk_len + swa_len) + + +@triton.jit +def _combine_topk_swa_indices_ragged_kernel( + combined_ragged_ptr, + combined_indptr_ptr, + topk_indices_ptr, + topk_indices_stride, + query_start_loc_ptr, + seq_lens_ptr, + gather_lens_ptr, + M, + N, + topk_width, + TOP_K: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + WINDOW_SIZE: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + batch_idx = tl.program_id(0) + worker_id = tl.program_id(1) + block_idx = tl.program_id(2) + num_workers = tl.num_programs(1) + + base = tl.load(query_start_loc_ptr) + query_start = tl.load(query_start_loc_ptr + batch_idx) - base + query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base + query_len = query_end - query_start + seq_len = tl.load(seq_lens_ptr + batch_idx) + gather_len = tl.load(gather_lens_ptr + batch_idx) + start_pos = seq_len - query_len + gather_start = seq_len - gather_len + + for token_idx in range(query_start + worker_id, query_end, num_workers): + token_idx_in_query = token_idx - query_start + pos = start_pos + token_idx_in_query + topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K) + swa_len = tl.minimum(pos + 1, WINDOW_SIZE) + combined_len = topk_len + swa_len + + offset = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + if block_idx * BLOCK_SIZE < combined_len: + out_start = tl.load(combined_indptr_ptr + token_idx) + topk_mask = (offset < topk_len) & (offset < topk_width) + topk_vals = tl.load( + topk_indices_ptr + token_idx * topk_indices_stride + offset, + mask=topk_mask, + other=-1, + ) + tl.store( + combined_ragged_ptr + out_start + offset, + topk_vals + M * batch_idx, + mask=topk_mask, + ) + + swa_offset = offset - topk_len + swa_mask = (offset >= topk_len) & (swa_offset < swa_len) + tl.store( + combined_ragged_ptr + out_start + offset, + M * batch_idx + N + swa_offset + pos - swa_len + 1 - gather_start, + mask=swa_mask, + ) + + +def combine_topk_swa_indices_ragged( + topk_indices: torch.Tensor, + query_start_loc: torch.Tensor, + seq_lens: torch.Tensor, + gather_lens: torch.Tensor, + window_size: int, + compress_ratio: int, + topk: int, + M: int, + N: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous() + num_tokens = topk_indices.shape[0] + num_reqs = seq_lens.shape[0] + combined_lens = torch.empty( + num_tokens, dtype=torch.int32, device=topk_indices.device + ) + + num_workers = 128 + _compute_combined_lens_kernel[(num_reqs, num_workers)]( + combined_lens, + query_start_loc, + seq_lens, + TOP_K=topk, + COMPRESS_RATIO=compress_ratio, + WINDOW_SIZE=window_size, + ) + + combined_indptr = _build_indptr_from_lengths(combined_lens) + combined_ragged = torch.empty( + num_tokens * (topk + window_size), + dtype=torch.int32, + device=topk_indices.device, + ) + if combined_ragged.numel() > 0: + block = 128 + _combine_topk_swa_indices_ragged_kernel[ + (num_reqs, num_workers, triton.cdiv(topk + window_size, block)) + ]( + combined_ragged, + combined_indptr, + topk_indices, + topk_indices.stride(0), + query_start_loc, + seq_lens, + gather_lens, + M, + N, + topk_indices.shape[-1], + TOP_K=topk, + COMPRESS_RATIO=compress_ratio, + WINDOW_SIZE=window_size, + BLOCK_SIZE=block, + ) + return combined_ragged, combined_indptr, combined_lens + + +@dataclass +class DeepseekV4ROCMAiterMLASparseMetadata(FlashMLASparseMetadata): + """ROCm-specific DeepSeek V4 metadata carrying ragged decode topk.""" + + c128a_decode_topk_ragged_indices: torch.Tensor | None = None + c128a_decode_topk_ragged_indptr: torch.Tensor | None = None + + +@dataclass +class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata): + decode_swa_ragged_indices: torch.Tensor | None = None + decode_swa_ragged_indptr: torch.Tensor | None = None + + +class DeepseekV4ROCMAiterMLASparseMetadataBuilder(FlashMLASparseMetadataBuilder): + def build( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + ) -> DeepseekV4ROCMAiterMLASparseMetadata: + base = super().build( + common_prefix_len=common_prefix_len, + common_attn_metadata=common_attn_metadata, + fast_build=fast_build, + ) + + ragged_indices = None + ragged_indptr = None + dense_decode = base.c128a_global_decode_topk_indices + decode_lens = base.c128a_decode_topk_lens + if dense_decode is not None and decode_lens is not None: + ragged_indices, ragged_indptr = build_ragged_indices_from_dense( + dense_decode.reshape(dense_decode.shape[0], -1), + decode_lens, + ) + + return DeepseekV4ROCMAiterMLASparseMetadata( + **vars(base), + c128a_decode_topk_ragged_indices=ragged_indices, + c128a_decode_topk_ragged_indptr=ragged_indptr, + ) + + +class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder): + def build( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + ) -> DeepseekV4ROCMAiterSparseSWAMetadata: + base = super().build( + common_prefix_len=common_prefix_len, + common_attn_metadata=common_attn_metadata, + fast_build=fast_build, + ) + + ragged_indices = None + ragged_indptr = None + if ( + base.num_decode_tokens > 0 + and base.decode_swa_indices is not None + and base.decode_swa_lens is not None + ): + ragged_indices, ragged_indptr = build_ragged_indices_from_dense( + base.decode_swa_indices.reshape(base.num_decode_tokens, -1), + base.decode_swa_lens, + ) + + return DeepseekV4ROCMAiterSparseSWAMetadata( + **vars(base), + decode_swa_ragged_indices=ragged_indices, + decode_swa_ragged_indptr=ragged_indptr, + ) + + +class DeepseekV4ROCMAiterMLASparseImpl( + SparseMLAAttentionImpl[DeepseekV4ROCMAiterMLASparseMetadata] +): + """ROCm sparse MLA implementation used by DeepSeek V4's custom MLA layer.""" + + _PREFILL_CHUNK_SIZE = 4 + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: list[float] | None, + sliding_window: int | None, + kv_cache_dtype: str, + logits_soft_cap: float | None, + attn_type: str, + kv_sharing_target_layer_name: str | None, + **_: object, + ) -> None: + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_kv_heads + self.kv_cache_dtype = kv_cache_dtype + + def forward_mqa( + self, + q: torch.Tensor | tuple[torch.Tensor, torch.Tensor], + kv_c_and_k_pe_cache: torch.Tensor, + attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata, + layer: AttentionLayer, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + raise NotImplementedError( + "DeepseekV4ROCMAiterMLASparseImpl is driven by " + "DeepseekV4MLAAttention.forward." + ) + + @classmethod + def forward( + cls, + layer: "DeepseekV4MLAAttention", + q: torch.Tensor, + kv: torch.Tensor, + positions: torch.Tensor, + output: torch.Tensor, + ) -> None: + assert output.shape == q.shape, ( + f"output buffer shape {output.shape} must match q shape {q.shape}" + ) + assert output.dtype == q.dtype, ( + f"output buffer dtype {output.dtype} must match q dtype {q.dtype}" + ) + + forward_context = get_forward_context() + attn_metadata = forward_context.attn_metadata + assert isinstance(attn_metadata, dict) + rocm_metadata = cast( + DeepseekV4ROCMAiterMLASparseMetadata | None, + attn_metadata.get(layer.prefix), + ) + swa_metadata = cast( + DeepseekV4ROCMAiterSparseSWAMetadata | None, + attn_metadata.get(layer.swa_cache_layer.prefix), + ) + assert swa_metadata is not None + + swa_only = layer.compress_ratio <= 1 + self_kv_cache = layer.kv_cache if not swa_only else None + swa_kv_cache = layer.swa_cache_layer.kv_cache + + num_decodes = swa_metadata.num_decodes + num_prefills = swa_metadata.num_prefills + num_decode_tokens = swa_metadata.num_decode_tokens + + if num_prefills > 0: + cls._forward_prefill( + layer=layer, + q=q[num_decode_tokens:], + positions=positions[num_decode_tokens:], + compressed_k_cache=self_kv_cache, + swa_k_cache=swa_kv_cache, + output=output[num_decode_tokens:], + attn_metadata=rocm_metadata, + swa_metadata=swa_metadata, + ) + if num_decodes > 0: + cls._forward_decode( + layer=layer, + q=q[:num_decode_tokens], + kv_cache=self_kv_cache, + swa_metadata=swa_metadata, + attn_metadata=rocm_metadata, + swa_only=swa_only, + output=output[:num_decode_tokens], + ) + + @classmethod + def _forward_decode( + cls, + layer: "DeepseekV4MLAAttention", + q: torch.Tensor, + kv_cache: torch.Tensor | None, + swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, + attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None, + swa_only: bool, + output: torch.Tensor, + ) -> None: + num_decodes = swa_metadata.num_decodes + num_decode_tokens = swa_metadata.num_decode_tokens + + topk_indices = None + topk_lens = None + topk_ragged_indices = None + topk_ragged_indptr = None + if not swa_only: + assert attn_metadata is not None + assert swa_metadata.is_valid_token is not None + block_size = attn_metadata.block_size // layer.compress_ratio + is_valid = swa_metadata.is_valid_token[:num_decode_tokens] + if layer.compress_ratio == 4: + assert layer.topk_indices_buffer is not None + ( + topk_ragged_indices, + topk_ragged_indptr, + topk_lens, + ) = compute_global_topk_ragged_indices_and_indptr( + layer.topk_indices_buffer[:num_decode_tokens], + swa_metadata.token_to_req_indices, + attn_metadata.block_table[:num_decodes], + block_size, + is_valid, + ) + else: + topk_indices = attn_metadata.c128a_global_decode_topk_indices + topk_lens = attn_metadata.c128a_decode_topk_lens + topk_ragged_indices = attn_metadata.c128a_decode_topk_ragged_indices + topk_ragged_indptr = attn_metadata.c128a_decode_topk_ragged_indptr + + rocm_sparse_attn_decode( + q=q, + kv_cache=kv_cache, + swa_k_cache=layer.swa_cache_layer.kv_cache, + swa_only=swa_only, + topk_indices=topk_indices, + topk_lens=topk_lens, + swa_indices=swa_metadata.decode_swa_indices, + swa_lens=swa_metadata.decode_swa_lens, + swa_ragged_indices=swa_metadata.decode_swa_ragged_indices, + swa_ragged_indptr=swa_metadata.decode_swa_ragged_indptr, + topk_ragged_indices=topk_ragged_indices, + topk_ragged_indptr=topk_ragged_indptr, + attn_sink=layer.attn_sink, + scale=layer.scale, + head_dim=layer.head_dim, + nope_head_dim=layer.nope_head_dim, + rope_head_dim=layer.rope_head_dim, + output=output, + ) + + @classmethod + def _forward_prefill( + cls, + layer: "DeepseekV4MLAAttention", + q: torch.Tensor, + positions: torch.Tensor, + compressed_k_cache: torch.Tensor | None, + swa_k_cache: torch.Tensor, + output: torch.Tensor, + attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None, + swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata, + ) -> None: + swa_only = attn_metadata is None + + num_prefills = swa_metadata.num_prefills + num_prefill_tokens = swa_metadata.num_prefill_tokens + num_decodes = swa_metadata.num_decodes + num_decode_tokens = swa_metadata.num_decode_tokens + + seq_lens = swa_metadata.prefill_seq_lens + gather_lens = swa_metadata.prefill_gather_lens + assert seq_lens is not None + assert gather_lens is not None + + query_start_loc_cpu = swa_metadata.query_start_loc_cpu + query_start_loc = swa_metadata.query_start_loc + assert query_start_loc_cpu is not None + assert query_start_loc is not None + prefill_token_base = query_start_loc_cpu[num_decodes] + + if not swa_only: + if layer.compress_ratio == 4: + assert layer.topk_indices_buffer is not None + topk_indices = layer.topk_indices_buffer[num_decode_tokens:] + topk_indices = topk_indices[:num_prefill_tokens] + else: + assert attn_metadata is not None + topk_indices = attn_metadata.c128a_prefill_topk_indices + assert topk_indices is not None + top_k = topk_indices.shape[-1] + N = (layer.max_model_len + layer.compress_ratio - 1) // layer.compress_ratio + else: + assert layer.topk_indices_buffer is not None + topk_indices = layer.topk_indices_buffer[num_decode_tokens:] + top_k = 0 + N = 0 + + M = N + layer.window_size + layer.max_num_batched_tokens + num_chunks = (num_prefills + cls._PREFILL_CHUNK_SIZE - 1) // ( + cls._PREFILL_CHUNK_SIZE + ) + + workspace_manager = current_workspace_manager() + kv = workspace_manager.get_simultaneous( + ((cls._PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), + )[0] + for chunk_idx in range(num_chunks): + chunk_start = chunk_idx * cls._PREFILL_CHUNK_SIZE + chunk_end = min(chunk_start + cls._PREFILL_CHUNK_SIZE, num_prefills) + chunk_size = chunk_end - chunk_start + if not swa_only: + assert attn_metadata is not None + assert compressed_k_cache is not None + block_table = attn_metadata.block_table[num_decodes:] + dequantize_and_gather_k_cache( + kv[:chunk_size], + compressed_k_cache, + seq_lens=seq_lens[chunk_start:chunk_end] // layer.compress_ratio, + gather_lens=None, + block_table=block_table[chunk_start:chunk_end], + block_size=attn_metadata.block_size // layer.compress_ratio, + offset=0, + ) + + swa_block_table = swa_metadata.block_table[num_decodes:] + dequantize_and_gather_k_cache( + kv[:chunk_size], + swa_k_cache, + seq_lens=seq_lens[chunk_start:chunk_end], + gather_lens=gather_lens[chunk_start:chunk_end], + block_table=swa_block_table[chunk_start:chunk_end], + block_size=swa_metadata.block_size, + offset=N, + ) + + query_start = ( + query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base + ) + query_end = ( + query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base + ) + + combined_ragged_indices, combined_ragged_indptr, combined_lens = ( + combine_topk_swa_indices_ragged( + topk_indices[query_start:query_end], + query_start_loc[ + num_decodes + chunk_start : num_decodes + chunk_end + 1 + ], + seq_lens[chunk_start:chunk_end], + gather_lens[chunk_start:chunk_end], + layer.window_size, + layer.compress_ratio, + top_k, + M, + N, + ) + ) + rocm_sparse_attn_prefill( + q=q[query_start:query_end], + kv=kv.view(-1, 1, q.shape[-1]), + indices=torch.empty( + q[query_start:query_end].shape[0], + 1, + 0, + dtype=torch.int32, + device=q.device, + ), + topk_length=combined_lens, + scale=layer.scale, + head_dim=layer.head_dim, + nope_head_dim=layer.nope_head_dim, + rope_head_dim=layer.rope_head_dim, + attn_sink=layer.attn_sink, + output=output[query_start:query_end], + ragged_indices=combined_ragged_indices, + ragged_indptr=combined_ragged_indptr, + ) + + +class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4FlashMLASparseBackend): + @staticmethod + def get_name() -> str: + return "ROCM_V4_FLASHMLA_SPARSE" + + @staticmethod + def get_builder_cls() -> type["DeepseekV4ROCMAiterMLASparseMetadataBuilder"]: + return DeepseekV4ROCMAiterMLASparseMetadataBuilder + + @staticmethod + def get_impl_cls() -> type["DeepseekV4ROCMAiterMLASparseImpl"]: + return DeepseekV4ROCMAiterMLASparseImpl diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index 28564e6a97..bfa3b7285d 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -112,6 +112,12 @@ class DeepseekSparseSWABackend(AttentionBackend): @staticmethod def get_builder_cls() -> type["DeepseekSparseSWAMetadataBuilder"]: + if current_platform.is_rocm(): + from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import ( + DeepseekV4ROCMAiterSparseSWAMetadataBuilder, + ) + + return DeepseekV4ROCMAiterSparseSWAMetadataBuilder return DeepseekSparseSWAMetadataBuilder @staticmethod diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 5d0343ffd6..936839b457 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -905,50 +905,705 @@ def rocm_inv_rope_einsum( return torch.einsum("tgd,grd->tgr", o_ref, wo_a_weight) -def rocm_ref_sparse_attn_prefill( +_DSV4_SPARSE_NOPE_DIM = 448 +_DSV4_SPARSE_ROPE_DIM = 64 + + +def _validate_dsv4_sparse_dims( + head_dim: int, + nope_head_dim: int, + rope_head_dim: int, + op_name: str, +) -> None: + assert head_dim == nope_head_dim + rope_head_dim, ( + f"{op_name} expected head_dim={nope_head_dim + rope_head_dim}, got {head_dim}" + ) + assert ( + nope_head_dim == _DSV4_SPARSE_NOPE_DIM + and rope_head_dim == _DSV4_SPARSE_ROPE_DIM + ), ( + f"{op_name} expects {_DSV4_SPARSE_NOPE_DIM} NoPE dims and " + f"{_DSV4_SPARSE_ROPE_DIM} RoPE dims" + ) + + +@triton.jit +def _pack_dense_prefix_to_ragged_kernel( + indices_ptr, + lengths_ptr, + indptr_ptr, + out_ptr, + indices_stride0, + num_rows_limit, + row_width, + BLOCK_SIZE: tl.constexpr, +): + row_idx = tl.program_id(0) + block_idx = tl.program_id(1) + offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + + row_len = tl.load(lengths_ptr + row_idx) + if block_idx * BLOCK_SIZE >= row_len: + return + + mask = offsets < row_len + vals = tl.load( + indices_ptr + row_idx * indices_stride0 + offsets, + mask=mask & (offsets < row_width), + other=-1, + ).to(tl.int32) + if num_rows_limit >= 0: + vals = tl.where((vals >= 0) & (vals < num_rows_limit), vals, -1) + + out_start = tl.load(indptr_ptr + row_idx) + tl.store(out_ptr + out_start + offsets, vals, mask=mask) + + +def build_ragged_indices_from_dense( + indices: torch.Tensor, + lengths: torch.Tensor, + num_rows: int = -1, +) -> tuple[torch.Tensor, torch.Tensor]: + indices = indices.reshape(indices.shape[0], -1) + lengths = lengths.to(device=indices.device, dtype=torch.int32).reshape(-1) + assert lengths.numel() == indices.shape[0], ( + f"Expected one length per row, got {lengths.shape} for indices {indices.shape}" + ) + + max_width = indices.shape[1] if indices.ndim == 2 else 0 + lengths = lengths.clamp(min=0, max=max_width).contiguous() + + indptr = torch.empty(indices.shape[0] + 1, dtype=torch.int32, device=indices.device) + indptr[0] = 0 + torch.cumsum(lengths, dim=0, out=indptr[1:]) + + if indices.numel() == 0: + flat = torch.empty(0, dtype=torch.int32, device=indices.device) + else: + flat = torch.empty( + int(indptr[-1].item()), dtype=torch.int32, device=indices.device + ) + if flat.numel() > 0: + block_size = 128 + _pack_dense_prefix_to_ragged_kernel[ + (indices.shape[0], triton.cdiv(max_width, block_size)) + ]( + indices, + lengths, + indptr, + flat, + indices.stride(0), + int(num_rows), + max_width, + BLOCK_SIZE=block_size, + ) + + return flat, indptr + + +def _as_int32_contiguous_1d(x: torch.Tensor) -> torch.Tensor: + if x.dtype == torch.int32 and x.ndim == 1 and x.is_contiguous(): + return x + return x.to(torch.int32).contiguous() + + +@triton.jit +def _sparse_attn_prefill_ragged_kernel( + q_ptr, + kv_ptr, + kv_indices_ptr, + kv_indptr_ptr, + attn_sink_ptr, + out_ptr, + q_stride_t, + q_stride_h, + q_stride_d, + kv_stride_n, + kv_stride_d, + out_stride_t, + out_stride_h, + out_stride_d, + num_heads, + head_dim, + num_kv, + scale, + HAS_ATTN_SINK: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_K: tl.constexpr, +): + query_idx = tl.program_id(0) + pid_h = tl.program_id(1) + + head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + dim_offsets = tl.arange(0, BLOCK_D) + head_mask = head_offsets < num_heads + dim_mask = dim_offsets < head_dim + + q = tl.load( + q_ptr + + query_idx * q_stride_t + + head_offsets[:, None] * q_stride_h + + dim_offsets[None, :] * q_stride_d, + mask=head_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + + neg_large = -3.4028234663852886e38 + m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32) + l_i = tl.zeros((BLOCK_H,), dtype=tl.float32) + acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32) + + kv_start = tl.load(kv_indptr_ptr + query_idx) + kv_end = tl.load(kv_indptr_ptr + query_idx + 1) + kv_len = kv_end - kv_start + + k_offsets = tl.arange(0, BLOCK_K) + for k_start in tl.range(0, kv_len, BLOCK_K): + k_pos = k_start + k_offsets + in_range = k_pos < kv_len + slot = tl.load(kv_indices_ptr + kv_start + k_pos, mask=in_range, other=-1) + valid = in_range & (slot >= 0) & (slot < num_kv) + + kv = tl.load( + kv_ptr + slot[:, None] * kv_stride_n + dim_offsets[None, :] * kv_stride_d, + mask=valid[:, None] & dim_mask[None, :], + other=0.0, + ) + kv = tl.where(valid[:, None] & dim_mask[None, :], kv, 0.0) + + scores = tl.dot(q, tl.trans(kv)) * scale + scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large) + + m_block = tl.max(scores, axis=1) + m_new = tl.maximum(m_i, m_block) + alpha = tl.exp(m_i - m_new) + p = tl.exp(scores - m_new[:, None]) + p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0) + l_new = l_i * alpha + tl.sum(p, axis=1) + + acc = acc * alpha[:, None] + tl.dot(p.to(kv.dtype), kv) + m_i = m_new + l_i = l_new + + if HAS_ATTN_SINK: + sink = tl.load( + attn_sink_ptr + head_offsets, mask=head_mask, other=neg_large + ).to(tl.float32) + m_final = tl.maximum(m_i, sink) + alpha = tl.exp(m_i - m_final) + l_final = l_i * alpha + tl.exp(sink - m_final) + denom = tl.maximum(l_final, 1.0e-30) + out = tl.where( + l_final[:, None] > 0.0, + (acc * alpha[:, None]) / denom[:, None], + 0.0, + ) + else: + denom = tl.maximum(l_i, 1.0e-30) + out = tl.where(l_i[:, None] > 0.0, acc / denom[:, None], 0.0) + + tl.store( + out_ptr + + query_idx * out_stride_t + + head_offsets[:, None] * out_stride_h + + dim_offsets[None, :] * out_stride_d, + out, + mask=head_mask[:, None] & dim_mask[None, :], + ) + + +@triton.jit +def _sparse_attn_decode_ragged_kernel( + q_ptr, + main_cache_ptr, + main_indices_ptr, + main_indptr_ptr, + extra_cache_ptr, + extra_indices_ptr, + extra_indptr_ptr, + attn_sink_ptr, + out_ptr, + q_stride0, + q_stride1, + out_stride0, + out_stride1, + main_cache_stride0, + extra_cache_stride0, + main_num_rows, + extra_num_rows, + main_block_size, + extra_block_size, + scale, + num_heads, + HAS_ATTN_SINK: tl.constexpr, + HAS_EXTRA: tl.constexpr, + NOPE_DIM: tl.constexpr, + NOPE_BLOCK: tl.constexpr, + ROPE_DIM: tl.constexpr, + IS_FNUZ: tl.constexpr, + BLOCK_H: tl.constexpr, + BLOCK_K: tl.constexpr, +): + query_idx = tl.program_id(0) + pid_h = tl.program_id(1) + + head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + head_mask = head_offsets < num_heads + nope_offsets = tl.arange(0, NOPE_BLOCK) + nope_mask = nope_offsets < NOPE_DIM + rope_offsets = tl.arange(0, ROPE_DIM) + + q_row_ptr = q_ptr + query_idx * q_stride0 + head_offsets[:, None] * q_stride1 + q_nope = tl.load( + q_row_ptr + nope_offsets[None, :], + mask=head_mask[:, None] & nope_mask[None, :], + other=0.0, + ) + q_rope = tl.load( + q_row_ptr + NOPE_DIM + rope_offsets[None, :], + mask=head_mask[:, None], + other=0.0, + ) + + neg_large = -3.4028234663852886e38 + m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32) + l_i = tl.zeros((BLOCK_H,), dtype=tl.float32) + acc_nope = tl.zeros((BLOCK_H, NOPE_BLOCK), dtype=tl.float32) + acc_rope = tl.zeros((BLOCK_H, ROPE_DIM), dtype=tl.float32) + k_offsets = tl.arange(0, BLOCK_K) + + main_start = tl.load(main_indptr_ptr + query_idx) + main_end = tl.load(main_indptr_ptr + query_idx + 1) + main_len = main_end - main_start + + zero_nope = tl.zeros((BLOCK_K, NOPE_BLOCK), dtype=tl.bfloat16) + zero_rope = tl.zeros((BLOCK_K, ROPE_DIM), dtype=tl.bfloat16) + + for k_start in tl.range(0, main_len, BLOCK_K): + k_pos = k_start + k_offsets + in_range = k_pos < main_len + slot = tl.load(main_indices_ptr + main_start + k_pos, mask=in_range, other=-1) + valid = in_range & (slot >= 0) & (slot < main_num_rows) + safe_slot = tl.where(valid, slot, 0) + + block_idx = safe_slot // main_block_size + pos_in_block = safe_slot % main_block_size + cache_block_ptr = main_cache_ptr + block_idx.to(tl.int64) * main_cache_stride0 + token_data_ptr = cache_block_ptr + pos_in_block * 576 + token_scale_ptr = cache_block_ptr + main_block_size * 576 + pos_in_block * 8 + + x_uint8 = tl.load( + token_data_ptr[:, None] + nope_offsets[None, :], + mask=valid[:, None] & nope_mask[None, :], + other=0, + ) + if IS_FNUZ: + x_fp8 = x_uint8.to(tl.float8e4b15, bitcast=True) + else: + x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True) + encoded_scales = tl.load( + token_scale_ptr[:, None] + nope_offsets[None, :] // 64, + mask=valid[:, None] & nope_mask[None, :], + other=127, + ) + scales = tl.exp2(encoded_scales.to(tl.float32) - 127.0) + k_nope = x_fp8.to(tl.bfloat16) * scales.to(tl.bfloat16) + k_nope = tl.where(valid[:, None] & nope_mask[None, :], k_nope, zero_nope) + k_nope = tl.where(k_nope == k_nope, k_nope, zero_nope) + + rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16)) + k_rope = tl.load( + rope_ptr[:, None] + rope_offsets[None, :], + mask=valid[:, None], + other=0.0, + ) + k_rope = tl.where(valid[:, None], k_rope, zero_rope) + k_rope = tl.where(k_rope == k_rope, k_rope, zero_rope) + + scores = tl.dot(q_nope, tl.trans(k_nope)) + tl.dot(q_rope, tl.trans(k_rope)) + scores *= scale + scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large) + + m_block = tl.max(scores, axis=1) + m_new = tl.maximum(m_i, m_block) + alpha = tl.exp(m_i - m_new) + p = tl.exp(scores - m_new[:, None]) + p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0) + l_new = l_i * alpha + tl.sum(p, axis=1) + + acc_nope = acc_nope * alpha[:, None] + tl.dot(p.to(k_nope.dtype), k_nope) + acc_rope = acc_rope * alpha[:, None] + tl.dot(p.to(k_rope.dtype), k_rope) + m_i = m_new + l_i = l_new + + if HAS_EXTRA: + extra_start = tl.load(extra_indptr_ptr + query_idx) + extra_end = tl.load(extra_indptr_ptr + query_idx + 1) + extra_len = extra_end - extra_start + + for k_start in tl.range(0, extra_len, BLOCK_K): + k_pos = k_start + k_offsets + in_range = k_pos < extra_len + slot = tl.load( + extra_indices_ptr + extra_start + k_pos, mask=in_range, other=-1 + ) + valid = in_range & (slot >= 0) & (slot < extra_num_rows) + safe_slot = tl.where(valid, slot, 0) + + block_idx = safe_slot // extra_block_size + pos_in_block = safe_slot % extra_block_size + cache_block_ptr = ( + extra_cache_ptr + block_idx.to(tl.int64) * extra_cache_stride0 + ) + token_data_ptr = cache_block_ptr + pos_in_block * 576 + token_scale_ptr = ( + cache_block_ptr + extra_block_size * 576 + pos_in_block * 8 + ) + + x_uint8 = tl.load( + token_data_ptr[:, None] + nope_offsets[None, :], + mask=valid[:, None] & nope_mask[None, :], + other=0, + ) + if IS_FNUZ: + x_fp8 = x_uint8.to(tl.float8e4b15, bitcast=True) + else: + x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True) + encoded_scales = tl.load( + token_scale_ptr[:, None] + nope_offsets[None, :] // 64, + mask=valid[:, None] & nope_mask[None, :], + other=127, + ) + scales = tl.exp2(encoded_scales.to(tl.float32) - 127.0) + k_nope = x_fp8.to(tl.bfloat16) * scales.to(tl.bfloat16) + k_nope = tl.where(valid[:, None] & nope_mask[None, :], k_nope, zero_nope) + k_nope = tl.where(k_nope == k_nope, k_nope, zero_nope) + + rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16)) + k_rope = tl.load( + rope_ptr[:, None] + rope_offsets[None, :], + mask=valid[:, None], + other=0.0, + ) + k_rope = tl.where(valid[:, None], k_rope, zero_rope) + k_rope = tl.where(k_rope == k_rope, k_rope, zero_rope) + + scores = tl.dot(q_nope, tl.trans(k_nope)) + tl.dot( + q_rope, + tl.trans(k_rope), + ) + scores *= scale + scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large) + + m_block = tl.max(scores, axis=1) + m_new = tl.maximum(m_i, m_block) + alpha = tl.exp(m_i - m_new) + p = tl.exp(scores - m_new[:, None]) + p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0) + l_new = l_i * alpha + tl.sum(p, axis=1) + + acc_nope = acc_nope * alpha[:, None] + tl.dot(p.to(k_nope.dtype), k_nope) + acc_rope = acc_rope * alpha[:, None] + tl.dot(p.to(k_rope.dtype), k_rope) + m_i = m_new + l_i = l_new + + if HAS_ATTN_SINK: + sink = tl.load( + attn_sink_ptr + head_offsets, mask=head_mask, other=neg_large + ).to(tl.float32) + m_final = tl.maximum(m_i, sink) + alpha = tl.exp(m_i - m_final) + l_final = l_i * alpha + tl.exp(sink - m_final) + denom = tl.maximum(l_final, 1.0e-30) + out_nope = tl.where( + l_final[:, None] > 0.0, + (acc_nope * alpha[:, None]) / denom[:, None], + 0.0, + ) + out_rope = tl.where( + l_final[:, None] > 0.0, + (acc_rope * alpha[:, None]) / denom[:, None], + 0.0, + ) + else: + denom = tl.maximum(l_i, 1.0e-30) + out_nope = tl.where(l_i[:, None] > 0.0, acc_nope / denom[:, None], 0.0) + out_rope = tl.where(l_i[:, None] > 0.0, acc_rope / denom[:, None], 0.0) + + out_row_ptr = ( + out_ptr + query_idx * out_stride0 + head_offsets[:, None] * out_stride1 + ) + tl.store( + out_row_ptr + nope_offsets[None, :], + out_nope, + mask=head_mask[:, None] & nope_mask[None, :], + ) + tl.store( + out_row_ptr + NOPE_DIM + rope_offsets[None, :], + out_rope, + mask=head_mask[:, None], + ) + + +def _rocm_sparse_attn_prefill_ragged_triton( q: torch.Tensor, kv: torch.Tensor, indices: torch.Tensor, - topk_length: torch.Tensor | None, + indptr: torch.Tensor, scale: float, - head_dim: int, attn_sink: torch.Tensor | None, + nope_head_dim: int, + rope_head_dim: int, ) -> torch.Tensor: - indices = indices.clone().squeeze(1) - s_q, h_q, d_qk = q.shape - topk = indices.shape[-1] - s_kv = kv.shape[0] - if topk_length is not None: - mask = torch.arange(topk, device=indices.device).unsqueeze( - 0 - ) >= topk_length.unsqueeze(1) - indices[mask] = -1 - invalid_mask = (indices < 0) | (indices >= s_kv) - indices[invalid_mask] = 0 + assert q.ndim == 3, f"expected q=[sq,h,d], got {q.shape}" + assert kv.ndim == 2, f"expected kv=[skv,d], got {kv.shape}" + assert indices.ndim == 1, f"expected indices=[nnz], got {indices.shape}" + assert indptr.ndim == 1, f"expected indptr=[sq+1], got {indptr.shape}" + assert q.is_cuda and kv.is_cuda and indices.is_cuda and indptr.is_cuda - qf = q.float() - gathered_kv = kv.index_select(0, indices.flatten()).reshape(s_q, topk, d_qk).float() - scores = qf @ gathered_kv.transpose(1, 2) - scores *= scale - scores[invalid_mask.unsqueeze(1).expand_as(scores)] = float("-inf") + indices = _as_int32_contiguous_1d(indices) + indptr = _as_int32_contiguous_1d(indptr) + has_attn_sink = attn_sink is not None + if attn_sink is None: + attn_sink = torch.empty(1, device=q.device, dtype=torch.float32) + else: + attn_sink = attn_sink.contiguous() - orig_lse = torch.logsumexp(scores, dim=-1) - lse_for_o = orig_lse - if attn_sink is not None: - lse_for_o = torch.logsumexp( - torch.stack( - [orig_lse, attn_sink[:h_q].view(1, h_q).expand_as(orig_lse)], - dim=0, - ), - dim=0, + num_queries, num_heads, head_dim = q.shape + assert indptr.numel() == num_queries + 1, ( + f"expected indptr shape [{num_queries + 1}], got {indptr.shape}" + ) + _validate_dsv4_sparse_dims( + head_dim, + nope_head_dim, + rope_head_dim, + "_rocm_sparse_attn_prefill_ragged_triton", + ) + + block_h = 16 + block_d = triton.next_power_of_2(head_dim) + block_k = 16 if head_dim >= 256 else 32 + out = torch.empty_like(q, dtype=torch.bfloat16) + _sparse_attn_prefill_ragged_kernel[(num_queries, triton.cdiv(num_heads, block_h))]( + q, + kv, + indices, + indptr, + attn_sink, + out, + q.stride(0), + q.stride(1), + q.stride(2), + kv.stride(0), + kv.stride(1), + out.stride(0), + out.stride(1), + out.stride(2), + num_heads, + head_dim, + kv.shape[0], + float(scale), + HAS_ATTN_SINK=has_attn_sink, + BLOCK_H=block_h, + BLOCK_D=block_d, + BLOCK_K=block_k, + num_warps=8, + ) + return out + + +def _rocm_sparse_attn_prefill_triton( + q: torch.Tensor, + kv: torch.Tensor, + indices: torch.Tensor, + scale: float, + attn_sink: torch.Tensor | None, + nope_head_dim: int, + rope_head_dim: int, + topk_length: torch.Tensor | None = None, +) -> torch.Tensor: + ragged_indices, ragged_indptr = build_ragged_indices_from_dense( + indices, + topk_length + if topk_length is not None + else (indices >= 0).sum(dim=-1, dtype=torch.int32), + num_rows=kv.shape[0], + ) + return _rocm_sparse_attn_prefill_ragged_triton( + q=q, + kv=kv, + indices=ragged_indices, + indptr=ragged_indptr, + scale=scale, + attn_sink=attn_sink, + nope_head_dim=nope_head_dim, + rope_head_dim=rope_head_dim, + ) + + +def _rocm_sparse_attn_decode_ragged_triton( + q: torch.Tensor, + main_cache: torch.Tensor, + main_indices: torch.Tensor, + main_indptr: torch.Tensor, + scale: float, + attn_sink: torch.Tensor | None, + nope_head_dim: int, + rope_head_dim: int, + extra_cache: torch.Tensor | None = None, + extra_indices: torch.Tensor | None = None, + extra_indptr: torch.Tensor | None = None, +) -> torch.Tensor: + assert q.ndim == 3, f"expected q=[b,h,d], got {q.shape}" + assert main_cache.ndim == 3, ( + f"expected main_cache=[blocks,block,bytes], got {main_cache.shape}" + ) + assert main_indices.ndim == 1, ( + f"expected main_indices=[nnz], got {main_indices.shape}" + ) + assert main_indptr.ndim == 1, f"expected main_indptr=[b+1], got {main_indptr.shape}" + assert ( + q.is_cuda + and main_cache.is_cuda + and main_indices.is_cuda + and main_indptr.is_cuda + ) + + main_indices = _as_int32_contiguous_1d(main_indices) + main_indptr = _as_int32_contiguous_1d(main_indptr) + has_attn_sink = attn_sink is not None + if attn_sink is None: + attn_sink = torch.empty(1, device=q.device, dtype=torch.float32) + else: + attn_sink = attn_sink.contiguous() + + num_queries, num_heads, head_dim = q.shape + assert main_indptr.numel() == num_queries + 1, ( + f"expected main_indptr shape [{num_queries + 1}], got {main_indptr.shape}" + ) + _validate_dsv4_sparse_dims( + head_dim, + nope_head_dim, + rope_head_dim, + "_rocm_sparse_attn_decode_ragged_triton", + ) + + has_extra = ( + extra_cache is not None + and extra_indices is not None + and extra_indptr is not None + ) + if has_extra: + assert extra_cache is not None + assert extra_indices is not None + assert extra_indptr is not None + assert extra_indices.ndim == 1, ( + f"expected extra_indices=[nnz], got {extra_indices.shape}" ) - lse_for_o = lse_for_o.clone() - lse_for_o[lse_for_o == float("-inf")] = float("+inf") - probs = torch.exp(scores - lse_for_o.unsqueeze(-1)) - out = probs @ gathered_kv[..., :head_dim] - lonely_q_mask = orig_lse == float("-inf") - out[lonely_q_mask.unsqueeze(-1).expand_as(out)] = 0.0 - return out.to(torch.bfloat16) + assert extra_indptr.ndim == 1, ( + f"expected extra_indptr=[b+1], got {extra_indptr.shape}" + ) + extra_indices = _as_int32_contiguous_1d(extra_indices) + extra_indptr = _as_int32_contiguous_1d(extra_indptr) + assert extra_indptr.numel() == num_queries + 1, ( + f"expected extra_indptr shape [{num_queries + 1}], got {extra_indptr.shape}" + ) + else: + extra_cache = main_cache + extra_indices = torch.empty(0, device=q.device, dtype=torch.int32) + extra_indptr = torch.zeros(num_queries + 1, device=q.device, dtype=torch.int32) + + block_h = 16 + block_k = 16 if head_dim >= 256 else 32 + out = torch.empty_like(q, dtype=torch.bfloat16) + _sparse_attn_decode_ragged_kernel[(num_queries, triton.cdiv(num_heads, block_h))]( + q, + main_cache, + main_indices, + main_indptr, + extra_cache, + extra_indices, + extra_indptr, + attn_sink, + out, + q.stride(0), + q.stride(1), + out.stride(0), + out.stride(1), + main_cache.stride(0), + extra_cache.stride(0), + main_cache.shape[0] * main_cache.shape[1], + extra_cache.shape[0] * extra_cache.shape[1], + main_cache.shape[1], + extra_cache.shape[1], + scale, + num_heads, + HAS_ATTN_SINK=has_attn_sink, + HAS_EXTRA=has_extra, + NOPE_DIM=nope_head_dim, + NOPE_BLOCK=triton.next_power_of_2(nope_head_dim), + ROPE_DIM=rope_head_dim, + IS_FNUZ=current_platform.is_fp8_fnuz(), + BLOCK_H=block_h, + BLOCK_K=block_k, + num_warps=8, + ) + return out + + +def _rocm_sparse_attn_decode_triton( + q: torch.Tensor, + main_cache: torch.Tensor, + main_indices: torch.Tensor, + scale: float, + attn_sink: torch.Tensor | None, + nope_head_dim: int, + rope_head_dim: int, + extra_cache: torch.Tensor | None = None, + extra_indices: torch.Tensor | None = None, + main_lengths: torch.Tensor | None = None, + extra_lengths: torch.Tensor | None = None, + main_ragged_indices: torch.Tensor | None = None, + main_ragged_indptr: torch.Tensor | None = None, + extra_ragged_indices: torch.Tensor | None = None, + extra_ragged_indptr: torch.Tensor | None = None, +) -> torch.Tensor: + if main_ragged_indices is None or main_ragged_indptr is None: + main_ragged_indices, main_ragged_indptr = build_ragged_indices_from_dense( + main_indices, + main_lengths + if main_lengths is not None + else (main_indices >= 0).sum(dim=-1, dtype=torch.int32), + num_rows=main_cache.shape[0] * main_cache.shape[1], + ) + + if ( + (extra_ragged_indices is None or extra_ragged_indptr is None) + and extra_cache is not None + and extra_indices is not None + ): + extra_ragged_indices, extra_ragged_indptr = build_ragged_indices_from_dense( + extra_indices, + extra_lengths + if extra_lengths is not None + else (extra_indices >= 0).sum(dim=-1, dtype=torch.int32), + num_rows=extra_cache.shape[0] * extra_cache.shape[1], + ) + + return _rocm_sparse_attn_decode_ragged_triton( + q=q, + main_cache=main_cache, + main_indices=main_ragged_indices, + main_indptr=main_ragged_indptr, + scale=scale, + attn_sink=attn_sink, + nope_head_dim=nope_head_dim, + rope_head_dim=rope_head_dim, + extra_cache=extra_cache, + extra_indices=extra_ragged_indices, + extra_indptr=extra_ragged_indptr, + ) def rocm_sparse_attn_prefill( @@ -958,132 +1613,49 @@ def rocm_sparse_attn_prefill( topk_length: torch.Tensor | None, scale: float, head_dim: int, + nope_head_dim: int, + rope_head_dim: int, attn_sink: torch.Tensor | None, output: torch.Tensor, + ragged_indices: torch.Tensor | None = None, + ragged_indptr: torch.Tensor | None = None, ) -> None: - output_chunk = rocm_ref_sparse_attn_prefill( - q=q, - kv=kv, - indices=indices, - topk_length=topk_length, - scale=scale, - head_dim=head_dim, - attn_sink=attn_sink, + assert kv.ndim == 3 and kv.shape[1] == 1, ( + f"ROCm Triton sparse prefill expects kv=[skv,1,d], got {kv.shape}" ) + _validate_dsv4_sparse_dims( + head_dim, + nope_head_dim, + rope_head_dim, + "rocm_sparse_attn_prefill", + ) + if ragged_indices is not None and ragged_indptr is not None: + output_chunk = _rocm_sparse_attn_prefill_ragged_triton( + q=q, + kv=kv.squeeze(1), + indices=ragged_indices, + indptr=ragged_indptr, + scale=scale, + attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]], + nope_head_dim=nope_head_dim, + rope_head_dim=rope_head_dim, + ) + else: + indices_2d = indices.reshape(indices.shape[0], -1) + output_chunk = _rocm_sparse_attn_prefill_triton( + q=q, + kv=kv.squeeze(1), + indices=indices_2d, + scale=scale, + attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]], + nope_head_dim=nope_head_dim, + rope_head_dim=rope_head_dim, + topk_length=topk_length, + ) output.copy_(output_chunk.to(output.dtype)) -def rocm_dequantize_blocked_k_cache( - quant_k_cache: torch.Tensor, - head_dim: int, - nope_head_dim: int, - rope_head_dim: int, -) -> torch.Tensor: - fp8_dtype = current_platform.fp8_dtype() - tile_size = 64 - num_tiles = nope_head_dim // tile_size - - num_blocks, block_size, _ = quant_k_cache.shape - quant_k_cache = quant_k_cache.view(num_blocks, -1) - input_nope_rope = quant_k_cache[ - :, : block_size * (nope_head_dim + 2 * rope_head_dim) - ].view(num_blocks, block_size, nope_head_dim + 2 * rope_head_dim) - input_nope = input_nope_rope[:, :, :nope_head_dim].view(fp8_dtype) - input_rope = input_nope_rope[:, :, nope_head_dim:].view(torch.bfloat16) - input_scale = ( - quant_k_cache[:, block_size * (nope_head_dim + 2 * rope_head_dim) :] - .view(num_blocks, block_size, 8)[:, :, :num_tiles] - .view(torch.float8_e8m0fnu) - ) - - result = torch.empty( - (num_blocks, block_size, 1, head_dim), - dtype=torch.bfloat16, - device=quant_k_cache.device, - ) - result[..., nope_head_dim:] = input_rope.unsqueeze(2) - for tile_idx in range(num_tiles): - cur_nope = input_nope[ - ..., tile_idx * tile_size : (tile_idx + 1) * tile_size - ].to(torch.bfloat16) - cur_scales = input_scale[:, :, tile_idx].to(torch.bfloat16).unsqueeze(-1) - result[..., tile_idx * tile_size : (tile_idx + 1) * tile_size] = ( - cur_nope * cur_scales - ).unsqueeze(2) - return result - - -def rocm_ref_sparse_attn_decode( - q: torch.Tensor, - blocked_k: torch.Tensor, - indices_in_kvcache: torch.Tensor, - topk_length: torch.Tensor | None, - scale: float, - head_dim: int, - attn_sink: torch.Tensor | None, - extra_blocked_k: torch.Tensor | None = None, - extra_indices_in_kvcache: torch.Tensor | None = None, - extra_topk_length: torch.Tensor | None = None, -) -> torch.Tensor: - b, s_q, h_q, d_qk = q.shape - - def process_scope( - cur_blocked_k: torch.Tensor, - cur_indices: torch.Tensor, - cur_topk_length: torch.Tensor | None, - ) -> tuple[torch.Tensor, torch.Tensor]: - cur_indices = cur_indices.reshape(b, s_q, -1) - topk = cur_indices.size(-1) - fixed_indices = torch.clamp_min(cur_indices, 0) - gathered_kv = ( - cur_blocked_k.view(-1, d_qk) - .index_select(0, fixed_indices.view(-1)) - .view(b, s_q, topk, d_qk) - ) - invalid_mask = cur_indices == -1 - if cur_topk_length is not None: - cur_topk_length = cur_topk_length.reshape(b) - invalid_mask |= torch.arange(0, topk, device=invalid_mask.device).view( - 1, 1, topk - ) >= cur_topk_length.view(b, 1, 1) - return gathered_kv, invalid_mask - - gathered_kv, invalid_mask = process_scope( - blocked_k, indices_in_kvcache, topk_length - ) - if extra_blocked_k is not None: - assert extra_indices_in_kvcache is not None - gathered_kv1, invalid_mask1 = process_scope( - extra_blocked_k, extra_indices_in_kvcache, extra_topk_length - ) - gathered_kv = torch.cat([gathered_kv, gathered_kv1], dim=2) - invalid_mask = torch.cat([invalid_mask, invalid_mask1], dim=2) - - gathered_kv = gathered_kv.view(b * s_q, -1, d_qk).float() - gathered_kv[gathered_kv != gathered_kv] = 0.0 - qf = q.float().view(b * s_q, h_q, d_qk) - attn_weight = qf @ gathered_kv.transpose(-1, -2) - attn_weight *= scale - attn_weight[ - invalid_mask.view(b * s_q, 1, -1).expand(b * s_q, h_q, invalid_mask.size(-1)) - ] = float("-inf") - lse = attn_weight.logsumexp(dim=-1) - attn_weight = torch.exp(attn_weight - lse.unsqueeze(-1)) - output = attn_weight @ gathered_kv[..., :head_dim] - output = output.view(b, s_q, h_q, head_dim) - lse = lse.view(b, s_q, h_q) - - if attn_sink is not None: - output *= (1.0 / (1.0 + torch.exp(attn_sink.view(1, 1, h_q) - lse))).unsqueeze( - -1 - ) - - lonely_q_mask = lse == float("-inf") - output[lonely_q_mask.unsqueeze(-1).expand_as(output)] = 0.0 - return output.squeeze(1).to(torch.bfloat16) - - -def rocm_forward_decode_fallback( +def rocm_sparse_attn_decode( q: torch.Tensor, kv_cache: torch.Tensor | None, swa_k_cache: torch.Tensor, @@ -1092,6 +1664,10 @@ def rocm_forward_decode_fallback( topk_lens: torch.Tensor | None, swa_indices: torch.Tensor, swa_lens: torch.Tensor, + swa_ragged_indices: torch.Tensor | None, + swa_ragged_indptr: torch.Tensor | None, + topk_ragged_indices: torch.Tensor | None, + topk_ragged_indptr: torch.Tensor | None, attn_sink: torch.Tensor | None, scale: float, head_dim: int, @@ -1099,31 +1675,49 @@ def rocm_forward_decode_fallback( rope_head_dim: int, output: torch.Tensor, ) -> None: - blocked_swa = rocm_dequantize_blocked_k_cache( - swa_k_cache, - head_dim=head_dim, - nope_head_dim=nope_head_dim, - rope_head_dim=rope_head_dim, + assert swa_k_cache.dtype == torch.uint8, ( + "ROCm Triton sparse decode expects uint8 fp8_ds_mla SWA cache, " + f"got {swa_k_cache.dtype}" ) - blocked_extra = None + _validate_dsv4_sparse_dims( + head_dim, + nope_head_dim, + rope_head_dim, + "rocm_sparse_attn_decode", + ) + + main_indices = swa_indices.reshape(swa_indices.shape[0], -1) + + extra_cache = None + extra_indices = None if not swa_only: assert kv_cache is not None - blocked_extra = rocm_dequantize_blocked_k_cache( - kv_cache, - head_dim=head_dim, - nope_head_dim=nope_head_dim, - rope_head_dim=rope_head_dim, + assert topk_indices is not None or ( + topk_ragged_indices is not None and topk_ragged_indptr is not None ) - attn_out = rocm_ref_sparse_attn_decode( - q=q.unsqueeze(1), - blocked_k=blocked_swa, - indices_in_kvcache=swa_indices.unsqueeze(1), - topk_length=swa_lens, + assert kv_cache.dtype == torch.uint8, ( + "ROCm Triton sparse decode expects uint8 fp8_ds_mla extra cache, " + f"got {kv_cache.dtype}" + ) + extra_cache = kv_cache + if topk_indices is not None: + extra_indices = topk_indices.reshape(topk_indices.shape[0], -1) + + attn_out = _rocm_sparse_attn_decode_triton( + q=q, + main_cache=swa_k_cache, + main_indices=main_indices, scale=scale, - head_dim=head_dim, - attn_sink=attn_sink[: q.shape[1]] if attn_sink is not None else None, - extra_blocked_k=blocked_extra, - extra_indices_in_kvcache=topk_indices, - extra_topk_length=topk_lens, + attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]], + nope_head_dim=nope_head_dim, + rope_head_dim=rope_head_dim, + extra_cache=extra_cache, + extra_indices=extra_indices, + main_lengths=swa_lens, + extra_lengths=topk_lens, + main_ragged_indices=swa_ragged_indices, + main_ragged_indptr=swa_ragged_indptr, + extra_ragged_indices=topk_ragged_indices, + extra_ragged_indptr=topk_ragged_indptr, ) output.copy_(attn_out.to(output.dtype))