[ROCm][DSv4] implement flash sparse mla with triton kernels (#41812)
Signed-off-by: whx-sjtu <xiaowang990929@gmail.com>
This commit is contained in:
@@ -0,0 +1,377 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="Only used by ROCm"
|
||||
)
|
||||
|
||||
NOPE_HEAD_DIM = 448
|
||||
ROPE_HEAD_DIM = 64
|
||||
HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM
|
||||
|
||||
|
||||
def _ref_global_topk_ragged(
|
||||
topk_indices: torch.Tensor,
|
||||
token_to_req_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
is_valid_token: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
topk = topk_indices.reshape(topk_indices.shape[0], -1)
|
||||
valid = (topk >= 0) & is_valid_token[:, None]
|
||||
lens = valid.sum(dim=1, dtype=torch.int32)
|
||||
indptr = torch.zeros(lens.shape[0] + 1, dtype=torch.int32, device=topk.device)
|
||||
torch.cumsum(lens, dim=0, out=indptr[1:])
|
||||
|
||||
safe_topk = torch.clamp(topk, min=0)
|
||||
block_indices = safe_topk // block_size
|
||||
block_offsets = safe_topk % block_size
|
||||
req_indices = token_to_req_indices[:, None].expand_as(topk)
|
||||
slot_ids = block_table[req_indices, block_indices] * block_size + block_offsets
|
||||
|
||||
offsets = torch.arange(topk.shape[1], dtype=torch.int32, device=topk.device)
|
||||
positions = indptr[:-1, None] + offsets[None, :]
|
||||
return slot_ids[valid], positions[valid].to(torch.long), indptr, lens
|
||||
|
||||
|
||||
def _ref_sparse_prefill_ragged(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
rows: list[list[int]],
|
||||
scale: float,
|
||||
attn_sink: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
q_f32 = q.float()
|
||||
kv_f32 = kv.float()
|
||||
out = torch.empty_like(q_f32)
|
||||
|
||||
for query_idx in range(q.shape[0]):
|
||||
row_indices = rows[query_idx]
|
||||
for head_idx in range(q.shape[1]):
|
||||
if row_indices:
|
||||
selected_kv = kv_f32[row_indices]
|
||||
scores = torch.mv(selected_kv, q_f32[query_idx, head_idx]) * scale
|
||||
if attn_sink is not None:
|
||||
scores_with_sink = torch.cat(
|
||||
[scores, attn_sink[head_idx].float().reshape(1)]
|
||||
)
|
||||
probs = torch.softmax(scores_with_sink, dim=0)[:-1]
|
||||
else:
|
||||
probs = torch.softmax(scores, dim=0)
|
||||
out[query_idx, head_idx] = torch.sum(
|
||||
probs[:, None] * selected_kv, dim=0
|
||||
)
|
||||
else:
|
||||
out[query_idx, head_idx] = 0
|
||||
return out.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _pack_fp8_ds_mla_cache(kv: torch.Tensor, block_size: int) -> torch.Tensor:
|
||||
assert kv.shape[-1] == HEAD_DIM
|
||||
num_tokens = kv.shape[0]
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size
|
||||
cache = torch.zeros(
|
||||
(num_blocks, block_size, 584),
|
||||
dtype=torch.uint8,
|
||||
device=kv.device,
|
||||
)
|
||||
cache_flat = cache.view(torch.uint8).flatten()
|
||||
kv_nope_fp8 = (
|
||||
kv[:, :NOPE_HEAD_DIM].to(current_platform.fp8_dtype()).view(torch.uint8)
|
||||
)
|
||||
kv_rope_u8 = kv[:, NOPE_HEAD_DIM:].contiguous().view(torch.uint8)
|
||||
|
||||
for slot in range(num_tokens):
|
||||
block_idx = slot // block_size
|
||||
pos = slot % block_size
|
||||
block_base = block_idx * cache.stride(0)
|
||||
token_base = block_base + pos * 576
|
||||
scale_base = block_base + block_size * 576 + pos * 8
|
||||
cache_flat[token_base : token_base + NOPE_HEAD_DIM].copy_(kv_nope_fp8[slot])
|
||||
cache_flat[
|
||||
token_base + NOPE_HEAD_DIM : token_base + NOPE_HEAD_DIM + ROPE_HEAD_DIM * 2
|
||||
].copy_(kv_rope_u8[slot])
|
||||
cache_flat[scale_base : scale_base + 7].fill_(127)
|
||||
return cache
|
||||
|
||||
|
||||
def _read_fp8_ds_mla_cache(
|
||||
cache: torch.Tensor, slot: int, block_size: int
|
||||
) -> torch.Tensor:
|
||||
cache_flat = cache.view(torch.uint8).flatten()
|
||||
block_idx = slot // block_size
|
||||
pos = slot % block_size
|
||||
block_base = block_idx * cache.stride(0)
|
||||
token_base = block_base + pos * 576
|
||||
|
||||
nope_u8 = cache_flat[token_base : token_base + NOPE_HEAD_DIM]
|
||||
nope = nope_u8.view(current_platform.fp8_dtype()).to(torch.float32)
|
||||
rope_u8 = cache_flat[
|
||||
token_base + NOPE_HEAD_DIM : token_base + NOPE_HEAD_DIM + ROPE_HEAD_DIM * 2
|
||||
]
|
||||
rope = rope_u8.view(torch.bfloat16).to(torch.float32)
|
||||
return torch.cat([nope, rope])
|
||||
|
||||
|
||||
def _ref_sparse_decode_ragged(
|
||||
q: torch.Tensor,
|
||||
main_cache: torch.Tensor,
|
||||
main_rows: list[list[int]],
|
||||
scale: float,
|
||||
attn_sink: torch.Tensor | None,
|
||||
block_size: int,
|
||||
extra_cache: torch.Tensor | None = None,
|
||||
extra_rows: list[list[int]] | None = None,
|
||||
) -> torch.Tensor:
|
||||
q_f32 = q.float()
|
||||
out = torch.empty_like(q_f32)
|
||||
|
||||
for query_idx in range(q.shape[0]):
|
||||
row_kv = [
|
||||
_read_fp8_ds_mla_cache(main_cache, int(slot), block_size)
|
||||
for slot in main_rows[query_idx]
|
||||
]
|
||||
if extra_cache is not None and extra_rows is not None:
|
||||
row_kv.extend(
|
||||
_read_fp8_ds_mla_cache(extra_cache, int(slot), block_size)
|
||||
for slot in extra_rows[query_idx]
|
||||
)
|
||||
|
||||
kv = torch.stack(row_kv).to(q.device)
|
||||
for head_idx in range(q.shape[1]):
|
||||
scores = torch.mv(kv, q_f32[query_idx, head_idx]) * scale
|
||||
if attn_sink is not None:
|
||||
scores_with_sink = torch.cat(
|
||||
[scores, attn_sink[head_idx].float().reshape(1)]
|
||||
)
|
||||
probs = torch.softmax(scores_with_sink, dim=0)[:-1]
|
||||
else:
|
||||
probs = torch.softmax(scores, dim=0)
|
||||
out[query_idx, head_idx] = torch.sum(probs[:, None] * kv, dim=0)
|
||||
return out.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _ref_combine_topk_swa_ragged(
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
expected_ragged = torch.tensor(
|
||||
[
|
||||
100,
|
||||
101,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
110,
|
||||
111,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
120,
|
||||
121,
|
||||
122,
|
||||
9,
|
||||
10,
|
||||
11,
|
||||
150,
|
||||
27,
|
||||
28,
|
||||
29,
|
||||
160,
|
||||
161,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
expected_lens = torch.tensor([5, 5, 6, 4, 5], dtype=torch.int32, device=device)
|
||||
expected_indptr = torch.zeros(6, dtype=torch.int32, device=device)
|
||||
torch.cumsum(expected_lens, dim=0, out=expected_indptr[1:])
|
||||
return expected_ragged, expected_indptr, expected_lens
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_compute_global_topk_ragged_indices_and_indptr() -> None:
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
|
||||
compute_global_topk_ragged_indices_and_indptr,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
block_size = 4
|
||||
topk_indices = torch.tensor(
|
||||
[
|
||||
[0, 3, 4, -1],
|
||||
[5, 8, -1, -1],
|
||||
[2, 7, 9, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
token_to_req_indices = torch.tensor([0, 1, 1], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor(
|
||||
[
|
||||
[10, 11, 12],
|
||||
[20, 21, 22],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
is_valid_token = torch.tensor([True, False, True], dtype=torch.bool, device=device)
|
||||
|
||||
actual_ragged, actual_indptr, actual_lens = (
|
||||
compute_global_topk_ragged_indices_and_indptr(
|
||||
topk_indices,
|
||||
token_to_req_indices,
|
||||
block_table,
|
||||
block_size,
|
||||
is_valid_token,
|
||||
)
|
||||
)
|
||||
expected_values, expected_positions, expected_indptr, expected_lens = (
|
||||
_ref_global_topk_ragged(
|
||||
topk_indices,
|
||||
token_to_req_indices,
|
||||
block_table,
|
||||
block_size,
|
||||
is_valid_token,
|
||||
)
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ragged[expected_positions], expected_values)
|
||||
torch.testing.assert_close(actual_indptr, expected_indptr)
|
||||
torch.testing.assert_close(actual_lens, expected_lens)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_ragged_kernel() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
q = torch.randn(3, 3, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
kv = torch.randn(5, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
indices = torch.tensor([0, 2, 1, 3, 4], dtype=torch.int32, device=device)
|
||||
indptr = torch.tensor([0, 2, 5, 5], dtype=torch.int32, device=device)
|
||||
attn_sink = torch.tensor([-0.25, 0.0, 0.25], dtype=torch.float32, device=device)
|
||||
scale = HEAD_DIM**-0.5
|
||||
|
||||
actual = _rocm_sparse_attn_prefill_ragged_triton(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
indptr=indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
)
|
||||
expected = _ref_sparse_prefill_ragged(
|
||||
q, kv, [[0, 2], [1, 3, 4], []], scale, attn_sink
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_ragged_kernel() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_decode_ragged_triton,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(1)
|
||||
block_size = 4
|
||||
q = torch.randn(2, 3, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
main_kv = torch.randn(6, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
extra_kv = torch.randn(5, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
main_cache = _pack_fp8_ds_mla_cache(main_kv, block_size)
|
||||
extra_cache = _pack_fp8_ds_mla_cache(extra_kv, block_size)
|
||||
main_indices = torch.tensor([0, 2, 4, 1], dtype=torch.int32, device=device)
|
||||
main_indptr = torch.tensor([0, 2, 4], dtype=torch.int32, device=device)
|
||||
extra_indices = torch.tensor([1, 3, 0], dtype=torch.int32, device=device)
|
||||
extra_indptr = torch.tensor([0, 1, 3], dtype=torch.int32, device=device)
|
||||
attn_sink = torch.tensor([-0.1, 0.0, 0.1], dtype=torch.float32, device=device)
|
||||
scale = HEAD_DIM**-0.5
|
||||
|
||||
actual = _rocm_sparse_attn_decode_ragged_triton(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
main_indices=main_indices,
|
||||
main_indptr=main_indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
extra_cache=extra_cache,
|
||||
extra_indices=extra_indices,
|
||||
extra_indptr=extra_indptr,
|
||||
)
|
||||
expected = _ref_sparse_decode_ragged(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
main_rows=[[0, 2], [4, 1]],
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
block_size=block_size,
|
||||
extra_cache=extra_cache,
|
||||
extra_rows=[[1], [3, 0]],
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_combine_topk_swa_indices_ragged() -> None:
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
|
||||
combine_topk_swa_indices_ragged,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
topk_indices = torch.tensor(
|
||||
[
|
||||
[100, 101, 102, 103],
|
||||
[110, 111, 112, 113],
|
||||
[120, 121, 122, 123],
|
||||
[130, 131, 132, 133],
|
||||
[140, 141, 142, 143],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
query_start_loc = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
|
||||
seq_lens = torch.tensor([6, 4], dtype=torch.int32, device=device)
|
||||
gather_lens = torch.tensor([4, 3], dtype=torch.int32, device=device)
|
||||
window_size = 3
|
||||
compress_ratio = 2
|
||||
topk = 4
|
||||
M = 20
|
||||
N = 8
|
||||
|
||||
actual_ragged, actual_indptr, actual_lens = combine_topk_swa_indices_ragged(
|
||||
topk_indices,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
gather_lens,
|
||||
window_size,
|
||||
compress_ratio,
|
||||
topk,
|
||||
M,
|
||||
N,
|
||||
)
|
||||
expected_ragged, expected_indptr, expected_lens = _ref_combine_topk_swa_ragged(
|
||||
device
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual_ragged[: expected_ragged.numel()], expected_ragged
|
||||
)
|
||||
torch.testing.assert_close(actual_indptr, expected_indptr)
|
||||
torch.testing.assert_close(actual_lens, expected_lens)
|
||||
@@ -28,11 +28,7 @@ from vllm.v1.attention.ops.deepseek_v4_ops import (
|
||||
fused_inv_rope_fp8_quant,
|
||||
fused_q_kv_rmsnorm,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_forward_decode_fallback,
|
||||
rocm_inv_rope_einsum,
|
||||
rocm_sparse_attn_prefill,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import rocm_inv_rope_einsum
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
@@ -725,6 +721,12 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.kv_cache = torch.tensor([])
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
if current_platform.is_rocm():
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
|
||||
DeepseekV4ROCMAiterMLASparseBackend,
|
||||
)
|
||||
|
||||
return DeepseekV4ROCMAiterMLASparseBackend
|
||||
return DeepseekV4FlashMLASparseBackend
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
@@ -757,6 +759,14 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}"
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
|
||||
DeepseekV4ROCMAiterMLASparseImpl,
|
||||
)
|
||||
|
||||
DeepseekV4ROCMAiterMLASparseImpl.forward(self, q, kv, positions, output)
|
||||
return
|
||||
|
||||
# Get SWA and indexer metadata from forward context
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
@@ -839,25 +849,6 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
swa_indices = swa_metadata.decode_swa_indices
|
||||
swa_lens = swa_metadata.decode_swa_lens
|
||||
|
||||
if current_platform.is_rocm():
|
||||
rocm_forward_decode_fallback(
|
||||
q=q,
|
||||
kv_cache=kv_cache,
|
||||
swa_k_cache=self.swa_cache_layer.kv_cache,
|
||||
swa_only=swa_only,
|
||||
topk_indices=topk_indices,
|
||||
topk_lens=topk_lens,
|
||||
swa_indices=swa_indices,
|
||||
swa_lens=swa_lens,
|
||||
attn_sink=self.attn_sink,
|
||||
scale=self.scale,
|
||||
head_dim=self.head_dim,
|
||||
nope_head_dim=self.nope_head_dim,
|
||||
rope_head_dim=self.rope_head_dim,
|
||||
output=output,
|
||||
)
|
||||
return
|
||||
|
||||
# We treat queries in the same seq as different queries
|
||||
# and later we only attend by generated indices.
|
||||
# q arrives pre-padded to self.padded_heads by the outer wrapper.
|
||||
@@ -1021,28 +1012,15 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
M,
|
||||
N,
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
rocm_sparse_attn_prefill(
|
||||
q=q[query_start:query_end],
|
||||
kv=kv.view(-1, 1, q.shape[-1]),
|
||||
indices=combined_indices.unsqueeze(1),
|
||||
topk_length=combined_lens,
|
||||
scale=self.scale,
|
||||
head_dim=self.head_dim,
|
||||
attn_sink=self.attn_sink,
|
||||
output=output[query_start:query_end],
|
||||
)
|
||||
else:
|
||||
output_chunk, _, _ = flash_mla_sparse_fwd(
|
||||
q=q[query_start:query_end],
|
||||
kv=kv.view(-1, 1, q.shape[-1]),
|
||||
indices=combined_indices.unsqueeze(1),
|
||||
sm_scale=self.scale,
|
||||
attn_sink=self.attn_sink,
|
||||
topk_length=combined_lens,
|
||||
out=output[query_start:query_end],
|
||||
)
|
||||
flash_mla_sparse_fwd(
|
||||
q=q[query_start:query_end],
|
||||
kv=kv.view(-1, 1, q.shape[-1]),
|
||||
indices=combined_indices.unsqueeze(1),
|
||||
sm_scale=self.scale,
|
||||
attn_sink=self.attn_sink,
|
||||
topk_length=combined_lens,
|
||||
out=output[query_start:query_end],
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -112,7 +112,7 @@ class FlashMLASparseBackend(AttentionBackend):
|
||||
return FlashMLASparseMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["FlashMLASparseImpl"]:
|
||||
def get_impl_cls() -> type[SparseMLAAttentionImpl[Any]]:
|
||||
return FlashMLASparseImpl
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -0,0 +1,682 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionLayer,
|
||||
CommonAttentionMetadata,
|
||||
SparseMLAAttentionImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
DeepseekV4FlashMLASparseBackend,
|
||||
FlashMLASparseMetadata,
|
||||
FlashMLASparseMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
DeepseekSparseSWAMetadata,
|
||||
DeepseekSparseSWAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.deepseek_v4_ops import dequantize_and_gather_k_cache
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
build_ragged_indices_from_dense,
|
||||
rocm_sparse_attn_decode,
|
||||
rocm_sparse_attn_prefill,
|
||||
)
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.deepseek_v4_attention import (
|
||||
DeepseekV4MLAAttention,
|
||||
)
|
||||
|
||||
|
||||
def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
|
||||
lengths = lengths.to(dtype=torch.int32).contiguous()
|
||||
indptr = torch.zeros(lengths.shape[0] + 1, dtype=torch.int32, device=lengths.device)
|
||||
torch.cumsum(lengths, dim=0, out=indptr[1:])
|
||||
return indptr
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compute_topk_lens_kernel(
|
||||
topk_lens_ptr,
|
||||
topk_indices_ptr,
|
||||
topk_indices_stride,
|
||||
topk,
|
||||
is_valid_token_ptr,
|
||||
TRITON_BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
is_valid_token = tl.load(is_valid_token_ptr + token_idx)
|
||||
|
||||
count = tl.zeros((), dtype=tl.int32)
|
||||
for i in range(0, topk, TRITON_BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, TRITON_BLOCK_SIZE)
|
||||
mask = offset < topk
|
||||
local_idx = tl.load(
|
||||
topk_indices_ptr + token_idx * topk_indices_stride + offset,
|
||||
mask=mask,
|
||||
other=-1,
|
||||
)
|
||||
count += tl.sum((local_idx >= 0).to(tl.int32), axis=0)
|
||||
|
||||
tl.store(topk_lens_ptr + token_idx, tl.where(is_valid_token, count, 0))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_global_topk_ragged_kernel(
|
||||
global_topk_ragged_ptr,
|
||||
topk_indptr_ptr,
|
||||
topk_indices_ptr,
|
||||
topk_indices_stride,
|
||||
token_to_req_indices_ptr,
|
||||
block_table_ptr,
|
||||
block_table_stride,
|
||||
block_size,
|
||||
topk,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
block_idx = tl.program_id(1)
|
||||
offset = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
|
||||
out_start = tl.load(topk_indptr_ptr + token_idx)
|
||||
out_end = tl.load(topk_indptr_ptr + token_idx + 1)
|
||||
out_len = out_end - out_start
|
||||
if block_idx * BLOCK_SIZE >= out_len:
|
||||
return
|
||||
|
||||
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
|
||||
mask = (offset < out_len) & (offset < topk)
|
||||
local_idx = tl.load(
|
||||
topk_indices_ptr + token_idx * topk_indices_stride + offset,
|
||||
mask=mask,
|
||||
other=-1,
|
||||
)
|
||||
valid = mask & (local_idx >= 0)
|
||||
block_indices = local_idx // block_size
|
||||
block_numbers = tl.load(
|
||||
block_table_ptr + req_idx * block_table_stride + block_indices,
|
||||
mask=valid,
|
||||
other=0,
|
||||
)
|
||||
block_offsets = local_idx % block_size
|
||||
slot_ids = tl.where(valid, block_numbers * block_size + block_offsets, -1)
|
||||
tl.store(global_topk_ragged_ptr + out_start + offset, slot_ids, mask=mask)
|
||||
|
||||
|
||||
def compute_global_topk_ragged_indices_and_indptr(
|
||||
topk_indices: torch.Tensor,
|
||||
token_to_req_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
is_valid_token: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous()
|
||||
num_tokens = topk_indices.shape[0]
|
||||
topk = topk_indices.shape[1]
|
||||
|
||||
topk_lens = torch.empty(num_tokens, dtype=torch.int32, device=topk_indices.device)
|
||||
_compute_topk_lens_kernel[(num_tokens,)](
|
||||
topk_lens,
|
||||
topk_indices,
|
||||
topk_indices.stride(0),
|
||||
topk,
|
||||
is_valid_token,
|
||||
TRITON_BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
topk_indptr = _build_indptr_from_lengths(topk_lens)
|
||||
global_topk_ragged = torch.empty(
|
||||
num_tokens * topk,
|
||||
dtype=torch.int32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
if global_topk_ragged.numel() > 0:
|
||||
block = 128
|
||||
_pack_global_topk_ragged_kernel[(num_tokens, triton.cdiv(topk, block))](
|
||||
global_topk_ragged,
|
||||
topk_indptr,
|
||||
topk_indices,
|
||||
topk_indices.stride(0),
|
||||
token_to_req_indices,
|
||||
block_table,
|
||||
block_table.stride(0),
|
||||
block_size,
|
||||
topk,
|
||||
BLOCK_SIZE=block,
|
||||
)
|
||||
return global_topk_ragged, topk_indptr, topk_lens
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compute_combined_lens_kernel(
|
||||
combined_lens_ptr,
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
TOP_K: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
num_workers = tl.num_programs(1)
|
||||
|
||||
base = tl.load(query_start_loc_ptr)
|
||||
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
|
||||
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
|
||||
query_len = query_end - query_start
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
start_pos = seq_len - query_len
|
||||
|
||||
for token_idx in range(query_start + worker_id, query_end, num_workers):
|
||||
token_idx_in_query = token_idx - query_start
|
||||
pos = start_pos + token_idx_in_query
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
|
||||
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
|
||||
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _combine_topk_swa_indices_ragged_kernel(
|
||||
combined_ragged_ptr,
|
||||
combined_indptr_ptr,
|
||||
topk_indices_ptr,
|
||||
topk_indices_stride,
|
||||
query_start_loc_ptr,
|
||||
seq_lens_ptr,
|
||||
gather_lens_ptr,
|
||||
M,
|
||||
N,
|
||||
topk_width,
|
||||
TOP_K: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
block_idx = tl.program_id(2)
|
||||
num_workers = tl.num_programs(1)
|
||||
|
||||
base = tl.load(query_start_loc_ptr)
|
||||
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
|
||||
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
|
||||
query_len = query_end - query_start
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx)
|
||||
start_pos = seq_len - query_len
|
||||
gather_start = seq_len - gather_len
|
||||
|
||||
for token_idx in range(query_start + worker_id, query_end, num_workers):
|
||||
token_idx_in_query = token_idx - query_start
|
||||
pos = start_pos + token_idx_in_query
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
|
||||
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
|
||||
combined_len = topk_len + swa_len
|
||||
|
||||
offset = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
if block_idx * BLOCK_SIZE < combined_len:
|
||||
out_start = tl.load(combined_indptr_ptr + token_idx)
|
||||
topk_mask = (offset < topk_len) & (offset < topk_width)
|
||||
topk_vals = tl.load(
|
||||
topk_indices_ptr + token_idx * topk_indices_stride + offset,
|
||||
mask=topk_mask,
|
||||
other=-1,
|
||||
)
|
||||
tl.store(
|
||||
combined_ragged_ptr + out_start + offset,
|
||||
topk_vals + M * batch_idx,
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
swa_offset = offset - topk_len
|
||||
swa_mask = (offset >= topk_len) & (swa_offset < swa_len)
|
||||
tl.store(
|
||||
combined_ragged_ptr + out_start + offset,
|
||||
M * batch_idx + N + swa_offset + pos - swa_len + 1 - gather_start,
|
||||
mask=swa_mask,
|
||||
)
|
||||
|
||||
|
||||
def combine_topk_swa_indices_ragged(
|
||||
topk_indices: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
gather_lens: torch.Tensor,
|
||||
window_size: int,
|
||||
compress_ratio: int,
|
||||
topk: int,
|
||||
M: int,
|
||||
N: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous()
|
||||
num_tokens = topk_indices.shape[0]
|
||||
num_reqs = seq_lens.shape[0]
|
||||
combined_lens = torch.empty(
|
||||
num_tokens, dtype=torch.int32, device=topk_indices.device
|
||||
)
|
||||
|
||||
num_workers = 128
|
||||
_compute_combined_lens_kernel[(num_reqs, num_workers)](
|
||||
combined_lens,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
TOP_K=topk,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
WINDOW_SIZE=window_size,
|
||||
)
|
||||
|
||||
combined_indptr = _build_indptr_from_lengths(combined_lens)
|
||||
combined_ragged = torch.empty(
|
||||
num_tokens * (topk + window_size),
|
||||
dtype=torch.int32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
if combined_ragged.numel() > 0:
|
||||
block = 128
|
||||
_combine_topk_swa_indices_ragged_kernel[
|
||||
(num_reqs, num_workers, triton.cdiv(topk + window_size, block))
|
||||
](
|
||||
combined_ragged,
|
||||
combined_indptr,
|
||||
topk_indices,
|
||||
topk_indices.stride(0),
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
gather_lens,
|
||||
M,
|
||||
N,
|
||||
topk_indices.shape[-1],
|
||||
TOP_K=topk,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
WINDOW_SIZE=window_size,
|
||||
BLOCK_SIZE=block,
|
||||
)
|
||||
return combined_ragged, combined_indptr, combined_lens
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeepseekV4ROCMAiterMLASparseMetadata(FlashMLASparseMetadata):
|
||||
"""ROCm-specific DeepSeek V4 metadata carrying ragged decode topk."""
|
||||
|
||||
c128a_decode_topk_ragged_indices: torch.Tensor | None = None
|
||||
c128a_decode_topk_ragged_indptr: torch.Tensor | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
|
||||
decode_swa_ragged_indices: torch.Tensor | None = None
|
||||
decode_swa_ragged_indptr: torch.Tensor | None = None
|
||||
|
||||
|
||||
class DeepseekV4ROCMAiterMLASparseMetadataBuilder(FlashMLASparseMetadataBuilder):
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> DeepseekV4ROCMAiterMLASparseMetadata:
|
||||
base = super().build(
|
||||
common_prefix_len=common_prefix_len,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
fast_build=fast_build,
|
||||
)
|
||||
|
||||
ragged_indices = None
|
||||
ragged_indptr = None
|
||||
dense_decode = base.c128a_global_decode_topk_indices
|
||||
decode_lens = base.c128a_decode_topk_lens
|
||||
if dense_decode is not None and decode_lens is not None:
|
||||
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
|
||||
dense_decode.reshape(dense_decode.shape[0], -1),
|
||||
decode_lens,
|
||||
)
|
||||
|
||||
return DeepseekV4ROCMAiterMLASparseMetadata(
|
||||
**vars(base),
|
||||
c128a_decode_topk_ragged_indices=ragged_indices,
|
||||
c128a_decode_topk_ragged_indptr=ragged_indptr,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
fast_build: bool = False,
|
||||
) -> DeepseekV4ROCMAiterSparseSWAMetadata:
|
||||
base = super().build(
|
||||
common_prefix_len=common_prefix_len,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
fast_build=fast_build,
|
||||
)
|
||||
|
||||
ragged_indices = None
|
||||
ragged_indptr = None
|
||||
if (
|
||||
base.num_decode_tokens > 0
|
||||
and base.decode_swa_indices is not None
|
||||
and base.decode_swa_lens is not None
|
||||
):
|
||||
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
|
||||
base.decode_swa_indices.reshape(base.num_decode_tokens, -1),
|
||||
base.decode_swa_lens,
|
||||
)
|
||||
|
||||
return DeepseekV4ROCMAiterSparseSWAMetadata(
|
||||
**vars(base),
|
||||
decode_swa_ragged_indices=ragged_indices,
|
||||
decode_swa_ragged_indptr=ragged_indptr,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4ROCMAiterMLASparseImpl(
|
||||
SparseMLAAttentionImpl[DeepseekV4ROCMAiterMLASparseMetadata]
|
||||
):
|
||||
"""ROCm sparse MLA implementation used by DeepSeek V4's custom MLA layer."""
|
||||
|
||||
_PREFILL_CHUNK_SIZE = 4
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
scale: float,
|
||||
num_kv_heads: int,
|
||||
alibi_slopes: list[float] | None,
|
||||
sliding_window: int | None,
|
||||
kv_cache_dtype: str,
|
||||
logits_soft_cap: float | None,
|
||||
attn_type: str,
|
||||
kv_sharing_target_layer_name: str | None,
|
||||
**_: object,
|
||||
) -> None:
|
||||
self.num_heads = num_heads
|
||||
self.head_size = head_size
|
||||
self.scale = float(scale)
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
kv_c_and_k_pe_cache: torch.Tensor,
|
||||
attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata,
|
||||
layer: AttentionLayer,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
raise NotImplementedError(
|
||||
"DeepseekV4ROCMAiterMLASparseImpl is driven by "
|
||||
"DeepseekV4MLAAttention.forward."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def forward(
|
||||
cls,
|
||||
layer: "DeepseekV4MLAAttention",
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
assert output.shape == q.shape, (
|
||||
f"output buffer shape {output.shape} must match q shape {q.shape}"
|
||||
)
|
||||
assert output.dtype == q.dtype, (
|
||||
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}"
|
||||
)
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
assert isinstance(attn_metadata, dict)
|
||||
rocm_metadata = cast(
|
||||
DeepseekV4ROCMAiterMLASparseMetadata | None,
|
||||
attn_metadata.get(layer.prefix),
|
||||
)
|
||||
swa_metadata = cast(
|
||||
DeepseekV4ROCMAiterSparseSWAMetadata | None,
|
||||
attn_metadata.get(layer.swa_cache_layer.prefix),
|
||||
)
|
||||
assert swa_metadata is not None
|
||||
|
||||
swa_only = layer.compress_ratio <= 1
|
||||
self_kv_cache = layer.kv_cache if not swa_only else None
|
||||
swa_kv_cache = layer.swa_cache_layer.kv_cache
|
||||
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
if num_prefills > 0:
|
||||
cls._forward_prefill(
|
||||
layer=layer,
|
||||
q=q[num_decode_tokens:],
|
||||
positions=positions[num_decode_tokens:],
|
||||
compressed_k_cache=self_kv_cache,
|
||||
swa_k_cache=swa_kv_cache,
|
||||
output=output[num_decode_tokens:],
|
||||
attn_metadata=rocm_metadata,
|
||||
swa_metadata=swa_metadata,
|
||||
)
|
||||
if num_decodes > 0:
|
||||
cls._forward_decode(
|
||||
layer=layer,
|
||||
q=q[:num_decode_tokens],
|
||||
kv_cache=self_kv_cache,
|
||||
swa_metadata=swa_metadata,
|
||||
attn_metadata=rocm_metadata,
|
||||
swa_only=swa_only,
|
||||
output=output[:num_decode_tokens],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _forward_decode(
|
||||
cls,
|
||||
layer: "DeepseekV4MLAAttention",
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor | None,
|
||||
swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata,
|
||||
attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None,
|
||||
swa_only: bool,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
topk_indices = None
|
||||
topk_lens = None
|
||||
topk_ragged_indices = None
|
||||
topk_ragged_indptr = None
|
||||
if not swa_only:
|
||||
assert attn_metadata is not None
|
||||
assert swa_metadata.is_valid_token is not None
|
||||
block_size = attn_metadata.block_size // layer.compress_ratio
|
||||
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
|
||||
if layer.compress_ratio == 4:
|
||||
assert layer.topk_indices_buffer is not None
|
||||
(
|
||||
topk_ragged_indices,
|
||||
topk_ragged_indptr,
|
||||
topk_lens,
|
||||
) = compute_global_topk_ragged_indices_and_indptr(
|
||||
layer.topk_indices_buffer[:num_decode_tokens],
|
||||
swa_metadata.token_to_req_indices,
|
||||
attn_metadata.block_table[:num_decodes],
|
||||
block_size,
|
||||
is_valid,
|
||||
)
|
||||
else:
|
||||
topk_indices = attn_metadata.c128a_global_decode_topk_indices
|
||||
topk_lens = attn_metadata.c128a_decode_topk_lens
|
||||
topk_ragged_indices = attn_metadata.c128a_decode_topk_ragged_indices
|
||||
topk_ragged_indptr = attn_metadata.c128a_decode_topk_ragged_indptr
|
||||
|
||||
rocm_sparse_attn_decode(
|
||||
q=q,
|
||||
kv_cache=kv_cache,
|
||||
swa_k_cache=layer.swa_cache_layer.kv_cache,
|
||||
swa_only=swa_only,
|
||||
topk_indices=topk_indices,
|
||||
topk_lens=topk_lens,
|
||||
swa_indices=swa_metadata.decode_swa_indices,
|
||||
swa_lens=swa_metadata.decode_swa_lens,
|
||||
swa_ragged_indices=swa_metadata.decode_swa_ragged_indices,
|
||||
swa_ragged_indptr=swa_metadata.decode_swa_ragged_indptr,
|
||||
topk_ragged_indices=topk_ragged_indices,
|
||||
topk_ragged_indptr=topk_ragged_indptr,
|
||||
attn_sink=layer.attn_sink,
|
||||
scale=layer.scale,
|
||||
head_dim=layer.head_dim,
|
||||
nope_head_dim=layer.nope_head_dim,
|
||||
rope_head_dim=layer.rope_head_dim,
|
||||
output=output,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _forward_prefill(
|
||||
cls,
|
||||
layer: "DeepseekV4MLAAttention",
|
||||
q: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
compressed_k_cache: torch.Tensor | None,
|
||||
swa_k_cache: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None,
|
||||
swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata,
|
||||
) -> None:
|
||||
swa_only = attn_metadata is None
|
||||
|
||||
num_prefills = swa_metadata.num_prefills
|
||||
num_prefill_tokens = swa_metadata.num_prefill_tokens
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
|
||||
seq_lens = swa_metadata.prefill_seq_lens
|
||||
gather_lens = swa_metadata.prefill_gather_lens
|
||||
assert seq_lens is not None
|
||||
assert gather_lens is not None
|
||||
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
query_start_loc = swa_metadata.query_start_loc
|
||||
assert query_start_loc_cpu is not None
|
||||
assert query_start_loc is not None
|
||||
prefill_token_base = query_start_loc_cpu[num_decodes]
|
||||
|
||||
if not swa_only:
|
||||
if layer.compress_ratio == 4:
|
||||
assert layer.topk_indices_buffer is not None
|
||||
topk_indices = layer.topk_indices_buffer[num_decode_tokens:]
|
||||
topk_indices = topk_indices[:num_prefill_tokens]
|
||||
else:
|
||||
assert attn_metadata is not None
|
||||
topk_indices = attn_metadata.c128a_prefill_topk_indices
|
||||
assert topk_indices is not None
|
||||
top_k = topk_indices.shape[-1]
|
||||
N = (layer.max_model_len + layer.compress_ratio - 1) // layer.compress_ratio
|
||||
else:
|
||||
assert layer.topk_indices_buffer is not None
|
||||
topk_indices = layer.topk_indices_buffer[num_decode_tokens:]
|
||||
top_k = 0
|
||||
N = 0
|
||||
|
||||
M = N + layer.window_size + layer.max_num_batched_tokens
|
||||
num_chunks = (num_prefills + cls._PREFILL_CHUNK_SIZE - 1) // (
|
||||
cls._PREFILL_CHUNK_SIZE
|
||||
)
|
||||
|
||||
workspace_manager = current_workspace_manager()
|
||||
kv = workspace_manager.get_simultaneous(
|
||||
((cls._PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16),
|
||||
)[0]
|
||||
for chunk_idx in range(num_chunks):
|
||||
chunk_start = chunk_idx * cls._PREFILL_CHUNK_SIZE
|
||||
chunk_end = min(chunk_start + cls._PREFILL_CHUNK_SIZE, num_prefills)
|
||||
chunk_size = chunk_end - chunk_start
|
||||
if not swa_only:
|
||||
assert attn_metadata is not None
|
||||
assert compressed_k_cache is not None
|
||||
block_table = attn_metadata.block_table[num_decodes:]
|
||||
dequantize_and_gather_k_cache(
|
||||
kv[:chunk_size],
|
||||
compressed_k_cache,
|
||||
seq_lens=seq_lens[chunk_start:chunk_end] // layer.compress_ratio,
|
||||
gather_lens=None,
|
||||
block_table=block_table[chunk_start:chunk_end],
|
||||
block_size=attn_metadata.block_size // layer.compress_ratio,
|
||||
offset=0,
|
||||
)
|
||||
|
||||
swa_block_table = swa_metadata.block_table[num_decodes:]
|
||||
dequantize_and_gather_k_cache(
|
||||
kv[:chunk_size],
|
||||
swa_k_cache,
|
||||
seq_lens=seq_lens[chunk_start:chunk_end],
|
||||
gather_lens=gather_lens[chunk_start:chunk_end],
|
||||
block_table=swa_block_table[chunk_start:chunk_end],
|
||||
block_size=swa_metadata.block_size,
|
||||
offset=N,
|
||||
)
|
||||
|
||||
query_start = (
|
||||
query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base
|
||||
)
|
||||
query_end = (
|
||||
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
|
||||
)
|
||||
|
||||
combined_ragged_indices, combined_ragged_indptr, combined_lens = (
|
||||
combine_topk_swa_indices_ragged(
|
||||
topk_indices[query_start:query_end],
|
||||
query_start_loc[
|
||||
num_decodes + chunk_start : num_decodes + chunk_end + 1
|
||||
],
|
||||
seq_lens[chunk_start:chunk_end],
|
||||
gather_lens[chunk_start:chunk_end],
|
||||
layer.window_size,
|
||||
layer.compress_ratio,
|
||||
top_k,
|
||||
M,
|
||||
N,
|
||||
)
|
||||
)
|
||||
rocm_sparse_attn_prefill(
|
||||
q=q[query_start:query_end],
|
||||
kv=kv.view(-1, 1, q.shape[-1]),
|
||||
indices=torch.empty(
|
||||
q[query_start:query_end].shape[0],
|
||||
1,
|
||||
0,
|
||||
dtype=torch.int32,
|
||||
device=q.device,
|
||||
),
|
||||
topk_length=combined_lens,
|
||||
scale=layer.scale,
|
||||
head_dim=layer.head_dim,
|
||||
nope_head_dim=layer.nope_head_dim,
|
||||
rope_head_dim=layer.rope_head_dim,
|
||||
attn_sink=layer.attn_sink,
|
||||
output=output[query_start:query_end],
|
||||
ragged_indices=combined_ragged_indices,
|
||||
ragged_indptr=combined_ragged_indptr,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4FlashMLASparseBackend):
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "ROCM_V4_FLASHMLA_SPARSE"
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["DeepseekV4ROCMAiterMLASparseMetadataBuilder"]:
|
||||
return DeepseekV4ROCMAiterMLASparseMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["DeepseekV4ROCMAiterMLASparseImpl"]:
|
||||
return DeepseekV4ROCMAiterMLASparseImpl
|
||||
@@ -112,6 +112,12 @@ class DeepseekSparseSWABackend(AttentionBackend):
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["DeepseekSparseSWAMetadataBuilder"]:
|
||||
if current_platform.is_rocm():
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
|
||||
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
)
|
||||
|
||||
return DeepseekV4ROCMAiterSparseSWAMetadataBuilder
|
||||
return DeepseekSparseSWAMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -905,50 +905,705 @@ def rocm_inv_rope_einsum(
|
||||
return torch.einsum("tgd,grd->tgr", o_ref, wo_a_weight)
|
||||
|
||||
|
||||
def rocm_ref_sparse_attn_prefill(
|
||||
_DSV4_SPARSE_NOPE_DIM = 448
|
||||
_DSV4_SPARSE_ROPE_DIM = 64
|
||||
|
||||
|
||||
def _validate_dsv4_sparse_dims(
|
||||
head_dim: int,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
op_name: str,
|
||||
) -> None:
|
||||
assert head_dim == nope_head_dim + rope_head_dim, (
|
||||
f"{op_name} expected head_dim={nope_head_dim + rope_head_dim}, got {head_dim}"
|
||||
)
|
||||
assert (
|
||||
nope_head_dim == _DSV4_SPARSE_NOPE_DIM
|
||||
and rope_head_dim == _DSV4_SPARSE_ROPE_DIM
|
||||
), (
|
||||
f"{op_name} expects {_DSV4_SPARSE_NOPE_DIM} NoPE dims and "
|
||||
f"{_DSV4_SPARSE_ROPE_DIM} RoPE dims"
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_dense_prefix_to_ragged_kernel(
|
||||
indices_ptr,
|
||||
lengths_ptr,
|
||||
indptr_ptr,
|
||||
out_ptr,
|
||||
indices_stride0,
|
||||
num_rows_limit,
|
||||
row_width,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row_idx = tl.program_id(0)
|
||||
block_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
|
||||
row_len = tl.load(lengths_ptr + row_idx)
|
||||
if block_idx * BLOCK_SIZE >= row_len:
|
||||
return
|
||||
|
||||
mask = offsets < row_len
|
||||
vals = tl.load(
|
||||
indices_ptr + row_idx * indices_stride0 + offsets,
|
||||
mask=mask & (offsets < row_width),
|
||||
other=-1,
|
||||
).to(tl.int32)
|
||||
if num_rows_limit >= 0:
|
||||
vals = tl.where((vals >= 0) & (vals < num_rows_limit), vals, -1)
|
||||
|
||||
out_start = tl.load(indptr_ptr + row_idx)
|
||||
tl.store(out_ptr + out_start + offsets, vals, mask=mask)
|
||||
|
||||
|
||||
def build_ragged_indices_from_dense(
|
||||
indices: torch.Tensor,
|
||||
lengths: torch.Tensor,
|
||||
num_rows: int = -1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
indices = indices.reshape(indices.shape[0], -1)
|
||||
lengths = lengths.to(device=indices.device, dtype=torch.int32).reshape(-1)
|
||||
assert lengths.numel() == indices.shape[0], (
|
||||
f"Expected one length per row, got {lengths.shape} for indices {indices.shape}"
|
||||
)
|
||||
|
||||
max_width = indices.shape[1] if indices.ndim == 2 else 0
|
||||
lengths = lengths.clamp(min=0, max=max_width).contiguous()
|
||||
|
||||
indptr = torch.empty(indices.shape[0] + 1, dtype=torch.int32, device=indices.device)
|
||||
indptr[0] = 0
|
||||
torch.cumsum(lengths, dim=0, out=indptr[1:])
|
||||
|
||||
if indices.numel() == 0:
|
||||
flat = torch.empty(0, dtype=torch.int32, device=indices.device)
|
||||
else:
|
||||
flat = torch.empty(
|
||||
int(indptr[-1].item()), dtype=torch.int32, device=indices.device
|
||||
)
|
||||
if flat.numel() > 0:
|
||||
block_size = 128
|
||||
_pack_dense_prefix_to_ragged_kernel[
|
||||
(indices.shape[0], triton.cdiv(max_width, block_size))
|
||||
](
|
||||
indices,
|
||||
lengths,
|
||||
indptr,
|
||||
flat,
|
||||
indices.stride(0),
|
||||
int(num_rows),
|
||||
max_width,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
|
||||
return flat, indptr
|
||||
|
||||
|
||||
def _as_int32_contiguous_1d(x: torch.Tensor) -> torch.Tensor:
|
||||
if x.dtype == torch.int32 and x.ndim == 1 and x.is_contiguous():
|
||||
return x
|
||||
return x.to(torch.int32).contiguous()
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_attn_prefill_ragged_kernel(
|
||||
q_ptr,
|
||||
kv_ptr,
|
||||
kv_indices_ptr,
|
||||
kv_indptr_ptr,
|
||||
attn_sink_ptr,
|
||||
out_ptr,
|
||||
q_stride_t,
|
||||
q_stride_h,
|
||||
q_stride_d,
|
||||
kv_stride_n,
|
||||
kv_stride_d,
|
||||
out_stride_t,
|
||||
out_stride_h,
|
||||
out_stride_d,
|
||||
num_heads,
|
||||
head_dim,
|
||||
num_kv,
|
||||
scale,
|
||||
HAS_ATTN_SINK: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
query_idx = tl.program_id(0)
|
||||
pid_h = tl.program_id(1)
|
||||
|
||||
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
dim_offsets = tl.arange(0, BLOCK_D)
|
||||
head_mask = head_offsets < num_heads
|
||||
dim_mask = dim_offsets < head_dim
|
||||
|
||||
q = tl.load(
|
||||
q_ptr
|
||||
+ query_idx * q_stride_t
|
||||
+ head_offsets[:, None] * q_stride_h
|
||||
+ dim_offsets[None, :] * q_stride_d,
|
||||
mask=head_mask[:, None] & dim_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
neg_large = -3.4028234663852886e38
|
||||
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
|
||||
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||||
acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32)
|
||||
|
||||
kv_start = tl.load(kv_indptr_ptr + query_idx)
|
||||
kv_end = tl.load(kv_indptr_ptr + query_idx + 1)
|
||||
kv_len = kv_end - kv_start
|
||||
|
||||
k_offsets = tl.arange(0, BLOCK_K)
|
||||
for k_start in tl.range(0, kv_len, BLOCK_K):
|
||||
k_pos = k_start + k_offsets
|
||||
in_range = k_pos < kv_len
|
||||
slot = tl.load(kv_indices_ptr + kv_start + k_pos, mask=in_range, other=-1)
|
||||
valid = in_range & (slot >= 0) & (slot < num_kv)
|
||||
|
||||
kv = tl.load(
|
||||
kv_ptr + slot[:, None] * kv_stride_n + dim_offsets[None, :] * kv_stride_d,
|
||||
mask=valid[:, None] & dim_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
kv = tl.where(valid[:, None] & dim_mask[None, :], kv, 0.0)
|
||||
|
||||
scores = tl.dot(q, tl.trans(kv)) * scale
|
||||
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
|
||||
|
||||
m_block = tl.max(scores, axis=1)
|
||||
m_new = tl.maximum(m_i, m_block)
|
||||
alpha = tl.exp(m_i - m_new)
|
||||
p = tl.exp(scores - m_new[:, None])
|
||||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||||
|
||||
acc = acc * alpha[:, None] + tl.dot(p.to(kv.dtype), kv)
|
||||
m_i = m_new
|
||||
l_i = l_new
|
||||
|
||||
if HAS_ATTN_SINK:
|
||||
sink = tl.load(
|
||||
attn_sink_ptr + head_offsets, mask=head_mask, other=neg_large
|
||||
).to(tl.float32)
|
||||
m_final = tl.maximum(m_i, sink)
|
||||
alpha = tl.exp(m_i - m_final)
|
||||
l_final = l_i * alpha + tl.exp(sink - m_final)
|
||||
denom = tl.maximum(l_final, 1.0e-30)
|
||||
out = tl.where(
|
||||
l_final[:, None] > 0.0,
|
||||
(acc * alpha[:, None]) / denom[:, None],
|
||||
0.0,
|
||||
)
|
||||
else:
|
||||
denom = tl.maximum(l_i, 1.0e-30)
|
||||
out = tl.where(l_i[:, None] > 0.0, acc / denom[:, None], 0.0)
|
||||
|
||||
tl.store(
|
||||
out_ptr
|
||||
+ query_idx * out_stride_t
|
||||
+ head_offsets[:, None] * out_stride_h
|
||||
+ dim_offsets[None, :] * out_stride_d,
|
||||
out,
|
||||
mask=head_mask[:, None] & dim_mask[None, :],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_attn_decode_ragged_kernel(
|
||||
q_ptr,
|
||||
main_cache_ptr,
|
||||
main_indices_ptr,
|
||||
main_indptr_ptr,
|
||||
extra_cache_ptr,
|
||||
extra_indices_ptr,
|
||||
extra_indptr_ptr,
|
||||
attn_sink_ptr,
|
||||
out_ptr,
|
||||
q_stride0,
|
||||
q_stride1,
|
||||
out_stride0,
|
||||
out_stride1,
|
||||
main_cache_stride0,
|
||||
extra_cache_stride0,
|
||||
main_num_rows,
|
||||
extra_num_rows,
|
||||
main_block_size,
|
||||
extra_block_size,
|
||||
scale,
|
||||
num_heads,
|
||||
HAS_ATTN_SINK: tl.constexpr,
|
||||
HAS_EXTRA: tl.constexpr,
|
||||
NOPE_DIM: tl.constexpr,
|
||||
NOPE_BLOCK: tl.constexpr,
|
||||
ROPE_DIM: tl.constexpr,
|
||||
IS_FNUZ: tl.constexpr,
|
||||
BLOCK_H: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
):
|
||||
query_idx = tl.program_id(0)
|
||||
pid_h = tl.program_id(1)
|
||||
|
||||
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
head_mask = head_offsets < num_heads
|
||||
nope_offsets = tl.arange(0, NOPE_BLOCK)
|
||||
nope_mask = nope_offsets < NOPE_DIM
|
||||
rope_offsets = tl.arange(0, ROPE_DIM)
|
||||
|
||||
q_row_ptr = q_ptr + query_idx * q_stride0 + head_offsets[:, None] * q_stride1
|
||||
q_nope = tl.load(
|
||||
q_row_ptr + nope_offsets[None, :],
|
||||
mask=head_mask[:, None] & nope_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
q_rope = tl.load(
|
||||
q_row_ptr + NOPE_DIM + rope_offsets[None, :],
|
||||
mask=head_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
neg_large = -3.4028234663852886e38
|
||||
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
|
||||
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||||
acc_nope = tl.zeros((BLOCK_H, NOPE_BLOCK), dtype=tl.float32)
|
||||
acc_rope = tl.zeros((BLOCK_H, ROPE_DIM), dtype=tl.float32)
|
||||
k_offsets = tl.arange(0, BLOCK_K)
|
||||
|
||||
main_start = tl.load(main_indptr_ptr + query_idx)
|
||||
main_end = tl.load(main_indptr_ptr + query_idx + 1)
|
||||
main_len = main_end - main_start
|
||||
|
||||
zero_nope = tl.zeros((BLOCK_K, NOPE_BLOCK), dtype=tl.bfloat16)
|
||||
zero_rope = tl.zeros((BLOCK_K, ROPE_DIM), dtype=tl.bfloat16)
|
||||
|
||||
for k_start in tl.range(0, main_len, BLOCK_K):
|
||||
k_pos = k_start + k_offsets
|
||||
in_range = k_pos < main_len
|
||||
slot = tl.load(main_indices_ptr + main_start + k_pos, mask=in_range, other=-1)
|
||||
valid = in_range & (slot >= 0) & (slot < main_num_rows)
|
||||
safe_slot = tl.where(valid, slot, 0)
|
||||
|
||||
block_idx = safe_slot // main_block_size
|
||||
pos_in_block = safe_slot % main_block_size
|
||||
cache_block_ptr = main_cache_ptr + block_idx.to(tl.int64) * main_cache_stride0
|
||||
token_data_ptr = cache_block_ptr + pos_in_block * 576
|
||||
token_scale_ptr = cache_block_ptr + main_block_size * 576 + pos_in_block * 8
|
||||
|
||||
x_uint8 = tl.load(
|
||||
token_data_ptr[:, None] + nope_offsets[None, :],
|
||||
mask=valid[:, None] & nope_mask[None, :],
|
||||
other=0,
|
||||
)
|
||||
if IS_FNUZ:
|
||||
x_fp8 = x_uint8.to(tl.float8e4b15, bitcast=True)
|
||||
else:
|
||||
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
|
||||
encoded_scales = tl.load(
|
||||
token_scale_ptr[:, None] + nope_offsets[None, :] // 64,
|
||||
mask=valid[:, None] & nope_mask[None, :],
|
||||
other=127,
|
||||
)
|
||||
scales = tl.exp2(encoded_scales.to(tl.float32) - 127.0)
|
||||
k_nope = x_fp8.to(tl.bfloat16) * scales.to(tl.bfloat16)
|
||||
k_nope = tl.where(valid[:, None] & nope_mask[None, :], k_nope, zero_nope)
|
||||
k_nope = tl.where(k_nope == k_nope, k_nope, zero_nope)
|
||||
|
||||
rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16))
|
||||
k_rope = tl.load(
|
||||
rope_ptr[:, None] + rope_offsets[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
k_rope = tl.where(valid[:, None], k_rope, zero_rope)
|
||||
k_rope = tl.where(k_rope == k_rope, k_rope, zero_rope)
|
||||
|
||||
scores = tl.dot(q_nope, tl.trans(k_nope)) + tl.dot(q_rope, tl.trans(k_rope))
|
||||
scores *= scale
|
||||
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
|
||||
|
||||
m_block = tl.max(scores, axis=1)
|
||||
m_new = tl.maximum(m_i, m_block)
|
||||
alpha = tl.exp(m_i - m_new)
|
||||
p = tl.exp(scores - m_new[:, None])
|
||||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||||
|
||||
acc_nope = acc_nope * alpha[:, None] + tl.dot(p.to(k_nope.dtype), k_nope)
|
||||
acc_rope = acc_rope * alpha[:, None] + tl.dot(p.to(k_rope.dtype), k_rope)
|
||||
m_i = m_new
|
||||
l_i = l_new
|
||||
|
||||
if HAS_EXTRA:
|
||||
extra_start = tl.load(extra_indptr_ptr + query_idx)
|
||||
extra_end = tl.load(extra_indptr_ptr + query_idx + 1)
|
||||
extra_len = extra_end - extra_start
|
||||
|
||||
for k_start in tl.range(0, extra_len, BLOCK_K):
|
||||
k_pos = k_start + k_offsets
|
||||
in_range = k_pos < extra_len
|
||||
slot = tl.load(
|
||||
extra_indices_ptr + extra_start + k_pos, mask=in_range, other=-1
|
||||
)
|
||||
valid = in_range & (slot >= 0) & (slot < extra_num_rows)
|
||||
safe_slot = tl.where(valid, slot, 0)
|
||||
|
||||
block_idx = safe_slot // extra_block_size
|
||||
pos_in_block = safe_slot % extra_block_size
|
||||
cache_block_ptr = (
|
||||
extra_cache_ptr + block_idx.to(tl.int64) * extra_cache_stride0
|
||||
)
|
||||
token_data_ptr = cache_block_ptr + pos_in_block * 576
|
||||
token_scale_ptr = (
|
||||
cache_block_ptr + extra_block_size * 576 + pos_in_block * 8
|
||||
)
|
||||
|
||||
x_uint8 = tl.load(
|
||||
token_data_ptr[:, None] + nope_offsets[None, :],
|
||||
mask=valid[:, None] & nope_mask[None, :],
|
||||
other=0,
|
||||
)
|
||||
if IS_FNUZ:
|
||||
x_fp8 = x_uint8.to(tl.float8e4b15, bitcast=True)
|
||||
else:
|
||||
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
|
||||
encoded_scales = tl.load(
|
||||
token_scale_ptr[:, None] + nope_offsets[None, :] // 64,
|
||||
mask=valid[:, None] & nope_mask[None, :],
|
||||
other=127,
|
||||
)
|
||||
scales = tl.exp2(encoded_scales.to(tl.float32) - 127.0)
|
||||
k_nope = x_fp8.to(tl.bfloat16) * scales.to(tl.bfloat16)
|
||||
k_nope = tl.where(valid[:, None] & nope_mask[None, :], k_nope, zero_nope)
|
||||
k_nope = tl.where(k_nope == k_nope, k_nope, zero_nope)
|
||||
|
||||
rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16))
|
||||
k_rope = tl.load(
|
||||
rope_ptr[:, None] + rope_offsets[None, :],
|
||||
mask=valid[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
k_rope = tl.where(valid[:, None], k_rope, zero_rope)
|
||||
k_rope = tl.where(k_rope == k_rope, k_rope, zero_rope)
|
||||
|
||||
scores = tl.dot(q_nope, tl.trans(k_nope)) + tl.dot(
|
||||
q_rope,
|
||||
tl.trans(k_rope),
|
||||
)
|
||||
scores *= scale
|
||||
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
|
||||
|
||||
m_block = tl.max(scores, axis=1)
|
||||
m_new = tl.maximum(m_i, m_block)
|
||||
alpha = tl.exp(m_i - m_new)
|
||||
p = tl.exp(scores - m_new[:, None])
|
||||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||||
|
||||
acc_nope = acc_nope * alpha[:, None] + tl.dot(p.to(k_nope.dtype), k_nope)
|
||||
acc_rope = acc_rope * alpha[:, None] + tl.dot(p.to(k_rope.dtype), k_rope)
|
||||
m_i = m_new
|
||||
l_i = l_new
|
||||
|
||||
if HAS_ATTN_SINK:
|
||||
sink = tl.load(
|
||||
attn_sink_ptr + head_offsets, mask=head_mask, other=neg_large
|
||||
).to(tl.float32)
|
||||
m_final = tl.maximum(m_i, sink)
|
||||
alpha = tl.exp(m_i - m_final)
|
||||
l_final = l_i * alpha + tl.exp(sink - m_final)
|
||||
denom = tl.maximum(l_final, 1.0e-30)
|
||||
out_nope = tl.where(
|
||||
l_final[:, None] > 0.0,
|
||||
(acc_nope * alpha[:, None]) / denom[:, None],
|
||||
0.0,
|
||||
)
|
||||
out_rope = tl.where(
|
||||
l_final[:, None] > 0.0,
|
||||
(acc_rope * alpha[:, None]) / denom[:, None],
|
||||
0.0,
|
||||
)
|
||||
else:
|
||||
denom = tl.maximum(l_i, 1.0e-30)
|
||||
out_nope = tl.where(l_i[:, None] > 0.0, acc_nope / denom[:, None], 0.0)
|
||||
out_rope = tl.where(l_i[:, None] > 0.0, acc_rope / denom[:, None], 0.0)
|
||||
|
||||
out_row_ptr = (
|
||||
out_ptr + query_idx * out_stride0 + head_offsets[:, None] * out_stride1
|
||||
)
|
||||
tl.store(
|
||||
out_row_ptr + nope_offsets[None, :],
|
||||
out_nope,
|
||||
mask=head_mask[:, None] & nope_mask[None, :],
|
||||
)
|
||||
tl.store(
|
||||
out_row_ptr + NOPE_DIM + rope_offsets[None, :],
|
||||
out_rope,
|
||||
mask=head_mask[:, None],
|
||||
)
|
||||
|
||||
|
||||
def _rocm_sparse_attn_prefill_ragged_triton(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
topk_length: torch.Tensor | None,
|
||||
indptr: torch.Tensor,
|
||||
scale: float,
|
||||
head_dim: int,
|
||||
attn_sink: torch.Tensor | None,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
) -> torch.Tensor:
|
||||
indices = indices.clone().squeeze(1)
|
||||
s_q, h_q, d_qk = q.shape
|
||||
topk = indices.shape[-1]
|
||||
s_kv = kv.shape[0]
|
||||
if topk_length is not None:
|
||||
mask = torch.arange(topk, device=indices.device).unsqueeze(
|
||||
0
|
||||
) >= topk_length.unsqueeze(1)
|
||||
indices[mask] = -1
|
||||
invalid_mask = (indices < 0) | (indices >= s_kv)
|
||||
indices[invalid_mask] = 0
|
||||
assert q.ndim == 3, f"expected q=[sq,h,d], got {q.shape}"
|
||||
assert kv.ndim == 2, f"expected kv=[skv,d], got {kv.shape}"
|
||||
assert indices.ndim == 1, f"expected indices=[nnz], got {indices.shape}"
|
||||
assert indptr.ndim == 1, f"expected indptr=[sq+1], got {indptr.shape}"
|
||||
assert q.is_cuda and kv.is_cuda and indices.is_cuda and indptr.is_cuda
|
||||
|
||||
qf = q.float()
|
||||
gathered_kv = kv.index_select(0, indices.flatten()).reshape(s_q, topk, d_qk).float()
|
||||
scores = qf @ gathered_kv.transpose(1, 2)
|
||||
scores *= scale
|
||||
scores[invalid_mask.unsqueeze(1).expand_as(scores)] = float("-inf")
|
||||
indices = _as_int32_contiguous_1d(indices)
|
||||
indptr = _as_int32_contiguous_1d(indptr)
|
||||
has_attn_sink = attn_sink is not None
|
||||
if attn_sink is None:
|
||||
attn_sink = torch.empty(1, device=q.device, dtype=torch.float32)
|
||||
else:
|
||||
attn_sink = attn_sink.contiguous()
|
||||
|
||||
orig_lse = torch.logsumexp(scores, dim=-1)
|
||||
lse_for_o = orig_lse
|
||||
if attn_sink is not None:
|
||||
lse_for_o = torch.logsumexp(
|
||||
torch.stack(
|
||||
[orig_lse, attn_sink[:h_q].view(1, h_q).expand_as(orig_lse)],
|
||||
dim=0,
|
||||
),
|
||||
dim=0,
|
||||
num_queries, num_heads, head_dim = q.shape
|
||||
assert indptr.numel() == num_queries + 1, (
|
||||
f"expected indptr shape [{num_queries + 1}], got {indptr.shape}"
|
||||
)
|
||||
_validate_dsv4_sparse_dims(
|
||||
head_dim,
|
||||
nope_head_dim,
|
||||
rope_head_dim,
|
||||
"_rocm_sparse_attn_prefill_ragged_triton",
|
||||
)
|
||||
|
||||
block_h = 16
|
||||
block_d = triton.next_power_of_2(head_dim)
|
||||
block_k = 16 if head_dim >= 256 else 32
|
||||
out = torch.empty_like(q, dtype=torch.bfloat16)
|
||||
_sparse_attn_prefill_ragged_kernel[(num_queries, triton.cdiv(num_heads, block_h))](
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
indptr,
|
||||
attn_sink,
|
||||
out,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
kv.stride(0),
|
||||
kv.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
out.stride(2),
|
||||
num_heads,
|
||||
head_dim,
|
||||
kv.shape[0],
|
||||
float(scale),
|
||||
HAS_ATTN_SINK=has_attn_sink,
|
||||
BLOCK_H=block_h,
|
||||
BLOCK_D=block_d,
|
||||
BLOCK_K=block_k,
|
||||
num_warps=8,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _rocm_sparse_attn_prefill_triton(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
scale: float,
|
||||
attn_sink: torch.Tensor | None,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
topk_length: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
|
||||
indices,
|
||||
topk_length
|
||||
if topk_length is not None
|
||||
else (indices >= 0).sum(dim=-1, dtype=torch.int32),
|
||||
num_rows=kv.shape[0],
|
||||
)
|
||||
return _rocm_sparse_attn_prefill_ragged_triton(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=ragged_indices,
|
||||
indptr=ragged_indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=nope_head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
)
|
||||
|
||||
|
||||
def _rocm_sparse_attn_decode_ragged_triton(
|
||||
q: torch.Tensor,
|
||||
main_cache: torch.Tensor,
|
||||
main_indices: torch.Tensor,
|
||||
main_indptr: torch.Tensor,
|
||||
scale: float,
|
||||
attn_sink: torch.Tensor | None,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
extra_cache: torch.Tensor | None = None,
|
||||
extra_indices: torch.Tensor | None = None,
|
||||
extra_indptr: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert q.ndim == 3, f"expected q=[b,h,d], got {q.shape}"
|
||||
assert main_cache.ndim == 3, (
|
||||
f"expected main_cache=[blocks,block,bytes], got {main_cache.shape}"
|
||||
)
|
||||
assert main_indices.ndim == 1, (
|
||||
f"expected main_indices=[nnz], got {main_indices.shape}"
|
||||
)
|
||||
assert main_indptr.ndim == 1, f"expected main_indptr=[b+1], got {main_indptr.shape}"
|
||||
assert (
|
||||
q.is_cuda
|
||||
and main_cache.is_cuda
|
||||
and main_indices.is_cuda
|
||||
and main_indptr.is_cuda
|
||||
)
|
||||
|
||||
main_indices = _as_int32_contiguous_1d(main_indices)
|
||||
main_indptr = _as_int32_contiguous_1d(main_indptr)
|
||||
has_attn_sink = attn_sink is not None
|
||||
if attn_sink is None:
|
||||
attn_sink = torch.empty(1, device=q.device, dtype=torch.float32)
|
||||
else:
|
||||
attn_sink = attn_sink.contiguous()
|
||||
|
||||
num_queries, num_heads, head_dim = q.shape
|
||||
assert main_indptr.numel() == num_queries + 1, (
|
||||
f"expected main_indptr shape [{num_queries + 1}], got {main_indptr.shape}"
|
||||
)
|
||||
_validate_dsv4_sparse_dims(
|
||||
head_dim,
|
||||
nope_head_dim,
|
||||
rope_head_dim,
|
||||
"_rocm_sparse_attn_decode_ragged_triton",
|
||||
)
|
||||
|
||||
has_extra = (
|
||||
extra_cache is not None
|
||||
and extra_indices is not None
|
||||
and extra_indptr is not None
|
||||
)
|
||||
if has_extra:
|
||||
assert extra_cache is not None
|
||||
assert extra_indices is not None
|
||||
assert extra_indptr is not None
|
||||
assert extra_indices.ndim == 1, (
|
||||
f"expected extra_indices=[nnz], got {extra_indices.shape}"
|
||||
)
|
||||
lse_for_o = lse_for_o.clone()
|
||||
lse_for_o[lse_for_o == float("-inf")] = float("+inf")
|
||||
probs = torch.exp(scores - lse_for_o.unsqueeze(-1))
|
||||
out = probs @ gathered_kv[..., :head_dim]
|
||||
lonely_q_mask = orig_lse == float("-inf")
|
||||
out[lonely_q_mask.unsqueeze(-1).expand_as(out)] = 0.0
|
||||
return out.to(torch.bfloat16)
|
||||
assert extra_indptr.ndim == 1, (
|
||||
f"expected extra_indptr=[b+1], got {extra_indptr.shape}"
|
||||
)
|
||||
extra_indices = _as_int32_contiguous_1d(extra_indices)
|
||||
extra_indptr = _as_int32_contiguous_1d(extra_indptr)
|
||||
assert extra_indptr.numel() == num_queries + 1, (
|
||||
f"expected extra_indptr shape [{num_queries + 1}], got {extra_indptr.shape}"
|
||||
)
|
||||
else:
|
||||
extra_cache = main_cache
|
||||
extra_indices = torch.empty(0, device=q.device, dtype=torch.int32)
|
||||
extra_indptr = torch.zeros(num_queries + 1, device=q.device, dtype=torch.int32)
|
||||
|
||||
block_h = 16
|
||||
block_k = 16 if head_dim >= 256 else 32
|
||||
out = torch.empty_like(q, dtype=torch.bfloat16)
|
||||
_sparse_attn_decode_ragged_kernel[(num_queries, triton.cdiv(num_heads, block_h))](
|
||||
q,
|
||||
main_cache,
|
||||
main_indices,
|
||||
main_indptr,
|
||||
extra_cache,
|
||||
extra_indices,
|
||||
extra_indptr,
|
||||
attn_sink,
|
||||
out,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
main_cache.stride(0),
|
||||
extra_cache.stride(0),
|
||||
main_cache.shape[0] * main_cache.shape[1],
|
||||
extra_cache.shape[0] * extra_cache.shape[1],
|
||||
main_cache.shape[1],
|
||||
extra_cache.shape[1],
|
||||
scale,
|
||||
num_heads,
|
||||
HAS_ATTN_SINK=has_attn_sink,
|
||||
HAS_EXTRA=has_extra,
|
||||
NOPE_DIM=nope_head_dim,
|
||||
NOPE_BLOCK=triton.next_power_of_2(nope_head_dim),
|
||||
ROPE_DIM=rope_head_dim,
|
||||
IS_FNUZ=current_platform.is_fp8_fnuz(),
|
||||
BLOCK_H=block_h,
|
||||
BLOCK_K=block_k,
|
||||
num_warps=8,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _rocm_sparse_attn_decode_triton(
|
||||
q: torch.Tensor,
|
||||
main_cache: torch.Tensor,
|
||||
main_indices: torch.Tensor,
|
||||
scale: float,
|
||||
attn_sink: torch.Tensor | None,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
extra_cache: torch.Tensor | None = None,
|
||||
extra_indices: torch.Tensor | None = None,
|
||||
main_lengths: torch.Tensor | None = None,
|
||||
extra_lengths: torch.Tensor | None = None,
|
||||
main_ragged_indices: torch.Tensor | None = None,
|
||||
main_ragged_indptr: torch.Tensor | None = None,
|
||||
extra_ragged_indices: torch.Tensor | None = None,
|
||||
extra_ragged_indptr: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if main_ragged_indices is None or main_ragged_indptr is None:
|
||||
main_ragged_indices, main_ragged_indptr = build_ragged_indices_from_dense(
|
||||
main_indices,
|
||||
main_lengths
|
||||
if main_lengths is not None
|
||||
else (main_indices >= 0).sum(dim=-1, dtype=torch.int32),
|
||||
num_rows=main_cache.shape[0] * main_cache.shape[1],
|
||||
)
|
||||
|
||||
if (
|
||||
(extra_ragged_indices is None or extra_ragged_indptr is None)
|
||||
and extra_cache is not None
|
||||
and extra_indices is not None
|
||||
):
|
||||
extra_ragged_indices, extra_ragged_indptr = build_ragged_indices_from_dense(
|
||||
extra_indices,
|
||||
extra_lengths
|
||||
if extra_lengths is not None
|
||||
else (extra_indices >= 0).sum(dim=-1, dtype=torch.int32),
|
||||
num_rows=extra_cache.shape[0] * extra_cache.shape[1],
|
||||
)
|
||||
|
||||
return _rocm_sparse_attn_decode_ragged_triton(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
main_indices=main_ragged_indices,
|
||||
main_indptr=main_ragged_indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=nope_head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
extra_cache=extra_cache,
|
||||
extra_indices=extra_ragged_indices,
|
||||
extra_indptr=extra_ragged_indptr,
|
||||
)
|
||||
|
||||
|
||||
def rocm_sparse_attn_prefill(
|
||||
@@ -958,132 +1613,49 @@ def rocm_sparse_attn_prefill(
|
||||
topk_length: torch.Tensor | None,
|
||||
scale: float,
|
||||
head_dim: int,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
attn_sink: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
ragged_indices: torch.Tensor | None = None,
|
||||
ragged_indptr: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
output_chunk = rocm_ref_sparse_attn_prefill(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
topk_length=topk_length,
|
||||
scale=scale,
|
||||
head_dim=head_dim,
|
||||
attn_sink=attn_sink,
|
||||
assert kv.ndim == 3 and kv.shape[1] == 1, (
|
||||
f"ROCm Triton sparse prefill expects kv=[skv,1,d], got {kv.shape}"
|
||||
)
|
||||
_validate_dsv4_sparse_dims(
|
||||
head_dim,
|
||||
nope_head_dim,
|
||||
rope_head_dim,
|
||||
"rocm_sparse_attn_prefill",
|
||||
)
|
||||
if ragged_indices is not None and ragged_indptr is not None:
|
||||
output_chunk = _rocm_sparse_attn_prefill_ragged_triton(
|
||||
q=q,
|
||||
kv=kv.squeeze(1),
|
||||
indices=ragged_indices,
|
||||
indptr=ragged_indptr,
|
||||
scale=scale,
|
||||
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
|
||||
nope_head_dim=nope_head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
)
|
||||
else:
|
||||
indices_2d = indices.reshape(indices.shape[0], -1)
|
||||
output_chunk = _rocm_sparse_attn_prefill_triton(
|
||||
q=q,
|
||||
kv=kv.squeeze(1),
|
||||
indices=indices_2d,
|
||||
scale=scale,
|
||||
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
|
||||
nope_head_dim=nope_head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
topk_length=topk_length,
|
||||
)
|
||||
output.copy_(output_chunk.to(output.dtype))
|
||||
|
||||
|
||||
def rocm_dequantize_blocked_k_cache(
|
||||
quant_k_cache: torch.Tensor,
|
||||
head_dim: int,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
) -> torch.Tensor:
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
tile_size = 64
|
||||
num_tiles = nope_head_dim // tile_size
|
||||
|
||||
num_blocks, block_size, _ = quant_k_cache.shape
|
||||
quant_k_cache = quant_k_cache.view(num_blocks, -1)
|
||||
input_nope_rope = quant_k_cache[
|
||||
:, : block_size * (nope_head_dim + 2 * rope_head_dim)
|
||||
].view(num_blocks, block_size, nope_head_dim + 2 * rope_head_dim)
|
||||
input_nope = input_nope_rope[:, :, :nope_head_dim].view(fp8_dtype)
|
||||
input_rope = input_nope_rope[:, :, nope_head_dim:].view(torch.bfloat16)
|
||||
input_scale = (
|
||||
quant_k_cache[:, block_size * (nope_head_dim + 2 * rope_head_dim) :]
|
||||
.view(num_blocks, block_size, 8)[:, :, :num_tiles]
|
||||
.view(torch.float8_e8m0fnu)
|
||||
)
|
||||
|
||||
result = torch.empty(
|
||||
(num_blocks, block_size, 1, head_dim),
|
||||
dtype=torch.bfloat16,
|
||||
device=quant_k_cache.device,
|
||||
)
|
||||
result[..., nope_head_dim:] = input_rope.unsqueeze(2)
|
||||
for tile_idx in range(num_tiles):
|
||||
cur_nope = input_nope[
|
||||
..., tile_idx * tile_size : (tile_idx + 1) * tile_size
|
||||
].to(torch.bfloat16)
|
||||
cur_scales = input_scale[:, :, tile_idx].to(torch.bfloat16).unsqueeze(-1)
|
||||
result[..., tile_idx * tile_size : (tile_idx + 1) * tile_size] = (
|
||||
cur_nope * cur_scales
|
||||
).unsqueeze(2)
|
||||
return result
|
||||
|
||||
|
||||
def rocm_ref_sparse_attn_decode(
|
||||
q: torch.Tensor,
|
||||
blocked_k: torch.Tensor,
|
||||
indices_in_kvcache: torch.Tensor,
|
||||
topk_length: torch.Tensor | None,
|
||||
scale: float,
|
||||
head_dim: int,
|
||||
attn_sink: torch.Tensor | None,
|
||||
extra_blocked_k: torch.Tensor | None = None,
|
||||
extra_indices_in_kvcache: torch.Tensor | None = None,
|
||||
extra_topk_length: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
b, s_q, h_q, d_qk = q.shape
|
||||
|
||||
def process_scope(
|
||||
cur_blocked_k: torch.Tensor,
|
||||
cur_indices: torch.Tensor,
|
||||
cur_topk_length: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
cur_indices = cur_indices.reshape(b, s_q, -1)
|
||||
topk = cur_indices.size(-1)
|
||||
fixed_indices = torch.clamp_min(cur_indices, 0)
|
||||
gathered_kv = (
|
||||
cur_blocked_k.view(-1, d_qk)
|
||||
.index_select(0, fixed_indices.view(-1))
|
||||
.view(b, s_q, topk, d_qk)
|
||||
)
|
||||
invalid_mask = cur_indices == -1
|
||||
if cur_topk_length is not None:
|
||||
cur_topk_length = cur_topk_length.reshape(b)
|
||||
invalid_mask |= torch.arange(0, topk, device=invalid_mask.device).view(
|
||||
1, 1, topk
|
||||
) >= cur_topk_length.view(b, 1, 1)
|
||||
return gathered_kv, invalid_mask
|
||||
|
||||
gathered_kv, invalid_mask = process_scope(
|
||||
blocked_k, indices_in_kvcache, topk_length
|
||||
)
|
||||
if extra_blocked_k is not None:
|
||||
assert extra_indices_in_kvcache is not None
|
||||
gathered_kv1, invalid_mask1 = process_scope(
|
||||
extra_blocked_k, extra_indices_in_kvcache, extra_topk_length
|
||||
)
|
||||
gathered_kv = torch.cat([gathered_kv, gathered_kv1], dim=2)
|
||||
invalid_mask = torch.cat([invalid_mask, invalid_mask1], dim=2)
|
||||
|
||||
gathered_kv = gathered_kv.view(b * s_q, -1, d_qk).float()
|
||||
gathered_kv[gathered_kv != gathered_kv] = 0.0
|
||||
qf = q.float().view(b * s_q, h_q, d_qk)
|
||||
attn_weight = qf @ gathered_kv.transpose(-1, -2)
|
||||
attn_weight *= scale
|
||||
attn_weight[
|
||||
invalid_mask.view(b * s_q, 1, -1).expand(b * s_q, h_q, invalid_mask.size(-1))
|
||||
] = float("-inf")
|
||||
lse = attn_weight.logsumexp(dim=-1)
|
||||
attn_weight = torch.exp(attn_weight - lse.unsqueeze(-1))
|
||||
output = attn_weight @ gathered_kv[..., :head_dim]
|
||||
output = output.view(b, s_q, h_q, head_dim)
|
||||
lse = lse.view(b, s_q, h_q)
|
||||
|
||||
if attn_sink is not None:
|
||||
output *= (1.0 / (1.0 + torch.exp(attn_sink.view(1, 1, h_q) - lse))).unsqueeze(
|
||||
-1
|
||||
)
|
||||
|
||||
lonely_q_mask = lse == float("-inf")
|
||||
output[lonely_q_mask.unsqueeze(-1).expand_as(output)] = 0.0
|
||||
return output.squeeze(1).to(torch.bfloat16)
|
||||
|
||||
|
||||
def rocm_forward_decode_fallback(
|
||||
def rocm_sparse_attn_decode(
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor | None,
|
||||
swa_k_cache: torch.Tensor,
|
||||
@@ -1092,6 +1664,10 @@ def rocm_forward_decode_fallback(
|
||||
topk_lens: torch.Tensor | None,
|
||||
swa_indices: torch.Tensor,
|
||||
swa_lens: torch.Tensor,
|
||||
swa_ragged_indices: torch.Tensor | None,
|
||||
swa_ragged_indptr: torch.Tensor | None,
|
||||
topk_ragged_indices: torch.Tensor | None,
|
||||
topk_ragged_indptr: torch.Tensor | None,
|
||||
attn_sink: torch.Tensor | None,
|
||||
scale: float,
|
||||
head_dim: int,
|
||||
@@ -1099,31 +1675,49 @@ def rocm_forward_decode_fallback(
|
||||
rope_head_dim: int,
|
||||
output: torch.Tensor,
|
||||
) -> None:
|
||||
blocked_swa = rocm_dequantize_blocked_k_cache(
|
||||
swa_k_cache,
|
||||
head_dim=head_dim,
|
||||
nope_head_dim=nope_head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
assert swa_k_cache.dtype == torch.uint8, (
|
||||
"ROCm Triton sparse decode expects uint8 fp8_ds_mla SWA cache, "
|
||||
f"got {swa_k_cache.dtype}"
|
||||
)
|
||||
blocked_extra = None
|
||||
_validate_dsv4_sparse_dims(
|
||||
head_dim,
|
||||
nope_head_dim,
|
||||
rope_head_dim,
|
||||
"rocm_sparse_attn_decode",
|
||||
)
|
||||
|
||||
main_indices = swa_indices.reshape(swa_indices.shape[0], -1)
|
||||
|
||||
extra_cache = None
|
||||
extra_indices = None
|
||||
if not swa_only:
|
||||
assert kv_cache is not None
|
||||
blocked_extra = rocm_dequantize_blocked_k_cache(
|
||||
kv_cache,
|
||||
head_dim=head_dim,
|
||||
nope_head_dim=nope_head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
assert topk_indices is not None or (
|
||||
topk_ragged_indices is not None and topk_ragged_indptr is not None
|
||||
)
|
||||
attn_out = rocm_ref_sparse_attn_decode(
|
||||
q=q.unsqueeze(1),
|
||||
blocked_k=blocked_swa,
|
||||
indices_in_kvcache=swa_indices.unsqueeze(1),
|
||||
topk_length=swa_lens,
|
||||
assert kv_cache.dtype == torch.uint8, (
|
||||
"ROCm Triton sparse decode expects uint8 fp8_ds_mla extra cache, "
|
||||
f"got {kv_cache.dtype}"
|
||||
)
|
||||
extra_cache = kv_cache
|
||||
if topk_indices is not None:
|
||||
extra_indices = topk_indices.reshape(topk_indices.shape[0], -1)
|
||||
|
||||
attn_out = _rocm_sparse_attn_decode_triton(
|
||||
q=q,
|
||||
main_cache=swa_k_cache,
|
||||
main_indices=main_indices,
|
||||
scale=scale,
|
||||
head_dim=head_dim,
|
||||
attn_sink=attn_sink[: q.shape[1]] if attn_sink is not None else None,
|
||||
extra_blocked_k=blocked_extra,
|
||||
extra_indices_in_kvcache=topk_indices,
|
||||
extra_topk_length=topk_lens,
|
||||
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
|
||||
nope_head_dim=nope_head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
extra_cache=extra_cache,
|
||||
extra_indices=extra_indices,
|
||||
main_lengths=swa_lens,
|
||||
extra_lengths=topk_lens,
|
||||
main_ragged_indices=swa_ragged_indices,
|
||||
main_ragged_indptr=swa_ragged_indptr,
|
||||
extra_ragged_indices=topk_ragged_indices,
|
||||
extra_ragged_indptr=topk_ragged_indptr,
|
||||
)
|
||||
output.copy_(attn_out.to(output.dtype))
|
||||
|
||||
Reference in New Issue
Block a user