[Attention][DSA] support dcp for FLASHINFER_MLA_SPARSE (#46076)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Jingyi Yang <girasoleyang@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: GirasoleY <girasoleyang@gmail.com>
Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jiangyun Zhu
2026-07-01 00:32:20 -04:00
committed by GitHub
co-authored by Jingyi Yang Lucas Wilkinson Claude Opus 4.8
parent 3c1396bab6
commit c5200d3565
12 changed files with 1995 additions and 97 deletions
+1 -1
View File
@@ -222,7 +222,7 @@ MLA decode backends are selected using the standard
| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ |
| `CUTLASS_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 128 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 10.x |
| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
| `FLASHINFER_MLA_SPARSE_SM120` | bf16 | `auto`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 64, 256 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 12.x |
| `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x |
| `FLASHMLA_SPARSE` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla` | 64 | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |
@@ -0,0 +1,951 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_cutedsl
from vllm.v1.attention.backends.mla.indexer import build_prefill_chunk_metadata
from vllm.v1.attention.backends.mla.sparse_utils import (
triton_filter_and_convert_dcp_index,
)
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.v1.attention.ops.common import CPTritonContext, correct_attn_out
def _local_count(length: int, rank: int, world: int, interleave: int) -> int:
return sum(1 for pos in range(length) if (pos // interleave) % world == rank)
def _global_to_local_indices(
global_indices: torch.Tensor,
rank: int,
world: int,
interleave: int,
) -> torch.Tensor:
valid = global_indices >= 0
global_i64 = global_indices.to(torch.int64).clamp_min(0)
owner = (global_i64 // interleave) % world
local = (global_i64 // (world * interleave)) * interleave + global_i64 % interleave
return torch.where(valid & (owner == rank), local, -1).to(torch.int64)
def _local_to_global_indices(
local_indices: torch.Tensor,
rank: int,
world: int,
interleave: int,
) -> torch.Tensor:
valid = local_indices >= 0
local = local_indices.to(torch.int64).clamp_min(0)
global_indices = (
(local // interleave) * (world * interleave)
+ rank * interleave
+ local % interleave
)
return torch.where(valid, global_indices, -1).to(torch.int64)
def _ref_stable_topk_from_candidates_fp64(
candidate_scores: torch.Tensor,
candidate_token_ids: torch.Tensor,
k: int,
) -> torch.Tensor:
"""Pure-PyTorch reference for the CuteDSL stable-topk selector order
(score desc, then lowest global token id). Selects the same SET as the
kernel; only the set is compared in tests."""
num_rows, num_candidates = candidate_scores.shape
device = candidate_scores.device
select_k = min(k, num_candidates)
valid = candidate_token_ids >= 0
bits = (
candidate_scores.to(torch.float32).view(torch.int32).to(torch.int64)
& 0xFFFFFFFF
)
sign = (bits >> 31) & 1
score_key = (
torch.where(sign.bool(), bits ^ 0xFFFFFFFF, bits ^ 0x80000000) & 0xFFFFFFFF
)
id_key = (~candidate_token_ids.to(torch.int64)) & 0xFFFFFFFF
key = (score_key << 32) | id_key
key = torch.where(valid, key, torch.zeros_like(key))
topk_key = key ^ torch.iinfo(torch.int64).min
_, topk_pos = topk_key.topk(select_k, dim=-1)
selected = candidate_token_ids.gather(1, topk_pos).to(torch.int32)
selected_valid = valid.gather(1, topk_pos)
selected = torch.where(selected_valid, selected, selected.new_full((), -1))
if select_k == k:
return selected
pad = torch.full((num_rows, k - select_k), -1, dtype=torch.int32, device=device)
return torch.cat((selected, pad), dim=1)
def _attention_from_indices(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
indices: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
valid = indices >= 0
safe_indices = indices.clamp_min(0)
selected_k = k[safe_indices]
selected_v = v[safe_indices]
scores = torch.einsum("td,tkd->tk", q, selected_k)
scores = scores.masked_fill(~valid, float("-inf"))
lse = torch.logsumexp(scores, dim=-1)
probs = torch.softmax(scores, dim=-1).masked_fill(~valid, 0.0)
out = torch.einsum("tk,tkd->td", probs, selected_v)
empty_rows = ~valid.any(dim=-1)
out[empty_rows] = 0
lse[empty_rows] = float("-inf")
return out, lse
def _dcp_lse_merge(
local_outs: list[torch.Tensor],
local_lses: list[torch.Tensor],
) -> tuple[torch.Tensor, torch.Tensor]:
outs = torch.stack(local_outs, dim=0)
lses = torch.stack(local_lses, dim=0)
merged_lse = torch.logsumexp(lses, dim=0)
weights = torch.exp(lses - merged_lse.unsqueeze(0))
weights = torch.where(torch.isfinite(weights), weights, torch.zeros_like(weights))
merged_out = (outs * weights.unsqueeze(-1)).sum(dim=0)
return merged_out, merged_lse
class _FakeDCPGroup:
"""Single-process stand-in: ``all_gather`` returns the pre-built
concatenation of every rank's packed ``(score, global_id)`` candidates,
mirroring the one packed all-gather the merge issues."""
def __init__(self, gathered_packed: torch.Tensor) -> None:
self.gathered_packed = gathered_packed
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
assert dim == 1
return self.gathered_packed.clone()
def _run_decode_topk(
logits: torch.Tensor,
seq_lens: torch.Tensor,
next_n: int,
topk: int,
) -> torch.Tensor:
indices = torch.empty(
(logits.shape[0], topk), dtype=torch.int32, device=logits.device
)
torch.ops._C.top_k_per_row_decode(
logits,
next_n,
seq_lens,
indices,
logits.shape[0],
logits.stride(0),
logits.stride(1),
topk,
)
return indices
def _run_persistent_topk(
logits: torch.Tensor,
seq_lens: torch.Tensor,
topk: int,
max_seq_len: int,
) -> torch.Tensor:
indices = torch.empty(
(logits.shape[0], topk), dtype=torch.int32, device=logits.device
)
workspace = torch.empty(1024 * 1024, dtype=torch.uint8, device=logits.device)
torch.ops._C.persistent_topk(
logits,
seq_lens,
indices,
workspace,
topk,
max_seq_len,
)
return indices
def _dcp_attention_from_local_topks(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
local_topks: list[torch.Tensor],
world: int,
interleave: int,
) -> tuple[torch.Tensor, torch.Tensor]:
local_outs = []
local_lses = []
for rank, local_topk in enumerate(local_topks):
owned = [
pos for pos in range(k.shape[0]) if (pos // interleave) % world == rank
]
local_out, local_lse = _attention_from_indices(
q, k[owned], v[owned], local_topk.to(torch.int64)
)
local_outs.append(local_out)
local_lses.append(local_lse)
return _dcp_lse_merge(local_outs, local_lses)
def _merge_local_topks_global_with_fake_dcp(
local_logits: list[torch.Tensor],
local_topks: list[torch.Tensor],
topk: int,
world: int,
interleave: int,
row_starts: list[torch.Tensor | None] | None = None,
) -> list[torch.Tensor]:
"""Run ``_merge_dcp_topk_global`` for every rank against a faked
all-gather and return each rank's global-index result (all ranks should
agree). The fake pre-builds the packed candidate concatenation the merge's
single ``all_gather`` would return."""
# The merge is now CuteDSL-only (no PyTorch fallback), so it runs the real
# Triton pack + CuteDSL selector kernels even behind the faked all-gather.
if not current_platform.is_cuda() or not has_cutedsl():
pytest.skip("DCP merge requires CUDA and CuteDSL")
packed_per_rank = []
for rank, (logits, indices) in enumerate(zip(local_logits, local_topks)):
score_indices = indices.clamp_min(0).to(torch.long)
if row_starts is not None:
rs = row_starts[rank]
assert rs is not None
score_indices = score_indices + rs.to(torch.long).view(-1, 1)
if logits.shape[1] == 0:
scores = torch.full_like(indices, float("-inf"), dtype=torch.float32)
else:
score_indices = score_indices.clamp_max(logits.shape[1] - 1)
scores = logits.gather(1, score_indices).masked_fill(
indices < 0, float("-inf")
)
global_ids = _local_to_global_indices(indices, rank, world, interleave)
packed_per_rank.append(
torch.stack((scores.float(), global_ids.to(torch.float32)), dim=-1)
)
fake_group = _FakeDCPGroup(torch.cat(packed_per_rank, dim=1).contiguous())
original_get_dcp_group = sparse_indexer.get_dcp_group
sparse_indexer.get_dcp_group = lambda: fake_group
try:
merged = []
for rank, (logits, indices) in enumerate(zip(local_logits, local_topks)):
rank_indices = indices.clone()
result = sparse_indexer._merge_dcp_topk_global(
logits,
rank_indices,
topk,
rank,
world,
interleave,
row_starts=None if row_starts is None else row_starts[rank],
)
assert result is None
merged.append(rank_indices)
return merged
finally:
sparse_indexer.get_dcp_group = original_get_dcp_group
@pytest.mark.parametrize("world", [1, 2, 4])
@pytest.mark.parametrize("interleave", [1, 2, 4])
def test_get_dcp_local_seq_lens_matches_naive(world: int, interleave: int):
seq_lens = torch.arange(0, 33, dtype=torch.int32)
for rank in range(world):
actual = get_dcp_local_seq_lens(seq_lens, world, rank, interleave)
expected = torch.tensor(
[
_local_count(int(seq_len), rank, world, interleave)
for seq_len in seq_lens
],
dtype=torch.int32,
)
torch.testing.assert_close(actual, expected)
def test_get_dcp_local_seq_lens_can_localize_per_token_bounds():
seq_lens = torch.tensor([0, 1, 2, 3, 4, 7, 8, 17], dtype=torch.int32)
world = 4
interleave = 2
for rank in range(world):
actual = get_dcp_local_seq_lens(seq_lens, world, rank, interleave)
expected = torch.tensor(
[
_local_count(int(seq_len), rank, world, interleave)
for seq_len in seq_lens
],
dtype=torch.int32,
)
torch.testing.assert_close(actual, expected)
def test_get_dcp_local_seq_lens_preserves_mtp_bounds_shape():
seq_lens = torch.tensor([[8, 9, 10], [11, 12, 13]], dtype=torch.int32)
world = 2
rank = 1
interleave = 1
actual = get_dcp_local_seq_lens(seq_lens, world, rank, interleave)
expected = torch.tensor(
[
[_local_count(int(seq_len), rank, world, interleave) for seq_len in row]
for row in seq_lens
],
dtype=torch.int32,
)
assert actual.shape == seq_lens.shape
torch.testing.assert_close(actual, expected)
def test_get_dcp_local_seq_lens_must_run_after_decode_expansion():
world = 2
rank = 1
interleave = 1
expanded_bounds = torch.tensor([8, 9, 10], dtype=torch.int32)
localized_after_expansion = get_dcp_local_seq_lens(
expanded_bounds, world, rank, interleave
)
localized_request_len_minus_offsets = get_dcp_local_seq_lens(
torch.tensor([10], dtype=torch.int32), world, rank
) - torch.tensor([2, 1, 0], dtype=torch.int32)
assert not torch.equal(
localized_after_expansion, localized_request_len_minus_offsets
)
torch.testing.assert_close(
localized_after_expansion, torch.tensor([4, 4, 5], dtype=torch.int32)
)
@pytest.mark.parametrize("interleave", [1, 2])
def test_sparse_dcp_attention_matches_global_topk_attention(interleave: int):
torch.manual_seed(0)
world = 2
topk = 3
num_queries = 4
max_seq_len = 13
head_dim = 8
q = torch.randn(num_queries, head_dim)
k = torch.randn(max_seq_len, head_dim)
v = torch.randn(max_seq_len, head_dim)
seq_lens = torch.tensor([6, 8, 11, 13], dtype=torch.int64)
scores = q @ k.T
global_topk = torch.full((num_queries, topk), -1, dtype=torch.int64)
for row, seq_len in enumerate(seq_lens.tolist()):
global_topk[row, :topk] = scores[row, :seq_len].topk(topk).indices
ref_out, ref_lse = _attention_from_indices(q, k, v, global_topk)
local_outs = []
local_lses = []
local_topks = []
for rank in range(world):
owned = [
pos for pos in range(max_seq_len) if (pos // interleave) % world == rank
]
k_local = k[owned]
v_local = v[owned]
local_topk = _global_to_local_indices(global_topk, rank, world, interleave)
local_topks.append(local_topk)
local_out, local_lse = _attention_from_indices(q, k_local, v_local, local_topk)
local_outs.append(local_out)
local_lses.append(local_lse)
dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses)
torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5)
torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5)
gathered_global = torch.cat(
[
_local_to_global_indices(local_topks[rank], rank, world, interleave)
for rank in range(world)
],
dim=1,
)
assert set(gathered_global[gathered_global >= 0].tolist()) == set(
global_topk.flatten().tolist()
)
def test_local_topk_union_is_not_equivalent_to_global_topk_attention():
world = 2
interleave = 1
topk = 2
q = torch.tensor([[1.0]])
k = torch.tensor(
[
[1.00],
[0.90],
[0.95],
[0.85],
]
)
v = torch.tensor([[0.0], [1000.0], [0.0], [1000.0]])
scores = q @ k.T
global_topk = scores.topk(topk, dim=-1).indices
ref_out, ref_lse = _attention_from_indices(q, k, v, global_topk)
local_topks = []
for rank in range(world):
owned = [
pos for pos in range(k.shape[0]) if (pos // interleave) % world == rank
]
local_topks.append(scores[:, owned].topk(topk, dim=-1).indices)
local_union_out, local_union_lse = _dcp_attention_from_local_topks(
q, k, v, local_topks, world, interleave
)
assert not torch.allclose(local_union_out, ref_out)
assert not torch.allclose(local_union_lse, ref_lse)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
def test_sparse_decode_dcp_persistent_topk_matches_non_dcp():
torch.manual_seed(3)
device = torch.device("cuda")
world = 2
interleave = 1
topk = 512
num_rows = 2
max_seq_len = 1025
head_dim = 16
q = torch.randn(num_rows, head_dim, device=device)
k = torch.randn(max_seq_len, head_dim, device=device)
v = torch.randn(max_seq_len, head_dim, device=device)
logits = q @ k.T
seq_lens = torch.tensor([[1024], [1025]], dtype=torch.int32, device=device)
non_dcp_topk = torch.empty((num_rows, topk), dtype=torch.int64, device=device)
for row, seq_len in enumerate(seq_lens.flatten().tolist()):
non_dcp_topk[row] = logits[row, :seq_len].topk(topk).indices
ref_out, ref_lse = _attention_from_indices(q, k, v, non_dcp_topk)
local_logits = []
local_topks = []
for rank in range(world):
owned = [
pos for pos in range(max_seq_len) if (pos // interleave) % world == rank
]
rank_logits = logits[:, owned].contiguous()
rank_seq_lens = get_dcp_local_seq_lens(
seq_lens, world, rank, interleave
).contiguous()
local_logits.append(rank_logits)
local_topks.append(
_run_persistent_topk(
rank_logits,
rank_seq_lens,
topk,
max_seq_len=rank_logits.shape[1],
)
)
merged_global_topks = _merge_local_topks_global_with_fake_dcp(
local_logits, local_topks, topk, world, interleave
)
# The radix top-K kernel selects a deterministic SET but writes it in
# nondeterministic (atomicAdd) order; the production path is permutation-
# invariant (compaction + softmax), so all ranks must agree on the set, not
# the array order. (The fp64 fallback happens to return sorted order.)
ref_topk = merged_global_topks[0]
for rank_topk in merged_global_topks[1:]:
for row in range(rank_topk.shape[0]):
assert set(rank_topk[row].tolist()) == set(ref_topk[row].tolist())
local_outs = []
local_lses = []
for rank, global_topk in enumerate(merged_global_topks):
owned = [
pos for pos in range(max_seq_len) if (pos // interleave) % world == rank
]
local_topk = _global_to_local_indices(
global_topk.to(torch.int64),
rank,
world,
interleave,
)
local_out, local_lse = _attention_from_indices(
q, k[owned], v[owned], local_topk
)
local_outs.append(local_out)
local_lses.append(local_lse)
dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses)
torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5)
torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5)
@pytest.mark.skipif(
not current_platform.is_cuda() or not has_cutedsl(),
reason="This test requires CUDA and CuteDSL",
)
@pytest.mark.parametrize("use_row_starts", [False, True])
def test_cutedsl_dcp_candidate_pack_and_select_matches_reference(
use_row_starts: bool,
):
from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import (
pack_dcp_topk_candidates_cutedsl,
stable_topk_from_gathered_candidates_cutedsl,
)
torch.manual_seed(13)
device = torch.device("cuda")
rows = 4
valid_width = 1024
width = valid_width + (8 if use_row_starts else 0)
topk = 512
world = 2
row_starts = (
torch.tensor([0, 2, 4, 1], device=device, dtype=torch.int32)
if use_row_starts
else None
)
row_offsets = (
row_starts
if row_starts is not None
else torch.zeros(rows, device=device, dtype=torch.int32)
)
packed_by_rank = []
for rank in range(world):
logits = torch.randn((rows, width), device=device, dtype=torch.float32)
local_topks = []
for row in range(rows):
start = int(row_offsets[row].item())
local_topks.append(
logits[row, start : start + valid_width].topk(topk).indices
)
topk_indices = torch.stack(local_topks).to(torch.int32)
packed = torch.empty((rows, topk, 2), device=device, dtype=torch.float32)
pack_dcp_topk_candidates_cutedsl(
logits,
topk_indices,
packed,
rank,
world,
1,
row_starts,
)
expected_scores = logits.gather(
1, topk_indices.to(torch.long) + row_offsets.to(torch.long).view(-1, 1)
)
expected_ids = (topk_indices * world + rank).to(torch.float32)
torch.testing.assert_close(packed[..., 0], expected_scores)
torch.testing.assert_close(packed[..., 1], expected_ids)
packed_by_rank.append(packed)
gathered = torch.cat(packed_by_rank, dim=1).contiguous()
actual = torch.empty((rows, topk), device=device, dtype=torch.int32)
returned = stable_topk_from_gathered_candidates_cutedsl(gathered, topk, out=actual)
assert returned is actual
expected = _ref_stable_topk_from_candidates_fp64(
gathered[..., 0],
gathered[..., 1].to(torch.int32),
topk,
)
for row in range(rows):
assert set(actual[row].cpu().tolist()) == set(expected[row].cpu().tolist())
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
def test_sparse_prefill_dcp_metadata_localizes_causal_bounds():
device = torch.device("cuda")
seq_len = 8
query_start_loc = torch.tensor([0, seq_len], dtype=torch.int32, device=device)
query_start_loc_cpu = torch.tensor([0, seq_len], dtype=torch.int32)
seq_lens = torch.tensor([seq_len], dtype=torch.int32, device=device)
seq_lens_cpu = torch.tensor([seq_len], dtype=torch.int32)
block_table = torch.zeros((1, 1), dtype=torch.int32, device=device)
def build(dcp_world_size, dcp_rank, interleave=1):
chunk = build_prefill_chunk_metadata(
start_idx=0,
end_idx=1,
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc_cpu,
uncompressed_seq_lens=seq_lens,
compressed_seq_lens=seq_lens,
compressed_seq_lens_cpu=seq_lens_cpu,
block_table=block_table,
compress_ratio=1,
dcp_rank=dcp_rank,
dcp_world_size=dcp_world_size,
cp_kv_cache_interleave_size=interleave,
)
assert chunk is not None
torch.accelerator.synchronize()
return chunk
# Non-DCP: local_cu_seq_lens aliases the global cu_seq_lens, and
# cu_seqlen_ks/ke carry the global causal bounds.
chunk = build(dcp_world_size=1, dcp_rank=0)
assert chunk.local_cu_seq_lens is chunk.cu_seq_lens
torch.testing.assert_close(
chunk.cu_seqlen_ks.cpu(),
torch.zeros(seq_len, dtype=torch.int32),
)
torch.testing.assert_close(
chunk.cu_seqlen_ke.cpu(),
torch.arange(1, seq_len + 1, dtype=torch.int32),
)
# DCP: cu_seqlen_ks/ke are localized in place to this rank's shard.
chunk = build(dcp_world_size=4, dcp_rank=0)
assert chunk.local_cu_seq_lens is not None
torch.testing.assert_close(
chunk.local_cu_seq_lens.cpu(),
torch.tensor([0, 2], dtype=torch.int32),
)
torch.testing.assert_close(
chunk.cu_seqlen_ks.cpu(),
torch.zeros(seq_len, dtype=torch.int32),
)
torch.testing.assert_close(
chunk.cu_seqlen_ke.cpu(),
torch.tensor([1, 1, 1, 1, 2, 2, 2, 2], dtype=torch.int32),
)
# DCP with interleave=2: per-token causal bounds localize differently from
# interleave=1 (groups of 2 consecutive tokens are owned together). For
# world=4, rank=0, K=2, per-token global len L=1..8 -> local len
# [1,2,2,2,2,2,2,2] (matches get_dcp_local_seq_lens).
chunk = build(dcp_world_size=4, dcp_rank=0, interleave=2)
assert chunk.local_cu_seq_lens is not None
torch.testing.assert_close(
chunk.cu_seqlen_ks.cpu(),
torch.zeros(seq_len, dtype=torch.int32),
)
torch.testing.assert_close(
chunk.cu_seqlen_ke.cpu(),
torch.tensor([1, 2, 2, 2, 2, 2, 2, 2], dtype=torch.int32),
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
def test_dcp_filter_compacts_valid_slots_for_sparse_kernel():
block_size = 4
num_topk = 128
dcp_size = 2
req_id = torch.zeros(1, dtype=torch.int32, device="cuda")
token_indices = torch.full((1, num_topk), -1, dtype=torch.int32, device="cuda")
token_indices[0, :8] = torch.arange(8, dtype=torch.int32, device="cuda")
block_table = torch.tensor([[10]], dtype=torch.int32, device="cuda")
out, valid_counts = triton_filter_and_convert_dcp_index(
req_id,
block_table,
token_indices,
dcp_size=dcp_size,
dcp_rank=0,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=num_topk,
return_valid_counts=True,
)
valid = int(valid_counts.item())
assert valid == 4
assert (out[0, :valid] >= 0).all()
assert (out[0, valid:] == -1).all()
# In-kernel compaction packs valid slots to the front; prefix order is
# unspecified, so compare as a set.
assert set(out[0, :valid].cpu().tolist()) == {40, 41, 42, 43}
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@pytest.mark.parametrize("interleave", [1, 2])
@pytest.mark.parametrize("dcp_rank", [0, 1])
def test_dcp_filter_compaction_matches_reference(interleave: int, dcp_rank: int):
"""In-kernel compaction (atomic slot allocator across multiple column tiles)
must, for every row, produce exactly the rank-owned physical slots packed
into [0, valid_count) with -1 in the tail -- the same SET a reference filter
+ sort/gather produces. Uses wide rows (> BLOCK_N valid slots) so the
cross-tile atomic allocation is exercised, with interior -1 gaps."""
device = torch.device("cuda")
torch.manual_seed(7)
dcp_size = 2
block_size = 8
num_topk = 1024 # > BLOCK_N(128) -> multiple tiles per row
num_rows = 5
max_blocks = 64
seq = max_blocks * block_size
req_id = torch.randint(0, 3, (num_rows,), dtype=torch.int32, device=device)
block_table = torch.randint(
0, 1000, (3, max_blocks), dtype=torch.int32, device=device
)
# Each row: a dense valid prefix of distinct global token ids, then -1 pad.
token_indices = torch.full(
(num_rows, num_topk), -1, dtype=torch.int32, device=device
)
for r in range(num_rows):
n_valid = int(torch.randint(200, 600, (1,)).item())
perm = torch.randperm(seq, device=device)[:n_valid].to(torch.int32)
token_indices[r, :n_valid] = perm
out, valid_counts = triton_filter_and_convert_dcp_index(
req_id,
block_table,
token_indices,
dcp_size=dcp_size,
dcp_rank=dcp_rank,
cp_kv_cache_interleave_size=interleave,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=num_topk,
return_valid_counts=True,
)
for r in range(num_rows):
toks = token_indices[r]
toks = toks[toks >= 0]
owner = (toks // interleave) % dcp_size
owned = toks[owner == dcp_rank]
local = (owned // (dcp_size * interleave)) * interleave + owned % interleave
blk = local // block_size
off = local % block_size
expected = (
block_table[int(req_id[r].item()), blk].to(torch.int64) * block_size + off
)
n = int(valid_counts[r].item())
assert n == owned.numel()
assert (out[r, :n] >= 0).all()
assert (out[r, n:] == -1).all()
assert set(out[r, :n].cpu().tolist()) == set(expected.cpu().tolist())
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@pytest.mark.parametrize("interleave", [1, 2, 4])
def test_dcp_global_topk_physical_attention_matches_non_dcp(interleave: int):
torch.manual_seed(2)
device = torch.device("cuda")
dcp_size = 2
block_size = 4
num_topk = 128
selected_k = 8
seq_len = 16
head_dim = 16
num_queries = 3
q = torch.randn(num_queries, head_dim, device=device)
k_global = torch.randn(seq_len, head_dim, device=device)
v_global = torch.randn(seq_len, head_dim, device=device)
scores = q @ k_global.T
global_topk = torch.full(
(num_queries, num_topk), -1, dtype=torch.int32, device=device
)
global_topk[:, :selected_k] = scores.topk(selected_k, dim=-1).indices.to(
torch.int32
)
ref_out, ref_lse = _attention_from_indices(
q, k_global, v_global, global_topk[:, :selected_k].to(torch.int64)
)
local_outs = []
local_lses = []
for rank in range(dcp_size):
block_ids = torch.tensor(
[[rank * 10 + 1, rank * 10 + 2]], dtype=torch.int32, device=device
)
num_slots = int((block_ids.max().item() + 1) * block_size)
k_cache = torch.zeros(num_slots, head_dim, device=device)
v_cache = torch.zeros(num_slots, head_dim, device=device)
for global_idx in range(seq_len):
if (global_idx // interleave) % dcp_size != rank:
continue
local_idx = (
global_idx // (dcp_size * interleave)
) * interleave + global_idx % interleave
block = local_idx // block_size
offset = local_idx % block_size
slot = int(block_ids[0, block].item()) * block_size + offset
k_cache[slot] = k_global[global_idx]
v_cache[slot] = v_global[global_idx]
slots, valid_counts = triton_filter_and_convert_dcp_index(
torch.zeros(num_queries, dtype=torch.int32, device=device),
block_ids,
global_topk,
dcp_size=dcp_size,
dcp_rank=rank,
cp_kv_cache_interleave_size=interleave,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=num_topk,
return_valid_counts=True,
)
row_ids = torch.arange(num_queries, device=device)
assert (slots[row_ids, valid_counts] == -1).all()
local_out, local_lse = _attention_from_indices(
q, k_cache, v_cache, slots.to(torch.int64)
)
local_outs.append(local_out)
local_lses.append(local_lse)
dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses)
torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5)
torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@pytest.mark.parametrize("is_lse_base_on_e", [True, False])
def test_correct_attn_out_zeroes_empty_nan_partial(is_lse_base_on_e: bool):
out = torch.full((1, 1, 4), float("nan"), device="cuda")
lses = torch.tensor(
[[[0.0]], [[float("-inf")]]],
dtype=torch.float32,
device="cuda",
)
corrected, final_lse = correct_attn_out(
out,
lses,
cp_rank=1,
ctx=CPTritonContext(),
is_lse_base_on_e=is_lse_base_on_e,
)
torch.accelerator.synchronize()
torch.testing.assert_close(corrected, torch.zeros_like(corrected))
torch.testing.assert_close(final_lse, torch.zeros_like(final_lse))
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
def test_decode_topk_pads_surplus_with_negative_one():
"""When a row's valid length < topk the top-k kernel must pad the surplus
slots with -1: the DCP merge (`topk_indices >= 0`) and
`triton_filter_and_convert_dcp_index` (`tok < 0`) both treat <0 as invalid,
so a non-(-1) pad would be silently attended. This is the common case under
DCP, where each rank's local seq_len = global / world is usually << topk.
Checked on the *set* of valid indices (the kernel may order them by score,
not position)."""
torch.manual_seed(0)
device = torch.device("cuda")
topk = 8
next_n = 1
seq_lens_list = [0, 3, 5, 8]
num_rows = len(seq_lens_list)
logits = torch.randn(num_rows, 16, device=device)
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device=device).view(
num_rows, next_n
)
idx = _run_decode_topk(logits, seq_lens, next_n, topk)
for r, sl in enumerate(seq_lens_list):
valid = idx[r][idx[r] >= 0]
# seq_len <= topk, so top-k selects exactly the whole valid range.
assert valid.numel() == min(sl, topk)
assert set(valid.tolist()) == set(range(sl))
assert (idx[r] == -1).sum().item() == topk - min(sl, topk)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
def test_persistent_topk_pads_surplus_with_negative_one():
"""Same surplus=-1 invariant for the persistent_topk kernel (k>=512)."""
torch.manual_seed(0)
device = torch.device("cuda")
topk = 512 # persistent_topk requires k in {512, 1024, 2048}
seq_lens_list = [100, 300, 512]
num_rows = len(seq_lens_list)
max_seq_len = 600
logits = torch.randn(num_rows, max_seq_len, device=device)
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device=device).view(
num_rows, 1
)
idx = _run_persistent_topk(logits, seq_lens, topk, max_seq_len)
for r, sl in enumerate(seq_lens_list):
valid = idx[r][idx[r] >= 0]
assert valid.numel() == min(sl, topk)
assert set(valid.tolist()) == set(range(sl))
assert (idx[r] == -1).sum().item() == topk - min(sl, topk)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
def test_sparse_decode_dcp_short_context_matches_non_dcp():
"""End-to-end DCP decode where the global seq_len < topk (so every rank's
local top-k is surplus-padded). Exercises the kernel surplus -> merge mask
-> global top-k -> physical localize -> LSE merge chain for the common
short-context decode case, vs the non-DCP reference."""
torch.manual_seed(4)
device = torch.device("cuda")
world = 2
interleave = 1
topk = 512
num_rows = 2
max_seq_len = 300 # < topk -> surplus everywhere
head_dim = 16
q = torch.randn(num_rows, head_dim, device=device)
k = torch.randn(max_seq_len, head_dim, device=device)
v = torch.randn(max_seq_len, head_dim, device=device)
logits = q @ k.T
seq_lens = torch.tensor([[250], [300]], dtype=torch.int32, device=device)
non_dcp_topk = torch.empty((num_rows, topk), dtype=torch.int64, device=device)
for row, seq_len in enumerate(seq_lens.flatten().tolist()):
sel = logits[row, :seq_len].topk(min(topk, seq_len)).indices
non_dcp_topk[row, : sel.numel()] = sel
non_dcp_topk[row, sel.numel() :] = -1
ref_out, ref_lse = _attention_from_indices(q, k, v, non_dcp_topk)
local_logits = []
local_topks = []
for rank in range(world):
owned = [p for p in range(max_seq_len) if (p // interleave) % world == rank]
rank_logits = logits[:, owned].contiguous()
rank_seq_lens = get_dcp_local_seq_lens(seq_lens, world, rank, interleave)
local_logits.append(rank_logits)
local_topks.append(
_run_persistent_topk(
rank_logits,
rank_seq_lens.contiguous(),
topk,
max_seq_len=rank_logits.shape[1],
)
)
merged_global_topks = _merge_local_topks_global_with_fake_dcp(
local_logits, local_topks, topk, world, interleave
)
# The radix top-K kernel selects a deterministic SET but writes it in
# nondeterministic (atomicAdd) order; the production path is permutation-
# invariant (compaction + softmax), so all ranks must agree on the set, not
# the array order. (The fp64 fallback happens to return sorted order.)
ref_topk = merged_global_topks[0]
for rank_topk in merged_global_topks[1:]:
for row in range(rank_topk.shape[0]):
assert set(rank_topk[row].tolist()) == set(ref_topk[row].tolist())
local_outs = []
local_lses = []
for rank, global_topk in enumerate(merged_global_topks):
owned = [p for p in range(max_seq_len) if (p // interleave) % world == rank]
local_topk = _global_to_local_indices(
global_topk.to(torch.int64), rank, world, interleave
)
local_out, local_lse = _attention_from_indices(
q, k[owned], v[owned], local_topk
)
local_outs.append(local_out)
local_lses.append(local_lse)
dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses)
torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5)
torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,420 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings.driver import CUstream
from cutlass import Float32, Int32, Uint32, Uint64
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import recast_val
from vllm.triton_utils import tl, triton
def stable_topk_from_gathered_candidates_cutedsl(
gathered: torch.Tensor,
topk: int,
out: torch.Tensor | None = None,
) -> torch.Tensor:
if out is None:
out = torch.empty(
(gathered.shape[0], topk),
dtype=torch.int32,
device=gathered.device,
)
StableTopKFromGatheredCandidatesKernel.compile(topk, gathered.shape[1])(
gathered, out
)
return out
def pack_dcp_topk_candidates_cutedsl(
logits: torch.Tensor,
topk_indices: torch.Tensor,
packed: torch.Tensor,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
row_starts: torch.Tensor | None,
) -> None:
topk = topk_indices.shape[1]
grid = (topk_indices.shape[0], triton.cdiv(topk, 512))
row_starts_arg = row_starts if row_starts is not None else topk_indices
_pack_dcp_topk_candidates_triton_kernel[grid](
logits,
topk_indices,
packed,
row_starts_arg,
logits.stride(0),
logits.stride(1),
topk_indices.stride(0),
topk_indices.stride(1),
packed.stride(0),
packed.stride(1),
packed.stride(2),
logits.shape[1],
DCP_RANK=dcp_rank,
DCP_WORLD_SIZE=dcp_world_size,
CP_INTERLEAVE=cp_interleave,
HAS_ROW_STARTS=row_starts is not None,
TOPK=topk,
BLOCK_SIZE=512,
num_warps=8,
)
@triton.jit
def _pack_dcp_topk_candidates_triton_kernel(
logits,
topk_indices,
packed,
row_starts,
logits_stride0: tl.constexpr,
logits_stride1: tl.constexpr,
topk_stride0: tl.constexpr,
topk_stride1: tl.constexpr,
packed_stride0: tl.constexpr,
packed_stride1: tl.constexpr,
packed_stride2: tl.constexpr,
num_cols,
DCP_RANK: tl.constexpr,
DCP_WORLD_SIZE: tl.constexpr,
CP_INTERLEAVE: tl.constexpr,
HAS_ROW_STARTS: tl.constexpr,
TOPK: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
tile = tl.program_id(1)
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = cols < TOPK
local_idx = tl.load(
topk_indices + row * topk_stride0 + cols * topk_stride1,
mask=mask,
other=-1,
)
valid = local_idx >= 0
safe_local_idx = tl.maximum(local_idx, 0)
row_start = 0
if HAS_ROW_STARTS:
row_start = tl.load(row_starts + row)
score_col = safe_local_idx + row_start
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
score = tl.load(
logits + row * logits_stride0 + score_col * logits_stride1,
mask=mask & valid,
other=-float("inf"),
)
global_id = (
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
+ DCP_RANK * CP_INTERLEAVE
+ safe_local_idx % CP_INTERLEAVE
)
global_id = tl.where(valid, global_id, -1)
packed_base = packed + row * packed_stride0 + cols * packed_stride1
tl.store(packed_base, score, mask=mask)
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
@cute.jit
def _warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
offset = 1 << i
partial = cute.arch.shuffle_sync_up(val, offset=offset, mask_and_clamp=0)
if lane >= offset:
val += partial
return val
@cute.jit
def _block_scan_inclusive_i32(
val: Int32,
lane: Int32,
warp_id: Int32,
warp_scratch: cute.Tensor,
warps_per_block: int,
) -> Int32:
prefix = _warp_scan_inclusive_i32(val, lane)
if lane == Int32(cute.arch.WARP_SIZE - 1):
warp_scratch[0, warp_id] = prefix
cute.arch.sync_threads()
if warp_id == Int32(0):
warp_total = Int32(0)
if lane < Int32(warps_per_block):
warp_total = warp_scratch[0, lane]
warp_prefix = _warp_scan_inclusive_i32(warp_total, lane)
if lane < Int32(warps_per_block):
warp_scratch[0, lane] = warp_prefix - warp_total
cute.arch.sync_threads()
return prefix + warp_scratch[0, warp_id]
class StableTopKFromGatheredCandidatesKernel:
tb_size = 512
hist_bins = 2048
radix_bits = (hist_bins - 1).bit_length()
assert hist_bins == 1 << radix_bits
key_bits = Uint64.width
radix_passes = (key_bits + radix_bits - 1) // radix_bits
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
hist_chunks = (hist_bins + tb_size - 1) // tb_size
warps_per_block = tb_size // cute.arch.WARP_SIZE
def __init__(self, topk: int, num_candidates: int):
assert num_candidates % self.tb_size == 0, (
"StableTopKFromGatheredCandidatesKernel requires candidate count "
f"to be a multiple of {self.tb_size}, got {num_candidates}"
)
self.topk = topk
self.keys_per_thread = num_candidates // self.tb_size
@cute.struct
class SharedStorage:
hist: cute.struct.MemRange[Int32, self.hist_bins]
committed_count: cute.struct.MemRange[Int32, 1]
running_count: cute.struct.MemRange[Int32, 1]
threshold_bin: cute.struct.MemRange[Int32, 1]
threshold_found: cute.struct.MemRange[Int32, 1]
include_threshold_bin: cute.struct.MemRange[Int32, 1]
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
warp_totals: cute.struct.MemRange[Int32, self.warps_per_block]
self.shared_storage = SharedStorage
@cute.jit
def __call__(
self,
gathered: cute.Tensor,
out: cute.Tensor,
stream: CUstream,
):
grid = (gathered.shape[0], 1, 1)
self.kernel(gathered, out).launch(
grid=grid,
block=(self.tb_size, 1, 1),
stream=stream,
)
@cute.jit
def _stable_key(self, score: Float32, token_id: Int32) -> Uint64:
bits = recast_val(score, Uint32)
mask = Uint32(0x80000000)
if (bits & Uint32(0x80000000)) != Uint32(0):
mask = Uint32(0xFFFFFFFF)
score_key = Uint64(bits ^ mask) << Uint64(32)
id_key = Uint64(~Uint32(token_id))
key = score_key | id_key
if token_id < Int32(0):
key = Uint64(0)
return key
@cute.jit
def _prefix_matches(
self,
key: Uint64,
prefix: Uint64,
prefix_bits: Int32,
):
matches = prefix_bits == Int32(0)
if prefix_bits != Int32(0):
shift = Int32(self.key_bits) - prefix_bits
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
return matches
@cute.jit
def _radix_pass(
self,
keys: cute.Tensor,
output: cute.Tensor,
storage,
tid: Int32,
step: Int32,
bits: int,
is_final_pass: bool,
):
hist_smem = storage.hist.get_tensor(cute.make_layout((self.hist_bins,)))
committed_count_smem = storage.committed_count.data_ptr()
running_count_smem = storage.running_count.data_ptr()
threshold_bin_smem = storage.threshold_bin.data_ptr()
threshold_found_smem = storage.threshold_found.data_ptr()
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
warp_totals_smem = storage.warp_totals.get_tensor(
cute.make_layout((1, self.warps_per_block))
)
prefix_bits = step * Int32(self.radix_bits)
num_bins = 1 << bits
block_scan_iterations = (num_bins + self.tb_size - 1) // self.tb_size
shift = Int32(self.key_bits) - prefix_bits - Int32(bits)
bin_mask = Uint64(num_bins - 1)
prefix = prefix_smem.load()
for chunk in cutlass.range_constexpr(self.hist_chunks):
hist_smem[tid + Int32(chunk * self.tb_size)] = Int32(0)
if tid == Int32(0):
running_count_smem.store(committed_count_smem.load())
include_threshold_bin_smem.store(Int32(0))
threshold_found_smem.store(Int32(0))
cute.arch.sync_threads()
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
key = keys[key_idx]
if self._prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
cute.arch.atomic_add(
hist_smem.iterator + bin_idx,
Int32(1),
sem="relaxed",
scope="cta",
)
cute.arch.sync_threads()
lane = cute.arch.lane_idx()
warp_id = cute.arch.warp_idx()
# Each iteration scans one tb_size-wide slice of bins, high to low.
iter = Int32(0)
threshold_found = threshold_found_smem.load()
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
bin_idx = Int32(num_bins - 1) - (iter * Int32(self.tb_size) + tid)
count = hist_smem[bin_idx]
chunk_inclusive = _block_scan_inclusive_i32(
count,
lane,
warp_id,
warp_totals_smem,
self.warps_per_block,
)
running_count = running_count_smem.load()
prior_in_scan_slice = chunk_inclusive - count
remaining = Int32(self.topk) - running_count - prior_in_scan_slice
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
threshold_bin_smem.store(bin_idx)
if count <= remaining or cutlass.const_expr(is_final_pass):
include_threshold_bin_smem.store(Int32(1))
threshold_found_smem.store(Int32(1))
# Barrier: every thread must finish reading running_count for this
# slice before tb_size-1 advances it, else a warp racing ahead to
# the store makes a lagging thread double-count the slice total
# (-> remaining too small -> threshold too high -> under-fill).
cute.arch.sync_threads()
if tid == Int32(self.tb_size - 1):
running_count_smem.store(running_count + chunk_inclusive)
cute.arch.sync_threads()
threshold_found = threshold_found_smem.load()
iter += Int32(1)
threshold = threshold_bin_smem.load()
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
key = keys[key_idx]
if self._prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
selected = bin_idx > threshold
if should_include_threshold:
selected = selected or bin_idx == threshold
if selected:
dst = cute.arch.atomic_add(
committed_count_smem,
Int32(1),
sem="relaxed",
scope="cta",
)
if dst < Int32(self.topk):
output[dst] = recast_val(~Uint32(key), Int32)
cute.arch.sync_threads()
pass_finished = include_threshold_bin_smem.load()
if tid == Int32(0) and pass_finished == Int32(0):
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
cute.arch.sync_threads()
return pass_finished
@cute.kernel
def kernel(
self,
input: cute.Tensor,
out: cute.Tensor,
):
row, _, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
input_row = input[row, None, None]
output_row = out[row, None]
keys = cute.make_rmem_tensor((self.keys_per_thread,), Uint64)
smem = cutlass.utils.SmemAllocator()
storage = smem.allocate(self.shared_storage, 8)
committed_count_smem = storage.committed_count.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
for i in range(tid, self.topk, self.tb_size):
output_row[i] = Int32(-1)
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
col = tid + Int32(key_idx * self.tb_size)
score = Float32(input_row[col, 0])
token_id = Int32(input_row[col, 1])
keys[key_idx] = self._stable_key(score, token_id)
if tid == Int32(0):
committed_count_smem.store(Int32(0))
prefix_smem.store(Uint64(0))
cute.arch.sync_threads()
step = Int32(0)
finished = Int32(0)
while finished == Int32(0) and step < Int32(self.radix_passes - 1):
finished = self._radix_pass(
keys,
output_row,
storage,
tid,
step,
self.radix_bits,
False,
)
step += Int32(1)
if finished == Int32(0):
self._radix_pass(
keys,
output_row,
storage,
tid,
Int32(self.radix_passes - 1),
self.final_radix_bits,
True,
)
@cache
@staticmethod
def compile(topk: int, num_candidates: int):
num_rows = cute.sym_int()
gathered = cute.runtime.make_fake_tensor(
Float32,
(num_rows, num_candidates, 2),
stride=(cute.sym_int64(divisibility=2), 2, 1),
assumed_align=8,
)
out = make_fake_tensor(Int32, (num_rows, topk), divisibility=1)
kernel = StableTopKFromGatheredCandidatesKernel(topk, num_candidates)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
gathered,
out,
stream,
options="--enable-tvm-ffi",
)
+167 -44
View File
@@ -8,6 +8,8 @@ import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.distributed import get_dcp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
@@ -21,6 +23,7 @@ from vllm.utils.deep_gemm import (
fp8_fp4_paged_mqa_logits,
has_deep_gemm,
)
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
@@ -41,6 +44,85 @@ RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
MXFP4_BLOCK_SIZE = 32
def _assert_cutedsl_dcp_merge_supported(
logits: torch.Tensor,
topk_indices: torch.Tensor,
k: int,
) -> None:
# The DCP merge only supports the CuteDSL path (Triton pack kernel + CuteDSL
# stable-topk selector); there is no PyTorch fallback. The first cut targets
# Blackwell/Hopper with index_topk in (512, 1024, 2048) (the selector's radix
# sizing); the Triton pack itself has no shape/topk constraints.
if not has_cutedsl():
raise RuntimeError(
"DCP sparse-indexer merge requires CuteDSL; install it or disable DCP."
)
if logits.device.type != "cuda":
raise RuntimeError("DCP sparse-indexer merge requires CUDA tensors.")
if logits.dtype != torch.float32 or topk_indices.dtype != torch.int32:
raise RuntimeError(
"DCP sparse-indexer merge requires fp32 logits and int32 indices."
)
if k not in (512, 1024, 2048):
raise RuntimeError(
f"DCP sparse-indexer merge requires index_topk in (512, 1024, 2048); "
f"got {k}."
)
def _merge_dcp_topk_global(
logits: torch.Tensor,
topk_indices: torch.Tensor,
topk_tokens: int,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
row_starts: torch.Tensor | None = None,
) -> None:
"""Merge each DCP rank's local top-K into the global top-K.
``topk_indices`` are this rank's local top-K positions into its 1/N KV
shard. A token in the global top-K must also be in its owning rank's local
top-K (at most ``topk_tokens - 1`` tokens rank globally above it, hence at
most that many on its own rank), so exchanging only the per-rank local
candidates is exact -- equivalent to all-gathering the full logit matrix,
but it ships ``dcp_world_size * topk_tokens`` candidates instead of the whole
score row. Overwrites ``topk_indices`` with global token ids (``-1`` for
padding); the attention backend localizes them back to physical slots per
rank.
"""
if dcp_world_size <= 1:
return
# CuteDSL-only path (no PyTorch fallback): Triton-pack each rank's
# (score, global_id) candidates on-device, all-gather, then the CuteDSL
# stable-topk selector.
_assert_cutedsl_dcp_merge_supported(logits, topk_indices, topk_tokens)
from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import (
pack_dcp_topk_candidates_cutedsl,
stable_topk_from_gathered_candidates_cutedsl,
)
packed = torch.empty(
(*topk_indices.shape, 2),
dtype=torch.float32,
device=topk_indices.device,
)
pack_dcp_topk_candidates_cutedsl(
logits,
topk_indices,
packed,
dcp_rank,
dcp_world_size,
cp_interleave,
row_starts,
)
gathered = get_dcp_group().all_gather(packed, dim=1)
stable_topk_from_gathered_candidates_cutedsl(
gathered, topk_tokens, out=topk_indices
)
@triton.jit
def _fused_indexer_q_rope_quant_kernel(
positions,
@@ -227,6 +309,9 @@ def sparse_attn_indexer(
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
dcp_world_size: int = 1,
cp_kv_cache_interleave_size: int = 1,
skip_topk_buffer_clear: bool = False,
) -> torch.Tensor:
# careful! this will be None in dummy run
@@ -328,16 +413,18 @@ def sparse_attn_indexer(
scales_spec,
)
for chunk in prefill_metadata.chunks:
k_quant = k_quant_full[: chunk.total_seq_lens]
k_scale = k_scale_full[: chunk.total_seq_lens]
if not chunk.skip_kv_gather:
cu_seqlen_ks = chunk.cu_seqlen_ks
cu_seqlen_ke = chunk.cu_seqlen_ke
assert chunk.local_cu_seq_lens is not None
k_quant = k_quant_full[: chunk.max_local_total_seq_lens]
k_scale = k_scale_full[: chunk.max_local_total_seq_lens]
if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0:
ops.cp_gather_indexer_k_quant_cache(
kv_cache,
k_quant,
k_scale,
chunk.block_table,
chunk.cu_seq_lens,
chunk.local_cu_seq_lens,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
@@ -346,51 +433,64 @@ def sparse_attn_indexer(
if q_scale is not None
else None
)
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
if use_fp4_cache:
q_slice_cast = q_slice.view(torch.int8)
k_quant_cast = k_quant.view(torch.int8)
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
else:
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
if current_platform.is_xpu():
if q_scale_slice is not None:
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
q_slice_cast,
k_quant_cast,
k_scale_cast,
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
)
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
clean_logits=False,
)
num_rows = logits.shape[0]
topk_indices = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
ops.top_k_per_row_prefill(
if chunk.local_total_seq_lens == 0:
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
topk_indices.fill_(-1)
else:
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
if use_fp4_cache:
q_slice_cast = q_slice.view(torch.int8)
k_quant_cast = k_quant.view(torch.int8)
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
else:
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
if current_platform.is_xpu():
if q_scale_slice is not None:
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
q_slice_cast,
k_quant_cast,
k_scale_cast,
weights[chunk.token_start : chunk.token_end],
cu_seqlen_ks,
cu_seqlen_ke,
)
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
cu_seqlen_ks,
cu_seqlen_ke,
clean_logits=False,
)
num_rows = logits.shape[0]
ops.top_k_per_row_prefill(
logits,
cu_seqlen_ks,
cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
_merge_dcp_topk_global(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
row_starts=chunk.cu_seqlen_ks,
)
if has_decode:
@@ -508,7 +608,7 @@ def sparse_attn_indexer(
topk_indices,
topk_workspace,
topk_tokens,
attn_metadata_narrowed.max_seq_len,
logits.shape[1],
)
else:
ops.top_k_per_row_decode(
@@ -522,6 +622,16 @@ def sparse_attn_indexer(
topk_tokens,
)
if decode_metadata.global_seq_lens is not None:
_merge_dcp_topk_global(
logits,
topk_indices,
topk_tokens,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
)
if decode_metadata.requires_padding:
# if padded, we need to unpack
# the topk indices removing padded tokens
@@ -553,6 +663,9 @@ def sparse_attn_indexer_fake(
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
dcp_world_size: int = 1,
cp_kv_cache_interleave_size: int = 1,
skip_topk_buffer_clear: bool = False,
) -> torch.Tensor:
return topk_indices_buffer
@@ -604,6 +717,13 @@ class SparseAttnIndexer(CustomOp):
self.topk_indices_buffer = topk_indices_buffer
self.skip_k_cache_insert = skip_k_cache_insert
self.use_fp4_cache = use_fp4_cache
# DCP scalars are constant for the run; resolve them here (config is set
# during model construction) and pass them into the custom op, rather
# than threading them through per-step metadata.
parallel_config = get_current_vllm_config().parallel_config
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
if current_platform.is_cuda() and not has_deep_gemm():
raise RuntimeError(
"Sparse Attention Indexer CUDA op requires DeepGEMM support in "
@@ -657,6 +777,9 @@ class SparseAttnIndexer(CustomOp):
self.topk_indices_buffer,
self.skip_k_cache_insert,
self.use_fp4_cache,
self.dcp_rank,
self.dcp_world_size,
self.cp_kv_cache_interleave_size,
)
def forward_xpu(
+2 -1
View File
@@ -496,7 +496,8 @@ class DeepseekV32Attention(MLAAttention):
self.topk_indices_buffer,
True, # skip_k_cache_insert
False, # use_fp4_cache
True, # skip_topk_buffer_clear (fused_norm_rope already did it)
# fused_norm_rope already cleared the topk buffer this forward.
skip_topk_buffer_clear=True,
)
if attn_metadata is None:
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, ClassVar
import numpy as np
import torch
from vllm import envs
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
@@ -29,8 +30,12 @@ from vllm.v1.attention.backend import (
)
from vllm.v1.attention.backends.mla.sparse_utils import (
triton_convert_req_index_to_global_index,
triton_filter_and_convert_dcp_index,
)
from vllm.v1.attention.backends.utils import (
KVCacheLayoutType,
split_decodes_and_prefills,
)
from vllm.v1.attention.backends.utils import KVCacheLayoutType
from vllm.v1.kv_cache_interface import AttentionSpec
if TYPE_CHECKING:
@@ -38,8 +43,6 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
FLASHINFER_MLA_SPARSE_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
class _FlashInferMLASparseBackendBase(AttentionBackend):
"""Common metadata for concrete FlashInfer sparse MLA backends."""
@@ -252,10 +255,13 @@ class FlashInferMLASparseMetadata(AttentionMetadata):
# Sequence lengths for all requests (context + query)
seq_lens: torch.Tensor
num_decodes: int
num_decode_tokens: int
# Sparse-specific
block_size: int = 64
topk_tokens: int = 2048
cp_kv_cache_interleave_size: int = 1
class FlashInferMLASparseMetadataBuilder(
@@ -281,6 +287,12 @@ class FlashInferMLASparseMetadataBuilder(
self.mla_dims = get_mla_dims(self.model_config)
self.topk_tokens = vllm_config.model_config.hf_config.index_topk
self._init_reorder_batch_threshold(
1,
supports_spec_as_decode=True,
supports_dcp_with_varlen=True,
)
self.req_id_per_token_buffer = torch.empty(
(vllm_config.scheduler_config.max_num_batched_tokens,),
dtype=torch.int32,
@@ -295,6 +307,12 @@ class FlashInferMLASparseMetadataBuilder(
) -> FlashInferMLASparseMetadata:
cm = common_attn_metadata
num_tokens = cm.num_actual_tokens
assert self.reorder_batch_threshold is not None
num_decodes, _, num_decode_tokens, _ = split_decodes_and_prefills(
cm,
decode_threshold=self.reorder_batch_threshold,
treat_short_extends_as_decodes=True,
)
# Build req_id_per_token mapping
starts = np.asarray(cm.query_start_loc_cpu, dtype=np.int32)
@@ -320,8 +338,13 @@ class FlashInferMLASparseMetadataBuilder(
block_table=cm.block_table_tensor,
req_id_per_token=req_id_per_token_tensor,
seq_lens=cm.seq_lens,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
block_size=self.kv_cache_spec.block_size,
topk_tokens=self.topk_tokens,
cp_kv_cache_interleave_size=(
self.vllm_config.parallel_config.cp_kv_cache_interleave_size
),
)
@@ -333,7 +356,7 @@ def _get_workspace_buffer(device: torch.device) -> torch.Tensor:
global _fi_sparse_workspace
if _fi_sparse_workspace is None:
_fi_sparse_workspace = torch.zeros(
FLASHINFER_MLA_SPARSE_WORKSPACE_BUFFER_SIZE,
envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,
dtype=torch.uint8,
device=device,
)
@@ -347,6 +370,9 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata
sparse attention computation.
"""
can_return_lse_for_decode: bool = True
lse_base_on_e: bool = False
def __init__(
self,
num_heads: int,
@@ -421,14 +447,27 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata
assert self.topk_indices_buffer is not None
topk_indices = self.topk_indices_buffer[:num_actual_toks]
topk_indices_physical, seq_lens = triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token[:num_actual_toks],
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
NUM_TOPK_TOKENS=topk_indices.shape[1],
return_valid_counts=True,
)
if self.dcp_world_size > 1:
topk_indices_physical, seq_lens = triton_filter_and_convert_dcp_index(
attn_metadata.req_id_per_token[:num_actual_toks],
attn_metadata.block_table,
topk_indices,
dcp_size=self.dcp_world_size,
dcp_rank=self.dcp_rank,
cp_kv_cache_interleave_size=(attn_metadata.cp_kv_cache_interleave_size),
BLOCK_SIZE=attn_metadata.block_size,
NUM_TOPK_TOKENS=topk_indices.shape[1],
return_valid_counts=True,
)
else:
topk_indices_physical, seq_lens = triton_convert_req_index_to_global_index(
attn_metadata.req_id_per_token[:num_actual_toks],
attn_metadata.block_table,
topk_indices,
BLOCK_SIZE=attn_metadata.block_size,
NUM_TOPK_TOKENS=topk_indices.shape[1],
return_valid_counts=True,
)
if self._workspace_buffer is None:
self._workspace_buffer = _get_workspace_buffer(q.device)
@@ -444,18 +483,66 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
o = trtllm_batch_decode_with_kv_cache_mla(
query=q.unsqueeze(1),
# Single-token sparse decode. trtllm-gen requires the q_len_per_request
# dim, but the sparse attention mask is fully per-token (each query token
# carries its own top-k index row), so unsqueeze is sufficient and
# correct. The MTP/multi-token q_len grouping is a perf-only layout and is
# deferred until MTP is validated end-to-end for this backend.
query = q.unsqueeze(1)
block_tables = topk_indices_physical.unsqueeze(1)
seq_lens_arg = seq_lens
kernel_out = trtllm_batch_decode_with_kv_cache_mla(
query=query,
kv_cache=kv_c_and_k_pe_cache.unsqueeze(1),
workspace_buffer=self._workspace_buffer,
qk_nope_head_dim=self.qk_nope_head_dim,
kv_lora_rank=self.kv_lora_rank,
qk_rope_head_dim=self.qk_rope_head_dim,
block_tables=topk_indices_physical.unsqueeze(1),
seq_lens=seq_lens,
block_tables=block_tables,
seq_lens=seq_lens_arg,
max_seq_len=attn_metadata.topk_tokens,
bmm1_scale=self.bmm1_scale,
bmm2_scale=self.bmm2_scale,
sparse_mla_top_k=attn_metadata.topk_tokens,
return_lse=self.need_to_return_lse_for_decode,
)
return o.view(-1, o.shape[-2], o.shape[-1]), None
if self.need_to_return_lse_for_decode:
assert isinstance(kernel_out, tuple)
o, lse = kernel_out
else:
assert isinstance(kernel_out, torch.Tensor)
o = kernel_out
lse = None
out = o.view(-1, o.shape[-2], o.shape[-1])
if lse is not None:
lse = self._normalize_lse(lse, out.shape[0], out.shape[1])
empty_rows = (topk_indices_physical == -1).all(dim=-1)
out.masked_fill_(empty_rows.view(-1, 1, 1), 0.0)
lse.masked_fill_(empty_rows.view(-1, 1), float("-inf"))
return out, lse
@staticmethod
def _normalize_lse(
lse: torch.Tensor,
num_tokens: int,
num_heads: int,
) -> torch.Tensor:
# FlashInfer returns the decode LSE either as 2D (num_tokens, num_heads)
# or 3D ((num_tokens, num_heads, 1) / (num_tokens, 1, num_heads)).
# Collapse all of these to the (num_tokens, num_heads) the shared DCP
# reducer expects.
if lse.dim() == 3:
if lse.shape[-1] == 1:
lse = lse.squeeze(-1)
elif lse.shape[1] == 1:
lse = lse.squeeze(1)
elif lse.shape[0] * lse.shape[1] == num_tokens:
lse = lse.reshape(num_tokens, lse.shape[-1])
if lse.shape != (num_tokens, num_heads):
raise RuntimeError(
"Unexpected FlashInfer sparse MLA LSE shape: "
f"{tuple(lse.shape)}, expected ({num_tokens}, {num_heads})."
)
return lse
+175 -11
View File
@@ -6,6 +6,7 @@ import torch
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed import get_dcp_group
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@@ -24,6 +25,7 @@ from vllm.v1.attention.backend import (
)
from vllm.v1.attention.backends.mla.compressor_utils import get_compressed_slot_mapping
from vllm.v1.attention.backends.utils import (
get_dcp_local_seq_lens,
split_decodes_and_prefills,
)
from vllm.v1.kv_cache_interface import AttentionSpec, MLAAttentionSpec
@@ -168,6 +170,8 @@ class DeepseekV4IndexerBackend(DeepseekV32IndexerBackend):
@dataclass
class DeepseekV32IndexerPrefillChunkMetadata:
block_table: torch.Tensor
# Under DCP (dcp_world_size > 1) these hold this rank's local row bounds;
# otherwise they hold the global bounds.
cu_seqlen_ks: torch.Tensor
cu_seqlen_ke: torch.Tensor
cu_seq_lens: torch.Tensor
@@ -177,6 +181,9 @@ class DeepseekV32IndexerPrefillChunkMetadata:
token_end: int
num_reqs: int
skip_kv_gather: bool = False
local_cu_seq_lens: torch.Tensor | None = None
local_total_seq_lens: int = 0
max_local_total_seq_lens: int = 0
@dataclass
@@ -195,6 +202,7 @@ class DeepSeekV32IndexerDecodeMetadata:
decode_lens: torch.Tensor
requires_padding: bool
schedule_metadata: torch.Tensor
global_seq_lens: torch.Tensor | None = None
@dataclass
@@ -243,6 +251,19 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
scheduler_config = self.vllm_config.scheduler_config
parallel_config = self.vllm_config.parallel_config
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
# The DCP sparse-indexer code is parameterized by interleave size, but
# interleave > 1 is not yet validated end-to-end (gsm8k parity fails),
# so fail closed here rather than silently produce wrong output.
if self.dcp_world_size > 1 and self.cp_kv_cache_interleave_size > 1:
raise NotImplementedError(
"DCP sparse indexer currently supports only "
f"cp_kv_cache_interleave_size=1 (got "
f"{self.cp_kv_cache_interleave_size})."
)
# NOTE(Chen):an estimated max size of flattened_kv. Need to double check.
self.max_prefill_buffer_size = get_max_prefill_buffer_size(self.vllm_config)
self.num_speculative_tokens = (
@@ -300,6 +321,11 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
dtype=torch.int32,
device=self.device,
)
self.global_decode_seq_lens_buffer = torch.zeros(
(scheduler_config.max_num_batched_tokens,),
dtype=torch.int32,
device=self.device,
)
self.arange_buffer = torch.arange(
max(
scheduler_config.max_num_seqs * next_n,
@@ -331,6 +357,11 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
# Get compress_ratio for DeepseekV4 support
if isinstance(self.kv_cache_spec, MLAAttentionSpec):
self.compress_ratio = self.kv_cache_spec.compress_ratio
if self.dcp_world_size > 1 and self.compress_ratio > 1:
raise NotImplementedError(
"DCP is not supported with sparse indexer KV compression "
f"(compress_ratio={self.compress_ratio})."
)
# Pre-allocate buffers for CUDA graph compatibility when
if self.compress_ratio > 1:
@@ -348,6 +379,26 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
device=self.device,
)
def _dcp_localize_decode_seq_lens(
self,
seq_lens: torch.Tensor,
num_decodes: int,
seq_lens_is_buffer_view: bool,
) -> torch.Tensor:
local_seq_lens = get_dcp_local_seq_lens(
seq_lens,
self.dcp_world_size,
self.dcp_rank,
self.cp_kv_cache_interleave_size,
)
if seq_lens_is_buffer_view:
seq_lens.copy_(local_seq_lens)
return seq_lens
out = self.decode_seq_lens_buffer[:num_decodes]
out.copy_(local_seq_lens)
return out
def _prepare_decode_tensors(
self,
seq_lens: torch.Tensor,
@@ -468,6 +519,34 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
seq_lens = seq_lens_buffer
return seq_lens, block_table, decode_lens, num_decodes, requires_padding
def _prepare_global_decode_seq_lens(
self,
global_seq_lens: torch.Tensor | None,
decode_lens: torch.Tensor,
decode_lens_cpu: torch.Tensor,
query_start_loc: torch.Tensor,
num_decode_tokens: int,
use_native: bool,
max_decode_len: int,
) -> torch.Tensor | None:
if global_seq_lens is None:
return None
if use_native or max_decode_len <= 1:
return global_seq_lens
actual_expanded = int(decode_lens_cpu.sum().item())
if actual_expanded > 0:
expanded_offsets = torch.repeat_interleave(
global_seq_lens - decode_lens - query_start_loc,
decode_lens,
output_size=actual_expanded,
)
self.global_decode_seq_lens_buffer[:actual_expanded] = (
expanded_offsets + self.arange_buffer[:actual_expanded] + 1
)
self.global_decode_seq_lens_buffer[actual_expanded:num_decode_tokens] = 0
return self.global_decode_seq_lens_buffer[:num_decode_tokens]
def build(
self,
common_prefix_len: int,
@@ -481,6 +560,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
seq_lens = common_attn_metadata.seq_lens
slot_mapping = common_attn_metadata.slot_mapping
block_table = common_attn_metadata.block_table_tensor
dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
split_decodes_and_prefills(
@@ -549,6 +629,9 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
self.compress_ratio,
query_slice=query_slice,
skip_kv_gather=query_slice.start > 0,
dcp_rank=self.dcp_rank,
dcp_world_size=self.dcp_world_size,
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
)
# Skip when total_seq_lens is 0 (i.e., no compressed token).
if metadata is not None:
@@ -566,6 +649,17 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
common_attn_metadata.query_start_loc_cpu[: num_decodes + 1]
)
# Under DCP the per-token decode bounds must be localized AFTER the
# per-token expansion below, not before. Expanding from a
# request-level localized length subtracts decode offsets in local
# space and yields too-short bounds (e.g. world=2, rank=1, global
# per-token bounds [8, 9, 10] -> [3, 4, 5] instead of [4, 4, 5]), so
# the first decode token would run top-k against too short a local KV
# range and miss valid tokens. Keep the global seq_lens here and
# localize the expanded bounds further down.
global_seq_lens_for_decode: torch.Tensor | None = None
if dcp_local_seq_lens is not None:
global_seq_lens_for_decode = common_attn_metadata.seq_lens[:num_decodes]
seq_lens = common_attn_metadata.seq_lens[:num_decodes]
block_table = common_attn_metadata.block_table_tensor[:num_decodes, ...]
@@ -573,6 +667,16 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
next_n = 1 + self.num_speculative_tokens
use_native = not self.use_flattening and max_decode_len <= next_n
global_seq_lens_for_decode = self._prepare_global_decode_seq_lens(
global_seq_lens=global_seq_lens_for_decode,
decode_lens=decode_lens,
decode_lens_cpu=decode_lens_cpu,
query_start_loc=common_attn_metadata.query_start_loc[:num_decodes],
num_decode_tokens=num_decode_tokens,
use_native=use_native,
max_decode_len=max_decode_len,
)
seq_lens, block_table, decode_lens, batch_size, requires_padding = (
self._prepare_decode_tensors(
seq_lens=seq_lens,
@@ -588,15 +692,22 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
)
)
seq_lens_is_buffer_view = (use_native and next_n > 1) or (
not use_native and max_decode_len > 1
)
# DCP: localize the now-expanded per-token global bounds to this
# rank's owned KV. Done here (after expansion) so each token's global
# causal length is localized individually; see the comment above.
if dcp_local_seq_lens is not None:
seq_lens = self._dcp_localize_decode_seq_lens(
seq_lens, num_decodes, seq_lens_is_buffer_view
)
# For DeepseekV4 (compress_ratio > 1), the indexer KV cache stores
# compressed tokens. Convert uncompressed seq_lens to compressed.
if self.compress_ratio > 1:
# True iff seq_lens aliases decode_seq_lens_buffer (flatten or
# native wrote it); False iff it aliases common_attn_metadata.
seq_lens_is_local_view = (use_native and next_n > 1) or (
not use_native and max_decode_len > 1
)
if seq_lens_is_local_view:
if seq_lens_is_buffer_view:
seq_lens //= self.compress_ratio
else:
# Copy to avoid mutating shared state; keeps CG address stable.
@@ -626,6 +737,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
decode_lens=decode_lens,
requires_padding=requires_padding,
schedule_metadata=self.scheduler_metadata_buffer,
global_seq_lens=global_seq_lens_for_decode,
)
attn_metadata = DeepseekV32IndexerMetadata(
@@ -655,6 +767,9 @@ def build_prefill_chunk_metadata(
compress_ratio: int,
query_slice: slice | None = None,
skip_kv_gather: bool = False,
dcp_rank: int = 0,
dcp_world_size: int = 1,
cp_kv_cache_interleave_size: int = 1,
) -> DeepseekV32IndexerPrefillChunkMetadata | None:
total_seq_lens = compressed_seq_lens_cpu[start_idx:end_idx].sum().item()
if total_seq_lens == 0:
@@ -669,6 +784,25 @@ def build_prefill_chunk_metadata(
cu_seq_lens[:1] = 0
torch.cumsum(compressed_seq_lens[start_idx:end_idx], dim=0, out=cu_seq_lens[1:])
local_cu_seq_lens = cu_seq_lens
local_total_seq_lens = total_seq_lens
max_local_total_seq_lens = total_seq_lens
if dcp_world_size > 1:
# Per-rank local KV length under interleave-aware DCP sharding, shape
# [num_reqs, dcp_world_size]. Reuse the canonical CP helper so the
# sharding matches the rest of the DCP pipeline (decode/prefill).
local_seq_lens = get_dcp_local_seq_lens(
compressed_seq_lens[start_idx:end_idx],
dcp_world_size,
None,
cp_kv_cache_interleave_size,
)
this_rank_counts = local_seq_lens[:, dcp_rank].to(torch.int32)
local_cu_seq_lens = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
torch.cumsum(this_rank_counts, dim=0, out=local_cu_seq_lens[1:])
local_total_seq_lens = int(local_cu_seq_lens[-1].item())
max_local_total_seq_lens = int(local_seq_lens.sum(dim=0).max().item())
query_start_loc = (
query_start_loc[start_idx : end_idx + 1] - query_start_loc[start_idx]
)
@@ -687,15 +821,21 @@ def build_prefill_chunk_metadata(
cu_seq_len_ks = torch.empty(output_query_len, dtype=torch.int32, device=device)
cu_seq_len_ke = torch.empty(output_query_len, dtype=torch.int32, device=device)
# Under DCP the kernel writes this rank's local row bounds into
# cu_seq_len_ks/ke; otherwise local_cu_seq_lens aliases cu_seq_lens.
_build_prefill_chunk_metadata_kernel[(num_reqs,)](
query_start_loc,
uncompressed_seq_lens[start_idx:end_idx],
cu_seq_lens,
local_cu_seq_lens,
token_to_seq,
cu_seq_len_ks,
cu_seq_len_ke,
qs_start,
qs_stop,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
BLOCK_SIZE=1024,
COMPRESS_RATIO=compress_ratio,
)
@@ -719,6 +859,9 @@ def build_prefill_chunk_metadata(
token_end=token_end,
num_reqs=num_reqs,
skip_kv_gather=skip_kv_gather,
local_cu_seq_lens=local_cu_seq_lens,
local_total_seq_lens=local_total_seq_lens,
max_local_total_seq_lens=max_local_total_seq_lens,
)
@@ -728,12 +871,18 @@ def _build_prefill_chunk_metadata_kernel(
query_start_loc_ptr,
uncompressed_seq_lens_ptr,
cu_compressed_seq_lens_ptr,
# Row-start base for cu_seq_len_ks/ke: local cumulative lens under DCP,
# aliases cu_compressed_seq_lens_ptr otherwise.
row_start_cu_compressed_seq_lens_ptr,
# Outputs
token_to_seq_ptr,
cu_compressed_seq_len_ks_ptr,
cu_compressed_seq_len_ke_ptr,
query_slice_start,
query_slice_stop,
DCP_RANK,
DCP_WORLD,
DCP_INTERLEAVE,
BLOCK_SIZE: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
):
@@ -747,6 +896,10 @@ def _build_prefill_chunk_metadata_kernel(
seq_end = tl.load(cu_compressed_seq_lens_ptr + batch_idx + 1)
compressed_seq_len = seq_end - seq_start
# Row start for the (possibly localized) cu_seq_len_ks/ke. Equals seq_start
# when DCP is disabled (the pointer aliases cu_compressed_seq_lens_ptr).
row_start = tl.load(row_start_cu_compressed_seq_lens_ptr + batch_idx)
uncompressed_seq_len = tl.load(uncompressed_seq_lens_ptr + batch_idx)
start_pos = uncompressed_seq_len - query_len
@@ -760,14 +913,25 @@ def _build_prefill_chunk_metadata_kernel(
)
out_pos = abs_pos - query_slice_start
# Compute cu_seq_len_ks
tl.store(cu_compressed_seq_len_ks_ptr + out_pos, seq_start, mask=mask)
# cu_seq_len_ks: row start in the gathered K buffer.
tl.store(cu_compressed_seq_len_ks_ptr + out_pos, row_start, mask=mask)
# Compute cu_seq_len_ke
seq_len_per_token = (start_pos + 1 + offset) // COMPRESS_RATIO
# cu_seq_len_ke: row start + per-token context length. Under DCP the
# global per-token length is sharded across ranks.
global_ctx = start_pos + 1 + offset
len_per_token = global_ctx // COMPRESS_RATIO
if DCP_WORLD > 1:
# Per-rank local context length under interleave-aware DCP, matching
# get_dcp_local_seq_lens. K == 1 reduces to (len + world-1-rank)//world.
base = (len_per_token // DCP_INTERLEAVE // DCP_WORLD) * DCP_INTERLEAVE
remainder = len_per_token - base * DCP_WORLD
remainder = tl.minimum(
tl.maximum(remainder - DCP_RANK * DCP_INTERLEAVE, 0), DCP_INTERLEAVE
)
len_per_token = base + remainder
tl.store(
cu_compressed_seq_len_ke_ptr + out_pos,
seq_start + seq_len_per_token,
row_start + len_per_token,
mask=mask,
)
+158 -11
View File
@@ -23,6 +23,16 @@ def _convert_req_index_to_global_index_kernel(
BLOCK_N: tl.constexpr, # tile width along columns
HAS_PREFILL: tl.constexpr,
COUNT_VALID: tl.constexpr, # whether to count valid indices
# When set, scatter valid slots to a contiguous prefix [0, valid_count) using
# valid_count_ptr as an atomic slot allocator (DCP filtering leaves interior
# -1 gaps; the trtllm-gen sparse kernel reads the first valid_count entries).
# Requires COUNT_VALID and an out buffer pre-filled with -1. Order within the
# prefix is unspecified (only the selected set matters).
COMPACT_TO_FRONT: tl.constexpr,
# DCP de-interleave: with DCP_SIZE == 1 these are an exact no-op
DCP_SIZE: tl.constexpr,
DCP_RANK: tl.constexpr,
DCP_INTERLEAVE: tl.constexpr,
# strides (in elements)
bt_stride0,
bt_stride1,
@@ -52,15 +62,27 @@ def _convert_req_index_to_global_index_kernel(
if HAS_PREFILL:
prefill_req_id = tl.load(prefill_request_id_ptr + token_id)
is_prefill = prefill_req_id >= 0
# DCP de-interleave the global token id into this rank's local slot.
# Tokens are interleaved in groups of DCP_INTERLEAVE across ranks. With
# DCP_SIZE == 1 (and any interleave) owning_rank == 0 == DCP_RANK (never
# remote) and local_idx == tok, so this reduces to the non-DCP path; with
# DCP_INTERLEAVE == 1 it reduces to plain round-robin (tok % / // DCP_SIZE).
owning_rank = (tok // DCP_INTERLEAVE) % DCP_SIZE
is_remote = owning_rank != DCP_RANK
local_idx = (
tok // (DCP_SIZE * DCP_INTERLEAVE)
) * DCP_INTERLEAVE + tok % DCP_INTERLEAVE
# Compute block id and in-block offset
block_id = tok // BLOCK_SIZE
inblock_off = tok % BLOCK_SIZE
block_id = local_idx // BLOCK_SIZE
inblock_off = local_idx % BLOCK_SIZE
# Guard block_table access
valid_block = (block_id < max_num_blocks_per_req) & (block_id >= 0)
bt_ptr = block_table_ptr + req * bt_stride0 + block_id * bt_stride1
is_invalid_tok |= ~valid_block
base = tl.load(bt_ptr, mask=valid_block & ~is_prefill, other=0)
is_invalid_tok |= ~valid_block | is_remote
base = tl.load(bt_ptr, mask=valid_block & ~is_prefill & ~is_remote, other=0)
out_val = base * BLOCK_SIZE + inblock_off
# Override with prefill output if prefill is enabled
@@ -72,14 +94,27 @@ def _convert_req_index_to_global_index_kernel(
out_val = tl.where(is_prefill, prefill_out, out_val)
out_val = tl.where(is_invalid_tok, -1, out_val)
# Store results
out_ptr_ij = out_ptr + token_id * out_stride0 + indice_id * out_stride1
tl.store(out_ptr_ij, out_val)
if COMPACT_TO_FRONT:
# Scatter valid slots to a contiguous prefix. A per-tile exclusive prefix
# sum gives each valid lane a distinct local offset; one atomic add of the
# tile's valid count reserves a contiguous base across racing tiles. The
# out buffer is pre-filled with -1, so unwritten tail slots stay -1.
is_valid = (~is_invalid_tok).to(tl.int32)
local_offset = tl.cumsum(is_valid) - is_valid
tile_valid_count = tl.sum(is_valid)
base = tl.atomic_add(valid_count_ptr + token_id, tile_valid_count)
dest = base + local_offset
out_ptr_dest = out_ptr + token_id * out_stride0 + dest * out_stride1
tl.store(out_ptr_dest, out_val, mask=is_valid == 1)
else:
# Store results in place (input column == output column).
out_ptr_ij = out_ptr + token_id * out_stride0 + indice_id * out_stride1
tl.store(out_ptr_ij, out_val)
# Count valid indices in this tile and atomically add to row total
if COUNT_VALID:
tile_valid_count = tl.sum((~is_invalid_tok).to(tl.int32))
tl.atomic_add(valid_count_ptr + token_id, tile_valid_count)
# Count valid indices in this tile and atomically add to row total
if COUNT_VALID:
tile_valid_count = tl.sum((~is_invalid_tok).to(tl.int32))
tl.atomic_add(valid_count_ptr + token_id, tile_valid_count)
def triton_convert_req_index_to_global_index(
@@ -176,6 +211,11 @@ def triton_convert_req_index_to_global_index(
BLOCK_N,
HAS_PREFILL_WORKSPACE,
return_valid_counts,
False, # COMPACT_TO_FRONT: keep input column == output column
# DCP disabled (no-op de-interleave)
1,
0,
1,
# strides
bt_stride0,
bt_stride1,
@@ -189,3 +229,110 @@ def triton_convert_req_index_to_global_index(
assert valid_counts is not None
return out, valid_counts
return out
def triton_filter_and_convert_dcp_index(
req_id: torch.Tensor,
block_table: torch.Tensor,
token_indices: torch.Tensor,
dcp_size: int,
dcp_rank: int,
cp_kv_cache_interleave_size: int = 1,
BLOCK_SIZE: int = 64,
NUM_TOPK_TOKENS: int = 2048,
BLOCK_N: int = 128,
return_valid_counts: bool = False,
compact_valid_to_front: bool = True,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Filter global per-request indices to this DCP rank's local slots.
With ``compact_valid_to_front`` (default), the conversion kernel scatters
this rank's owned slots to a contiguous prefix ``[0, valid_count)`` and
leaves the rest ``-1``. DCP filtering marks non-owned slots ``-1`` and so
creates interior gaps; the trtllm-gen sparse kernel reads the first
``valid_count`` entries of each row, so they must be a contiguous prefix.
Compaction is fused into the kernel (atomic slot allocator) rather than a
separate sort/gather pass. Prefix order is unspecified (only the set matters).
"""
assert dcp_size >= 1
assert 0 <= dcp_rank < dcp_size
# Interleave groups must align to KV blocks (globally enforced by
# VllmConfig: block_size % cp_kv_cache_interleave_size == 0); assert the
# local invariant so local_idx // BLOCK_SIZE never straddles a group.
assert BLOCK_SIZE % cp_kv_cache_interleave_size == 0, (
f"BLOCK_SIZE ({BLOCK_SIZE}) must be divisible by "
f"cp_kv_cache_interleave_size ({cp_kv_cache_interleave_size})."
)
assert req_id.dtype == torch.int32
assert block_table.dtype == torch.int32
assert token_indices.dtype == torch.int32
assert token_indices.shape[1] == NUM_TOPK_TOKENS
assert NUM_TOPK_TOKENS % BLOCK_N == 0
if dcp_size == 1:
return triton_convert_req_index_to_global_index(
req_id,
block_table,
token_indices,
BLOCK_SIZE=BLOCK_SIZE,
NUM_TOPK_TOKENS=NUM_TOPK_TOKENS,
BLOCK_N=BLOCK_N,
return_valid_counts=return_valid_counts,
)
num_tokens = req_id.shape[0]
max_num_blocks_per_req = block_table.shape[1]
tiles_per_row = NUM_TOPK_TOKENS // BLOCK_N
req_id_c = req_id.contiguous()
block_table_c = block_table.contiguous()
token_indices_c = token_indices.contiguous()
# The compaction uses the valid-count buffer as an atomic slot allocator, so
# it requires counting. Pre-fill out with -1 so the unwritten tail stays -1.
count_valid = return_valid_counts or compact_valid_to_front
if compact_valid_to_front:
out = torch.full_like(token_indices_c, -1)
else:
out = torch.empty_like(token_indices_c)
valid_counts: torch.Tensor | None = None
if count_valid:
valid_counts = torch.zeros(
num_tokens, dtype=torch.int32, device=token_indices.device
)
bt_stride0, bt_stride1 = block_table_c.stride()
ti_stride0, ti_stride1 = token_indices_c.stride()
out_stride0, out_stride1 = out.stride()
_convert_req_index_to_global_index_kernel[(num_tokens, tiles_per_row)](
req_id_c,
block_table_c,
token_indices_c,
out,
valid_counts,
# No prefill workspace on the DCP decode path.
None,
None,
max_num_blocks_per_req,
BLOCK_SIZE,
BLOCK_N,
False, # HAS_PREFILL
count_valid,
compact_valid_to_front,
dcp_size,
dcp_rank,
cp_kv_cache_interleave_size,
bt_stride0,
bt_stride1,
ti_stride0,
ti_stride1,
out_stride0,
out_stride1,
)
if return_valid_counts:
assert valid_counts is not None
return out, valid_counts
return out
+12 -12
View File
@@ -892,20 +892,20 @@ def get_dcp_local_seq_lens(
use this function to calculate split decode seq_lens of each dcp rank.
Only consider dcp now, we can extend the case of cp based on this.
"""
num_requests = seq_lens.size(0)
seq_lens_i32 = seq_lens.to(torch.int32)
if dcp_rank is None:
rank_offsets = (
torch.arange(dcp_size, dtype=torch.int32, device=seq_lens.device)
.unsqueeze(0)
.repeat(num_requests, 1)
rank_offsets = torch.arange(
dcp_size,
dtype=torch.int32,
device=seq_lens.device,
).view(
*((1,) * seq_lens_i32.dim()),
dcp_size,
)
seq_lens_tiled = seq_lens_i32.unsqueeze(-1)
else:
rank_offsets = torch.tensor(
[[dcp_rank]], dtype=torch.int32, device=seq_lens.device
)
seq_lens_tiled = (
seq_lens.to(torch.int32).unsqueeze(-1).repeat(1, rank_offsets.shape[1])
)
rank_offsets = torch.tensor(dcp_rank, dtype=torch.int32, device=seq_lens.device)
seq_lens_tiled = seq_lens_i32
base = (
seq_lens_tiled
// cp_kv_cache_interleave_size
@@ -919,7 +919,7 @@ def get_dcp_local_seq_lens(
cp_kv_cache_interleave_size,
)
dcp_local_seq_lens = base + remainder
return dcp_local_seq_lens.squeeze(1)
return dcp_local_seq_lens
def mamba_get_block_table_tensor(
+1
View File
@@ -90,6 +90,7 @@ def _correct_attn_cp_out_kernel(
factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally)
output = tl.load(outputs_ptr + output_offsets)
output = output * factor
output = tl.where(factor == 0.0, 0.0, output)
tl.store(new_output_ptr + output_offsets, output)