[Attention][DSA] support dcp for FLASHINFER_MLA_SPARSE (#46076)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Jingyi Yang <girasoleyang@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: GirasoleY <girasoleyang@gmail.com> Co-authored-by: Jingyi Yang <girasoleyang@gmail.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jingyi Yang
Lucas Wilkinson
Claude Opus 4.8
parent
3c1396bab6
commit
c5200d3565
@@ -222,7 +222,7 @@ MLA decode backends are selected using the standard
|
||||
| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ |
|
||||
| `CUTLASS_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 128 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
|
||||
| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
|
||||
| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 10.x |
|
||||
| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
|
||||
| `FLASHINFER_MLA_SPARSE_SM120` | bf16 | `auto`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 64, 256 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 12.x |
|
||||
| `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x |
|
||||
| `FLASHMLA_SPARSE` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla` | 64 | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |
|
||||
|
||||
@@ -0,0 +1,951 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.v1.attention.backends.mla.indexer import build_prefill_chunk_metadata
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
triton_filter_and_convert_dcp_index,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
|
||||
from vllm.v1.attention.ops.common import CPTritonContext, correct_attn_out
|
||||
|
||||
|
||||
def _local_count(length: int, rank: int, world: int, interleave: int) -> int:
|
||||
return sum(1 for pos in range(length) if (pos // interleave) % world == rank)
|
||||
|
||||
|
||||
def _global_to_local_indices(
|
||||
global_indices: torch.Tensor,
|
||||
rank: int,
|
||||
world: int,
|
||||
interleave: int,
|
||||
) -> torch.Tensor:
|
||||
valid = global_indices >= 0
|
||||
global_i64 = global_indices.to(torch.int64).clamp_min(0)
|
||||
owner = (global_i64 // interleave) % world
|
||||
local = (global_i64 // (world * interleave)) * interleave + global_i64 % interleave
|
||||
return torch.where(valid & (owner == rank), local, -1).to(torch.int64)
|
||||
|
||||
|
||||
def _local_to_global_indices(
|
||||
local_indices: torch.Tensor,
|
||||
rank: int,
|
||||
world: int,
|
||||
interleave: int,
|
||||
) -> torch.Tensor:
|
||||
valid = local_indices >= 0
|
||||
local = local_indices.to(torch.int64).clamp_min(0)
|
||||
global_indices = (
|
||||
(local // interleave) * (world * interleave)
|
||||
+ rank * interleave
|
||||
+ local % interleave
|
||||
)
|
||||
return torch.where(valid, global_indices, -1).to(torch.int64)
|
||||
|
||||
|
||||
def _ref_stable_topk_from_candidates_fp64(
|
||||
candidate_scores: torch.Tensor,
|
||||
candidate_token_ids: torch.Tensor,
|
||||
k: int,
|
||||
) -> torch.Tensor:
|
||||
"""Pure-PyTorch reference for the CuteDSL stable-topk selector order
|
||||
(score desc, then lowest global token id). Selects the same SET as the
|
||||
kernel; only the set is compared in tests."""
|
||||
num_rows, num_candidates = candidate_scores.shape
|
||||
device = candidate_scores.device
|
||||
select_k = min(k, num_candidates)
|
||||
valid = candidate_token_ids >= 0
|
||||
bits = (
|
||||
candidate_scores.to(torch.float32).view(torch.int32).to(torch.int64)
|
||||
& 0xFFFFFFFF
|
||||
)
|
||||
sign = (bits >> 31) & 1
|
||||
score_key = (
|
||||
torch.where(sign.bool(), bits ^ 0xFFFFFFFF, bits ^ 0x80000000) & 0xFFFFFFFF
|
||||
)
|
||||
id_key = (~candidate_token_ids.to(torch.int64)) & 0xFFFFFFFF
|
||||
key = (score_key << 32) | id_key
|
||||
key = torch.where(valid, key, torch.zeros_like(key))
|
||||
topk_key = key ^ torch.iinfo(torch.int64).min
|
||||
_, topk_pos = topk_key.topk(select_k, dim=-1)
|
||||
|
||||
selected = candidate_token_ids.gather(1, topk_pos).to(torch.int32)
|
||||
selected_valid = valid.gather(1, topk_pos)
|
||||
selected = torch.where(selected_valid, selected, selected.new_full((), -1))
|
||||
if select_k == k:
|
||||
return selected
|
||||
pad = torch.full((num_rows, k - select_k), -1, dtype=torch.int32, device=device)
|
||||
return torch.cat((selected, pad), dim=1)
|
||||
|
||||
|
||||
def _attention_from_indices(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
valid = indices >= 0
|
||||
safe_indices = indices.clamp_min(0)
|
||||
selected_k = k[safe_indices]
|
||||
selected_v = v[safe_indices]
|
||||
scores = torch.einsum("td,tkd->tk", q, selected_k)
|
||||
scores = scores.masked_fill(~valid, float("-inf"))
|
||||
lse = torch.logsumexp(scores, dim=-1)
|
||||
probs = torch.softmax(scores, dim=-1).masked_fill(~valid, 0.0)
|
||||
out = torch.einsum("tk,tkd->td", probs, selected_v)
|
||||
empty_rows = ~valid.any(dim=-1)
|
||||
out[empty_rows] = 0
|
||||
lse[empty_rows] = float("-inf")
|
||||
return out, lse
|
||||
|
||||
|
||||
def _dcp_lse_merge(
|
||||
local_outs: list[torch.Tensor],
|
||||
local_lses: list[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
outs = torch.stack(local_outs, dim=0)
|
||||
lses = torch.stack(local_lses, dim=0)
|
||||
merged_lse = torch.logsumexp(lses, dim=0)
|
||||
weights = torch.exp(lses - merged_lse.unsqueeze(0))
|
||||
weights = torch.where(torch.isfinite(weights), weights, torch.zeros_like(weights))
|
||||
merged_out = (outs * weights.unsqueeze(-1)).sum(dim=0)
|
||||
return merged_out, merged_lse
|
||||
|
||||
|
||||
class _FakeDCPGroup:
|
||||
"""Single-process stand-in: ``all_gather`` returns the pre-built
|
||||
concatenation of every rank's packed ``(score, global_id)`` candidates,
|
||||
mirroring the one packed all-gather the merge issues."""
|
||||
|
||||
def __init__(self, gathered_packed: torch.Tensor) -> None:
|
||||
self.gathered_packed = gathered_packed
|
||||
|
||||
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
|
||||
assert dim == 1
|
||||
return self.gathered_packed.clone()
|
||||
|
||||
|
||||
def _run_decode_topk(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
next_n: int,
|
||||
topk: int,
|
||||
) -> torch.Tensor:
|
||||
indices = torch.empty(
|
||||
(logits.shape[0], topk), dtype=torch.int32, device=logits.device
|
||||
)
|
||||
torch.ops._C.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
indices,
|
||||
logits.shape[0],
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk,
|
||||
)
|
||||
return indices
|
||||
|
||||
|
||||
def _run_persistent_topk(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
topk: int,
|
||||
max_seq_len: int,
|
||||
) -> torch.Tensor:
|
||||
indices = torch.empty(
|
||||
(logits.shape[0], topk), dtype=torch.int32, device=logits.device
|
||||
)
|
||||
workspace = torch.empty(1024 * 1024, dtype=torch.uint8, device=logits.device)
|
||||
torch.ops._C.persistent_topk(
|
||||
logits,
|
||||
seq_lens,
|
||||
indices,
|
||||
workspace,
|
||||
topk,
|
||||
max_seq_len,
|
||||
)
|
||||
return indices
|
||||
|
||||
|
||||
def _dcp_attention_from_local_topks(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
local_topks: list[torch.Tensor],
|
||||
world: int,
|
||||
interleave: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
local_outs = []
|
||||
local_lses = []
|
||||
for rank, local_topk in enumerate(local_topks):
|
||||
owned = [
|
||||
pos for pos in range(k.shape[0]) if (pos // interleave) % world == rank
|
||||
]
|
||||
local_out, local_lse = _attention_from_indices(
|
||||
q, k[owned], v[owned], local_topk.to(torch.int64)
|
||||
)
|
||||
local_outs.append(local_out)
|
||||
local_lses.append(local_lse)
|
||||
return _dcp_lse_merge(local_outs, local_lses)
|
||||
|
||||
|
||||
def _merge_local_topks_global_with_fake_dcp(
|
||||
local_logits: list[torch.Tensor],
|
||||
local_topks: list[torch.Tensor],
|
||||
topk: int,
|
||||
world: int,
|
||||
interleave: int,
|
||||
row_starts: list[torch.Tensor | None] | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Run ``_merge_dcp_topk_global`` for every rank against a faked
|
||||
all-gather and return each rank's global-index result (all ranks should
|
||||
agree). The fake pre-builds the packed candidate concatenation the merge's
|
||||
single ``all_gather`` would return."""
|
||||
# The merge is now CuteDSL-only (no PyTorch fallback), so it runs the real
|
||||
# Triton pack + CuteDSL selector kernels even behind the faked all-gather.
|
||||
if not current_platform.is_cuda() or not has_cutedsl():
|
||||
pytest.skip("DCP merge requires CUDA and CuteDSL")
|
||||
packed_per_rank = []
|
||||
for rank, (logits, indices) in enumerate(zip(local_logits, local_topks)):
|
||||
score_indices = indices.clamp_min(0).to(torch.long)
|
||||
if row_starts is not None:
|
||||
rs = row_starts[rank]
|
||||
assert rs is not None
|
||||
score_indices = score_indices + rs.to(torch.long).view(-1, 1)
|
||||
if logits.shape[1] == 0:
|
||||
scores = torch.full_like(indices, float("-inf"), dtype=torch.float32)
|
||||
else:
|
||||
score_indices = score_indices.clamp_max(logits.shape[1] - 1)
|
||||
scores = logits.gather(1, score_indices).masked_fill(
|
||||
indices < 0, float("-inf")
|
||||
)
|
||||
global_ids = _local_to_global_indices(indices, rank, world, interleave)
|
||||
packed_per_rank.append(
|
||||
torch.stack((scores.float(), global_ids.to(torch.float32)), dim=-1)
|
||||
)
|
||||
|
||||
fake_group = _FakeDCPGroup(torch.cat(packed_per_rank, dim=1).contiguous())
|
||||
original_get_dcp_group = sparse_indexer.get_dcp_group
|
||||
sparse_indexer.get_dcp_group = lambda: fake_group
|
||||
try:
|
||||
merged = []
|
||||
for rank, (logits, indices) in enumerate(zip(local_logits, local_topks)):
|
||||
rank_indices = indices.clone()
|
||||
result = sparse_indexer._merge_dcp_topk_global(
|
||||
logits,
|
||||
rank_indices,
|
||||
topk,
|
||||
rank,
|
||||
world,
|
||||
interleave,
|
||||
row_starts=None if row_starts is None else row_starts[rank],
|
||||
)
|
||||
assert result is None
|
||||
merged.append(rank_indices)
|
||||
return merged
|
||||
finally:
|
||||
sparse_indexer.get_dcp_group = original_get_dcp_group
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world", [1, 2, 4])
|
||||
@pytest.mark.parametrize("interleave", [1, 2, 4])
|
||||
def test_get_dcp_local_seq_lens_matches_naive(world: int, interleave: int):
|
||||
seq_lens = torch.arange(0, 33, dtype=torch.int32)
|
||||
|
||||
for rank in range(world):
|
||||
actual = get_dcp_local_seq_lens(seq_lens, world, rank, interleave)
|
||||
expected = torch.tensor(
|
||||
[
|
||||
_local_count(int(seq_len), rank, world, interleave)
|
||||
for seq_len in seq_lens
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
def test_get_dcp_local_seq_lens_can_localize_per_token_bounds():
|
||||
seq_lens = torch.tensor([0, 1, 2, 3, 4, 7, 8, 17], dtype=torch.int32)
|
||||
world = 4
|
||||
interleave = 2
|
||||
|
||||
for rank in range(world):
|
||||
actual = get_dcp_local_seq_lens(seq_lens, world, rank, interleave)
|
||||
expected = torch.tensor(
|
||||
[
|
||||
_local_count(int(seq_len), rank, world, interleave)
|
||||
for seq_len in seq_lens
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
def test_get_dcp_local_seq_lens_preserves_mtp_bounds_shape():
|
||||
seq_lens = torch.tensor([[8, 9, 10], [11, 12, 13]], dtype=torch.int32)
|
||||
world = 2
|
||||
rank = 1
|
||||
interleave = 1
|
||||
|
||||
actual = get_dcp_local_seq_lens(seq_lens, world, rank, interleave)
|
||||
expected = torch.tensor(
|
||||
[
|
||||
[_local_count(int(seq_len), rank, world, interleave) for seq_len in row]
|
||||
for row in seq_lens
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
assert actual.shape == seq_lens.shape
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
def test_get_dcp_local_seq_lens_must_run_after_decode_expansion():
|
||||
world = 2
|
||||
rank = 1
|
||||
interleave = 1
|
||||
expanded_bounds = torch.tensor([8, 9, 10], dtype=torch.int32)
|
||||
|
||||
localized_after_expansion = get_dcp_local_seq_lens(
|
||||
expanded_bounds, world, rank, interleave
|
||||
)
|
||||
localized_request_len_minus_offsets = get_dcp_local_seq_lens(
|
||||
torch.tensor([10], dtype=torch.int32), world, rank
|
||||
) - torch.tensor([2, 1, 0], dtype=torch.int32)
|
||||
|
||||
assert not torch.equal(
|
||||
localized_after_expansion, localized_request_len_minus_offsets
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
localized_after_expansion, torch.tensor([4, 4, 5], dtype=torch.int32)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interleave", [1, 2])
|
||||
def test_sparse_dcp_attention_matches_global_topk_attention(interleave: int):
|
||||
torch.manual_seed(0)
|
||||
world = 2
|
||||
topk = 3
|
||||
num_queries = 4
|
||||
max_seq_len = 13
|
||||
head_dim = 8
|
||||
|
||||
q = torch.randn(num_queries, head_dim)
|
||||
k = torch.randn(max_seq_len, head_dim)
|
||||
v = torch.randn(max_seq_len, head_dim)
|
||||
seq_lens = torch.tensor([6, 8, 11, 13], dtype=torch.int64)
|
||||
|
||||
scores = q @ k.T
|
||||
global_topk = torch.full((num_queries, topk), -1, dtype=torch.int64)
|
||||
for row, seq_len in enumerate(seq_lens.tolist()):
|
||||
global_topk[row, :topk] = scores[row, :seq_len].topk(topk).indices
|
||||
|
||||
ref_out, ref_lse = _attention_from_indices(q, k, v, global_topk)
|
||||
|
||||
local_outs = []
|
||||
local_lses = []
|
||||
local_topks = []
|
||||
for rank in range(world):
|
||||
owned = [
|
||||
pos for pos in range(max_seq_len) if (pos // interleave) % world == rank
|
||||
]
|
||||
k_local = k[owned]
|
||||
v_local = v[owned]
|
||||
local_topk = _global_to_local_indices(global_topk, rank, world, interleave)
|
||||
local_topks.append(local_topk)
|
||||
local_out, local_lse = _attention_from_indices(q, k_local, v_local, local_topk)
|
||||
local_outs.append(local_out)
|
||||
local_lses.append(local_lse)
|
||||
|
||||
dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses)
|
||||
|
||||
torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5)
|
||||
torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5)
|
||||
|
||||
gathered_global = torch.cat(
|
||||
[
|
||||
_local_to_global_indices(local_topks[rank], rank, world, interleave)
|
||||
for rank in range(world)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
assert set(gathered_global[gathered_global >= 0].tolist()) == set(
|
||||
global_topk.flatten().tolist()
|
||||
)
|
||||
|
||||
|
||||
def test_local_topk_union_is_not_equivalent_to_global_topk_attention():
|
||||
world = 2
|
||||
interleave = 1
|
||||
topk = 2
|
||||
q = torch.tensor([[1.0]])
|
||||
k = torch.tensor(
|
||||
[
|
||||
[1.00],
|
||||
[0.90],
|
||||
[0.95],
|
||||
[0.85],
|
||||
]
|
||||
)
|
||||
v = torch.tensor([[0.0], [1000.0], [0.0], [1000.0]])
|
||||
|
||||
scores = q @ k.T
|
||||
global_topk = scores.topk(topk, dim=-1).indices
|
||||
ref_out, ref_lse = _attention_from_indices(q, k, v, global_topk)
|
||||
|
||||
local_topks = []
|
||||
for rank in range(world):
|
||||
owned = [
|
||||
pos for pos in range(k.shape[0]) if (pos // interleave) % world == rank
|
||||
]
|
||||
local_topks.append(scores[:, owned].topk(topk, dim=-1).indices)
|
||||
|
||||
local_union_out, local_union_lse = _dcp_attention_from_local_topks(
|
||||
q, k, v, local_topks, world, interleave
|
||||
)
|
||||
|
||||
assert not torch.allclose(local_union_out, ref_out)
|
||||
assert not torch.allclose(local_union_lse, ref_lse)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
def test_sparse_decode_dcp_persistent_topk_matches_non_dcp():
|
||||
torch.manual_seed(3)
|
||||
device = torch.device("cuda")
|
||||
world = 2
|
||||
interleave = 1
|
||||
topk = 512
|
||||
num_rows = 2
|
||||
max_seq_len = 1025
|
||||
head_dim = 16
|
||||
|
||||
q = torch.randn(num_rows, head_dim, device=device)
|
||||
k = torch.randn(max_seq_len, head_dim, device=device)
|
||||
v = torch.randn(max_seq_len, head_dim, device=device)
|
||||
logits = q @ k.T
|
||||
seq_lens = torch.tensor([[1024], [1025]], dtype=torch.int32, device=device)
|
||||
|
||||
non_dcp_topk = torch.empty((num_rows, topk), dtype=torch.int64, device=device)
|
||||
for row, seq_len in enumerate(seq_lens.flatten().tolist()):
|
||||
non_dcp_topk[row] = logits[row, :seq_len].topk(topk).indices
|
||||
ref_out, ref_lse = _attention_from_indices(q, k, v, non_dcp_topk)
|
||||
|
||||
local_logits = []
|
||||
local_topks = []
|
||||
for rank in range(world):
|
||||
owned = [
|
||||
pos for pos in range(max_seq_len) if (pos // interleave) % world == rank
|
||||
]
|
||||
rank_logits = logits[:, owned].contiguous()
|
||||
rank_seq_lens = get_dcp_local_seq_lens(
|
||||
seq_lens, world, rank, interleave
|
||||
).contiguous()
|
||||
local_logits.append(rank_logits)
|
||||
local_topks.append(
|
||||
_run_persistent_topk(
|
||||
rank_logits,
|
||||
rank_seq_lens,
|
||||
topk,
|
||||
max_seq_len=rank_logits.shape[1],
|
||||
)
|
||||
)
|
||||
|
||||
merged_global_topks = _merge_local_topks_global_with_fake_dcp(
|
||||
local_logits, local_topks, topk, world, interleave
|
||||
)
|
||||
# The radix top-K kernel selects a deterministic SET but writes it in
|
||||
# nondeterministic (atomicAdd) order; the production path is permutation-
|
||||
# invariant (compaction + softmax), so all ranks must agree on the set, not
|
||||
# the array order. (The fp64 fallback happens to return sorted order.)
|
||||
ref_topk = merged_global_topks[0]
|
||||
for rank_topk in merged_global_topks[1:]:
|
||||
for row in range(rank_topk.shape[0]):
|
||||
assert set(rank_topk[row].tolist()) == set(ref_topk[row].tolist())
|
||||
|
||||
local_outs = []
|
||||
local_lses = []
|
||||
for rank, global_topk in enumerate(merged_global_topks):
|
||||
owned = [
|
||||
pos for pos in range(max_seq_len) if (pos // interleave) % world == rank
|
||||
]
|
||||
local_topk = _global_to_local_indices(
|
||||
global_topk.to(torch.int64),
|
||||
rank,
|
||||
world,
|
||||
interleave,
|
||||
)
|
||||
local_out, local_lse = _attention_from_indices(
|
||||
q, k[owned], v[owned], local_topk
|
||||
)
|
||||
local_outs.append(local_out)
|
||||
local_lses.append(local_lse)
|
||||
|
||||
dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses)
|
||||
torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5)
|
||||
torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not has_cutedsl(),
|
||||
reason="This test requires CUDA and CuteDSL",
|
||||
)
|
||||
@pytest.mark.parametrize("use_row_starts", [False, True])
|
||||
def test_cutedsl_dcp_candidate_pack_and_select_matches_reference(
|
||||
use_row_starts: bool,
|
||||
):
|
||||
from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import (
|
||||
pack_dcp_topk_candidates_cutedsl,
|
||||
stable_topk_from_gathered_candidates_cutedsl,
|
||||
)
|
||||
|
||||
torch.manual_seed(13)
|
||||
device = torch.device("cuda")
|
||||
rows = 4
|
||||
valid_width = 1024
|
||||
width = valid_width + (8 if use_row_starts else 0)
|
||||
topk = 512
|
||||
world = 2
|
||||
row_starts = (
|
||||
torch.tensor([0, 2, 4, 1], device=device, dtype=torch.int32)
|
||||
if use_row_starts
|
||||
else None
|
||||
)
|
||||
row_offsets = (
|
||||
row_starts
|
||||
if row_starts is not None
|
||||
else torch.zeros(rows, device=device, dtype=torch.int32)
|
||||
)
|
||||
|
||||
packed_by_rank = []
|
||||
for rank in range(world):
|
||||
logits = torch.randn((rows, width), device=device, dtype=torch.float32)
|
||||
local_topks = []
|
||||
for row in range(rows):
|
||||
start = int(row_offsets[row].item())
|
||||
local_topks.append(
|
||||
logits[row, start : start + valid_width].topk(topk).indices
|
||||
)
|
||||
topk_indices = torch.stack(local_topks).to(torch.int32)
|
||||
|
||||
packed = torch.empty((rows, topk, 2), device=device, dtype=torch.float32)
|
||||
pack_dcp_topk_candidates_cutedsl(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
rank,
|
||||
world,
|
||||
1,
|
||||
row_starts,
|
||||
)
|
||||
|
||||
expected_scores = logits.gather(
|
||||
1, topk_indices.to(torch.long) + row_offsets.to(torch.long).view(-1, 1)
|
||||
)
|
||||
expected_ids = (topk_indices * world + rank).to(torch.float32)
|
||||
torch.testing.assert_close(packed[..., 0], expected_scores)
|
||||
torch.testing.assert_close(packed[..., 1], expected_ids)
|
||||
packed_by_rank.append(packed)
|
||||
|
||||
gathered = torch.cat(packed_by_rank, dim=1).contiguous()
|
||||
actual = torch.empty((rows, topk), device=device, dtype=torch.int32)
|
||||
returned = stable_topk_from_gathered_candidates_cutedsl(gathered, topk, out=actual)
|
||||
assert returned is actual
|
||||
expected = _ref_stable_topk_from_candidates_fp64(
|
||||
gathered[..., 0],
|
||||
gathered[..., 1].to(torch.int32),
|
||||
topk,
|
||||
)
|
||||
|
||||
for row in range(rows):
|
||||
assert set(actual[row].cpu().tolist()) == set(expected[row].cpu().tolist())
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
def test_sparse_prefill_dcp_metadata_localizes_causal_bounds():
|
||||
device = torch.device("cuda")
|
||||
seq_len = 8
|
||||
|
||||
query_start_loc = torch.tensor([0, seq_len], dtype=torch.int32, device=device)
|
||||
query_start_loc_cpu = torch.tensor([0, seq_len], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([seq_len], dtype=torch.int32, device=device)
|
||||
seq_lens_cpu = torch.tensor([seq_len], dtype=torch.int32)
|
||||
block_table = torch.zeros((1, 1), dtype=torch.int32, device=device)
|
||||
|
||||
def build(dcp_world_size, dcp_rank, interleave=1):
|
||||
chunk = build_prefill_chunk_metadata(
|
||||
start_idx=0,
|
||||
end_idx=1,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
uncompressed_seq_lens=seq_lens,
|
||||
compressed_seq_lens=seq_lens,
|
||||
compressed_seq_lens_cpu=seq_lens_cpu,
|
||||
block_table=block_table,
|
||||
compress_ratio=1,
|
||||
dcp_rank=dcp_rank,
|
||||
dcp_world_size=dcp_world_size,
|
||||
cp_kv_cache_interleave_size=interleave,
|
||||
)
|
||||
assert chunk is not None
|
||||
torch.accelerator.synchronize()
|
||||
return chunk
|
||||
|
||||
# Non-DCP: local_cu_seq_lens aliases the global cu_seq_lens, and
|
||||
# cu_seqlen_ks/ke carry the global causal bounds.
|
||||
chunk = build(dcp_world_size=1, dcp_rank=0)
|
||||
assert chunk.local_cu_seq_lens is chunk.cu_seq_lens
|
||||
torch.testing.assert_close(
|
||||
chunk.cu_seqlen_ks.cpu(),
|
||||
torch.zeros(seq_len, dtype=torch.int32),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
chunk.cu_seqlen_ke.cpu(),
|
||||
torch.arange(1, seq_len + 1, dtype=torch.int32),
|
||||
)
|
||||
|
||||
# DCP: cu_seqlen_ks/ke are localized in place to this rank's shard.
|
||||
chunk = build(dcp_world_size=4, dcp_rank=0)
|
||||
assert chunk.local_cu_seq_lens is not None
|
||||
torch.testing.assert_close(
|
||||
chunk.local_cu_seq_lens.cpu(),
|
||||
torch.tensor([0, 2], dtype=torch.int32),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
chunk.cu_seqlen_ks.cpu(),
|
||||
torch.zeros(seq_len, dtype=torch.int32),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
chunk.cu_seqlen_ke.cpu(),
|
||||
torch.tensor([1, 1, 1, 1, 2, 2, 2, 2], dtype=torch.int32),
|
||||
)
|
||||
|
||||
# DCP with interleave=2: per-token causal bounds localize differently from
|
||||
# interleave=1 (groups of 2 consecutive tokens are owned together). For
|
||||
# world=4, rank=0, K=2, per-token global len L=1..8 -> local len
|
||||
# [1,2,2,2,2,2,2,2] (matches get_dcp_local_seq_lens).
|
||||
chunk = build(dcp_world_size=4, dcp_rank=0, interleave=2)
|
||||
assert chunk.local_cu_seq_lens is not None
|
||||
torch.testing.assert_close(
|
||||
chunk.cu_seqlen_ks.cpu(),
|
||||
torch.zeros(seq_len, dtype=torch.int32),
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
chunk.cu_seqlen_ke.cpu(),
|
||||
torch.tensor([1, 2, 2, 2, 2, 2, 2, 2], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
def test_dcp_filter_compacts_valid_slots_for_sparse_kernel():
|
||||
block_size = 4
|
||||
num_topk = 128
|
||||
dcp_size = 2
|
||||
req_id = torch.zeros(1, dtype=torch.int32, device="cuda")
|
||||
token_indices = torch.full((1, num_topk), -1, dtype=torch.int32, device="cuda")
|
||||
token_indices[0, :8] = torch.arange(8, dtype=torch.int32, device="cuda")
|
||||
block_table = torch.tensor([[10]], dtype=torch.int32, device="cuda")
|
||||
|
||||
out, valid_counts = triton_filter_and_convert_dcp_index(
|
||||
req_id,
|
||||
block_table,
|
||||
token_indices,
|
||||
dcp_size=dcp_size,
|
||||
dcp_rank=0,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=num_topk,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
|
||||
valid = int(valid_counts.item())
|
||||
assert valid == 4
|
||||
assert (out[0, :valid] >= 0).all()
|
||||
assert (out[0, valid:] == -1).all()
|
||||
# In-kernel compaction packs valid slots to the front; prefix order is
|
||||
# unspecified, so compare as a set.
|
||||
assert set(out[0, :valid].cpu().tolist()) == {40, 41, 42, 43}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
@pytest.mark.parametrize("interleave", [1, 2])
|
||||
@pytest.mark.parametrize("dcp_rank", [0, 1])
|
||||
def test_dcp_filter_compaction_matches_reference(interleave: int, dcp_rank: int):
|
||||
"""In-kernel compaction (atomic slot allocator across multiple column tiles)
|
||||
must, for every row, produce exactly the rank-owned physical slots packed
|
||||
into [0, valid_count) with -1 in the tail -- the same SET a reference filter
|
||||
+ sort/gather produces. Uses wide rows (> BLOCK_N valid slots) so the
|
||||
cross-tile atomic allocation is exercised, with interior -1 gaps."""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(7)
|
||||
dcp_size = 2
|
||||
block_size = 8
|
||||
num_topk = 1024 # > BLOCK_N(128) -> multiple tiles per row
|
||||
num_rows = 5
|
||||
max_blocks = 64
|
||||
seq = max_blocks * block_size
|
||||
|
||||
req_id = torch.randint(0, 3, (num_rows,), dtype=torch.int32, device=device)
|
||||
block_table = torch.randint(
|
||||
0, 1000, (3, max_blocks), dtype=torch.int32, device=device
|
||||
)
|
||||
# Each row: a dense valid prefix of distinct global token ids, then -1 pad.
|
||||
token_indices = torch.full(
|
||||
(num_rows, num_topk), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
for r in range(num_rows):
|
||||
n_valid = int(torch.randint(200, 600, (1,)).item())
|
||||
perm = torch.randperm(seq, device=device)[:n_valid].to(torch.int32)
|
||||
token_indices[r, :n_valid] = perm
|
||||
|
||||
out, valid_counts = triton_filter_and_convert_dcp_index(
|
||||
req_id,
|
||||
block_table,
|
||||
token_indices,
|
||||
dcp_size=dcp_size,
|
||||
dcp_rank=dcp_rank,
|
||||
cp_kv_cache_interleave_size=interleave,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=num_topk,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
|
||||
for r in range(num_rows):
|
||||
toks = token_indices[r]
|
||||
toks = toks[toks >= 0]
|
||||
owner = (toks // interleave) % dcp_size
|
||||
owned = toks[owner == dcp_rank]
|
||||
local = (owned // (dcp_size * interleave)) * interleave + owned % interleave
|
||||
blk = local // block_size
|
||||
off = local % block_size
|
||||
expected = (
|
||||
block_table[int(req_id[r].item()), blk].to(torch.int64) * block_size + off
|
||||
)
|
||||
n = int(valid_counts[r].item())
|
||||
assert n == owned.numel()
|
||||
assert (out[r, :n] >= 0).all()
|
||||
assert (out[r, n:] == -1).all()
|
||||
assert set(out[r, :n].cpu().tolist()) == set(expected.cpu().tolist())
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
@pytest.mark.parametrize("interleave", [1, 2, 4])
|
||||
def test_dcp_global_topk_physical_attention_matches_non_dcp(interleave: int):
|
||||
torch.manual_seed(2)
|
||||
device = torch.device("cuda")
|
||||
dcp_size = 2
|
||||
block_size = 4
|
||||
num_topk = 128
|
||||
selected_k = 8
|
||||
seq_len = 16
|
||||
head_dim = 16
|
||||
num_queries = 3
|
||||
|
||||
q = torch.randn(num_queries, head_dim, device=device)
|
||||
k_global = torch.randn(seq_len, head_dim, device=device)
|
||||
v_global = torch.randn(seq_len, head_dim, device=device)
|
||||
scores = q @ k_global.T
|
||||
global_topk = torch.full(
|
||||
(num_queries, num_topk), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
global_topk[:, :selected_k] = scores.topk(selected_k, dim=-1).indices.to(
|
||||
torch.int32
|
||||
)
|
||||
|
||||
ref_out, ref_lse = _attention_from_indices(
|
||||
q, k_global, v_global, global_topk[:, :selected_k].to(torch.int64)
|
||||
)
|
||||
|
||||
local_outs = []
|
||||
local_lses = []
|
||||
for rank in range(dcp_size):
|
||||
block_ids = torch.tensor(
|
||||
[[rank * 10 + 1, rank * 10 + 2]], dtype=torch.int32, device=device
|
||||
)
|
||||
num_slots = int((block_ids.max().item() + 1) * block_size)
|
||||
k_cache = torch.zeros(num_slots, head_dim, device=device)
|
||||
v_cache = torch.zeros(num_slots, head_dim, device=device)
|
||||
for global_idx in range(seq_len):
|
||||
if (global_idx // interleave) % dcp_size != rank:
|
||||
continue
|
||||
local_idx = (
|
||||
global_idx // (dcp_size * interleave)
|
||||
) * interleave + global_idx % interleave
|
||||
block = local_idx // block_size
|
||||
offset = local_idx % block_size
|
||||
slot = int(block_ids[0, block].item()) * block_size + offset
|
||||
k_cache[slot] = k_global[global_idx]
|
||||
v_cache[slot] = v_global[global_idx]
|
||||
|
||||
slots, valid_counts = triton_filter_and_convert_dcp_index(
|
||||
torch.zeros(num_queries, dtype=torch.int32, device=device),
|
||||
block_ids,
|
||||
global_topk,
|
||||
dcp_size=dcp_size,
|
||||
dcp_rank=rank,
|
||||
cp_kv_cache_interleave_size=interleave,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=num_topk,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
row_ids = torch.arange(num_queries, device=device)
|
||||
assert (slots[row_ids, valid_counts] == -1).all()
|
||||
local_out, local_lse = _attention_from_indices(
|
||||
q, k_cache, v_cache, slots.to(torch.int64)
|
||||
)
|
||||
local_outs.append(local_out)
|
||||
local_lses.append(local_lse)
|
||||
|
||||
dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses)
|
||||
torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5)
|
||||
torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
@pytest.mark.parametrize("is_lse_base_on_e", [True, False])
|
||||
def test_correct_attn_out_zeroes_empty_nan_partial(is_lse_base_on_e: bool):
|
||||
out = torch.full((1, 1, 4), float("nan"), device="cuda")
|
||||
lses = torch.tensor(
|
||||
[[[0.0]], [[float("-inf")]]],
|
||||
dtype=torch.float32,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
corrected, final_lse = correct_attn_out(
|
||||
out,
|
||||
lses,
|
||||
cp_rank=1,
|
||||
ctx=CPTritonContext(),
|
||||
is_lse_base_on_e=is_lse_base_on_e,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(corrected, torch.zeros_like(corrected))
|
||||
torch.testing.assert_close(final_lse, torch.zeros_like(final_lse))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
def test_decode_topk_pads_surplus_with_negative_one():
|
||||
"""When a row's valid length < topk the top-k kernel must pad the surplus
|
||||
slots with -1: the DCP merge (`topk_indices >= 0`) and
|
||||
`triton_filter_and_convert_dcp_index` (`tok < 0`) both treat <0 as invalid,
|
||||
so a non-(-1) pad would be silently attended. This is the common case under
|
||||
DCP, where each rank's local seq_len = global / world is usually << topk.
|
||||
Checked on the *set* of valid indices (the kernel may order them by score,
|
||||
not position)."""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
topk = 8
|
||||
next_n = 1
|
||||
seq_lens_list = [0, 3, 5, 8]
|
||||
num_rows = len(seq_lens_list)
|
||||
logits = torch.randn(num_rows, 16, device=device)
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device=device).view(
|
||||
num_rows, next_n
|
||||
)
|
||||
idx = _run_decode_topk(logits, seq_lens, next_n, topk)
|
||||
for r, sl in enumerate(seq_lens_list):
|
||||
valid = idx[r][idx[r] >= 0]
|
||||
# seq_len <= topk, so top-k selects exactly the whole valid range.
|
||||
assert valid.numel() == min(sl, topk)
|
||||
assert set(valid.tolist()) == set(range(sl))
|
||||
assert (idx[r] == -1).sum().item() == topk - min(sl, topk)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
def test_persistent_topk_pads_surplus_with_negative_one():
|
||||
"""Same surplus=-1 invariant for the persistent_topk kernel (k>=512)."""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
topk = 512 # persistent_topk requires k in {512, 1024, 2048}
|
||||
seq_lens_list = [100, 300, 512]
|
||||
num_rows = len(seq_lens_list)
|
||||
max_seq_len = 600
|
||||
logits = torch.randn(num_rows, max_seq_len, device=device)
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device=device).view(
|
||||
num_rows, 1
|
||||
)
|
||||
idx = _run_persistent_topk(logits, seq_lens, topk, max_seq_len)
|
||||
for r, sl in enumerate(seq_lens_list):
|
||||
valid = idx[r][idx[r] >= 0]
|
||||
assert valid.numel() == min(sl, topk)
|
||||
assert set(valid.tolist()) == set(range(sl))
|
||||
assert (idx[r] == -1).sum().item() == topk - min(sl, topk)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
def test_sparse_decode_dcp_short_context_matches_non_dcp():
|
||||
"""End-to-end DCP decode where the global seq_len < topk (so every rank's
|
||||
local top-k is surplus-padded). Exercises the kernel surplus -> merge mask
|
||||
-> global top-k -> physical localize -> LSE merge chain for the common
|
||||
short-context decode case, vs the non-DCP reference."""
|
||||
torch.manual_seed(4)
|
||||
device = torch.device("cuda")
|
||||
world = 2
|
||||
interleave = 1
|
||||
topk = 512
|
||||
num_rows = 2
|
||||
max_seq_len = 300 # < topk -> surplus everywhere
|
||||
head_dim = 16
|
||||
|
||||
q = torch.randn(num_rows, head_dim, device=device)
|
||||
k = torch.randn(max_seq_len, head_dim, device=device)
|
||||
v = torch.randn(max_seq_len, head_dim, device=device)
|
||||
logits = q @ k.T
|
||||
seq_lens = torch.tensor([[250], [300]], dtype=torch.int32, device=device)
|
||||
|
||||
non_dcp_topk = torch.empty((num_rows, topk), dtype=torch.int64, device=device)
|
||||
for row, seq_len in enumerate(seq_lens.flatten().tolist()):
|
||||
sel = logits[row, :seq_len].topk(min(topk, seq_len)).indices
|
||||
non_dcp_topk[row, : sel.numel()] = sel
|
||||
non_dcp_topk[row, sel.numel() :] = -1
|
||||
ref_out, ref_lse = _attention_from_indices(q, k, v, non_dcp_topk)
|
||||
|
||||
local_logits = []
|
||||
local_topks = []
|
||||
for rank in range(world):
|
||||
owned = [p for p in range(max_seq_len) if (p // interleave) % world == rank]
|
||||
rank_logits = logits[:, owned].contiguous()
|
||||
rank_seq_lens = get_dcp_local_seq_lens(seq_lens, world, rank, interleave)
|
||||
local_logits.append(rank_logits)
|
||||
local_topks.append(
|
||||
_run_persistent_topk(
|
||||
rank_logits,
|
||||
rank_seq_lens.contiguous(),
|
||||
topk,
|
||||
max_seq_len=rank_logits.shape[1],
|
||||
)
|
||||
)
|
||||
|
||||
merged_global_topks = _merge_local_topks_global_with_fake_dcp(
|
||||
local_logits, local_topks, topk, world, interleave
|
||||
)
|
||||
# The radix top-K kernel selects a deterministic SET but writes it in
|
||||
# nondeterministic (atomicAdd) order; the production path is permutation-
|
||||
# invariant (compaction + softmax), so all ranks must agree on the set, not
|
||||
# the array order. (The fp64 fallback happens to return sorted order.)
|
||||
ref_topk = merged_global_topks[0]
|
||||
for rank_topk in merged_global_topks[1:]:
|
||||
for row in range(rank_topk.shape[0]):
|
||||
assert set(rank_topk[row].tolist()) == set(ref_topk[row].tolist())
|
||||
|
||||
local_outs = []
|
||||
local_lses = []
|
||||
for rank, global_topk in enumerate(merged_global_topks):
|
||||
owned = [p for p in range(max_seq_len) if (p // interleave) % world == rank]
|
||||
local_topk = _global_to_local_indices(
|
||||
global_topk.to(torch.int64), rank, world, interleave
|
||||
)
|
||||
local_out, local_lse = _attention_from_indices(
|
||||
q, k[owned], v[owned], local_topk
|
||||
)
|
||||
local_outs.append(local_out)
|
||||
local_lses.append(local_lse)
|
||||
|
||||
dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses)
|
||||
torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5)
|
||||
torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,420 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import cache
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cuda.bindings.driver import CUstream
|
||||
from cutlass import Float32, Int32, Uint32, Uint64
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import recast_val
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def stable_topk_from_gathered_candidates_cutedsl(
|
||||
gathered: torch.Tensor,
|
||||
topk: int,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
(gathered.shape[0], topk),
|
||||
dtype=torch.int32,
|
||||
device=gathered.device,
|
||||
)
|
||||
StableTopKFromGatheredCandidatesKernel.compile(topk, gathered.shape[1])(
|
||||
gathered, out
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def pack_dcp_topk_candidates_cutedsl(
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
packed: torch.Tensor,
|
||||
dcp_rank: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
row_starts: torch.Tensor | None,
|
||||
) -> None:
|
||||
topk = topk_indices.shape[1]
|
||||
grid = (topk_indices.shape[0], triton.cdiv(topk, 512))
|
||||
row_starts_arg = row_starts if row_starts is not None else topk_indices
|
||||
_pack_dcp_topk_candidates_triton_kernel[grid](
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts_arg,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_indices.stride(0),
|
||||
topk_indices.stride(1),
|
||||
packed.stride(0),
|
||||
packed.stride(1),
|
||||
packed.stride(2),
|
||||
logits.shape[1],
|
||||
DCP_RANK=dcp_rank,
|
||||
DCP_WORLD_SIZE=dcp_world_size,
|
||||
CP_INTERLEAVE=cp_interleave,
|
||||
HAS_ROW_STARTS=row_starts is not None,
|
||||
TOPK=topk,
|
||||
BLOCK_SIZE=512,
|
||||
num_warps=8,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_dcp_topk_candidates_triton_kernel(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts,
|
||||
logits_stride0: tl.constexpr,
|
||||
logits_stride1: tl.constexpr,
|
||||
topk_stride0: tl.constexpr,
|
||||
topk_stride1: tl.constexpr,
|
||||
packed_stride0: tl.constexpr,
|
||||
packed_stride1: tl.constexpr,
|
||||
packed_stride2: tl.constexpr,
|
||||
num_cols,
|
||||
DCP_RANK: tl.constexpr,
|
||||
DCP_WORLD_SIZE: tl.constexpr,
|
||||
CP_INTERLEAVE: tl.constexpr,
|
||||
HAS_ROW_STARTS: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
tile = tl.program_id(1)
|
||||
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = cols < TOPK
|
||||
|
||||
local_idx = tl.load(
|
||||
topk_indices + row * topk_stride0 + cols * topk_stride1,
|
||||
mask=mask,
|
||||
other=-1,
|
||||
)
|
||||
valid = local_idx >= 0
|
||||
safe_local_idx = tl.maximum(local_idx, 0)
|
||||
|
||||
row_start = 0
|
||||
if HAS_ROW_STARTS:
|
||||
row_start = tl.load(row_starts + row)
|
||||
|
||||
score_col = safe_local_idx + row_start
|
||||
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
|
||||
score = tl.load(
|
||||
logits + row * logits_stride0 + score_col * logits_stride1,
|
||||
mask=mask & valid,
|
||||
other=-float("inf"),
|
||||
)
|
||||
|
||||
global_id = (
|
||||
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
|
||||
+ DCP_RANK * CP_INTERLEAVE
|
||||
+ safe_local_idx % CP_INTERLEAVE
|
||||
)
|
||||
global_id = tl.where(valid, global_id, -1)
|
||||
|
||||
packed_base = packed + row * packed_stride0 + cols * packed_stride1
|
||||
tl.store(packed_base, score, mask=mask)
|
||||
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
|
||||
|
||||
|
||||
@cute.jit
|
||||
def _warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
|
||||
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
|
||||
offset = 1 << i
|
||||
partial = cute.arch.shuffle_sync_up(val, offset=offset, mask_and_clamp=0)
|
||||
if lane >= offset:
|
||||
val += partial
|
||||
return val
|
||||
|
||||
|
||||
@cute.jit
|
||||
def _block_scan_inclusive_i32(
|
||||
val: Int32,
|
||||
lane: Int32,
|
||||
warp_id: Int32,
|
||||
warp_scratch: cute.Tensor,
|
||||
warps_per_block: int,
|
||||
) -> Int32:
|
||||
prefix = _warp_scan_inclusive_i32(val, lane)
|
||||
if lane == Int32(cute.arch.WARP_SIZE - 1):
|
||||
warp_scratch[0, warp_id] = prefix
|
||||
cute.arch.sync_threads()
|
||||
|
||||
if warp_id == Int32(0):
|
||||
warp_total = Int32(0)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_total = warp_scratch[0, lane]
|
||||
warp_prefix = _warp_scan_inclusive_i32(warp_total, lane)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_scratch[0, lane] = warp_prefix - warp_total
|
||||
cute.arch.sync_threads()
|
||||
|
||||
return prefix + warp_scratch[0, warp_id]
|
||||
|
||||
|
||||
class StableTopKFromGatheredCandidatesKernel:
|
||||
tb_size = 512
|
||||
hist_bins = 2048
|
||||
radix_bits = (hist_bins - 1).bit_length()
|
||||
assert hist_bins == 1 << radix_bits
|
||||
key_bits = Uint64.width
|
||||
radix_passes = (key_bits + radix_bits - 1) // radix_bits
|
||||
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
|
||||
hist_chunks = (hist_bins + tb_size - 1) // tb_size
|
||||
warps_per_block = tb_size // cute.arch.WARP_SIZE
|
||||
|
||||
def __init__(self, topk: int, num_candidates: int):
|
||||
assert num_candidates % self.tb_size == 0, (
|
||||
"StableTopKFromGatheredCandidatesKernel requires candidate count "
|
||||
f"to be a multiple of {self.tb_size}, got {num_candidates}"
|
||||
)
|
||||
self.topk = topk
|
||||
self.keys_per_thread = num_candidates // self.tb_size
|
||||
|
||||
@cute.struct
|
||||
class SharedStorage:
|
||||
hist: cute.struct.MemRange[Int32, self.hist_bins]
|
||||
committed_count: cute.struct.MemRange[Int32, 1]
|
||||
running_count: cute.struct.MemRange[Int32, 1]
|
||||
threshold_bin: cute.struct.MemRange[Int32, 1]
|
||||
threshold_found: cute.struct.MemRange[Int32, 1]
|
||||
include_threshold_bin: cute.struct.MemRange[Int32, 1]
|
||||
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
|
||||
warp_totals: cute.struct.MemRange[Int32, self.warps_per_block]
|
||||
|
||||
self.shared_storage = SharedStorage
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
gathered: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (gathered.shape[0], 1, 1)
|
||||
self.kernel(gathered, out).launch(
|
||||
grid=grid,
|
||||
block=(self.tb_size, 1, 1),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _stable_key(self, score: Float32, token_id: Int32) -> Uint64:
|
||||
bits = recast_val(score, Uint32)
|
||||
mask = Uint32(0x80000000)
|
||||
if (bits & Uint32(0x80000000)) != Uint32(0):
|
||||
mask = Uint32(0xFFFFFFFF)
|
||||
score_key = Uint64(bits ^ mask) << Uint64(32)
|
||||
id_key = Uint64(~Uint32(token_id))
|
||||
key = score_key | id_key
|
||||
if token_id < Int32(0):
|
||||
key = Uint64(0)
|
||||
return key
|
||||
|
||||
@cute.jit
|
||||
def _prefix_matches(
|
||||
self,
|
||||
key: Uint64,
|
||||
prefix: Uint64,
|
||||
prefix_bits: Int32,
|
||||
):
|
||||
matches = prefix_bits == Int32(0)
|
||||
if prefix_bits != Int32(0):
|
||||
shift = Int32(self.key_bits) - prefix_bits
|
||||
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
|
||||
return matches
|
||||
|
||||
@cute.jit
|
||||
def _radix_pass(
|
||||
self,
|
||||
keys: cute.Tensor,
|
||||
output: cute.Tensor,
|
||||
storage,
|
||||
tid: Int32,
|
||||
step: Int32,
|
||||
bits: int,
|
||||
is_final_pass: bool,
|
||||
):
|
||||
hist_smem = storage.hist.get_tensor(cute.make_layout((self.hist_bins,)))
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
running_count_smem = storage.running_count.data_ptr()
|
||||
threshold_bin_smem = storage.threshold_bin.data_ptr()
|
||||
threshold_found_smem = storage.threshold_found.data_ptr()
|
||||
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
warp_totals_smem = storage.warp_totals.get_tensor(
|
||||
cute.make_layout((1, self.warps_per_block))
|
||||
)
|
||||
|
||||
prefix_bits = step * Int32(self.radix_bits)
|
||||
num_bins = 1 << bits
|
||||
block_scan_iterations = (num_bins + self.tb_size - 1) // self.tb_size
|
||||
shift = Int32(self.key_bits) - prefix_bits - Int32(bits)
|
||||
bin_mask = Uint64(num_bins - 1)
|
||||
prefix = prefix_smem.load()
|
||||
|
||||
for chunk in cutlass.range_constexpr(self.hist_chunks):
|
||||
hist_smem[tid + Int32(chunk * self.tb_size)] = Int32(0)
|
||||
if tid == Int32(0):
|
||||
running_count_smem.store(committed_count_smem.load())
|
||||
include_threshold_bin_smem.store(Int32(0))
|
||||
threshold_found_smem.store(Int32(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if self._prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
cute.arch.atomic_add(
|
||||
hist_smem.iterator + bin_idx,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
lane = cute.arch.lane_idx()
|
||||
warp_id = cute.arch.warp_idx()
|
||||
# Each iteration scans one tb_size-wide slice of bins, high to low.
|
||||
iter = Int32(0)
|
||||
threshold_found = threshold_found_smem.load()
|
||||
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
|
||||
bin_idx = Int32(num_bins - 1) - (iter * Int32(self.tb_size) + tid)
|
||||
count = hist_smem[bin_idx]
|
||||
chunk_inclusive = _block_scan_inclusive_i32(
|
||||
count,
|
||||
lane,
|
||||
warp_id,
|
||||
warp_totals_smem,
|
||||
self.warps_per_block,
|
||||
)
|
||||
running_count = running_count_smem.load()
|
||||
prior_in_scan_slice = chunk_inclusive - count
|
||||
remaining = Int32(self.topk) - running_count - prior_in_scan_slice
|
||||
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
|
||||
threshold_bin_smem.store(bin_idx)
|
||||
if count <= remaining or cutlass.const_expr(is_final_pass):
|
||||
include_threshold_bin_smem.store(Int32(1))
|
||||
threshold_found_smem.store(Int32(1))
|
||||
# Barrier: every thread must finish reading running_count for this
|
||||
# slice before tb_size-1 advances it, else a warp racing ahead to
|
||||
# the store makes a lagging thread double-count the slice total
|
||||
# (-> remaining too small -> threshold too high -> under-fill).
|
||||
cute.arch.sync_threads()
|
||||
if tid == Int32(self.tb_size - 1):
|
||||
running_count_smem.store(running_count + chunk_inclusive)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
threshold_found = threshold_found_smem.load()
|
||||
iter += Int32(1)
|
||||
|
||||
threshold = threshold_bin_smem.load()
|
||||
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
|
||||
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if self._prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
selected = bin_idx > threshold
|
||||
if should_include_threshold:
|
||||
selected = selected or bin_idx == threshold
|
||||
if selected:
|
||||
dst = cute.arch.atomic_add(
|
||||
committed_count_smem,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
)
|
||||
if dst < Int32(self.topk):
|
||||
output[dst] = recast_val(~Uint32(key), Int32)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
pass_finished = include_threshold_bin_smem.load()
|
||||
if tid == Int32(0) and pass_finished == Int32(0):
|
||||
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
|
||||
cute.arch.sync_threads()
|
||||
return pass_finished
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
self,
|
||||
input: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
):
|
||||
row, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
input_row = input[row, None, None]
|
||||
output_row = out[row, None]
|
||||
keys = cute.make_rmem_tensor((self.keys_per_thread,), Uint64)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
storage = smem.allocate(self.shared_storage, 8)
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
for i in range(tid, self.topk, self.tb_size):
|
||||
output_row[i] = Int32(-1)
|
||||
|
||||
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
|
||||
col = tid + Int32(key_idx * self.tb_size)
|
||||
score = Float32(input_row[col, 0])
|
||||
token_id = Int32(input_row[col, 1])
|
||||
keys[key_idx] = self._stable_key(score, token_id)
|
||||
|
||||
if tid == Int32(0):
|
||||
committed_count_smem.store(Int32(0))
|
||||
prefix_smem.store(Uint64(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
step = Int32(0)
|
||||
finished = Int32(0)
|
||||
while finished == Int32(0) and step < Int32(self.radix_passes - 1):
|
||||
finished = self._radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
step,
|
||||
self.radix_bits,
|
||||
False,
|
||||
)
|
||||
step += Int32(1)
|
||||
|
||||
if finished == Int32(0):
|
||||
self._radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
Int32(self.radix_passes - 1),
|
||||
self.final_radix_bits,
|
||||
True,
|
||||
)
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(topk: int, num_candidates: int):
|
||||
num_rows = cute.sym_int()
|
||||
|
||||
gathered = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(num_rows, num_candidates, 2),
|
||||
stride=(cute.sym_int64(divisibility=2), 2, 1),
|
||||
assumed_align=8,
|
||||
)
|
||||
out = make_fake_tensor(Int32, (num_rows, topk), divisibility=1)
|
||||
|
||||
kernel = StableTopKFromGatheredCandidatesKernel(topk, num_candidates)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
gathered,
|
||||
out,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
)
|
||||
@@ -8,6 +8,8 @@ import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
@@ -21,6 +23,7 @@ from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
has_deep_gemm,
|
||||
)
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
@@ -41,6 +44,85 @@ RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
|
||||
MXFP4_BLOCK_SIZE = 32
|
||||
|
||||
|
||||
def _assert_cutedsl_dcp_merge_supported(
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
k: int,
|
||||
) -> None:
|
||||
# The DCP merge only supports the CuteDSL path (Triton pack kernel + CuteDSL
|
||||
# stable-topk selector); there is no PyTorch fallback. The first cut targets
|
||||
# Blackwell/Hopper with index_topk in (512, 1024, 2048) (the selector's radix
|
||||
# sizing); the Triton pack itself has no shape/topk constraints.
|
||||
if not has_cutedsl():
|
||||
raise RuntimeError(
|
||||
"DCP sparse-indexer merge requires CuteDSL; install it or disable DCP."
|
||||
)
|
||||
if logits.device.type != "cuda":
|
||||
raise RuntimeError("DCP sparse-indexer merge requires CUDA tensors.")
|
||||
if logits.dtype != torch.float32 or topk_indices.dtype != torch.int32:
|
||||
raise RuntimeError(
|
||||
"DCP sparse-indexer merge requires fp32 logits and int32 indices."
|
||||
)
|
||||
if k not in (512, 1024, 2048):
|
||||
raise RuntimeError(
|
||||
f"DCP sparse-indexer merge requires index_topk in (512, 1024, 2048); "
|
||||
f"got {k}."
|
||||
)
|
||||
|
||||
|
||||
def _merge_dcp_topk_global(
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
dcp_rank: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
row_starts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Merge each DCP rank's local top-K into the global top-K.
|
||||
|
||||
``topk_indices`` are this rank's local top-K positions into its 1/N KV
|
||||
shard. A token in the global top-K must also be in its owning rank's local
|
||||
top-K (at most ``topk_tokens - 1`` tokens rank globally above it, hence at
|
||||
most that many on its own rank), so exchanging only the per-rank local
|
||||
candidates is exact -- equivalent to all-gathering the full logit matrix,
|
||||
but it ships ``dcp_world_size * topk_tokens`` candidates instead of the whole
|
||||
score row. Overwrites ``topk_indices`` with global token ids (``-1`` for
|
||||
padding); the attention backend localizes them back to physical slots per
|
||||
rank.
|
||||
"""
|
||||
if dcp_world_size <= 1:
|
||||
return
|
||||
|
||||
# CuteDSL-only path (no PyTorch fallback): Triton-pack each rank's
|
||||
# (score, global_id) candidates on-device, all-gather, then the CuteDSL
|
||||
# stable-topk selector.
|
||||
_assert_cutedsl_dcp_merge_supported(logits, topk_indices, topk_tokens)
|
||||
from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import (
|
||||
pack_dcp_topk_candidates_cutedsl,
|
||||
stable_topk_from_gathered_candidates_cutedsl,
|
||||
)
|
||||
|
||||
packed = torch.empty(
|
||||
(*topk_indices.shape, 2),
|
||||
dtype=torch.float32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
pack_dcp_topk_candidates_cutedsl(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_interleave,
|
||||
row_starts,
|
||||
)
|
||||
gathered = get_dcp_group().all_gather(packed, dim=1)
|
||||
stable_topk_from_gathered_candidates_cutedsl(
|
||||
gathered, topk_tokens, out=topk_indices
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_indexer_q_rope_quant_kernel(
|
||||
positions,
|
||||
@@ -227,6 +309,9 @@ def sparse_attn_indexer(
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
skip_topk_buffer_clear: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
@@ -328,16 +413,18 @@ def sparse_attn_indexer(
|
||||
scales_spec,
|
||||
)
|
||||
for chunk in prefill_metadata.chunks:
|
||||
k_quant = k_quant_full[: chunk.total_seq_lens]
|
||||
k_scale = k_scale_full[: chunk.total_seq_lens]
|
||||
|
||||
if not chunk.skip_kv_gather:
|
||||
cu_seqlen_ks = chunk.cu_seqlen_ks
|
||||
cu_seqlen_ke = chunk.cu_seqlen_ke
|
||||
assert chunk.local_cu_seq_lens is not None
|
||||
k_quant = k_quant_full[: chunk.max_local_total_seq_lens]
|
||||
k_scale = k_scale_full[: chunk.max_local_total_seq_lens]
|
||||
if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0:
|
||||
ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache,
|
||||
k_quant,
|
||||
k_scale,
|
||||
chunk.block_table,
|
||||
chunk.cu_seq_lens,
|
||||
chunk.local_cu_seq_lens,
|
||||
)
|
||||
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
@@ -346,51 +433,64 @@ def sparse_attn_indexer(
|
||||
if q_scale is not None
|
||||
else None
|
||||
)
|
||||
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
|
||||
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
|
||||
if use_fp4_cache:
|
||||
q_slice_cast = q_slice.view(torch.int8)
|
||||
k_quant_cast = k_quant.view(torch.int8)
|
||||
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
|
||||
else:
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
if current_platform.is_xpu():
|
||||
if q_scale_slice is not None:
|
||||
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
|
||||
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
|
||||
q_slice_cast,
|
||||
k_quant_cast,
|
||||
k_scale_cast,
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
)
|
||||
else:
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
|
||||
topk_indices = topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
|
||||
ops.top_k_per_row_prefill(
|
||||
if chunk.local_total_seq_lens == 0:
|
||||
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
|
||||
topk_indices.fill_(-1)
|
||||
else:
|
||||
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
|
||||
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
|
||||
if use_fp4_cache:
|
||||
q_slice_cast = q_slice.view(torch.int8)
|
||||
k_quant_cast = k_quant.view(torch.int8)
|
||||
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
|
||||
else:
|
||||
q_slice_cast = q_slice
|
||||
k_quant_cast = k_quant
|
||||
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
|
||||
if current_platform.is_xpu():
|
||||
if q_scale_slice is not None:
|
||||
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
|
||||
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
|
||||
q_slice_cast,
|
||||
k_quant_cast,
|
||||
k_scale_cast,
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
)
|
||||
else:
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_slice_cast, q_scale_slice),
|
||||
(k_quant_cast, k_scale_cast),
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
clean_logits=False,
|
||||
)
|
||||
num_rows = logits.shape[0]
|
||||
ops.top_k_per_row_prefill(
|
||||
logits,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
_merge_dcp_topk_global(
|
||||
logits,
|
||||
chunk.cu_seqlen_ks,
|
||||
chunk.cu_seqlen_ke,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
row_starts=chunk.cu_seqlen_ks,
|
||||
)
|
||||
|
||||
if has_decode:
|
||||
@@ -508,7 +608,7 @@ def sparse_attn_indexer(
|
||||
topk_indices,
|
||||
topk_workspace,
|
||||
topk_tokens,
|
||||
attn_metadata_narrowed.max_seq_len,
|
||||
logits.shape[1],
|
||||
)
|
||||
else:
|
||||
ops.top_k_per_row_decode(
|
||||
@@ -522,6 +622,16 @@ def sparse_attn_indexer(
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
if decode_metadata.global_seq_lens is not None:
|
||||
_merge_dcp_topk_global(
|
||||
logits,
|
||||
topk_indices,
|
||||
topk_tokens,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
)
|
||||
|
||||
if decode_metadata.requires_padding:
|
||||
# if padded, we need to unpack
|
||||
# the topk indices removing padded tokens
|
||||
@@ -553,6 +663,9 @@ def sparse_attn_indexer_fake(
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
skip_topk_buffer_clear: bool = False,
|
||||
) -> torch.Tensor:
|
||||
return topk_indices_buffer
|
||||
@@ -604,6 +717,13 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self.skip_k_cache_insert = skip_k_cache_insert
|
||||
self.use_fp4_cache = use_fp4_cache
|
||||
# DCP scalars are constant for the run; resolve them here (config is set
|
||||
# during model construction) and pass them into the custom op, rather
|
||||
# than threading them through per-step metadata.
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
if current_platform.is_cuda() and not has_deep_gemm():
|
||||
raise RuntimeError(
|
||||
"Sparse Attention Indexer CUDA op requires DeepGEMM support in "
|
||||
@@ -657,6 +777,9 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.topk_indices_buffer,
|
||||
self.skip_k_cache_insert,
|
||||
self.use_fp4_cache,
|
||||
self.dcp_rank,
|
||||
self.dcp_world_size,
|
||||
self.cp_kv_cache_interleave_size,
|
||||
)
|
||||
|
||||
def forward_xpu(
|
||||
|
||||
@@ -496,7 +496,8 @@ class DeepseekV32Attention(MLAAttention):
|
||||
self.topk_indices_buffer,
|
||||
True, # skip_k_cache_insert
|
||||
False, # use_fp4_cache
|
||||
True, # skip_topk_buffer_clear (fused_norm_rope already did it)
|
||||
# fused_norm_rope already cleared the topk buffer this forward.
|
||||
skip_topk_buffer_clear=True,
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, ClassVar
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
@@ -29,8 +30,12 @@ from vllm.v1.attention.backend import (
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
triton_convert_req_index_to_global_index,
|
||||
triton_filter_and_convert_dcp_index,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
KVCacheLayoutType,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import KVCacheLayoutType
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -38,8 +43,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
FLASHINFER_MLA_SPARSE_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
|
||||
|
||||
|
||||
class _FlashInferMLASparseBackendBase(AttentionBackend):
|
||||
"""Common metadata for concrete FlashInfer sparse MLA backends."""
|
||||
@@ -252,10 +255,13 @@ class FlashInferMLASparseMetadata(AttentionMetadata):
|
||||
|
||||
# Sequence lengths for all requests (context + query)
|
||||
seq_lens: torch.Tensor
|
||||
num_decodes: int
|
||||
num_decode_tokens: int
|
||||
|
||||
# Sparse-specific
|
||||
block_size: int = 64
|
||||
topk_tokens: int = 2048
|
||||
cp_kv_cache_interleave_size: int = 1
|
||||
|
||||
|
||||
class FlashInferMLASparseMetadataBuilder(
|
||||
@@ -281,6 +287,12 @@ class FlashInferMLASparseMetadataBuilder(
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
self.topk_tokens = vllm_config.model_config.hf_config.index_topk
|
||||
|
||||
self._init_reorder_batch_threshold(
|
||||
1,
|
||||
supports_spec_as_decode=True,
|
||||
supports_dcp_with_varlen=True,
|
||||
)
|
||||
|
||||
self.req_id_per_token_buffer = torch.empty(
|
||||
(vllm_config.scheduler_config.max_num_batched_tokens,),
|
||||
dtype=torch.int32,
|
||||
@@ -295,6 +307,12 @@ class FlashInferMLASparseMetadataBuilder(
|
||||
) -> FlashInferMLASparseMetadata:
|
||||
cm = common_attn_metadata
|
||||
num_tokens = cm.num_actual_tokens
|
||||
assert self.reorder_batch_threshold is not None
|
||||
num_decodes, _, num_decode_tokens, _ = split_decodes_and_prefills(
|
||||
cm,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
treat_short_extends_as_decodes=True,
|
||||
)
|
||||
|
||||
# Build req_id_per_token mapping
|
||||
starts = np.asarray(cm.query_start_loc_cpu, dtype=np.int32)
|
||||
@@ -320,8 +338,13 @@ class FlashInferMLASparseMetadataBuilder(
|
||||
block_table=cm.block_table_tensor,
|
||||
req_id_per_token=req_id_per_token_tensor,
|
||||
seq_lens=cm.seq_lens,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
block_size=self.kv_cache_spec.block_size,
|
||||
topk_tokens=self.topk_tokens,
|
||||
cp_kv_cache_interleave_size=(
|
||||
self.vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -333,7 +356,7 @@ def _get_workspace_buffer(device: torch.device) -> torch.Tensor:
|
||||
global _fi_sparse_workspace
|
||||
if _fi_sparse_workspace is None:
|
||||
_fi_sparse_workspace = torch.zeros(
|
||||
FLASHINFER_MLA_SPARSE_WORKSPACE_BUFFER_SIZE,
|
||||
envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
@@ -347,6 +370,9 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata
|
||||
sparse attention computation.
|
||||
"""
|
||||
|
||||
can_return_lse_for_decode: bool = True
|
||||
lse_base_on_e: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
@@ -421,14 +447,27 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata
|
||||
assert self.topk_indices_buffer is not None
|
||||
topk_indices = self.topk_indices_buffer[:num_actual_toks]
|
||||
|
||||
topk_indices_physical, seq_lens = triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token[:num_actual_toks],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
if self.dcp_world_size > 1:
|
||||
topk_indices_physical, seq_lens = triton_filter_and_convert_dcp_index(
|
||||
attn_metadata.req_id_per_token[:num_actual_toks],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
dcp_size=self.dcp_world_size,
|
||||
dcp_rank=self.dcp_rank,
|
||||
cp_kv_cache_interleave_size=(attn_metadata.cp_kv_cache_interleave_size),
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
else:
|
||||
topk_indices_physical, seq_lens = triton_convert_req_index_to_global_index(
|
||||
attn_metadata.req_id_per_token[:num_actual_toks],
|
||||
attn_metadata.block_table,
|
||||
topk_indices,
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
|
||||
if self._workspace_buffer is None:
|
||||
self._workspace_buffer = _get_workspace_buffer(q.device)
|
||||
@@ -444,18 +483,66 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata
|
||||
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
|
||||
o = trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=q.unsqueeze(1),
|
||||
# Single-token sparse decode. trtllm-gen requires the q_len_per_request
|
||||
# dim, but the sparse attention mask is fully per-token (each query token
|
||||
# carries its own top-k index row), so unsqueeze is sufficient and
|
||||
# correct. The MTP/multi-token q_len grouping is a perf-only layout and is
|
||||
# deferred until MTP is validated end-to-end for this backend.
|
||||
query = q.unsqueeze(1)
|
||||
block_tables = topk_indices_physical.unsqueeze(1)
|
||||
seq_lens_arg = seq_lens
|
||||
|
||||
kernel_out = trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=query,
|
||||
kv_cache=kv_c_and_k_pe_cache.unsqueeze(1),
|
||||
workspace_buffer=self._workspace_buffer,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
block_tables=topk_indices_physical.unsqueeze(1),
|
||||
seq_lens=seq_lens,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens_arg,
|
||||
max_seq_len=attn_metadata.topk_tokens,
|
||||
bmm1_scale=self.bmm1_scale,
|
||||
bmm2_scale=self.bmm2_scale,
|
||||
sparse_mla_top_k=attn_metadata.topk_tokens,
|
||||
return_lse=self.need_to_return_lse_for_decode,
|
||||
)
|
||||
return o.view(-1, o.shape[-2], o.shape[-1]), None
|
||||
if self.need_to_return_lse_for_decode:
|
||||
assert isinstance(kernel_out, tuple)
|
||||
o, lse = kernel_out
|
||||
else:
|
||||
assert isinstance(kernel_out, torch.Tensor)
|
||||
o = kernel_out
|
||||
lse = None
|
||||
|
||||
out = o.view(-1, o.shape[-2], o.shape[-1])
|
||||
if lse is not None:
|
||||
lse = self._normalize_lse(lse, out.shape[0], out.shape[1])
|
||||
empty_rows = (topk_indices_physical == -1).all(dim=-1)
|
||||
out.masked_fill_(empty_rows.view(-1, 1, 1), 0.0)
|
||||
lse.masked_fill_(empty_rows.view(-1, 1), float("-inf"))
|
||||
return out, lse
|
||||
|
||||
@staticmethod
|
||||
def _normalize_lse(
|
||||
lse: torch.Tensor,
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
) -> torch.Tensor:
|
||||
# FlashInfer returns the decode LSE either as 2D (num_tokens, num_heads)
|
||||
# or 3D ((num_tokens, num_heads, 1) / (num_tokens, 1, num_heads)).
|
||||
# Collapse all of these to the (num_tokens, num_heads) the shared DCP
|
||||
# reducer expects.
|
||||
if lse.dim() == 3:
|
||||
if lse.shape[-1] == 1:
|
||||
lse = lse.squeeze(-1)
|
||||
elif lse.shape[1] == 1:
|
||||
lse = lse.squeeze(1)
|
||||
elif lse.shape[0] * lse.shape[1] == num_tokens:
|
||||
lse = lse.reshape(num_tokens, lse.shape[-1])
|
||||
if lse.shape != (num_tokens, num_heads):
|
||||
raise RuntimeError(
|
||||
"Unexpected FlashInfer sparse MLA LSE shape: "
|
||||
f"{tuple(lse.shape)}, expected ({num_tokens}, {num_heads})."
|
||||
)
|
||||
return lse
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
@@ -24,6 +25,7 @@ from vllm.v1.attention.backend import (
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import get_compressed_slot_mapping
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
get_dcp_local_seq_lens,
|
||||
split_decodes_and_prefills,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, MLAAttentionSpec
|
||||
@@ -168,6 +170,8 @@ class DeepseekV4IndexerBackend(DeepseekV32IndexerBackend):
|
||||
@dataclass
|
||||
class DeepseekV32IndexerPrefillChunkMetadata:
|
||||
block_table: torch.Tensor
|
||||
# Under DCP (dcp_world_size > 1) these hold this rank's local row bounds;
|
||||
# otherwise they hold the global bounds.
|
||||
cu_seqlen_ks: torch.Tensor
|
||||
cu_seqlen_ke: torch.Tensor
|
||||
cu_seq_lens: torch.Tensor
|
||||
@@ -177,6 +181,9 @@ class DeepseekV32IndexerPrefillChunkMetadata:
|
||||
token_end: int
|
||||
num_reqs: int
|
||||
skip_kv_gather: bool = False
|
||||
local_cu_seq_lens: torch.Tensor | None = None
|
||||
local_total_seq_lens: int = 0
|
||||
max_local_total_seq_lens: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -195,6 +202,7 @@ class DeepSeekV32IndexerDecodeMetadata:
|
||||
decode_lens: torch.Tensor
|
||||
requires_padding: bool
|
||||
schedule_metadata: torch.Tensor
|
||||
global_seq_lens: torch.Tensor | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -243,6 +251,19 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
scheduler_config = self.vllm_config.scheduler_config
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
|
||||
# The DCP sparse-indexer code is parameterized by interleave size, but
|
||||
# interleave > 1 is not yet validated end-to-end (gsm8k parity fails),
|
||||
# so fail closed here rather than silently produce wrong output.
|
||||
if self.dcp_world_size > 1 and self.cp_kv_cache_interleave_size > 1:
|
||||
raise NotImplementedError(
|
||||
"DCP sparse indexer currently supports only "
|
||||
f"cp_kv_cache_interleave_size=1 (got "
|
||||
f"{self.cp_kv_cache_interleave_size})."
|
||||
)
|
||||
# NOTE(Chen):an estimated max size of flattened_kv. Need to double check.
|
||||
self.max_prefill_buffer_size = get_max_prefill_buffer_size(self.vllm_config)
|
||||
self.num_speculative_tokens = (
|
||||
@@ -300,6 +321,11 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.global_decode_seq_lens_buffer = torch.zeros(
|
||||
(scheduler_config.max_num_batched_tokens,),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self.arange_buffer = torch.arange(
|
||||
max(
|
||||
scheduler_config.max_num_seqs * next_n,
|
||||
@@ -331,6 +357,11 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
# Get compress_ratio for DeepseekV4 support
|
||||
if isinstance(self.kv_cache_spec, MLAAttentionSpec):
|
||||
self.compress_ratio = self.kv_cache_spec.compress_ratio
|
||||
if self.dcp_world_size > 1 and self.compress_ratio > 1:
|
||||
raise NotImplementedError(
|
||||
"DCP is not supported with sparse indexer KV compression "
|
||||
f"(compress_ratio={self.compress_ratio})."
|
||||
)
|
||||
|
||||
# Pre-allocate buffers for CUDA graph compatibility when
|
||||
if self.compress_ratio > 1:
|
||||
@@ -348,6 +379,26 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def _dcp_localize_decode_seq_lens(
|
||||
self,
|
||||
seq_lens: torch.Tensor,
|
||||
num_decodes: int,
|
||||
seq_lens_is_buffer_view: bool,
|
||||
) -> torch.Tensor:
|
||||
local_seq_lens = get_dcp_local_seq_lens(
|
||||
seq_lens,
|
||||
self.dcp_world_size,
|
||||
self.dcp_rank,
|
||||
self.cp_kv_cache_interleave_size,
|
||||
)
|
||||
if seq_lens_is_buffer_view:
|
||||
seq_lens.copy_(local_seq_lens)
|
||||
return seq_lens
|
||||
|
||||
out = self.decode_seq_lens_buffer[:num_decodes]
|
||||
out.copy_(local_seq_lens)
|
||||
return out
|
||||
|
||||
def _prepare_decode_tensors(
|
||||
self,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -468,6 +519,34 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
seq_lens = seq_lens_buffer
|
||||
return seq_lens, block_table, decode_lens, num_decodes, requires_padding
|
||||
|
||||
def _prepare_global_decode_seq_lens(
|
||||
self,
|
||||
global_seq_lens: torch.Tensor | None,
|
||||
decode_lens: torch.Tensor,
|
||||
decode_lens_cpu: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
use_native: bool,
|
||||
max_decode_len: int,
|
||||
) -> torch.Tensor | None:
|
||||
if global_seq_lens is None:
|
||||
return None
|
||||
if use_native or max_decode_len <= 1:
|
||||
return global_seq_lens
|
||||
|
||||
actual_expanded = int(decode_lens_cpu.sum().item())
|
||||
if actual_expanded > 0:
|
||||
expanded_offsets = torch.repeat_interleave(
|
||||
global_seq_lens - decode_lens - query_start_loc,
|
||||
decode_lens,
|
||||
output_size=actual_expanded,
|
||||
)
|
||||
self.global_decode_seq_lens_buffer[:actual_expanded] = (
|
||||
expanded_offsets + self.arange_buffer[:actual_expanded] + 1
|
||||
)
|
||||
self.global_decode_seq_lens_buffer[actual_expanded:num_decode_tokens] = 0
|
||||
return self.global_decode_seq_lens_buffer[:num_decode_tokens]
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -481,6 +560,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
seq_lens = common_attn_metadata.seq_lens
|
||||
slot_mapping = common_attn_metadata.slot_mapping
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens
|
||||
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
@@ -549,6 +629,9 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
self.compress_ratio,
|
||||
query_slice=query_slice,
|
||||
skip_kv_gather=query_slice.start > 0,
|
||||
dcp_rank=self.dcp_rank,
|
||||
dcp_world_size=self.dcp_world_size,
|
||||
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
|
||||
)
|
||||
# Skip when total_seq_lens is 0 (i.e., no compressed token).
|
||||
if metadata is not None:
|
||||
@@ -566,6 +649,17 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
common_attn_metadata.query_start_loc_cpu[: num_decodes + 1]
|
||||
)
|
||||
|
||||
# Under DCP the per-token decode bounds must be localized AFTER the
|
||||
# per-token expansion below, not before. Expanding from a
|
||||
# request-level localized length subtracts decode offsets in local
|
||||
# space and yields too-short bounds (e.g. world=2, rank=1, global
|
||||
# per-token bounds [8, 9, 10] -> [3, 4, 5] instead of [4, 4, 5]), so
|
||||
# the first decode token would run top-k against too short a local KV
|
||||
# range and miss valid tokens. Keep the global seq_lens here and
|
||||
# localize the expanded bounds further down.
|
||||
global_seq_lens_for_decode: torch.Tensor | None = None
|
||||
if dcp_local_seq_lens is not None:
|
||||
global_seq_lens_for_decode = common_attn_metadata.seq_lens[:num_decodes]
|
||||
seq_lens = common_attn_metadata.seq_lens[:num_decodes]
|
||||
block_table = common_attn_metadata.block_table_tensor[:num_decodes, ...]
|
||||
|
||||
@@ -573,6 +667,16 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
next_n = 1 + self.num_speculative_tokens
|
||||
use_native = not self.use_flattening and max_decode_len <= next_n
|
||||
|
||||
global_seq_lens_for_decode = self._prepare_global_decode_seq_lens(
|
||||
global_seq_lens=global_seq_lens_for_decode,
|
||||
decode_lens=decode_lens,
|
||||
decode_lens_cpu=decode_lens_cpu,
|
||||
query_start_loc=common_attn_metadata.query_start_loc[:num_decodes],
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
use_native=use_native,
|
||||
max_decode_len=max_decode_len,
|
||||
)
|
||||
|
||||
seq_lens, block_table, decode_lens, batch_size, requires_padding = (
|
||||
self._prepare_decode_tensors(
|
||||
seq_lens=seq_lens,
|
||||
@@ -588,15 +692,22 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
)
|
||||
)
|
||||
|
||||
seq_lens_is_buffer_view = (use_native and next_n > 1) or (
|
||||
not use_native and max_decode_len > 1
|
||||
)
|
||||
|
||||
# DCP: localize the now-expanded per-token global bounds to this
|
||||
# rank's owned KV. Done here (after expansion) so each token's global
|
||||
# causal length is localized individually; see the comment above.
|
||||
if dcp_local_seq_lens is not None:
|
||||
seq_lens = self._dcp_localize_decode_seq_lens(
|
||||
seq_lens, num_decodes, seq_lens_is_buffer_view
|
||||
)
|
||||
|
||||
# For DeepseekV4 (compress_ratio > 1), the indexer KV cache stores
|
||||
# compressed tokens. Convert uncompressed seq_lens to compressed.
|
||||
if self.compress_ratio > 1:
|
||||
# True iff seq_lens aliases decode_seq_lens_buffer (flatten or
|
||||
# native wrote it); False iff it aliases common_attn_metadata.
|
||||
seq_lens_is_local_view = (use_native and next_n > 1) or (
|
||||
not use_native and max_decode_len > 1
|
||||
)
|
||||
if seq_lens_is_local_view:
|
||||
if seq_lens_is_buffer_view:
|
||||
seq_lens //= self.compress_ratio
|
||||
else:
|
||||
# Copy to avoid mutating shared state; keeps CG address stable.
|
||||
@@ -626,6 +737,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
decode_lens=decode_lens,
|
||||
requires_padding=requires_padding,
|
||||
schedule_metadata=self.scheduler_metadata_buffer,
|
||||
global_seq_lens=global_seq_lens_for_decode,
|
||||
)
|
||||
|
||||
attn_metadata = DeepseekV32IndexerMetadata(
|
||||
@@ -655,6 +767,9 @@ def build_prefill_chunk_metadata(
|
||||
compress_ratio: int,
|
||||
query_slice: slice | None = None,
|
||||
skip_kv_gather: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
) -> DeepseekV32IndexerPrefillChunkMetadata | None:
|
||||
total_seq_lens = compressed_seq_lens_cpu[start_idx:end_idx].sum().item()
|
||||
if total_seq_lens == 0:
|
||||
@@ -669,6 +784,25 @@ def build_prefill_chunk_metadata(
|
||||
cu_seq_lens[:1] = 0
|
||||
torch.cumsum(compressed_seq_lens[start_idx:end_idx], dim=0, out=cu_seq_lens[1:])
|
||||
|
||||
local_cu_seq_lens = cu_seq_lens
|
||||
local_total_seq_lens = total_seq_lens
|
||||
max_local_total_seq_lens = total_seq_lens
|
||||
if dcp_world_size > 1:
|
||||
# Per-rank local KV length under interleave-aware DCP sharding, shape
|
||||
# [num_reqs, dcp_world_size]. Reuse the canonical CP helper so the
|
||||
# sharding matches the rest of the DCP pipeline (decode/prefill).
|
||||
local_seq_lens = get_dcp_local_seq_lens(
|
||||
compressed_seq_lens[start_idx:end_idx],
|
||||
dcp_world_size,
|
||||
None,
|
||||
cp_kv_cache_interleave_size,
|
||||
)
|
||||
this_rank_counts = local_seq_lens[:, dcp_rank].to(torch.int32)
|
||||
local_cu_seq_lens = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
torch.cumsum(this_rank_counts, dim=0, out=local_cu_seq_lens[1:])
|
||||
local_total_seq_lens = int(local_cu_seq_lens[-1].item())
|
||||
max_local_total_seq_lens = int(local_seq_lens.sum(dim=0).max().item())
|
||||
|
||||
query_start_loc = (
|
||||
query_start_loc[start_idx : end_idx + 1] - query_start_loc[start_idx]
|
||||
)
|
||||
@@ -687,15 +821,21 @@ def build_prefill_chunk_metadata(
|
||||
cu_seq_len_ks = torch.empty(output_query_len, dtype=torch.int32, device=device)
|
||||
cu_seq_len_ke = torch.empty(output_query_len, dtype=torch.int32, device=device)
|
||||
|
||||
# Under DCP the kernel writes this rank's local row bounds into
|
||||
# cu_seq_len_ks/ke; otherwise local_cu_seq_lens aliases cu_seq_lens.
|
||||
_build_prefill_chunk_metadata_kernel[(num_reqs,)](
|
||||
query_start_loc,
|
||||
uncompressed_seq_lens[start_idx:end_idx],
|
||||
cu_seq_lens,
|
||||
local_cu_seq_lens,
|
||||
token_to_seq,
|
||||
cu_seq_len_ks,
|
||||
cu_seq_len_ke,
|
||||
qs_start,
|
||||
qs_stop,
|
||||
dcp_rank,
|
||||
dcp_world_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
BLOCK_SIZE=1024,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
)
|
||||
@@ -719,6 +859,9 @@ def build_prefill_chunk_metadata(
|
||||
token_end=token_end,
|
||||
num_reqs=num_reqs,
|
||||
skip_kv_gather=skip_kv_gather,
|
||||
local_cu_seq_lens=local_cu_seq_lens,
|
||||
local_total_seq_lens=local_total_seq_lens,
|
||||
max_local_total_seq_lens=max_local_total_seq_lens,
|
||||
)
|
||||
|
||||
|
||||
@@ -728,12 +871,18 @@ def _build_prefill_chunk_metadata_kernel(
|
||||
query_start_loc_ptr,
|
||||
uncompressed_seq_lens_ptr,
|
||||
cu_compressed_seq_lens_ptr,
|
||||
# Row-start base for cu_seq_len_ks/ke: local cumulative lens under DCP,
|
||||
# aliases cu_compressed_seq_lens_ptr otherwise.
|
||||
row_start_cu_compressed_seq_lens_ptr,
|
||||
# Outputs
|
||||
token_to_seq_ptr,
|
||||
cu_compressed_seq_len_ks_ptr,
|
||||
cu_compressed_seq_len_ke_ptr,
|
||||
query_slice_start,
|
||||
query_slice_stop,
|
||||
DCP_RANK,
|
||||
DCP_WORLD,
|
||||
DCP_INTERLEAVE,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
):
|
||||
@@ -747,6 +896,10 @@ def _build_prefill_chunk_metadata_kernel(
|
||||
seq_end = tl.load(cu_compressed_seq_lens_ptr + batch_idx + 1)
|
||||
compressed_seq_len = seq_end - seq_start
|
||||
|
||||
# Row start for the (possibly localized) cu_seq_len_ks/ke. Equals seq_start
|
||||
# when DCP is disabled (the pointer aliases cu_compressed_seq_lens_ptr).
|
||||
row_start = tl.load(row_start_cu_compressed_seq_lens_ptr + batch_idx)
|
||||
|
||||
uncompressed_seq_len = tl.load(uncompressed_seq_lens_ptr + batch_idx)
|
||||
start_pos = uncompressed_seq_len - query_len
|
||||
|
||||
@@ -760,14 +913,25 @@ def _build_prefill_chunk_metadata_kernel(
|
||||
)
|
||||
out_pos = abs_pos - query_slice_start
|
||||
|
||||
# Compute cu_seq_len_ks
|
||||
tl.store(cu_compressed_seq_len_ks_ptr + out_pos, seq_start, mask=mask)
|
||||
# cu_seq_len_ks: row start in the gathered K buffer.
|
||||
tl.store(cu_compressed_seq_len_ks_ptr + out_pos, row_start, mask=mask)
|
||||
|
||||
# Compute cu_seq_len_ke
|
||||
seq_len_per_token = (start_pos + 1 + offset) // COMPRESS_RATIO
|
||||
# cu_seq_len_ke: row start + per-token context length. Under DCP the
|
||||
# global per-token length is sharded across ranks.
|
||||
global_ctx = start_pos + 1 + offset
|
||||
len_per_token = global_ctx // COMPRESS_RATIO
|
||||
if DCP_WORLD > 1:
|
||||
# Per-rank local context length under interleave-aware DCP, matching
|
||||
# get_dcp_local_seq_lens. K == 1 reduces to (len + world-1-rank)//world.
|
||||
base = (len_per_token // DCP_INTERLEAVE // DCP_WORLD) * DCP_INTERLEAVE
|
||||
remainder = len_per_token - base * DCP_WORLD
|
||||
remainder = tl.minimum(
|
||||
tl.maximum(remainder - DCP_RANK * DCP_INTERLEAVE, 0), DCP_INTERLEAVE
|
||||
)
|
||||
len_per_token = base + remainder
|
||||
tl.store(
|
||||
cu_compressed_seq_len_ke_ptr + out_pos,
|
||||
seq_start + seq_len_per_token,
|
||||
row_start + len_per_token,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
@@ -23,6 +23,16 @@ def _convert_req_index_to_global_index_kernel(
|
||||
BLOCK_N: tl.constexpr, # tile width along columns
|
||||
HAS_PREFILL: tl.constexpr,
|
||||
COUNT_VALID: tl.constexpr, # whether to count valid indices
|
||||
# When set, scatter valid slots to a contiguous prefix [0, valid_count) using
|
||||
# valid_count_ptr as an atomic slot allocator (DCP filtering leaves interior
|
||||
# -1 gaps; the trtllm-gen sparse kernel reads the first valid_count entries).
|
||||
# Requires COUNT_VALID and an out buffer pre-filled with -1. Order within the
|
||||
# prefix is unspecified (only the selected set matters).
|
||||
COMPACT_TO_FRONT: tl.constexpr,
|
||||
# DCP de-interleave: with DCP_SIZE == 1 these are an exact no-op
|
||||
DCP_SIZE: tl.constexpr,
|
||||
DCP_RANK: tl.constexpr,
|
||||
DCP_INTERLEAVE: tl.constexpr,
|
||||
# strides (in elements)
|
||||
bt_stride0,
|
||||
bt_stride1,
|
||||
@@ -52,15 +62,27 @@ def _convert_req_index_to_global_index_kernel(
|
||||
if HAS_PREFILL:
|
||||
prefill_req_id = tl.load(prefill_request_id_ptr + token_id)
|
||||
is_prefill = prefill_req_id >= 0
|
||||
|
||||
# DCP de-interleave the global token id into this rank's local slot.
|
||||
# Tokens are interleaved in groups of DCP_INTERLEAVE across ranks. With
|
||||
# DCP_SIZE == 1 (and any interleave) owning_rank == 0 == DCP_RANK (never
|
||||
# remote) and local_idx == tok, so this reduces to the non-DCP path; with
|
||||
# DCP_INTERLEAVE == 1 it reduces to plain round-robin (tok % / // DCP_SIZE).
|
||||
owning_rank = (tok // DCP_INTERLEAVE) % DCP_SIZE
|
||||
is_remote = owning_rank != DCP_RANK
|
||||
local_idx = (
|
||||
tok // (DCP_SIZE * DCP_INTERLEAVE)
|
||||
) * DCP_INTERLEAVE + tok % DCP_INTERLEAVE
|
||||
|
||||
# Compute block id and in-block offset
|
||||
block_id = tok // BLOCK_SIZE
|
||||
inblock_off = tok % BLOCK_SIZE
|
||||
block_id = local_idx // BLOCK_SIZE
|
||||
inblock_off = local_idx % BLOCK_SIZE
|
||||
|
||||
# Guard block_table access
|
||||
valid_block = (block_id < max_num_blocks_per_req) & (block_id >= 0)
|
||||
bt_ptr = block_table_ptr + req * bt_stride0 + block_id * bt_stride1
|
||||
is_invalid_tok |= ~valid_block
|
||||
base = tl.load(bt_ptr, mask=valid_block & ~is_prefill, other=0)
|
||||
is_invalid_tok |= ~valid_block | is_remote
|
||||
base = tl.load(bt_ptr, mask=valid_block & ~is_prefill & ~is_remote, other=0)
|
||||
out_val = base * BLOCK_SIZE + inblock_off
|
||||
|
||||
# Override with prefill output if prefill is enabled
|
||||
@@ -72,14 +94,27 @@ def _convert_req_index_to_global_index_kernel(
|
||||
out_val = tl.where(is_prefill, prefill_out, out_val)
|
||||
out_val = tl.where(is_invalid_tok, -1, out_val)
|
||||
|
||||
# Store results
|
||||
out_ptr_ij = out_ptr + token_id * out_stride0 + indice_id * out_stride1
|
||||
tl.store(out_ptr_ij, out_val)
|
||||
if COMPACT_TO_FRONT:
|
||||
# Scatter valid slots to a contiguous prefix. A per-tile exclusive prefix
|
||||
# sum gives each valid lane a distinct local offset; one atomic add of the
|
||||
# tile's valid count reserves a contiguous base across racing tiles. The
|
||||
# out buffer is pre-filled with -1, so unwritten tail slots stay -1.
|
||||
is_valid = (~is_invalid_tok).to(tl.int32)
|
||||
local_offset = tl.cumsum(is_valid) - is_valid
|
||||
tile_valid_count = tl.sum(is_valid)
|
||||
base = tl.atomic_add(valid_count_ptr + token_id, tile_valid_count)
|
||||
dest = base + local_offset
|
||||
out_ptr_dest = out_ptr + token_id * out_stride0 + dest * out_stride1
|
||||
tl.store(out_ptr_dest, out_val, mask=is_valid == 1)
|
||||
else:
|
||||
# Store results in place (input column == output column).
|
||||
out_ptr_ij = out_ptr + token_id * out_stride0 + indice_id * out_stride1
|
||||
tl.store(out_ptr_ij, out_val)
|
||||
|
||||
# Count valid indices in this tile and atomically add to row total
|
||||
if COUNT_VALID:
|
||||
tile_valid_count = tl.sum((~is_invalid_tok).to(tl.int32))
|
||||
tl.atomic_add(valid_count_ptr + token_id, tile_valid_count)
|
||||
# Count valid indices in this tile and atomically add to row total
|
||||
if COUNT_VALID:
|
||||
tile_valid_count = tl.sum((~is_invalid_tok).to(tl.int32))
|
||||
tl.atomic_add(valid_count_ptr + token_id, tile_valid_count)
|
||||
|
||||
|
||||
def triton_convert_req_index_to_global_index(
|
||||
@@ -176,6 +211,11 @@ def triton_convert_req_index_to_global_index(
|
||||
BLOCK_N,
|
||||
HAS_PREFILL_WORKSPACE,
|
||||
return_valid_counts,
|
||||
False, # COMPACT_TO_FRONT: keep input column == output column
|
||||
# DCP disabled (no-op de-interleave)
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
# strides
|
||||
bt_stride0,
|
||||
bt_stride1,
|
||||
@@ -189,3 +229,110 @@ def triton_convert_req_index_to_global_index(
|
||||
assert valid_counts is not None
|
||||
return out, valid_counts
|
||||
return out
|
||||
|
||||
|
||||
def triton_filter_and_convert_dcp_index(
|
||||
req_id: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
token_indices: torch.Tensor,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
BLOCK_SIZE: int = 64,
|
||||
NUM_TOPK_TOKENS: int = 2048,
|
||||
BLOCK_N: int = 128,
|
||||
return_valid_counts: bool = False,
|
||||
compact_valid_to_front: bool = True,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Filter global per-request indices to this DCP rank's local slots.
|
||||
|
||||
With ``compact_valid_to_front`` (default), the conversion kernel scatters
|
||||
this rank's owned slots to a contiguous prefix ``[0, valid_count)`` and
|
||||
leaves the rest ``-1``. DCP filtering marks non-owned slots ``-1`` and so
|
||||
creates interior gaps; the trtllm-gen sparse kernel reads the first
|
||||
``valid_count`` entries of each row, so they must be a contiguous prefix.
|
||||
Compaction is fused into the kernel (atomic slot allocator) rather than a
|
||||
separate sort/gather pass. Prefix order is unspecified (only the set matters).
|
||||
"""
|
||||
assert dcp_size >= 1
|
||||
assert 0 <= dcp_rank < dcp_size
|
||||
# Interleave groups must align to KV blocks (globally enforced by
|
||||
# VllmConfig: block_size % cp_kv_cache_interleave_size == 0); assert the
|
||||
# local invariant so local_idx // BLOCK_SIZE never straddles a group.
|
||||
assert BLOCK_SIZE % cp_kv_cache_interleave_size == 0, (
|
||||
f"BLOCK_SIZE ({BLOCK_SIZE}) must be divisible by "
|
||||
f"cp_kv_cache_interleave_size ({cp_kv_cache_interleave_size})."
|
||||
)
|
||||
assert req_id.dtype == torch.int32
|
||||
assert block_table.dtype == torch.int32
|
||||
assert token_indices.dtype == torch.int32
|
||||
assert token_indices.shape[1] == NUM_TOPK_TOKENS
|
||||
assert NUM_TOPK_TOKENS % BLOCK_N == 0
|
||||
|
||||
if dcp_size == 1:
|
||||
return triton_convert_req_index_to_global_index(
|
||||
req_id,
|
||||
block_table,
|
||||
token_indices,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
NUM_TOPK_TOKENS=NUM_TOPK_TOKENS,
|
||||
BLOCK_N=BLOCK_N,
|
||||
return_valid_counts=return_valid_counts,
|
||||
)
|
||||
|
||||
num_tokens = req_id.shape[0]
|
||||
max_num_blocks_per_req = block_table.shape[1]
|
||||
tiles_per_row = NUM_TOPK_TOKENS // BLOCK_N
|
||||
|
||||
req_id_c = req_id.contiguous()
|
||||
block_table_c = block_table.contiguous()
|
||||
token_indices_c = token_indices.contiguous()
|
||||
|
||||
# The compaction uses the valid-count buffer as an atomic slot allocator, so
|
||||
# it requires counting. Pre-fill out with -1 so the unwritten tail stays -1.
|
||||
count_valid = return_valid_counts or compact_valid_to_front
|
||||
if compact_valid_to_front:
|
||||
out = torch.full_like(token_indices_c, -1)
|
||||
else:
|
||||
out = torch.empty_like(token_indices_c)
|
||||
|
||||
valid_counts: torch.Tensor | None = None
|
||||
if count_valid:
|
||||
valid_counts = torch.zeros(
|
||||
num_tokens, dtype=torch.int32, device=token_indices.device
|
||||
)
|
||||
|
||||
bt_stride0, bt_stride1 = block_table_c.stride()
|
||||
ti_stride0, ti_stride1 = token_indices_c.stride()
|
||||
out_stride0, out_stride1 = out.stride()
|
||||
|
||||
_convert_req_index_to_global_index_kernel[(num_tokens, tiles_per_row)](
|
||||
req_id_c,
|
||||
block_table_c,
|
||||
token_indices_c,
|
||||
out,
|
||||
valid_counts,
|
||||
# No prefill workspace on the DCP decode path.
|
||||
None,
|
||||
None,
|
||||
max_num_blocks_per_req,
|
||||
BLOCK_SIZE,
|
||||
BLOCK_N,
|
||||
False, # HAS_PREFILL
|
||||
count_valid,
|
||||
compact_valid_to_front,
|
||||
dcp_size,
|
||||
dcp_rank,
|
||||
cp_kv_cache_interleave_size,
|
||||
bt_stride0,
|
||||
bt_stride1,
|
||||
ti_stride0,
|
||||
ti_stride1,
|
||||
out_stride0,
|
||||
out_stride1,
|
||||
)
|
||||
|
||||
if return_valid_counts:
|
||||
assert valid_counts is not None
|
||||
return out, valid_counts
|
||||
return out
|
||||
|
||||
@@ -892,20 +892,20 @@ def get_dcp_local_seq_lens(
|
||||
use this function to calculate split decode seq_lens of each dcp rank.
|
||||
Only consider dcp now, we can extend the case of cp based on this.
|
||||
"""
|
||||
num_requests = seq_lens.size(0)
|
||||
seq_lens_i32 = seq_lens.to(torch.int32)
|
||||
if dcp_rank is None:
|
||||
rank_offsets = (
|
||||
torch.arange(dcp_size, dtype=torch.int32, device=seq_lens.device)
|
||||
.unsqueeze(0)
|
||||
.repeat(num_requests, 1)
|
||||
rank_offsets = torch.arange(
|
||||
dcp_size,
|
||||
dtype=torch.int32,
|
||||
device=seq_lens.device,
|
||||
).view(
|
||||
*((1,) * seq_lens_i32.dim()),
|
||||
dcp_size,
|
||||
)
|
||||
seq_lens_tiled = seq_lens_i32.unsqueeze(-1)
|
||||
else:
|
||||
rank_offsets = torch.tensor(
|
||||
[[dcp_rank]], dtype=torch.int32, device=seq_lens.device
|
||||
)
|
||||
seq_lens_tiled = (
|
||||
seq_lens.to(torch.int32).unsqueeze(-1).repeat(1, rank_offsets.shape[1])
|
||||
)
|
||||
rank_offsets = torch.tensor(dcp_rank, dtype=torch.int32, device=seq_lens.device)
|
||||
seq_lens_tiled = seq_lens_i32
|
||||
base = (
|
||||
seq_lens_tiled
|
||||
// cp_kv_cache_interleave_size
|
||||
@@ -919,7 +919,7 @@ def get_dcp_local_seq_lens(
|
||||
cp_kv_cache_interleave_size,
|
||||
)
|
||||
dcp_local_seq_lens = base + remainder
|
||||
return dcp_local_seq_lens.squeeze(1)
|
||||
return dcp_local_seq_lens
|
||||
|
||||
|
||||
def mamba_get_block_table_tensor(
|
||||
|
||||
@@ -90,6 +90,7 @@ def _correct_attn_cp_out_kernel(
|
||||
factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally)
|
||||
output = tl.load(outputs_ptr + output_offsets)
|
||||
output = output * factor
|
||||
output = tl.where(factor == 0.0, 0.0, output)
|
||||
|
||||
tl.store(new_output_ptr + output_offsets, output)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user