diff --git a/docs/design/attention_backends.md b/docs/design/attention_backends.md index bd3c6f72c9..77f0ede888 100644 --- a/docs/design/attention_backends.md +++ b/docs/design/attention_backends.md @@ -222,7 +222,7 @@ MLA decode backends are selected using the standard | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ | | `CUTLASS_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 128 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x | | `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x | -| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 10.x | +| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x | | `FLASHINFER_MLA_SPARSE_SM120` | bf16 | `auto`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 64, 256 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 12.x | | `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x | | `FLASHMLA_SPARSE` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla` | 64 | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x | diff --git a/tests/v1/attention/test_indexer_dcp_localize.py b/tests/v1/attention/test_indexer_dcp_localize.py new file mode 100644 index 0000000000..2809c17c4d --- /dev/null +++ b/tests/v1/attention/test_indexer_dcp_localize.py @@ -0,0 +1,951 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer +from vllm.platforms import current_platform +from vllm.utils.import_utils import has_cutedsl +from vllm.v1.attention.backends.mla.indexer import build_prefill_chunk_metadata +from vllm.v1.attention.backends.mla.sparse_utils import ( + triton_filter_and_convert_dcp_index, +) +from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens +from vllm.v1.attention.ops.common import CPTritonContext, correct_attn_out + + +def _local_count(length: int, rank: int, world: int, interleave: int) -> int: + return sum(1 for pos in range(length) if (pos // interleave) % world == rank) + + +def _global_to_local_indices( + global_indices: torch.Tensor, + rank: int, + world: int, + interleave: int, +) -> torch.Tensor: + valid = global_indices >= 0 + global_i64 = global_indices.to(torch.int64).clamp_min(0) + owner = (global_i64 // interleave) % world + local = (global_i64 // (world * interleave)) * interleave + global_i64 % interleave + return torch.where(valid & (owner == rank), local, -1).to(torch.int64) + + +def _local_to_global_indices( + local_indices: torch.Tensor, + rank: int, + world: int, + interleave: int, +) -> torch.Tensor: + valid = local_indices >= 0 + local = local_indices.to(torch.int64).clamp_min(0) + global_indices = ( + (local // interleave) * (world * interleave) + + rank * interleave + + local % interleave + ) + return torch.where(valid, global_indices, -1).to(torch.int64) + + +def _ref_stable_topk_from_candidates_fp64( + candidate_scores: torch.Tensor, + candidate_token_ids: torch.Tensor, + k: int, +) -> torch.Tensor: + """Pure-PyTorch reference for the CuteDSL stable-topk selector order + (score desc, then lowest global token id). Selects the same SET as the + kernel; only the set is compared in tests.""" + num_rows, num_candidates = candidate_scores.shape + device = candidate_scores.device + select_k = min(k, num_candidates) + valid = candidate_token_ids >= 0 + bits = ( + candidate_scores.to(torch.float32).view(torch.int32).to(torch.int64) + & 0xFFFFFFFF + ) + sign = (bits >> 31) & 1 + score_key = ( + torch.where(sign.bool(), bits ^ 0xFFFFFFFF, bits ^ 0x80000000) & 0xFFFFFFFF + ) + id_key = (~candidate_token_ids.to(torch.int64)) & 0xFFFFFFFF + key = (score_key << 32) | id_key + key = torch.where(valid, key, torch.zeros_like(key)) + topk_key = key ^ torch.iinfo(torch.int64).min + _, topk_pos = topk_key.topk(select_k, dim=-1) + + selected = candidate_token_ids.gather(1, topk_pos).to(torch.int32) + selected_valid = valid.gather(1, topk_pos) + selected = torch.where(selected_valid, selected, selected.new_full((), -1)) + if select_k == k: + return selected + pad = torch.full((num_rows, k - select_k), -1, dtype=torch.int32, device=device) + return torch.cat((selected, pad), dim=1) + + +def _attention_from_indices( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + indices: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + valid = indices >= 0 + safe_indices = indices.clamp_min(0) + selected_k = k[safe_indices] + selected_v = v[safe_indices] + scores = torch.einsum("td,tkd->tk", q, selected_k) + scores = scores.masked_fill(~valid, float("-inf")) + lse = torch.logsumexp(scores, dim=-1) + probs = torch.softmax(scores, dim=-1).masked_fill(~valid, 0.0) + out = torch.einsum("tk,tkd->td", probs, selected_v) + empty_rows = ~valid.any(dim=-1) + out[empty_rows] = 0 + lse[empty_rows] = float("-inf") + return out, lse + + +def _dcp_lse_merge( + local_outs: list[torch.Tensor], + local_lses: list[torch.Tensor], +) -> tuple[torch.Tensor, torch.Tensor]: + outs = torch.stack(local_outs, dim=0) + lses = torch.stack(local_lses, dim=0) + merged_lse = torch.logsumexp(lses, dim=0) + weights = torch.exp(lses - merged_lse.unsqueeze(0)) + weights = torch.where(torch.isfinite(weights), weights, torch.zeros_like(weights)) + merged_out = (outs * weights.unsqueeze(-1)).sum(dim=0) + return merged_out, merged_lse + + +class _FakeDCPGroup: + """Single-process stand-in: ``all_gather`` returns the pre-built + concatenation of every rank's packed ``(score, global_id)`` candidates, + mirroring the one packed all-gather the merge issues.""" + + def __init__(self, gathered_packed: torch.Tensor) -> None: + self.gathered_packed = gathered_packed + + def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor: + assert dim == 1 + return self.gathered_packed.clone() + + +def _run_decode_topk( + logits: torch.Tensor, + seq_lens: torch.Tensor, + next_n: int, + topk: int, +) -> torch.Tensor: + indices = torch.empty( + (logits.shape[0], topk), dtype=torch.int32, device=logits.device + ) + torch.ops._C.top_k_per_row_decode( + logits, + next_n, + seq_lens, + indices, + logits.shape[0], + logits.stride(0), + logits.stride(1), + topk, + ) + return indices + + +def _run_persistent_topk( + logits: torch.Tensor, + seq_lens: torch.Tensor, + topk: int, + max_seq_len: int, +) -> torch.Tensor: + indices = torch.empty( + (logits.shape[0], topk), dtype=torch.int32, device=logits.device + ) + workspace = torch.empty(1024 * 1024, dtype=torch.uint8, device=logits.device) + torch.ops._C.persistent_topk( + logits, + seq_lens, + indices, + workspace, + topk, + max_seq_len, + ) + return indices + + +def _dcp_attention_from_local_topks( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + local_topks: list[torch.Tensor], + world: int, + interleave: int, +) -> tuple[torch.Tensor, torch.Tensor]: + local_outs = [] + local_lses = [] + for rank, local_topk in enumerate(local_topks): + owned = [ + pos for pos in range(k.shape[0]) if (pos // interleave) % world == rank + ] + local_out, local_lse = _attention_from_indices( + q, k[owned], v[owned], local_topk.to(torch.int64) + ) + local_outs.append(local_out) + local_lses.append(local_lse) + return _dcp_lse_merge(local_outs, local_lses) + + +def _merge_local_topks_global_with_fake_dcp( + local_logits: list[torch.Tensor], + local_topks: list[torch.Tensor], + topk: int, + world: int, + interleave: int, + row_starts: list[torch.Tensor | None] | None = None, +) -> list[torch.Tensor]: + """Run ``_merge_dcp_topk_global`` for every rank against a faked + all-gather and return each rank's global-index result (all ranks should + agree). The fake pre-builds the packed candidate concatenation the merge's + single ``all_gather`` would return.""" + # The merge is now CuteDSL-only (no PyTorch fallback), so it runs the real + # Triton pack + CuteDSL selector kernels even behind the faked all-gather. + if not current_platform.is_cuda() or not has_cutedsl(): + pytest.skip("DCP merge requires CUDA and CuteDSL") + packed_per_rank = [] + for rank, (logits, indices) in enumerate(zip(local_logits, local_topks)): + score_indices = indices.clamp_min(0).to(torch.long) + if row_starts is not None: + rs = row_starts[rank] + assert rs is not None + score_indices = score_indices + rs.to(torch.long).view(-1, 1) + if logits.shape[1] == 0: + scores = torch.full_like(indices, float("-inf"), dtype=torch.float32) + else: + score_indices = score_indices.clamp_max(logits.shape[1] - 1) + scores = logits.gather(1, score_indices).masked_fill( + indices < 0, float("-inf") + ) + global_ids = _local_to_global_indices(indices, rank, world, interleave) + packed_per_rank.append( + torch.stack((scores.float(), global_ids.to(torch.float32)), dim=-1) + ) + + fake_group = _FakeDCPGroup(torch.cat(packed_per_rank, dim=1).contiguous()) + original_get_dcp_group = sparse_indexer.get_dcp_group + sparse_indexer.get_dcp_group = lambda: fake_group + try: + merged = [] + for rank, (logits, indices) in enumerate(zip(local_logits, local_topks)): + rank_indices = indices.clone() + result = sparse_indexer._merge_dcp_topk_global( + logits, + rank_indices, + topk, + rank, + world, + interleave, + row_starts=None if row_starts is None else row_starts[rank], + ) + assert result is None + merged.append(rank_indices) + return merged + finally: + sparse_indexer.get_dcp_group = original_get_dcp_group + + +@pytest.mark.parametrize("world", [1, 2, 4]) +@pytest.mark.parametrize("interleave", [1, 2, 4]) +def test_get_dcp_local_seq_lens_matches_naive(world: int, interleave: int): + seq_lens = torch.arange(0, 33, dtype=torch.int32) + + for rank in range(world): + actual = get_dcp_local_seq_lens(seq_lens, world, rank, interleave) + expected = torch.tensor( + [ + _local_count(int(seq_len), rank, world, interleave) + for seq_len in seq_lens + ], + dtype=torch.int32, + ) + torch.testing.assert_close(actual, expected) + + +def test_get_dcp_local_seq_lens_can_localize_per_token_bounds(): + seq_lens = torch.tensor([0, 1, 2, 3, 4, 7, 8, 17], dtype=torch.int32) + world = 4 + interleave = 2 + + for rank in range(world): + actual = get_dcp_local_seq_lens(seq_lens, world, rank, interleave) + expected = torch.tensor( + [ + _local_count(int(seq_len), rank, world, interleave) + for seq_len in seq_lens + ], + dtype=torch.int32, + ) + torch.testing.assert_close(actual, expected) + + +def test_get_dcp_local_seq_lens_preserves_mtp_bounds_shape(): + seq_lens = torch.tensor([[8, 9, 10], [11, 12, 13]], dtype=torch.int32) + world = 2 + rank = 1 + interleave = 1 + + actual = get_dcp_local_seq_lens(seq_lens, world, rank, interleave) + expected = torch.tensor( + [ + [_local_count(int(seq_len), rank, world, interleave) for seq_len in row] + for row in seq_lens + ], + dtype=torch.int32, + ) + + assert actual.shape == seq_lens.shape + torch.testing.assert_close(actual, expected) + + +def test_get_dcp_local_seq_lens_must_run_after_decode_expansion(): + world = 2 + rank = 1 + interleave = 1 + expanded_bounds = torch.tensor([8, 9, 10], dtype=torch.int32) + + localized_after_expansion = get_dcp_local_seq_lens( + expanded_bounds, world, rank, interleave + ) + localized_request_len_minus_offsets = get_dcp_local_seq_lens( + torch.tensor([10], dtype=torch.int32), world, rank + ) - torch.tensor([2, 1, 0], dtype=torch.int32) + + assert not torch.equal( + localized_after_expansion, localized_request_len_minus_offsets + ) + torch.testing.assert_close( + localized_after_expansion, torch.tensor([4, 4, 5], dtype=torch.int32) + ) + + +@pytest.mark.parametrize("interleave", [1, 2]) +def test_sparse_dcp_attention_matches_global_topk_attention(interleave: int): + torch.manual_seed(0) + world = 2 + topk = 3 + num_queries = 4 + max_seq_len = 13 + head_dim = 8 + + q = torch.randn(num_queries, head_dim) + k = torch.randn(max_seq_len, head_dim) + v = torch.randn(max_seq_len, head_dim) + seq_lens = torch.tensor([6, 8, 11, 13], dtype=torch.int64) + + scores = q @ k.T + global_topk = torch.full((num_queries, topk), -1, dtype=torch.int64) + for row, seq_len in enumerate(seq_lens.tolist()): + global_topk[row, :topk] = scores[row, :seq_len].topk(topk).indices + + ref_out, ref_lse = _attention_from_indices(q, k, v, global_topk) + + local_outs = [] + local_lses = [] + local_topks = [] + for rank in range(world): + owned = [ + pos for pos in range(max_seq_len) if (pos // interleave) % world == rank + ] + k_local = k[owned] + v_local = v[owned] + local_topk = _global_to_local_indices(global_topk, rank, world, interleave) + local_topks.append(local_topk) + local_out, local_lse = _attention_from_indices(q, k_local, v_local, local_topk) + local_outs.append(local_out) + local_lses.append(local_lse) + + dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses) + + torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5) + + gathered_global = torch.cat( + [ + _local_to_global_indices(local_topks[rank], rank, world, interleave) + for rank in range(world) + ], + dim=1, + ) + assert set(gathered_global[gathered_global >= 0].tolist()) == set( + global_topk.flatten().tolist() + ) + + +def test_local_topk_union_is_not_equivalent_to_global_topk_attention(): + world = 2 + interleave = 1 + topk = 2 + q = torch.tensor([[1.0]]) + k = torch.tensor( + [ + [1.00], + [0.90], + [0.95], + [0.85], + ] + ) + v = torch.tensor([[0.0], [1000.0], [0.0], [1000.0]]) + + scores = q @ k.T + global_topk = scores.topk(topk, dim=-1).indices + ref_out, ref_lse = _attention_from_indices(q, k, v, global_topk) + + local_topks = [] + for rank in range(world): + owned = [ + pos for pos in range(k.shape[0]) if (pos // interleave) % world == rank + ] + local_topks.append(scores[:, owned].topk(topk, dim=-1).indices) + + local_union_out, local_union_lse = _dcp_attention_from_local_topks( + q, k, v, local_topks, world, interleave + ) + + assert not torch.allclose(local_union_out, ref_out) + assert not torch.allclose(local_union_lse, ref_lse) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +def test_sparse_decode_dcp_persistent_topk_matches_non_dcp(): + torch.manual_seed(3) + device = torch.device("cuda") + world = 2 + interleave = 1 + topk = 512 + num_rows = 2 + max_seq_len = 1025 + head_dim = 16 + + q = torch.randn(num_rows, head_dim, device=device) + k = torch.randn(max_seq_len, head_dim, device=device) + v = torch.randn(max_seq_len, head_dim, device=device) + logits = q @ k.T + seq_lens = torch.tensor([[1024], [1025]], dtype=torch.int32, device=device) + + non_dcp_topk = torch.empty((num_rows, topk), dtype=torch.int64, device=device) + for row, seq_len in enumerate(seq_lens.flatten().tolist()): + non_dcp_topk[row] = logits[row, :seq_len].topk(topk).indices + ref_out, ref_lse = _attention_from_indices(q, k, v, non_dcp_topk) + + local_logits = [] + local_topks = [] + for rank in range(world): + owned = [ + pos for pos in range(max_seq_len) if (pos // interleave) % world == rank + ] + rank_logits = logits[:, owned].contiguous() + rank_seq_lens = get_dcp_local_seq_lens( + seq_lens, world, rank, interleave + ).contiguous() + local_logits.append(rank_logits) + local_topks.append( + _run_persistent_topk( + rank_logits, + rank_seq_lens, + topk, + max_seq_len=rank_logits.shape[1], + ) + ) + + merged_global_topks = _merge_local_topks_global_with_fake_dcp( + local_logits, local_topks, topk, world, interleave + ) + # The radix top-K kernel selects a deterministic SET but writes it in + # nondeterministic (atomicAdd) order; the production path is permutation- + # invariant (compaction + softmax), so all ranks must agree on the set, not + # the array order. (The fp64 fallback happens to return sorted order.) + ref_topk = merged_global_topks[0] + for rank_topk in merged_global_topks[1:]: + for row in range(rank_topk.shape[0]): + assert set(rank_topk[row].tolist()) == set(ref_topk[row].tolist()) + + local_outs = [] + local_lses = [] + for rank, global_topk in enumerate(merged_global_topks): + owned = [ + pos for pos in range(max_seq_len) if (pos // interleave) % world == rank + ] + local_topk = _global_to_local_indices( + global_topk.to(torch.int64), + rank, + world, + interleave, + ) + local_out, local_lse = _attention_from_indices( + q, k[owned], v[owned], local_topk + ) + local_outs.append(local_out) + local_lses.append(local_lse) + + dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses) + torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5) + + +@pytest.mark.skipif( + not current_platform.is_cuda() or not has_cutedsl(), + reason="This test requires CUDA and CuteDSL", +) +@pytest.mark.parametrize("use_row_starts", [False, True]) +def test_cutedsl_dcp_candidate_pack_and_select_matches_reference( + use_row_starts: bool, +): + from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import ( + pack_dcp_topk_candidates_cutedsl, + stable_topk_from_gathered_candidates_cutedsl, + ) + + torch.manual_seed(13) + device = torch.device("cuda") + rows = 4 + valid_width = 1024 + width = valid_width + (8 if use_row_starts else 0) + topk = 512 + world = 2 + row_starts = ( + torch.tensor([0, 2, 4, 1], device=device, dtype=torch.int32) + if use_row_starts + else None + ) + row_offsets = ( + row_starts + if row_starts is not None + else torch.zeros(rows, device=device, dtype=torch.int32) + ) + + packed_by_rank = [] + for rank in range(world): + logits = torch.randn((rows, width), device=device, dtype=torch.float32) + local_topks = [] + for row in range(rows): + start = int(row_offsets[row].item()) + local_topks.append( + logits[row, start : start + valid_width].topk(topk).indices + ) + topk_indices = torch.stack(local_topks).to(torch.int32) + + packed = torch.empty((rows, topk, 2), device=device, dtype=torch.float32) + pack_dcp_topk_candidates_cutedsl( + logits, + topk_indices, + packed, + rank, + world, + 1, + row_starts, + ) + + expected_scores = logits.gather( + 1, topk_indices.to(torch.long) + row_offsets.to(torch.long).view(-1, 1) + ) + expected_ids = (topk_indices * world + rank).to(torch.float32) + torch.testing.assert_close(packed[..., 0], expected_scores) + torch.testing.assert_close(packed[..., 1], expected_ids) + packed_by_rank.append(packed) + + gathered = torch.cat(packed_by_rank, dim=1).contiguous() + actual = torch.empty((rows, topk), device=device, dtype=torch.int32) + returned = stable_topk_from_gathered_candidates_cutedsl(gathered, topk, out=actual) + assert returned is actual + expected = _ref_stable_topk_from_candidates_fp64( + gathered[..., 0], + gathered[..., 1].to(torch.int32), + topk, + ) + + for row in range(rows): + assert set(actual[row].cpu().tolist()) == set(expected[row].cpu().tolist()) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +def test_sparse_prefill_dcp_metadata_localizes_causal_bounds(): + device = torch.device("cuda") + seq_len = 8 + + query_start_loc = torch.tensor([0, seq_len], dtype=torch.int32, device=device) + query_start_loc_cpu = torch.tensor([0, seq_len], dtype=torch.int32) + seq_lens = torch.tensor([seq_len], dtype=torch.int32, device=device) + seq_lens_cpu = torch.tensor([seq_len], dtype=torch.int32) + block_table = torch.zeros((1, 1), dtype=torch.int32, device=device) + + def build(dcp_world_size, dcp_rank, interleave=1): + chunk = build_prefill_chunk_metadata( + start_idx=0, + end_idx=1, + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc_cpu, + uncompressed_seq_lens=seq_lens, + compressed_seq_lens=seq_lens, + compressed_seq_lens_cpu=seq_lens_cpu, + block_table=block_table, + compress_ratio=1, + dcp_rank=dcp_rank, + dcp_world_size=dcp_world_size, + cp_kv_cache_interleave_size=interleave, + ) + assert chunk is not None + torch.accelerator.synchronize() + return chunk + + # Non-DCP: local_cu_seq_lens aliases the global cu_seq_lens, and + # cu_seqlen_ks/ke carry the global causal bounds. + chunk = build(dcp_world_size=1, dcp_rank=0) + assert chunk.local_cu_seq_lens is chunk.cu_seq_lens + torch.testing.assert_close( + chunk.cu_seqlen_ks.cpu(), + torch.zeros(seq_len, dtype=torch.int32), + ) + torch.testing.assert_close( + chunk.cu_seqlen_ke.cpu(), + torch.arange(1, seq_len + 1, dtype=torch.int32), + ) + + # DCP: cu_seqlen_ks/ke are localized in place to this rank's shard. + chunk = build(dcp_world_size=4, dcp_rank=0) + assert chunk.local_cu_seq_lens is not None + torch.testing.assert_close( + chunk.local_cu_seq_lens.cpu(), + torch.tensor([0, 2], dtype=torch.int32), + ) + torch.testing.assert_close( + chunk.cu_seqlen_ks.cpu(), + torch.zeros(seq_len, dtype=torch.int32), + ) + torch.testing.assert_close( + chunk.cu_seqlen_ke.cpu(), + torch.tensor([1, 1, 1, 1, 2, 2, 2, 2], dtype=torch.int32), + ) + + # DCP with interleave=2: per-token causal bounds localize differently from + # interleave=1 (groups of 2 consecutive tokens are owned together). For + # world=4, rank=0, K=2, per-token global len L=1..8 -> local len + # [1,2,2,2,2,2,2,2] (matches get_dcp_local_seq_lens). + chunk = build(dcp_world_size=4, dcp_rank=0, interleave=2) + assert chunk.local_cu_seq_lens is not None + torch.testing.assert_close( + chunk.cu_seqlen_ks.cpu(), + torch.zeros(seq_len, dtype=torch.int32), + ) + torch.testing.assert_close( + chunk.cu_seqlen_ke.cpu(), + torch.tensor([1, 2, 2, 2, 2, 2, 2, 2], dtype=torch.int32), + ) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +def test_dcp_filter_compacts_valid_slots_for_sparse_kernel(): + block_size = 4 + num_topk = 128 + dcp_size = 2 + req_id = torch.zeros(1, dtype=torch.int32, device="cuda") + token_indices = torch.full((1, num_topk), -1, dtype=torch.int32, device="cuda") + token_indices[0, :8] = torch.arange(8, dtype=torch.int32, device="cuda") + block_table = torch.tensor([[10]], dtype=torch.int32, device="cuda") + + out, valid_counts = triton_filter_and_convert_dcp_index( + req_id, + block_table, + token_indices, + dcp_size=dcp_size, + dcp_rank=0, + BLOCK_SIZE=block_size, + NUM_TOPK_TOKENS=num_topk, + return_valid_counts=True, + ) + + valid = int(valid_counts.item()) + assert valid == 4 + assert (out[0, :valid] >= 0).all() + assert (out[0, valid:] == -1).all() + # In-kernel compaction packs valid slots to the front; prefix order is + # unspecified, so compare as a set. + assert set(out[0, :valid].cpu().tolist()) == {40, 41, 42, 43} + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +@pytest.mark.parametrize("interleave", [1, 2]) +@pytest.mark.parametrize("dcp_rank", [0, 1]) +def test_dcp_filter_compaction_matches_reference(interleave: int, dcp_rank: int): + """In-kernel compaction (atomic slot allocator across multiple column tiles) + must, for every row, produce exactly the rank-owned physical slots packed + into [0, valid_count) with -1 in the tail -- the same SET a reference filter + + sort/gather produces. Uses wide rows (> BLOCK_N valid slots) so the + cross-tile atomic allocation is exercised, with interior -1 gaps.""" + device = torch.device("cuda") + torch.manual_seed(7) + dcp_size = 2 + block_size = 8 + num_topk = 1024 # > BLOCK_N(128) -> multiple tiles per row + num_rows = 5 + max_blocks = 64 + seq = max_blocks * block_size + + req_id = torch.randint(0, 3, (num_rows,), dtype=torch.int32, device=device) + block_table = torch.randint( + 0, 1000, (3, max_blocks), dtype=torch.int32, device=device + ) + # Each row: a dense valid prefix of distinct global token ids, then -1 pad. + token_indices = torch.full( + (num_rows, num_topk), -1, dtype=torch.int32, device=device + ) + for r in range(num_rows): + n_valid = int(torch.randint(200, 600, (1,)).item()) + perm = torch.randperm(seq, device=device)[:n_valid].to(torch.int32) + token_indices[r, :n_valid] = perm + + out, valid_counts = triton_filter_and_convert_dcp_index( + req_id, + block_table, + token_indices, + dcp_size=dcp_size, + dcp_rank=dcp_rank, + cp_kv_cache_interleave_size=interleave, + BLOCK_SIZE=block_size, + NUM_TOPK_TOKENS=num_topk, + return_valid_counts=True, + ) + + for r in range(num_rows): + toks = token_indices[r] + toks = toks[toks >= 0] + owner = (toks // interleave) % dcp_size + owned = toks[owner == dcp_rank] + local = (owned // (dcp_size * interleave)) * interleave + owned % interleave + blk = local // block_size + off = local % block_size + expected = ( + block_table[int(req_id[r].item()), blk].to(torch.int64) * block_size + off + ) + n = int(valid_counts[r].item()) + assert n == owned.numel() + assert (out[r, :n] >= 0).all() + assert (out[r, n:] == -1).all() + assert set(out[r, :n].cpu().tolist()) == set(expected.cpu().tolist()) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +@pytest.mark.parametrize("interleave", [1, 2, 4]) +def test_dcp_global_topk_physical_attention_matches_non_dcp(interleave: int): + torch.manual_seed(2) + device = torch.device("cuda") + dcp_size = 2 + block_size = 4 + num_topk = 128 + selected_k = 8 + seq_len = 16 + head_dim = 16 + num_queries = 3 + + q = torch.randn(num_queries, head_dim, device=device) + k_global = torch.randn(seq_len, head_dim, device=device) + v_global = torch.randn(seq_len, head_dim, device=device) + scores = q @ k_global.T + global_topk = torch.full( + (num_queries, num_topk), -1, dtype=torch.int32, device=device + ) + global_topk[:, :selected_k] = scores.topk(selected_k, dim=-1).indices.to( + torch.int32 + ) + + ref_out, ref_lse = _attention_from_indices( + q, k_global, v_global, global_topk[:, :selected_k].to(torch.int64) + ) + + local_outs = [] + local_lses = [] + for rank in range(dcp_size): + block_ids = torch.tensor( + [[rank * 10 + 1, rank * 10 + 2]], dtype=torch.int32, device=device + ) + num_slots = int((block_ids.max().item() + 1) * block_size) + k_cache = torch.zeros(num_slots, head_dim, device=device) + v_cache = torch.zeros(num_slots, head_dim, device=device) + for global_idx in range(seq_len): + if (global_idx // interleave) % dcp_size != rank: + continue + local_idx = ( + global_idx // (dcp_size * interleave) + ) * interleave + global_idx % interleave + block = local_idx // block_size + offset = local_idx % block_size + slot = int(block_ids[0, block].item()) * block_size + offset + k_cache[slot] = k_global[global_idx] + v_cache[slot] = v_global[global_idx] + + slots, valid_counts = triton_filter_and_convert_dcp_index( + torch.zeros(num_queries, dtype=torch.int32, device=device), + block_ids, + global_topk, + dcp_size=dcp_size, + dcp_rank=rank, + cp_kv_cache_interleave_size=interleave, + BLOCK_SIZE=block_size, + NUM_TOPK_TOKENS=num_topk, + return_valid_counts=True, + ) + row_ids = torch.arange(num_queries, device=device) + assert (slots[row_ids, valid_counts] == -1).all() + local_out, local_lse = _attention_from_indices( + q, k_cache, v_cache, slots.to(torch.int64) + ) + local_outs.append(local_out) + local_lses.append(local_lse) + + dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses) + torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +@pytest.mark.parametrize("is_lse_base_on_e", [True, False]) +def test_correct_attn_out_zeroes_empty_nan_partial(is_lse_base_on_e: bool): + out = torch.full((1, 1, 4), float("nan"), device="cuda") + lses = torch.tensor( + [[[0.0]], [[float("-inf")]]], + dtype=torch.float32, + device="cuda", + ) + + corrected, final_lse = correct_attn_out( + out, + lses, + cp_rank=1, + ctx=CPTritonContext(), + is_lse_base_on_e=is_lse_base_on_e, + ) + torch.accelerator.synchronize() + + torch.testing.assert_close(corrected, torch.zeros_like(corrected)) + torch.testing.assert_close(final_lse, torch.zeros_like(final_lse)) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +def test_decode_topk_pads_surplus_with_negative_one(): + """When a row's valid length < topk the top-k kernel must pad the surplus + slots with -1: the DCP merge (`topk_indices >= 0`) and + `triton_filter_and_convert_dcp_index` (`tok < 0`) both treat <0 as invalid, + so a non-(-1) pad would be silently attended. This is the common case under + DCP, where each rank's local seq_len = global / world is usually << topk. + Checked on the *set* of valid indices (the kernel may order them by score, + not position).""" + torch.manual_seed(0) + device = torch.device("cuda") + topk = 8 + next_n = 1 + seq_lens_list = [0, 3, 5, 8] + num_rows = len(seq_lens_list) + logits = torch.randn(num_rows, 16, device=device) + seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device=device).view( + num_rows, next_n + ) + idx = _run_decode_topk(logits, seq_lens, next_n, topk) + for r, sl in enumerate(seq_lens_list): + valid = idx[r][idx[r] >= 0] + # seq_len <= topk, so top-k selects exactly the whole valid range. + assert valid.numel() == min(sl, topk) + assert set(valid.tolist()) == set(range(sl)) + assert (idx[r] == -1).sum().item() == topk - min(sl, topk) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +def test_persistent_topk_pads_surplus_with_negative_one(): + """Same surplus=-1 invariant for the persistent_topk kernel (k>=512).""" + torch.manual_seed(0) + device = torch.device("cuda") + topk = 512 # persistent_topk requires k in {512, 1024, 2048} + seq_lens_list = [100, 300, 512] + num_rows = len(seq_lens_list) + max_seq_len = 600 + logits = torch.randn(num_rows, max_seq_len, device=device) + seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device=device).view( + num_rows, 1 + ) + idx = _run_persistent_topk(logits, seq_lens, topk, max_seq_len) + for r, sl in enumerate(seq_lens_list): + valid = idx[r][idx[r] >= 0] + assert valid.numel() == min(sl, topk) + assert set(valid.tolist()) == set(range(sl)) + assert (idx[r] == -1).sum().item() == topk - min(sl, topk) + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +def test_sparse_decode_dcp_short_context_matches_non_dcp(): + """End-to-end DCP decode where the global seq_len < topk (so every rank's + local top-k is surplus-padded). Exercises the kernel surplus -> merge mask + -> global top-k -> physical localize -> LSE merge chain for the common + short-context decode case, vs the non-DCP reference.""" + torch.manual_seed(4) + device = torch.device("cuda") + world = 2 + interleave = 1 + topk = 512 + num_rows = 2 + max_seq_len = 300 # < topk -> surplus everywhere + head_dim = 16 + + q = torch.randn(num_rows, head_dim, device=device) + k = torch.randn(max_seq_len, head_dim, device=device) + v = torch.randn(max_seq_len, head_dim, device=device) + logits = q @ k.T + seq_lens = torch.tensor([[250], [300]], dtype=torch.int32, device=device) + + non_dcp_topk = torch.empty((num_rows, topk), dtype=torch.int64, device=device) + for row, seq_len in enumerate(seq_lens.flatten().tolist()): + sel = logits[row, :seq_len].topk(min(topk, seq_len)).indices + non_dcp_topk[row, : sel.numel()] = sel + non_dcp_topk[row, sel.numel() :] = -1 + ref_out, ref_lse = _attention_from_indices(q, k, v, non_dcp_topk) + + local_logits = [] + local_topks = [] + for rank in range(world): + owned = [p for p in range(max_seq_len) if (p // interleave) % world == rank] + rank_logits = logits[:, owned].contiguous() + rank_seq_lens = get_dcp_local_seq_lens(seq_lens, world, rank, interleave) + local_logits.append(rank_logits) + local_topks.append( + _run_persistent_topk( + rank_logits, + rank_seq_lens.contiguous(), + topk, + max_seq_len=rank_logits.shape[1], + ) + ) + + merged_global_topks = _merge_local_topks_global_with_fake_dcp( + local_logits, local_topks, topk, world, interleave + ) + # The radix top-K kernel selects a deterministic SET but writes it in + # nondeterministic (atomicAdd) order; the production path is permutation- + # invariant (compaction + softmax), so all ranks must agree on the set, not + # the array order. (The fp64 fallback happens to return sorted order.) + ref_topk = merged_global_topks[0] + for rank_topk in merged_global_topks[1:]: + for row in range(rank_topk.shape[0]): + assert set(rank_topk[row].tolist()) == set(ref_topk[row].tolist()) + + local_outs = [] + local_lses = [] + for rank, global_topk in enumerate(merged_global_topks): + owned = [p for p in range(max_seq_len) if (p // interleave) % world == rank] + local_topk = _global_to_local_indices( + global_topk.to(torch.int64), rank, world, interleave + ) + local_out, local_lse = _attention_from_indices( + q, k[owned], v[owned], local_topk + ) + local_outs.append(local_out) + local_lses.append(local_lse) + + dcp_out, dcp_lse = _dcp_lse_merge(local_outs, local_lses) + torch.testing.assert_close(dcp_out, ref_out, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(dcp_lse, ref_lse, atol=1e-5, rtol=1e-5) diff --git a/vllm/model_executor/kernels/attention/__init__.py b/vllm/model_executor/kernels/attention/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/model_executor/kernels/attention/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/model_executor/kernels/attention/dsa/__init__.py b/vllm/model_executor/kernels/attention/dsa/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/model_executor/kernels/attention/dsa/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py b/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py new file mode 100644 index 0000000000..deb5a87bc0 --- /dev/null +++ b/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py @@ -0,0 +1,420 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from functools import cache + +import cutlass +import cutlass.cute as cute +import torch +from cuda.bindings.driver import CUstream +from cutlass import Float32, Int32, Uint32, Uint64 +from quack.compile_utils import make_fake_tensor + +from vllm.cute_utils import recast_val +from vllm.triton_utils import tl, triton + + +def stable_topk_from_gathered_candidates_cutedsl( + gathered: torch.Tensor, + topk: int, + out: torch.Tensor | None = None, +) -> torch.Tensor: + if out is None: + out = torch.empty( + (gathered.shape[0], topk), + dtype=torch.int32, + device=gathered.device, + ) + StableTopKFromGatheredCandidatesKernel.compile(topk, gathered.shape[1])( + gathered, out + ) + return out + + +def pack_dcp_topk_candidates_cutedsl( + logits: torch.Tensor, + topk_indices: torch.Tensor, + packed: torch.Tensor, + dcp_rank: int, + dcp_world_size: int, + cp_interleave: int, + row_starts: torch.Tensor | None, +) -> None: + topk = topk_indices.shape[1] + grid = (topk_indices.shape[0], triton.cdiv(topk, 512)) + row_starts_arg = row_starts if row_starts is not None else topk_indices + _pack_dcp_topk_candidates_triton_kernel[grid]( + logits, + topk_indices, + packed, + row_starts_arg, + logits.stride(0), + logits.stride(1), + topk_indices.stride(0), + topk_indices.stride(1), + packed.stride(0), + packed.stride(1), + packed.stride(2), + logits.shape[1], + DCP_RANK=dcp_rank, + DCP_WORLD_SIZE=dcp_world_size, + CP_INTERLEAVE=cp_interleave, + HAS_ROW_STARTS=row_starts is not None, + TOPK=topk, + BLOCK_SIZE=512, + num_warps=8, + ) + + +@triton.jit +def _pack_dcp_topk_candidates_triton_kernel( + logits, + topk_indices, + packed, + row_starts, + logits_stride0: tl.constexpr, + logits_stride1: tl.constexpr, + topk_stride0: tl.constexpr, + topk_stride1: tl.constexpr, + packed_stride0: tl.constexpr, + packed_stride1: tl.constexpr, + packed_stride2: tl.constexpr, + num_cols, + DCP_RANK: tl.constexpr, + DCP_WORLD_SIZE: tl.constexpr, + CP_INTERLEAVE: tl.constexpr, + HAS_ROW_STARTS: tl.constexpr, + TOPK: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + row = tl.program_id(0) + tile = tl.program_id(1) + cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = cols < TOPK + + local_idx = tl.load( + topk_indices + row * topk_stride0 + cols * topk_stride1, + mask=mask, + other=-1, + ) + valid = local_idx >= 0 + safe_local_idx = tl.maximum(local_idx, 0) + + row_start = 0 + if HAS_ROW_STARTS: + row_start = tl.load(row_starts + row) + + score_col = safe_local_idx + row_start + score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0)) + score = tl.load( + logits + row * logits_stride0 + score_col * logits_stride1, + mask=mask & valid, + other=-float("inf"), + ) + + global_id = ( + (safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE) + + DCP_RANK * CP_INTERLEAVE + + safe_local_idx % CP_INTERLEAVE + ) + global_id = tl.where(valid, global_id, -1) + + packed_base = packed + row * packed_stride0 + cols * packed_stride1 + tl.store(packed_base, score, mask=mask) + tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask) + + +@cute.jit +def _warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32: + for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1): + offset = 1 << i + partial = cute.arch.shuffle_sync_up(val, offset=offset, mask_and_clamp=0) + if lane >= offset: + val += partial + return val + + +@cute.jit +def _block_scan_inclusive_i32( + val: Int32, + lane: Int32, + warp_id: Int32, + warp_scratch: cute.Tensor, + warps_per_block: int, +) -> Int32: + prefix = _warp_scan_inclusive_i32(val, lane) + if lane == Int32(cute.arch.WARP_SIZE - 1): + warp_scratch[0, warp_id] = prefix + cute.arch.sync_threads() + + if warp_id == Int32(0): + warp_total = Int32(0) + if lane < Int32(warps_per_block): + warp_total = warp_scratch[0, lane] + warp_prefix = _warp_scan_inclusive_i32(warp_total, lane) + if lane < Int32(warps_per_block): + warp_scratch[0, lane] = warp_prefix - warp_total + cute.arch.sync_threads() + + return prefix + warp_scratch[0, warp_id] + + +class StableTopKFromGatheredCandidatesKernel: + tb_size = 512 + hist_bins = 2048 + radix_bits = (hist_bins - 1).bit_length() + assert hist_bins == 1 << radix_bits + key_bits = Uint64.width + radix_passes = (key_bits + radix_bits - 1) // radix_bits + final_radix_bits = key_bits - radix_bits * (radix_passes - 1) + hist_chunks = (hist_bins + tb_size - 1) // tb_size + warps_per_block = tb_size // cute.arch.WARP_SIZE + + def __init__(self, topk: int, num_candidates: int): + assert num_candidates % self.tb_size == 0, ( + "StableTopKFromGatheredCandidatesKernel requires candidate count " + f"to be a multiple of {self.tb_size}, got {num_candidates}" + ) + self.topk = topk + self.keys_per_thread = num_candidates // self.tb_size + + @cute.struct + class SharedStorage: + hist: cute.struct.MemRange[Int32, self.hist_bins] + committed_count: cute.struct.MemRange[Int32, 1] + running_count: cute.struct.MemRange[Int32, 1] + threshold_bin: cute.struct.MemRange[Int32, 1] + threshold_found: cute.struct.MemRange[Int32, 1] + include_threshold_bin: cute.struct.MemRange[Int32, 1] + prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8] + warp_totals: cute.struct.MemRange[Int32, self.warps_per_block] + + self.shared_storage = SharedStorage + + @cute.jit + def __call__( + self, + gathered: cute.Tensor, + out: cute.Tensor, + stream: CUstream, + ): + grid = (gathered.shape[0], 1, 1) + self.kernel(gathered, out).launch( + grid=grid, + block=(self.tb_size, 1, 1), + stream=stream, + ) + + @cute.jit + def _stable_key(self, score: Float32, token_id: Int32) -> Uint64: + bits = recast_val(score, Uint32) + mask = Uint32(0x80000000) + if (bits & Uint32(0x80000000)) != Uint32(0): + mask = Uint32(0xFFFFFFFF) + score_key = Uint64(bits ^ mask) << Uint64(32) + id_key = Uint64(~Uint32(token_id)) + key = score_key | id_key + if token_id < Int32(0): + key = Uint64(0) + return key + + @cute.jit + def _prefix_matches( + self, + key: Uint64, + prefix: Uint64, + prefix_bits: Int32, + ): + matches = prefix_bits == Int32(0) + if prefix_bits != Int32(0): + shift = Int32(self.key_bits) - prefix_bits + matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift)) + return matches + + @cute.jit + def _radix_pass( + self, + keys: cute.Tensor, + output: cute.Tensor, + storage, + tid: Int32, + step: Int32, + bits: int, + is_final_pass: bool, + ): + hist_smem = storage.hist.get_tensor(cute.make_layout((self.hist_bins,))) + committed_count_smem = storage.committed_count.data_ptr() + running_count_smem = storage.running_count.data_ptr() + threshold_bin_smem = storage.threshold_bin.data_ptr() + threshold_found_smem = storage.threshold_found.data_ptr() + include_threshold_bin_smem = storage.include_threshold_bin.data_ptr() + prefix_smem = storage.prefix_s.data_ptr() + warp_totals_smem = storage.warp_totals.get_tensor( + cute.make_layout((1, self.warps_per_block)) + ) + + prefix_bits = step * Int32(self.radix_bits) + num_bins = 1 << bits + block_scan_iterations = (num_bins + self.tb_size - 1) // self.tb_size + shift = Int32(self.key_bits) - prefix_bits - Int32(bits) + bin_mask = Uint64(num_bins - 1) + prefix = prefix_smem.load() + + for chunk in cutlass.range_constexpr(self.hist_chunks): + hist_smem[tid + Int32(chunk * self.tb_size)] = Int32(0) + if tid == Int32(0): + running_count_smem.store(committed_count_smem.load()) + include_threshold_bin_smem.store(Int32(0)) + threshold_found_smem.store(Int32(0)) + cute.arch.sync_threads() + + for key_idx in cutlass.range_constexpr(self.keys_per_thread): + key = keys[key_idx] + if self._prefix_matches(key, prefix, prefix_bits): + bin_idx = Int32((key >> Uint64(shift)) & bin_mask) + cute.arch.atomic_add( + hist_smem.iterator + bin_idx, + Int32(1), + sem="relaxed", + scope="cta", + ) + cute.arch.sync_threads() + + lane = cute.arch.lane_idx() + warp_id = cute.arch.warp_idx() + # Each iteration scans one tb_size-wide slice of bins, high to low. + iter = Int32(0) + threshold_found = threshold_found_smem.load() + while threshold_found == Int32(0) and iter < Int32(block_scan_iterations): + bin_idx = Int32(num_bins - 1) - (iter * Int32(self.tb_size) + tid) + count = hist_smem[bin_idx] + chunk_inclusive = _block_scan_inclusive_i32( + count, + lane, + warp_id, + warp_totals_smem, + self.warps_per_block, + ) + running_count = running_count_smem.load() + prior_in_scan_slice = chunk_inclusive - count + remaining = Int32(self.topk) - running_count - prior_in_scan_slice + if count > Int32(0) and remaining > Int32(0) and remaining <= count: + threshold_bin_smem.store(bin_idx) + if count <= remaining or cutlass.const_expr(is_final_pass): + include_threshold_bin_smem.store(Int32(1)) + threshold_found_smem.store(Int32(1)) + # Barrier: every thread must finish reading running_count for this + # slice before tb_size-1 advances it, else a warp racing ahead to + # the store makes a lagging thread double-count the slice total + # (-> remaining too small -> threshold too high -> under-fill). + cute.arch.sync_threads() + if tid == Int32(self.tb_size - 1): + running_count_smem.store(running_count + chunk_inclusive) + cute.arch.sync_threads() + + threshold_found = threshold_found_smem.load() + iter += Int32(1) + + threshold = threshold_bin_smem.load() + should_include_threshold = include_threshold_bin_smem.load() != Int32(0) + for key_idx in cutlass.range_constexpr(self.keys_per_thread): + key = keys[key_idx] + if self._prefix_matches(key, prefix, prefix_bits): + bin_idx = Int32((key >> Uint64(shift)) & bin_mask) + selected = bin_idx > threshold + if should_include_threshold: + selected = selected or bin_idx == threshold + if selected: + dst = cute.arch.atomic_add( + committed_count_smem, + Int32(1), + sem="relaxed", + scope="cta", + ) + if dst < Int32(self.topk): + output[dst] = recast_val(~Uint32(key), Int32) + cute.arch.sync_threads() + + pass_finished = include_threshold_bin_smem.load() + if tid == Int32(0) and pass_finished == Int32(0): + prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift))) + cute.arch.sync_threads() + return pass_finished + + @cute.kernel + def kernel( + self, + input: cute.Tensor, + out: cute.Tensor, + ): + row, _, _ = cute.arch.block_idx() + tid, _, _ = cute.arch.thread_idx() + input_row = input[row, None, None] + output_row = out[row, None] + keys = cute.make_rmem_tensor((self.keys_per_thread,), Uint64) + + smem = cutlass.utils.SmemAllocator() + storage = smem.allocate(self.shared_storage, 8) + committed_count_smem = storage.committed_count.data_ptr() + prefix_smem = storage.prefix_s.data_ptr() + for i in range(tid, self.topk, self.tb_size): + output_row[i] = Int32(-1) + + for key_idx in cutlass.range_constexpr(self.keys_per_thread): + col = tid + Int32(key_idx * self.tb_size) + score = Float32(input_row[col, 0]) + token_id = Int32(input_row[col, 1]) + keys[key_idx] = self._stable_key(score, token_id) + + if tid == Int32(0): + committed_count_smem.store(Int32(0)) + prefix_smem.store(Uint64(0)) + cute.arch.sync_threads() + + step = Int32(0) + finished = Int32(0) + while finished == Int32(0) and step < Int32(self.radix_passes - 1): + finished = self._radix_pass( + keys, + output_row, + storage, + tid, + step, + self.radix_bits, + False, + ) + step += Int32(1) + + if finished == Int32(0): + self._radix_pass( + keys, + output_row, + storage, + tid, + Int32(self.radix_passes - 1), + self.final_radix_bits, + True, + ) + + @cache + @staticmethod + def compile(topk: int, num_candidates: int): + num_rows = cute.sym_int() + + gathered = cute.runtime.make_fake_tensor( + Float32, + (num_rows, num_candidates, 2), + stride=(cute.sym_int64(divisibility=2), 2, 1), + assumed_align=8, + ) + out = make_fake_tensor(Int32, (num_rows, topk), divisibility=1) + + kernel = StableTopKFromGatheredCandidatesKernel(topk, num_candidates) + stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) + return cute.compile( + kernel, + gathered, + out, + stream, + options="--enable-tvm-ffi", + ) diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index 80c0c4ec36..678b66eab8 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -8,6 +8,8 @@ import vllm.envs as envs from vllm import _custom_ops as ops from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.breakable_cudagraph import eager_break_during_capture +from vllm.config import get_current_vllm_config +from vllm.distributed import get_dcp_group from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.custom_op import CustomOp @@ -21,6 +23,7 @@ from vllm.utils.deep_gemm import ( fp8_fp4_paged_mqa_logits, has_deep_gemm, ) +from vllm.utils.import_utils import has_cutedsl from vllm.utils.torch_utils import ( LayerNameType, _encode_layer_name, @@ -41,6 +44,85 @@ RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024 MXFP4_BLOCK_SIZE = 32 +def _assert_cutedsl_dcp_merge_supported( + logits: torch.Tensor, + topk_indices: torch.Tensor, + k: int, +) -> None: + # The DCP merge only supports the CuteDSL path (Triton pack kernel + CuteDSL + # stable-topk selector); there is no PyTorch fallback. The first cut targets + # Blackwell/Hopper with index_topk in (512, 1024, 2048) (the selector's radix + # sizing); the Triton pack itself has no shape/topk constraints. + if not has_cutedsl(): + raise RuntimeError( + "DCP sparse-indexer merge requires CuteDSL; install it or disable DCP." + ) + if logits.device.type != "cuda": + raise RuntimeError("DCP sparse-indexer merge requires CUDA tensors.") + if logits.dtype != torch.float32 or topk_indices.dtype != torch.int32: + raise RuntimeError( + "DCP sparse-indexer merge requires fp32 logits and int32 indices." + ) + if k not in (512, 1024, 2048): + raise RuntimeError( + f"DCP sparse-indexer merge requires index_topk in (512, 1024, 2048); " + f"got {k}." + ) + + +def _merge_dcp_topk_global( + logits: torch.Tensor, + topk_indices: torch.Tensor, + topk_tokens: int, + dcp_rank: int, + dcp_world_size: int, + cp_interleave: int, + row_starts: torch.Tensor | None = None, +) -> None: + """Merge each DCP rank's local top-K into the global top-K. + + ``topk_indices`` are this rank's local top-K positions into its 1/N KV + shard. A token in the global top-K must also be in its owning rank's local + top-K (at most ``topk_tokens - 1`` tokens rank globally above it, hence at + most that many on its own rank), so exchanging only the per-rank local + candidates is exact -- equivalent to all-gathering the full logit matrix, + but it ships ``dcp_world_size * topk_tokens`` candidates instead of the whole + score row. Overwrites ``topk_indices`` with global token ids (``-1`` for + padding); the attention backend localizes them back to physical slots per + rank. + """ + if dcp_world_size <= 1: + return + + # CuteDSL-only path (no PyTorch fallback): Triton-pack each rank's + # (score, global_id) candidates on-device, all-gather, then the CuteDSL + # stable-topk selector. + _assert_cutedsl_dcp_merge_supported(logits, topk_indices, topk_tokens) + from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import ( + pack_dcp_topk_candidates_cutedsl, + stable_topk_from_gathered_candidates_cutedsl, + ) + + packed = torch.empty( + (*topk_indices.shape, 2), + dtype=torch.float32, + device=topk_indices.device, + ) + pack_dcp_topk_candidates_cutedsl( + logits, + topk_indices, + packed, + dcp_rank, + dcp_world_size, + cp_interleave, + row_starts, + ) + gathered = get_dcp_group().all_gather(packed, dim=1) + stable_topk_from_gathered_candidates_cutedsl( + gathered, topk_tokens, out=topk_indices + ) + + @triton.jit def _fused_indexer_q_rope_quant_kernel( positions, @@ -227,6 +309,9 @@ def sparse_attn_indexer( topk_indices_buffer: torch.Tensor, skip_k_cache_insert: bool, use_fp4_cache: bool = False, + dcp_rank: int = 0, + dcp_world_size: int = 1, + cp_kv_cache_interleave_size: int = 1, skip_topk_buffer_clear: bool = False, ) -> torch.Tensor: # careful! this will be None in dummy run @@ -328,16 +413,18 @@ def sparse_attn_indexer( scales_spec, ) for chunk in prefill_metadata.chunks: - k_quant = k_quant_full[: chunk.total_seq_lens] - k_scale = k_scale_full[: chunk.total_seq_lens] - - if not chunk.skip_kv_gather: + cu_seqlen_ks = chunk.cu_seqlen_ks + cu_seqlen_ke = chunk.cu_seqlen_ke + assert chunk.local_cu_seq_lens is not None + k_quant = k_quant_full[: chunk.max_local_total_seq_lens] + k_scale = k_scale_full[: chunk.max_local_total_seq_lens] + if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0: ops.cp_gather_indexer_k_quant_cache( kv_cache, k_quant, k_scale, chunk.block_table, - chunk.cu_seq_lens, + chunk.local_cu_seq_lens, ) q_slice = q_quant[chunk.token_start : chunk.token_end] @@ -346,51 +433,64 @@ def sparse_attn_indexer( if q_scale is not None else None ) - # DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8 - # (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf. - if use_fp4_cache: - q_slice_cast = q_slice.view(torch.int8) - k_quant_cast = k_quant.view(torch.int8) - k_scale_cast = k_scale.view(torch.int32).squeeze(-1) - else: - q_slice_cast = q_slice - k_quant_cast = k_quant - k_scale_cast = k_scale.view(torch.float32).squeeze(-1) - if current_platform.is_xpu(): - if q_scale_slice is not None: - raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q") - logits = torch.ops.vllm.xpu_fp8_mqa_logits( - q_slice_cast, - k_quant_cast, - k_scale_cast, - weights[chunk.token_start : chunk.token_end], - chunk.cu_seqlen_ks, - chunk.cu_seqlen_ke, - ) - else: - logits = fp8_fp4_mqa_logits( - (q_slice_cast, q_scale_slice), - (k_quant_cast, k_scale_cast), - weights[chunk.token_start : chunk.token_end], - chunk.cu_seqlen_ks, - chunk.cu_seqlen_ke, - clean_logits=False, - ) - num_rows = logits.shape[0] - topk_indices = topk_indices_buffer[ chunk.token_start : chunk.token_end, :topk_tokens ] - ops.top_k_per_row_prefill( + if chunk.local_total_seq_lens == 0: + logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32) + topk_indices.fill_(-1) + else: + # DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8 + # (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf. + if use_fp4_cache: + q_slice_cast = q_slice.view(torch.int8) + k_quant_cast = k_quant.view(torch.int8) + k_scale_cast = k_scale.view(torch.int32).squeeze(-1) + else: + q_slice_cast = q_slice + k_quant_cast = k_quant + k_scale_cast = k_scale.view(torch.float32).squeeze(-1) + if current_platform.is_xpu(): + if q_scale_slice is not None: + raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q") + logits = torch.ops.vllm.xpu_fp8_mqa_logits( + q_slice_cast, + k_quant_cast, + k_scale_cast, + weights[chunk.token_start : chunk.token_end], + cu_seqlen_ks, + cu_seqlen_ke, + ) + else: + logits = fp8_fp4_mqa_logits( + (q_slice_cast, q_scale_slice), + (k_quant_cast, k_scale_cast), + weights[chunk.token_start : chunk.token_end], + cu_seqlen_ks, + cu_seqlen_ke, + clean_logits=False, + ) + num_rows = logits.shape[0] + ops.top_k_per_row_prefill( + logits, + cu_seqlen_ks, + cu_seqlen_ke, + topk_indices, + num_rows, + logits.stride(0), + logits.stride(1), + topk_tokens, + ) + + _merge_dcp_topk_global( logits, - chunk.cu_seqlen_ks, - chunk.cu_seqlen_ke, topk_indices, - num_rows, - logits.stride(0), - logits.stride(1), topk_tokens, + dcp_rank, + dcp_world_size, + cp_kv_cache_interleave_size, + row_starts=chunk.cu_seqlen_ks, ) if has_decode: @@ -508,7 +608,7 @@ def sparse_attn_indexer( topk_indices, topk_workspace, topk_tokens, - attn_metadata_narrowed.max_seq_len, + logits.shape[1], ) else: ops.top_k_per_row_decode( @@ -522,6 +622,16 @@ def sparse_attn_indexer( topk_tokens, ) + if decode_metadata.global_seq_lens is not None: + _merge_dcp_topk_global( + logits, + topk_indices, + topk_tokens, + dcp_rank, + dcp_world_size, + cp_kv_cache_interleave_size, + ) + if decode_metadata.requires_padding: # if padded, we need to unpack # the topk indices removing padded tokens @@ -553,6 +663,9 @@ def sparse_attn_indexer_fake( topk_indices_buffer: torch.Tensor | None, skip_k_cache_insert: bool, use_fp4_cache: bool = False, + dcp_rank: int = 0, + dcp_world_size: int = 1, + cp_kv_cache_interleave_size: int = 1, skip_topk_buffer_clear: bool = False, ) -> torch.Tensor: return topk_indices_buffer @@ -604,6 +717,13 @@ class SparseAttnIndexer(CustomOp): self.topk_indices_buffer = topk_indices_buffer self.skip_k_cache_insert = skip_k_cache_insert self.use_fp4_cache = use_fp4_cache + # DCP scalars are constant for the run; resolve them here (config is set + # during model construction) and pass them into the custom op, rather + # than threading them through per-step metadata. + parallel_config = get_current_vllm_config().parallel_config + self.dcp_world_size = parallel_config.decode_context_parallel_size + self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0 + self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size if current_platform.is_cuda() and not has_deep_gemm(): raise RuntimeError( "Sparse Attention Indexer CUDA op requires DeepGEMM support in " @@ -657,6 +777,9 @@ class SparseAttnIndexer(CustomOp): self.topk_indices_buffer, self.skip_k_cache_insert, self.use_fp4_cache, + self.dcp_rank, + self.dcp_world_size, + self.cp_kv_cache_interleave_size, ) def forward_xpu( diff --git a/vllm/models/deepseek_v32/nvidia/attention.py b/vllm/models/deepseek_v32/nvidia/attention.py index 953371060e..dcf955ad59 100644 --- a/vllm/models/deepseek_v32/nvidia/attention.py +++ b/vllm/models/deepseek_v32/nvidia/attention.py @@ -496,7 +496,8 @@ class DeepseekV32Attention(MLAAttention): self.topk_indices_buffer, True, # skip_k_cache_insert False, # use_fp4_cache - True, # skip_topk_buffer_clear (fused_norm_rope already did it) + # fused_norm_rope already cleared the topk buffer this forward. + skip_topk_buffer_clear=True, ) if attn_metadata is None: diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index 2a944d0618..97778dfea1 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, ClassVar import numpy as np import torch +from vllm import envs from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.logger import init_logger @@ -29,8 +30,12 @@ from vllm.v1.attention.backend import ( ) from vllm.v1.attention.backends.mla.sparse_utils import ( triton_convert_req_index_to_global_index, + triton_filter_and_convert_dcp_index, +) +from vllm.v1.attention.backends.utils import ( + KVCacheLayoutType, + split_decodes_and_prefills, ) -from vllm.v1.attention.backends.utils import KVCacheLayoutType from vllm.v1.kv_cache_interface import AttentionSpec if TYPE_CHECKING: @@ -38,8 +43,6 @@ if TYPE_CHECKING: logger = init_logger(__name__) -FLASHINFER_MLA_SPARSE_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024 - class _FlashInferMLASparseBackendBase(AttentionBackend): """Common metadata for concrete FlashInfer sparse MLA backends.""" @@ -252,10 +255,13 @@ class FlashInferMLASparseMetadata(AttentionMetadata): # Sequence lengths for all requests (context + query) seq_lens: torch.Tensor + num_decodes: int + num_decode_tokens: int # Sparse-specific block_size: int = 64 topk_tokens: int = 2048 + cp_kv_cache_interleave_size: int = 1 class FlashInferMLASparseMetadataBuilder( @@ -281,6 +287,12 @@ class FlashInferMLASparseMetadataBuilder( self.mla_dims = get_mla_dims(self.model_config) self.topk_tokens = vllm_config.model_config.hf_config.index_topk + self._init_reorder_batch_threshold( + 1, + supports_spec_as_decode=True, + supports_dcp_with_varlen=True, + ) + self.req_id_per_token_buffer = torch.empty( (vllm_config.scheduler_config.max_num_batched_tokens,), dtype=torch.int32, @@ -295,6 +307,12 @@ class FlashInferMLASparseMetadataBuilder( ) -> FlashInferMLASparseMetadata: cm = common_attn_metadata num_tokens = cm.num_actual_tokens + assert self.reorder_batch_threshold is not None + num_decodes, _, num_decode_tokens, _ = split_decodes_and_prefills( + cm, + decode_threshold=self.reorder_batch_threshold, + treat_short_extends_as_decodes=True, + ) # Build req_id_per_token mapping starts = np.asarray(cm.query_start_loc_cpu, dtype=np.int32) @@ -320,8 +338,13 @@ class FlashInferMLASparseMetadataBuilder( block_table=cm.block_table_tensor, req_id_per_token=req_id_per_token_tensor, seq_lens=cm.seq_lens, + num_decodes=num_decodes, + num_decode_tokens=num_decode_tokens, block_size=self.kv_cache_spec.block_size, topk_tokens=self.topk_tokens, + cp_kv_cache_interleave_size=( + self.vllm_config.parallel_config.cp_kv_cache_interleave_size + ), ) @@ -333,7 +356,7 @@ def _get_workspace_buffer(device: torch.device) -> torch.Tensor: global _fi_sparse_workspace if _fi_sparse_workspace is None: _fi_sparse_workspace = torch.zeros( - FLASHINFER_MLA_SPARSE_WORKSPACE_BUFFER_SIZE, + envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE, dtype=torch.uint8, device=device, ) @@ -347,6 +370,9 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata sparse attention computation. """ + can_return_lse_for_decode: bool = True + lse_base_on_e: bool = False + def __init__( self, num_heads: int, @@ -421,14 +447,27 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata assert self.topk_indices_buffer is not None topk_indices = self.topk_indices_buffer[:num_actual_toks] - topk_indices_physical, seq_lens = triton_convert_req_index_to_global_index( - attn_metadata.req_id_per_token[:num_actual_toks], - attn_metadata.block_table, - topk_indices, - BLOCK_SIZE=attn_metadata.block_size, - NUM_TOPK_TOKENS=topk_indices.shape[1], - return_valid_counts=True, - ) + if self.dcp_world_size > 1: + topk_indices_physical, seq_lens = triton_filter_and_convert_dcp_index( + attn_metadata.req_id_per_token[:num_actual_toks], + attn_metadata.block_table, + topk_indices, + dcp_size=self.dcp_world_size, + dcp_rank=self.dcp_rank, + cp_kv_cache_interleave_size=(attn_metadata.cp_kv_cache_interleave_size), + BLOCK_SIZE=attn_metadata.block_size, + NUM_TOPK_TOKENS=topk_indices.shape[1], + return_valid_counts=True, + ) + else: + topk_indices_physical, seq_lens = triton_convert_req_index_to_global_index( + attn_metadata.req_id_per_token[:num_actual_toks], + attn_metadata.block_table, + topk_indices, + BLOCK_SIZE=attn_metadata.block_size, + NUM_TOPK_TOKENS=topk_indices.shape[1], + return_valid_counts=True, + ) if self._workspace_buffer is None: self._workspace_buffer = _get_workspace_buffer(q.device) @@ -444,18 +483,66 @@ class FlashInferMLASparseImpl(SparseMLAAttentionImpl[FlashInferMLASparseMetadata from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla - o = trtllm_batch_decode_with_kv_cache_mla( - query=q.unsqueeze(1), + # Single-token sparse decode. trtllm-gen requires the q_len_per_request + # dim, but the sparse attention mask is fully per-token (each query token + # carries its own top-k index row), so unsqueeze is sufficient and + # correct. The MTP/multi-token q_len grouping is a perf-only layout and is + # deferred until MTP is validated end-to-end for this backend. + query = q.unsqueeze(1) + block_tables = topk_indices_physical.unsqueeze(1) + seq_lens_arg = seq_lens + + kernel_out = trtllm_batch_decode_with_kv_cache_mla( + query=query, kv_cache=kv_c_and_k_pe_cache.unsqueeze(1), workspace_buffer=self._workspace_buffer, qk_nope_head_dim=self.qk_nope_head_dim, kv_lora_rank=self.kv_lora_rank, qk_rope_head_dim=self.qk_rope_head_dim, - block_tables=topk_indices_physical.unsqueeze(1), - seq_lens=seq_lens, + block_tables=block_tables, + seq_lens=seq_lens_arg, max_seq_len=attn_metadata.topk_tokens, bmm1_scale=self.bmm1_scale, bmm2_scale=self.bmm2_scale, sparse_mla_top_k=attn_metadata.topk_tokens, + return_lse=self.need_to_return_lse_for_decode, ) - return o.view(-1, o.shape[-2], o.shape[-1]), None + if self.need_to_return_lse_for_decode: + assert isinstance(kernel_out, tuple) + o, lse = kernel_out + else: + assert isinstance(kernel_out, torch.Tensor) + o = kernel_out + lse = None + + out = o.view(-1, o.shape[-2], o.shape[-1]) + if lse is not None: + lse = self._normalize_lse(lse, out.shape[0], out.shape[1]) + empty_rows = (topk_indices_physical == -1).all(dim=-1) + out.masked_fill_(empty_rows.view(-1, 1, 1), 0.0) + lse.masked_fill_(empty_rows.view(-1, 1), float("-inf")) + return out, lse + + @staticmethod + def _normalize_lse( + lse: torch.Tensor, + num_tokens: int, + num_heads: int, + ) -> torch.Tensor: + # FlashInfer returns the decode LSE either as 2D (num_tokens, num_heads) + # or 3D ((num_tokens, num_heads, 1) / (num_tokens, 1, num_heads)). + # Collapse all of these to the (num_tokens, num_heads) the shared DCP + # reducer expects. + if lse.dim() == 3: + if lse.shape[-1] == 1: + lse = lse.squeeze(-1) + elif lse.shape[1] == 1: + lse = lse.squeeze(1) + elif lse.shape[0] * lse.shape[1] == num_tokens: + lse = lse.reshape(num_tokens, lse.shape[-1]) + if lse.shape != (num_tokens, num_heads): + raise RuntimeError( + "Unexpected FlashInfer sparse MLA LSE shape: " + f"{tuple(lse.shape)}, expected ({num_tokens}, {num_heads})." + ) + return lse diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 0bc7ca7aa4..b38445f9ff 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -6,6 +6,7 @@ import torch import vllm.envs as envs from vllm.config import VllmConfig +from vllm.distributed import get_dcp_group from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.triton_utils import tl, triton @@ -24,6 +25,7 @@ from vllm.v1.attention.backend import ( ) from vllm.v1.attention.backends.mla.compressor_utils import get_compressed_slot_mapping from vllm.v1.attention.backends.utils import ( + get_dcp_local_seq_lens, split_decodes_and_prefills, ) from vllm.v1.kv_cache_interface import AttentionSpec, MLAAttentionSpec @@ -168,6 +170,8 @@ class DeepseekV4IndexerBackend(DeepseekV32IndexerBackend): @dataclass class DeepseekV32IndexerPrefillChunkMetadata: block_table: torch.Tensor + # Under DCP (dcp_world_size > 1) these hold this rank's local row bounds; + # otherwise they hold the global bounds. cu_seqlen_ks: torch.Tensor cu_seqlen_ke: torch.Tensor cu_seq_lens: torch.Tensor @@ -177,6 +181,9 @@ class DeepseekV32IndexerPrefillChunkMetadata: token_end: int num_reqs: int skip_kv_gather: bool = False + local_cu_seq_lens: torch.Tensor | None = None + local_total_seq_lens: int = 0 + max_local_total_seq_lens: int = 0 @dataclass @@ -195,6 +202,7 @@ class DeepSeekV32IndexerDecodeMetadata: decode_lens: torch.Tensor requires_padding: bool schedule_metadata: torch.Tensor + global_seq_lens: torch.Tensor | None = None @dataclass @@ -243,6 +251,19 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) scheduler_config = self.vllm_config.scheduler_config + parallel_config = self.vllm_config.parallel_config + self.dcp_world_size = parallel_config.decode_context_parallel_size + self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0 + self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size + # The DCP sparse-indexer code is parameterized by interleave size, but + # interleave > 1 is not yet validated end-to-end (gsm8k parity fails), + # so fail closed here rather than silently produce wrong output. + if self.dcp_world_size > 1 and self.cp_kv_cache_interleave_size > 1: + raise NotImplementedError( + "DCP sparse indexer currently supports only " + f"cp_kv_cache_interleave_size=1 (got " + f"{self.cp_kv_cache_interleave_size})." + ) # NOTE(Chen):an estimated max size of flattened_kv. Need to double check. self.max_prefill_buffer_size = get_max_prefill_buffer_size(self.vllm_config) self.num_speculative_tokens = ( @@ -300,6 +321,11 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): dtype=torch.int32, device=self.device, ) + self.global_decode_seq_lens_buffer = torch.zeros( + (scheduler_config.max_num_batched_tokens,), + dtype=torch.int32, + device=self.device, + ) self.arange_buffer = torch.arange( max( scheduler_config.max_num_seqs * next_n, @@ -331,6 +357,11 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): # Get compress_ratio for DeepseekV4 support if isinstance(self.kv_cache_spec, MLAAttentionSpec): self.compress_ratio = self.kv_cache_spec.compress_ratio + if self.dcp_world_size > 1 and self.compress_ratio > 1: + raise NotImplementedError( + "DCP is not supported with sparse indexer KV compression " + f"(compress_ratio={self.compress_ratio})." + ) # Pre-allocate buffers for CUDA graph compatibility when if self.compress_ratio > 1: @@ -348,6 +379,26 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): device=self.device, ) + def _dcp_localize_decode_seq_lens( + self, + seq_lens: torch.Tensor, + num_decodes: int, + seq_lens_is_buffer_view: bool, + ) -> torch.Tensor: + local_seq_lens = get_dcp_local_seq_lens( + seq_lens, + self.dcp_world_size, + self.dcp_rank, + self.cp_kv_cache_interleave_size, + ) + if seq_lens_is_buffer_view: + seq_lens.copy_(local_seq_lens) + return seq_lens + + out = self.decode_seq_lens_buffer[:num_decodes] + out.copy_(local_seq_lens) + return out + def _prepare_decode_tensors( self, seq_lens: torch.Tensor, @@ -468,6 +519,34 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): seq_lens = seq_lens_buffer return seq_lens, block_table, decode_lens, num_decodes, requires_padding + def _prepare_global_decode_seq_lens( + self, + global_seq_lens: torch.Tensor | None, + decode_lens: torch.Tensor, + decode_lens_cpu: torch.Tensor, + query_start_loc: torch.Tensor, + num_decode_tokens: int, + use_native: bool, + max_decode_len: int, + ) -> torch.Tensor | None: + if global_seq_lens is None: + return None + if use_native or max_decode_len <= 1: + return global_seq_lens + + actual_expanded = int(decode_lens_cpu.sum().item()) + if actual_expanded > 0: + expanded_offsets = torch.repeat_interleave( + global_seq_lens - decode_lens - query_start_loc, + decode_lens, + output_size=actual_expanded, + ) + self.global_decode_seq_lens_buffer[:actual_expanded] = ( + expanded_offsets + self.arange_buffer[:actual_expanded] + 1 + ) + self.global_decode_seq_lens_buffer[actual_expanded:num_decode_tokens] = 0 + return self.global_decode_seq_lens_buffer[:num_decode_tokens] + def build( self, common_prefix_len: int, @@ -481,6 +560,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): seq_lens = common_attn_metadata.seq_lens slot_mapping = common_attn_metadata.slot_mapping block_table = common_attn_metadata.block_table_tensor + dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( split_decodes_and_prefills( @@ -549,6 +629,9 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): self.compress_ratio, query_slice=query_slice, skip_kv_gather=query_slice.start > 0, + dcp_rank=self.dcp_rank, + dcp_world_size=self.dcp_world_size, + cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size, ) # Skip when total_seq_lens is 0 (i.e., no compressed token). if metadata is not None: @@ -566,6 +649,17 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): common_attn_metadata.query_start_loc_cpu[: num_decodes + 1] ) + # Under DCP the per-token decode bounds must be localized AFTER the + # per-token expansion below, not before. Expanding from a + # request-level localized length subtracts decode offsets in local + # space and yields too-short bounds (e.g. world=2, rank=1, global + # per-token bounds [8, 9, 10] -> [3, 4, 5] instead of [4, 4, 5]), so + # the first decode token would run top-k against too short a local KV + # range and miss valid tokens. Keep the global seq_lens here and + # localize the expanded bounds further down. + global_seq_lens_for_decode: torch.Tensor | None = None + if dcp_local_seq_lens is not None: + global_seq_lens_for_decode = common_attn_metadata.seq_lens[:num_decodes] seq_lens = common_attn_metadata.seq_lens[:num_decodes] block_table = common_attn_metadata.block_table_tensor[:num_decodes, ...] @@ -573,6 +667,16 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): next_n = 1 + self.num_speculative_tokens use_native = not self.use_flattening and max_decode_len <= next_n + global_seq_lens_for_decode = self._prepare_global_decode_seq_lens( + global_seq_lens=global_seq_lens_for_decode, + decode_lens=decode_lens, + decode_lens_cpu=decode_lens_cpu, + query_start_loc=common_attn_metadata.query_start_loc[:num_decodes], + num_decode_tokens=num_decode_tokens, + use_native=use_native, + max_decode_len=max_decode_len, + ) + seq_lens, block_table, decode_lens, batch_size, requires_padding = ( self._prepare_decode_tensors( seq_lens=seq_lens, @@ -588,15 +692,22 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): ) ) + seq_lens_is_buffer_view = (use_native and next_n > 1) or ( + not use_native and max_decode_len > 1 + ) + + # DCP: localize the now-expanded per-token global bounds to this + # rank's owned KV. Done here (after expansion) so each token's global + # causal length is localized individually; see the comment above. + if dcp_local_seq_lens is not None: + seq_lens = self._dcp_localize_decode_seq_lens( + seq_lens, num_decodes, seq_lens_is_buffer_view + ) + # For DeepseekV4 (compress_ratio > 1), the indexer KV cache stores # compressed tokens. Convert uncompressed seq_lens to compressed. if self.compress_ratio > 1: - # True iff seq_lens aliases decode_seq_lens_buffer (flatten or - # native wrote it); False iff it aliases common_attn_metadata. - seq_lens_is_local_view = (use_native and next_n > 1) or ( - not use_native and max_decode_len > 1 - ) - if seq_lens_is_local_view: + if seq_lens_is_buffer_view: seq_lens //= self.compress_ratio else: # Copy to avoid mutating shared state; keeps CG address stable. @@ -626,6 +737,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): decode_lens=decode_lens, requires_padding=requires_padding, schedule_metadata=self.scheduler_metadata_buffer, + global_seq_lens=global_seq_lens_for_decode, ) attn_metadata = DeepseekV32IndexerMetadata( @@ -655,6 +767,9 @@ def build_prefill_chunk_metadata( compress_ratio: int, query_slice: slice | None = None, skip_kv_gather: bool = False, + dcp_rank: int = 0, + dcp_world_size: int = 1, + cp_kv_cache_interleave_size: int = 1, ) -> DeepseekV32IndexerPrefillChunkMetadata | None: total_seq_lens = compressed_seq_lens_cpu[start_idx:end_idx].sum().item() if total_seq_lens == 0: @@ -669,6 +784,25 @@ def build_prefill_chunk_metadata( cu_seq_lens[:1] = 0 torch.cumsum(compressed_seq_lens[start_idx:end_idx], dim=0, out=cu_seq_lens[1:]) + local_cu_seq_lens = cu_seq_lens + local_total_seq_lens = total_seq_lens + max_local_total_seq_lens = total_seq_lens + if dcp_world_size > 1: + # Per-rank local KV length under interleave-aware DCP sharding, shape + # [num_reqs, dcp_world_size]. Reuse the canonical CP helper so the + # sharding matches the rest of the DCP pipeline (decode/prefill). + local_seq_lens = get_dcp_local_seq_lens( + compressed_seq_lens[start_idx:end_idx], + dcp_world_size, + None, + cp_kv_cache_interleave_size, + ) + this_rank_counts = local_seq_lens[:, dcp_rank].to(torch.int32) + local_cu_seq_lens = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device) + torch.cumsum(this_rank_counts, dim=0, out=local_cu_seq_lens[1:]) + local_total_seq_lens = int(local_cu_seq_lens[-1].item()) + max_local_total_seq_lens = int(local_seq_lens.sum(dim=0).max().item()) + query_start_loc = ( query_start_loc[start_idx : end_idx + 1] - query_start_loc[start_idx] ) @@ -687,15 +821,21 @@ def build_prefill_chunk_metadata( cu_seq_len_ks = torch.empty(output_query_len, dtype=torch.int32, device=device) cu_seq_len_ke = torch.empty(output_query_len, dtype=torch.int32, device=device) + # Under DCP the kernel writes this rank's local row bounds into + # cu_seq_len_ks/ke; otherwise local_cu_seq_lens aliases cu_seq_lens. _build_prefill_chunk_metadata_kernel[(num_reqs,)]( query_start_loc, uncompressed_seq_lens[start_idx:end_idx], cu_seq_lens, + local_cu_seq_lens, token_to_seq, cu_seq_len_ks, cu_seq_len_ke, qs_start, qs_stop, + dcp_rank, + dcp_world_size, + cp_kv_cache_interleave_size, BLOCK_SIZE=1024, COMPRESS_RATIO=compress_ratio, ) @@ -719,6 +859,9 @@ def build_prefill_chunk_metadata( token_end=token_end, num_reqs=num_reqs, skip_kv_gather=skip_kv_gather, + local_cu_seq_lens=local_cu_seq_lens, + local_total_seq_lens=local_total_seq_lens, + max_local_total_seq_lens=max_local_total_seq_lens, ) @@ -728,12 +871,18 @@ def _build_prefill_chunk_metadata_kernel( query_start_loc_ptr, uncompressed_seq_lens_ptr, cu_compressed_seq_lens_ptr, + # Row-start base for cu_seq_len_ks/ke: local cumulative lens under DCP, + # aliases cu_compressed_seq_lens_ptr otherwise. + row_start_cu_compressed_seq_lens_ptr, # Outputs token_to_seq_ptr, cu_compressed_seq_len_ks_ptr, cu_compressed_seq_len_ke_ptr, query_slice_start, query_slice_stop, + DCP_RANK, + DCP_WORLD, + DCP_INTERLEAVE, BLOCK_SIZE: tl.constexpr, COMPRESS_RATIO: tl.constexpr, ): @@ -747,6 +896,10 @@ def _build_prefill_chunk_metadata_kernel( seq_end = tl.load(cu_compressed_seq_lens_ptr + batch_idx + 1) compressed_seq_len = seq_end - seq_start + # Row start for the (possibly localized) cu_seq_len_ks/ke. Equals seq_start + # when DCP is disabled (the pointer aliases cu_compressed_seq_lens_ptr). + row_start = tl.load(row_start_cu_compressed_seq_lens_ptr + batch_idx) + uncompressed_seq_len = tl.load(uncompressed_seq_lens_ptr + batch_idx) start_pos = uncompressed_seq_len - query_len @@ -760,14 +913,25 @@ def _build_prefill_chunk_metadata_kernel( ) out_pos = abs_pos - query_slice_start - # Compute cu_seq_len_ks - tl.store(cu_compressed_seq_len_ks_ptr + out_pos, seq_start, mask=mask) + # cu_seq_len_ks: row start in the gathered K buffer. + tl.store(cu_compressed_seq_len_ks_ptr + out_pos, row_start, mask=mask) - # Compute cu_seq_len_ke - seq_len_per_token = (start_pos + 1 + offset) // COMPRESS_RATIO + # cu_seq_len_ke: row start + per-token context length. Under DCP the + # global per-token length is sharded across ranks. + global_ctx = start_pos + 1 + offset + len_per_token = global_ctx // COMPRESS_RATIO + if DCP_WORLD > 1: + # Per-rank local context length under interleave-aware DCP, matching + # get_dcp_local_seq_lens. K == 1 reduces to (len + world-1-rank)//world. + base = (len_per_token // DCP_INTERLEAVE // DCP_WORLD) * DCP_INTERLEAVE + remainder = len_per_token - base * DCP_WORLD + remainder = tl.minimum( + tl.maximum(remainder - DCP_RANK * DCP_INTERLEAVE, 0), DCP_INTERLEAVE + ) + len_per_token = base + remainder tl.store( cu_compressed_seq_len_ke_ptr + out_pos, - seq_start + seq_len_per_token, + row_start + len_per_token, mask=mask, ) diff --git a/vllm/v1/attention/backends/mla/sparse_utils.py b/vllm/v1/attention/backends/mla/sparse_utils.py index e4bd0cf425..522b52b0dc 100644 --- a/vllm/v1/attention/backends/mla/sparse_utils.py +++ b/vllm/v1/attention/backends/mla/sparse_utils.py @@ -23,6 +23,16 @@ def _convert_req_index_to_global_index_kernel( BLOCK_N: tl.constexpr, # tile width along columns HAS_PREFILL: tl.constexpr, COUNT_VALID: tl.constexpr, # whether to count valid indices + # When set, scatter valid slots to a contiguous prefix [0, valid_count) using + # valid_count_ptr as an atomic slot allocator (DCP filtering leaves interior + # -1 gaps; the trtllm-gen sparse kernel reads the first valid_count entries). + # Requires COUNT_VALID and an out buffer pre-filled with -1. Order within the + # prefix is unspecified (only the selected set matters). + COMPACT_TO_FRONT: tl.constexpr, + # DCP de-interleave: with DCP_SIZE == 1 these are an exact no-op + DCP_SIZE: tl.constexpr, + DCP_RANK: tl.constexpr, + DCP_INTERLEAVE: tl.constexpr, # strides (in elements) bt_stride0, bt_stride1, @@ -52,15 +62,27 @@ def _convert_req_index_to_global_index_kernel( if HAS_PREFILL: prefill_req_id = tl.load(prefill_request_id_ptr + token_id) is_prefill = prefill_req_id >= 0 + + # DCP de-interleave the global token id into this rank's local slot. + # Tokens are interleaved in groups of DCP_INTERLEAVE across ranks. With + # DCP_SIZE == 1 (and any interleave) owning_rank == 0 == DCP_RANK (never + # remote) and local_idx == tok, so this reduces to the non-DCP path; with + # DCP_INTERLEAVE == 1 it reduces to plain round-robin (tok % / // DCP_SIZE). + owning_rank = (tok // DCP_INTERLEAVE) % DCP_SIZE + is_remote = owning_rank != DCP_RANK + local_idx = ( + tok // (DCP_SIZE * DCP_INTERLEAVE) + ) * DCP_INTERLEAVE + tok % DCP_INTERLEAVE + # Compute block id and in-block offset - block_id = tok // BLOCK_SIZE - inblock_off = tok % BLOCK_SIZE + block_id = local_idx // BLOCK_SIZE + inblock_off = local_idx % BLOCK_SIZE # Guard block_table access valid_block = (block_id < max_num_blocks_per_req) & (block_id >= 0) bt_ptr = block_table_ptr + req * bt_stride0 + block_id * bt_stride1 - is_invalid_tok |= ~valid_block - base = tl.load(bt_ptr, mask=valid_block & ~is_prefill, other=0) + is_invalid_tok |= ~valid_block | is_remote + base = tl.load(bt_ptr, mask=valid_block & ~is_prefill & ~is_remote, other=0) out_val = base * BLOCK_SIZE + inblock_off # Override with prefill output if prefill is enabled @@ -72,14 +94,27 @@ def _convert_req_index_to_global_index_kernel( out_val = tl.where(is_prefill, prefill_out, out_val) out_val = tl.where(is_invalid_tok, -1, out_val) - # Store results - out_ptr_ij = out_ptr + token_id * out_stride0 + indice_id * out_stride1 - tl.store(out_ptr_ij, out_val) + if COMPACT_TO_FRONT: + # Scatter valid slots to a contiguous prefix. A per-tile exclusive prefix + # sum gives each valid lane a distinct local offset; one atomic add of the + # tile's valid count reserves a contiguous base across racing tiles. The + # out buffer is pre-filled with -1, so unwritten tail slots stay -1. + is_valid = (~is_invalid_tok).to(tl.int32) + local_offset = tl.cumsum(is_valid) - is_valid + tile_valid_count = tl.sum(is_valid) + base = tl.atomic_add(valid_count_ptr + token_id, tile_valid_count) + dest = base + local_offset + out_ptr_dest = out_ptr + token_id * out_stride0 + dest * out_stride1 + tl.store(out_ptr_dest, out_val, mask=is_valid == 1) + else: + # Store results in place (input column == output column). + out_ptr_ij = out_ptr + token_id * out_stride0 + indice_id * out_stride1 + tl.store(out_ptr_ij, out_val) - # Count valid indices in this tile and atomically add to row total - if COUNT_VALID: - tile_valid_count = tl.sum((~is_invalid_tok).to(tl.int32)) - tl.atomic_add(valid_count_ptr + token_id, tile_valid_count) + # Count valid indices in this tile and atomically add to row total + if COUNT_VALID: + tile_valid_count = tl.sum((~is_invalid_tok).to(tl.int32)) + tl.atomic_add(valid_count_ptr + token_id, tile_valid_count) def triton_convert_req_index_to_global_index( @@ -176,6 +211,11 @@ def triton_convert_req_index_to_global_index( BLOCK_N, HAS_PREFILL_WORKSPACE, return_valid_counts, + False, # COMPACT_TO_FRONT: keep input column == output column + # DCP disabled (no-op de-interleave) + 1, + 0, + 1, # strides bt_stride0, bt_stride1, @@ -189,3 +229,110 @@ def triton_convert_req_index_to_global_index( assert valid_counts is not None return out, valid_counts return out + + +def triton_filter_and_convert_dcp_index( + req_id: torch.Tensor, + block_table: torch.Tensor, + token_indices: torch.Tensor, + dcp_size: int, + dcp_rank: int, + cp_kv_cache_interleave_size: int = 1, + BLOCK_SIZE: int = 64, + NUM_TOPK_TOKENS: int = 2048, + BLOCK_N: int = 128, + return_valid_counts: bool = False, + compact_valid_to_front: bool = True, +) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """Filter global per-request indices to this DCP rank's local slots. + + With ``compact_valid_to_front`` (default), the conversion kernel scatters + this rank's owned slots to a contiguous prefix ``[0, valid_count)`` and + leaves the rest ``-1``. DCP filtering marks non-owned slots ``-1`` and so + creates interior gaps; the trtllm-gen sparse kernel reads the first + ``valid_count`` entries of each row, so they must be a contiguous prefix. + Compaction is fused into the kernel (atomic slot allocator) rather than a + separate sort/gather pass. Prefix order is unspecified (only the set matters). + """ + assert dcp_size >= 1 + assert 0 <= dcp_rank < dcp_size + # Interleave groups must align to KV blocks (globally enforced by + # VllmConfig: block_size % cp_kv_cache_interleave_size == 0); assert the + # local invariant so local_idx // BLOCK_SIZE never straddles a group. + assert BLOCK_SIZE % cp_kv_cache_interleave_size == 0, ( + f"BLOCK_SIZE ({BLOCK_SIZE}) must be divisible by " + f"cp_kv_cache_interleave_size ({cp_kv_cache_interleave_size})." + ) + assert req_id.dtype == torch.int32 + assert block_table.dtype == torch.int32 + assert token_indices.dtype == torch.int32 + assert token_indices.shape[1] == NUM_TOPK_TOKENS + assert NUM_TOPK_TOKENS % BLOCK_N == 0 + + if dcp_size == 1: + return triton_convert_req_index_to_global_index( + req_id, + block_table, + token_indices, + BLOCK_SIZE=BLOCK_SIZE, + NUM_TOPK_TOKENS=NUM_TOPK_TOKENS, + BLOCK_N=BLOCK_N, + return_valid_counts=return_valid_counts, + ) + + num_tokens = req_id.shape[0] + max_num_blocks_per_req = block_table.shape[1] + tiles_per_row = NUM_TOPK_TOKENS // BLOCK_N + + req_id_c = req_id.contiguous() + block_table_c = block_table.contiguous() + token_indices_c = token_indices.contiguous() + + # The compaction uses the valid-count buffer as an atomic slot allocator, so + # it requires counting. Pre-fill out with -1 so the unwritten tail stays -1. + count_valid = return_valid_counts or compact_valid_to_front + if compact_valid_to_front: + out = torch.full_like(token_indices_c, -1) + else: + out = torch.empty_like(token_indices_c) + + valid_counts: torch.Tensor | None = None + if count_valid: + valid_counts = torch.zeros( + num_tokens, dtype=torch.int32, device=token_indices.device + ) + + bt_stride0, bt_stride1 = block_table_c.stride() + ti_stride0, ti_stride1 = token_indices_c.stride() + out_stride0, out_stride1 = out.stride() + + _convert_req_index_to_global_index_kernel[(num_tokens, tiles_per_row)]( + req_id_c, + block_table_c, + token_indices_c, + out, + valid_counts, + # No prefill workspace on the DCP decode path. + None, + None, + max_num_blocks_per_req, + BLOCK_SIZE, + BLOCK_N, + False, # HAS_PREFILL + count_valid, + compact_valid_to_front, + dcp_size, + dcp_rank, + cp_kv_cache_interleave_size, + bt_stride0, + bt_stride1, + ti_stride0, + ti_stride1, + out_stride0, + out_stride1, + ) + + if return_valid_counts: + assert valid_counts is not None + return out, valid_counts + return out diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index c00e80f752..1e12f43caa 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -892,20 +892,20 @@ def get_dcp_local_seq_lens( use this function to calculate split decode seq_lens of each dcp rank. Only consider dcp now, we can extend the case of cp based on this. """ - num_requests = seq_lens.size(0) + seq_lens_i32 = seq_lens.to(torch.int32) if dcp_rank is None: - rank_offsets = ( - torch.arange(dcp_size, dtype=torch.int32, device=seq_lens.device) - .unsqueeze(0) - .repeat(num_requests, 1) + rank_offsets = torch.arange( + dcp_size, + dtype=torch.int32, + device=seq_lens.device, + ).view( + *((1,) * seq_lens_i32.dim()), + dcp_size, ) + seq_lens_tiled = seq_lens_i32.unsqueeze(-1) else: - rank_offsets = torch.tensor( - [[dcp_rank]], dtype=torch.int32, device=seq_lens.device - ) - seq_lens_tiled = ( - seq_lens.to(torch.int32).unsqueeze(-1).repeat(1, rank_offsets.shape[1]) - ) + rank_offsets = torch.tensor(dcp_rank, dtype=torch.int32, device=seq_lens.device) + seq_lens_tiled = seq_lens_i32 base = ( seq_lens_tiled // cp_kv_cache_interleave_size @@ -919,7 +919,7 @@ def get_dcp_local_seq_lens( cp_kv_cache_interleave_size, ) dcp_local_seq_lens = base + remainder - return dcp_local_seq_lens.squeeze(1) + return dcp_local_seq_lens def mamba_get_block_table_tensor( diff --git a/vllm/v1/attention/ops/common.py b/vllm/v1/attention/ops/common.py index 98abc7790e..901d6bb30b 100644 --- a/vllm/v1/attention/ops/common.py +++ b/vllm/v1/attention/ops/common.py @@ -90,6 +90,7 @@ def _correct_attn_cp_out_kernel( factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally) output = tl.load(outputs_ptr + output_offsets) output = output * factor + output = tl.where(factor == 0.0, 0.0, output) tl.store(new_output_ptr + output_offsets, output)