From 6ee5bb0a0b3e32dd1a6d9fddb61b50905ccdd6e0 Mon Sep 17 00:00:00 2001 From: Rita Brugarolas Date: Thu, 10 Sep 2026 08:43:00 -0700 Subject: [PATCH] [DCP][Kernel][Perf] Fuse the empty-shard LSE mask into the A2A pack kernel (#54889) Signed-off-by: Rita Brugarolas Brufau Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/v1/attention/test_dcp_a2a_pack_mask.py | 151 +++++++++++++++++++ vllm/v1/attention/ops/dcp.py | 67 +++++++- 2 files changed, 213 insertions(+), 5 deletions(-) create mode 100644 tests/v1/attention/test_dcp_a2a_pack_mask.py diff --git a/tests/v1/attention/test_dcp_a2a_pack_mask.py b/tests/v1/attention/test_dcp_a2a_pack_mask.py new file mode 100644 index 0000000000..36032e7a69 --- /dev/null +++ b/tests/v1/attention/test_dcp_a2a_pack_mask.py @@ -0,0 +1,151 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The empty-shard mask fused into the DCP A2A pack kernel must be bit-exact +with the eager ``mask_dcp_empty_shards_`` pass it replaces.""" + +import pytest +import torch + +from vllm.platforms import current_platform +from vllm.v1.attention.ops.dcp import ( + _dcp_a2a_lse_pack_dim, + _dcp_a2a_pack_send, + mask_dcp_empty_shards_, +) + +pytestmark = pytest.mark.skipif( + not current_platform.is_cuda_alike(), + reason="needs a CUDA or ROCm device for the Triton pack kernel", +) + + +def _pack(out, lse, world_size, h_per_rank, head_dim, seq_lens, query_start_loc): + lse_pack_dim = _dcp_a2a_lse_pack_dim(out.dtype) + send = torch.zeros( + (world_size, out.shape[0], h_per_rank, head_dim + lse_pack_dim), + device=out.device, + dtype=out.dtype, + ) + _dcp_a2a_pack_send( + out, + lse, + send, + world_size, + h_per_rank, + head_dim, + lse_pack_dim, + seq_lens=seq_lens, + query_start_loc=query_start_loc, + ) + return send + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("world_size,h_per_rank", [(8, 2), (8, 16), (4, 4), (2, 1)]) +@pytest.mark.parametrize("tokens_per_req", [1, 2, 3]) +@pytest.mark.parametrize("num_pad_rows", [0, 2]) +@pytest.mark.parametrize( + "seq_lens_list", + [ + [16, 16, 16, 16], # no empty shards (the long-context decode case) + [16, 0, 16, 0], # interior empty shards + [0, 0, 0, 0], # every shard empty + [16, 16, 0, 0], # trailing empty shards, i.e. cudagraph padding + [0, 16, 16, 16], # leading empty shard + [16] * 52, # the profiled conc-52 decode batch + [7], # single request + ], +) +def test_fused_mask_matches_eager( + dtype, world_size, h_per_rank, tokens_per_req, num_pad_rows, seq_lens_list +): + torch.manual_seed(0) + device = "cuda" + head_dim = 512 + num_seqs = len(seq_lens_list) + # tokens_per_req > 1 is the MTP / multi-token-verify shape. + num_rows = num_seqs * tokens_per_req + num_pad_rows + num_heads = world_size * h_per_rank + + out = torch.randn(num_rows, num_heads, head_dim, device=device, dtype=dtype) + lse = torch.randn(num_rows, num_heads, device=device, dtype=torch.float32) + seq_lens = torch.tensor(seq_lens_list, device=device, dtype=torch.int32) + query_start_loc = torch.arange( + 0, + (num_seqs + 1) * tokens_per_req, + tokens_per_req, + device=device, + dtype=torch.int32, + ) # rows past query_start_loc[-1] are cudagraph padding + + # Reference: mask eagerly, then pack with masking disabled. + ref_lse = lse.clone() + mask_dcp_empty_shards_(ref_lse, seq_lens, query_start_loc) + expected = _pack(out, ref_lse, world_size, h_per_rank, head_dim, None, None) + + # Under test: pack with the mask fused in. + actual = _pack( + out, lse.clone(), world_size, h_per_rank, head_dim, seq_lens, query_start_loc + ) + + # The packed LSE slots hold halves of an fp32, not meaningful floats, so + # compare the send buffers bitwise: -inf's high half is a NaN in fp16. + assert torch.equal(actual.view(torch.int16), expected.view(torch.int16)), ( + "fused mask is not bit-exact with the eager mask" + ) + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("seed", range(8)) +def test_fused_mask_matches_eager_ragged(dtype, seed): + """Non-uniform per-request query lengths. + + The uniform-stride cases above never exercise an unequal gap between + consecutive ``query_start_loc`` entries, which is what the in-kernel + boundary search has to get right. MTP with partial acceptance produces + exactly this: some requests contribute one row, others several, in the + same batch. + """ + torch.manual_seed(seed) + device = "cuda" + world_size, h_per_rank, head_dim = 8, 2, 512 + num_seqs = 17 + + query_lens = torch.randint(1, 5, (num_seqs,), dtype=torch.int32) + query_start_loc = torch.zeros(num_seqs + 1, dtype=torch.int32) + query_start_loc[1:] = torch.cumsum(query_lens, 0) + # Mix real, empty, and (via padding rows) out-of-range shards. + seq_lens = torch.randint(0, 2, (num_seqs,), dtype=torch.int32) * 16 + num_rows = int(query_start_loc[-1]) + 3 + + query_start_loc = query_start_loc.to(device) + seq_lens = seq_lens.to(device) + num_heads = world_size * h_per_rank + out = torch.randn(num_rows, num_heads, head_dim, device=device, dtype=dtype) + lse = torch.randn(num_rows, num_heads, device=device, dtype=torch.float32) + + ref_lse = lse.clone() + mask_dcp_empty_shards_(ref_lse, seq_lens, query_start_loc) + expected = _pack(out, ref_lse, world_size, h_per_rank, head_dim, None, None) + actual = _pack( + out, lse.clone(), world_size, h_per_rank, head_dim, seq_lens, query_start_loc + ) + + assert torch.equal(actual.view(torch.int16), expected.view(torch.int16)), ( + f"ragged query_start_loc mismatch: query_lens={query_lens.tolist()}, " + f"seq_lens={seq_lens.tolist()}" + ) + + +def test_mask_disabled_is_unmasked(): + """Passing no seq_lens/query_start_loc must leave the LSE untouched.""" + torch.manual_seed(0) + device = "cuda" + world_size, h_per_rank, head_dim = 8, 2, 512 + out = torch.randn( + 4, world_size * h_per_rank, head_dim, device=device, dtype=torch.bfloat16 + ) + lse = torch.randn(4, world_size * h_per_rank, device=device, dtype=torch.float32) + + packed = _pack(out, lse, world_size, h_per_rank, head_dim, None, None) + assert torch.isfinite(packed.float()).all() diff --git a/vllm/v1/attention/ops/dcp.py b/vllm/v1/attention/ops/dcp.py index 96b25870b8..66a6bace46 100644 --- a/vllm/v1/attention/ops/dcp.py +++ b/vllm/v1/attention/ops/dcp.py @@ -45,13 +45,18 @@ if TYPE_CHECKING: # LSE/output combine -def mask_dcp_empty_shards_( - lse: torch.Tensor, +def _validate_dcp_empty_shard_args( seq_lens: torch.Tensor | None, query_start_loc: torch.Tensor | None, -) -> None: +) -> bool: + """Validate the empty-shard mask inputs. + + Returns True when masking is requested, False when it is disabled. Raises on + an inconsistent pair. Performs no device work so it is safe to call from a + CUDA-graph-captured region. + """ if seq_lens is None and query_start_loc is None: - return + return False if seq_lens is None or query_start_loc is None: raise ValueError("seq_lens and query_start_loc must be provided together") if ( @@ -60,6 +65,17 @@ def mask_dcp_empty_shards_( or query_start_loc.shape[0] != seq_lens.shape[0] + 1 ): raise ValueError("query_start_loc must contain one boundary per sequence") + return True + + +def mask_dcp_empty_shards_( + lse: torch.Tensor, + seq_lens: torch.Tensor | None, + query_start_loc: torch.Tensor | None, +) -> None: + if not _validate_dcp_empty_shard_args(seq_lens, query_start_loc): + return + assert seq_lens is not None and query_start_loc is not None row_indices = torch.arange( lse.shape[0], device=lse.device, dtype=query_start_loc.dtype @@ -603,6 +619,9 @@ def _dcp_a2a_pack_send_kernel( out_ptr, lse_ptr, send_ptr, + seq_lens_ptr, + query_start_loc_ptr, + num_seqs, out_stride_B, out_stride_H, out_stride_D, @@ -616,11 +635,33 @@ def _dcp_a2a_pack_send_kernel( HEAD_DIM: tl.constexpr, H_PER_RANK: tl.constexpr, LSE_PACK_DIM: tl.constexpr, + HAS_MASK: tl.constexpr, + NUM_SEQS_POW2: tl.constexpr, ): batch_idx = tl.program_id(0).to(tl.int64) local_head_idx = tl.program_id(1).to(tl.int64) d_offsets = tl.arange(0, HEAD_DIM) + # Empty-shard masking, fused in to avoid a separate per-layer pass over the + # LSE. Rows that belong to a request with no local KV shard, and rows past + # the end of the batch (CUDA-graph padding), must not contribute to the + # combine, which the reduction achieves by giving them an LSE of -inf. + row_is_empty = False + if HAS_MASK: + # searchsorted(query_start_loc[1:], batch_idx, right=True), i.e. the + # number of request boundaries at or below this row. + seq_offsets = tl.arange(0, NUM_SEQS_POW2) + boundaries = tl.load( + query_start_loc_ptr + 1 + seq_offsets, + mask=seq_offsets < num_seqs, + other=2**31 - 1, + ) + seq_idx = tl.sum((boundaries <= batch_idx).to(tl.int32), axis=0) + seq_idx = tl.minimum(seq_idx, num_seqs - 1) + num_rows = tl.load(query_start_loc_ptr + num_seqs) + seq_len = tl.load(seq_lens_ptr + seq_idx) + row_is_empty = (batch_idx >= num_rows) | (seq_len == 0) + for rank_idx in tl.static_range(N): src_head_idx = rank_idx * H_PER_RANK + local_head_idx send_base = ( @@ -642,6 +683,8 @@ def _dcp_a2a_pack_send_kernel( lse_val = tl.load( lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H ).to(tl.float32) + if HAS_MASK: + lse_val = tl.where(row_is_empty, -float("inf"), lse_val) if LSE_PACK_DIM == 1: tl.store( send_ptr + send_base + HEAD_DIM * send_stride_D, @@ -797,12 +840,21 @@ def _dcp_a2a_pack_send( seq_lens: torch.Tensor | None = None, query_start_loc: torch.Tensor | None = None, ) -> None: - mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc) + # The empty-shard mask is fused into the pack kernel below rather than + # applied as a separate pass: the mask is a pure function of seq_lens and + # query_start_loc, which are identical for every MLA layer in a step, while + # the eager form costs eight launch-bound kernels per layer. + has_mask = _validate_dcp_empty_shard_args(seq_lens, query_start_loc) + num_seqs = seq_lens.shape[0] if has_mask and seq_lens is not None else 0 + grid = (cp_attn_out.shape[0], h_per_rank, 1) _dcp_a2a_pack_send_kernel[grid]( cp_attn_out, cp_attn_lse, send_buffer, + seq_lens, + query_start_loc, + num_seqs, cp_attn_out.stride(0), cp_attn_out.stride(1), cp_attn_out.stride(2), @@ -816,6 +868,11 @@ def _dcp_a2a_pack_send( HEAD_DIM=head_dim, H_PER_RANK=h_per_rank, LSE_PACK_DIM=lse_pack_dim, + HAS_MASK=has_mask, + # Only the arange bound is a constexpr: num_seqs itself is a runtime + # argument so the kernel specializes on at most log2(max_num_seqs) + # variants instead of one per distinct batch size. + NUM_SEQS_POW2=triton.next_power_of_2(max(num_seqs, 1)), )