[DCP][Kernel][Perf] Fuse the empty-shard LSE mask into the A2A pack kernel (#54889)
Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
bdea2777ea
commit
6ee5bb0a0b
@@ -0,0 +1,151 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The empty-shard mask fused into the DCP A2A pack kernel must be bit-exact
|
||||
with the eager ``mask_dcp_empty_shards_`` pass it replaces."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.ops.dcp import (
|
||||
_dcp_a2a_lse_pack_dim,
|
||||
_dcp_a2a_pack_send,
|
||||
mask_dcp_empty_shards_,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="needs a CUDA or ROCm device for the Triton pack kernel",
|
||||
)
|
||||
|
||||
|
||||
def _pack(out, lse, world_size, h_per_rank, head_dim, seq_lens, query_start_loc):
|
||||
lse_pack_dim = _dcp_a2a_lse_pack_dim(out.dtype)
|
||||
send = torch.zeros(
|
||||
(world_size, out.shape[0], h_per_rank, head_dim + lse_pack_dim),
|
||||
device=out.device,
|
||||
dtype=out.dtype,
|
||||
)
|
||||
_dcp_a2a_pack_send(
|
||||
out,
|
||||
lse,
|
||||
send,
|
||||
world_size,
|
||||
h_per_rank,
|
||||
head_dim,
|
||||
lse_pack_dim,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=query_start_loc,
|
||||
)
|
||||
return send
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("world_size,h_per_rank", [(8, 2), (8, 16), (4, 4), (2, 1)])
|
||||
@pytest.mark.parametrize("tokens_per_req", [1, 2, 3])
|
||||
@pytest.mark.parametrize("num_pad_rows", [0, 2])
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens_list",
|
||||
[
|
||||
[16, 16, 16, 16], # no empty shards (the long-context decode case)
|
||||
[16, 0, 16, 0], # interior empty shards
|
||||
[0, 0, 0, 0], # every shard empty
|
||||
[16, 16, 0, 0], # trailing empty shards, i.e. cudagraph padding
|
||||
[0, 16, 16, 16], # leading empty shard
|
||||
[16] * 52, # the profiled conc-52 decode batch
|
||||
[7], # single request
|
||||
],
|
||||
)
|
||||
def test_fused_mask_matches_eager(
|
||||
dtype, world_size, h_per_rank, tokens_per_req, num_pad_rows, seq_lens_list
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
head_dim = 512
|
||||
num_seqs = len(seq_lens_list)
|
||||
# tokens_per_req > 1 is the MTP / multi-token-verify shape.
|
||||
num_rows = num_seqs * tokens_per_req + num_pad_rows
|
||||
num_heads = world_size * h_per_rank
|
||||
|
||||
out = torch.randn(num_rows, num_heads, head_dim, device=device, dtype=dtype)
|
||||
lse = torch.randn(num_rows, num_heads, device=device, dtype=torch.float32)
|
||||
seq_lens = torch.tensor(seq_lens_list, device=device, dtype=torch.int32)
|
||||
query_start_loc = torch.arange(
|
||||
0,
|
||||
(num_seqs + 1) * tokens_per_req,
|
||||
tokens_per_req,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
) # rows past query_start_loc[-1] are cudagraph padding
|
||||
|
||||
# Reference: mask eagerly, then pack with masking disabled.
|
||||
ref_lse = lse.clone()
|
||||
mask_dcp_empty_shards_(ref_lse, seq_lens, query_start_loc)
|
||||
expected = _pack(out, ref_lse, world_size, h_per_rank, head_dim, None, None)
|
||||
|
||||
# Under test: pack with the mask fused in.
|
||||
actual = _pack(
|
||||
out, lse.clone(), world_size, h_per_rank, head_dim, seq_lens, query_start_loc
|
||||
)
|
||||
|
||||
# The packed LSE slots hold halves of an fp32, not meaningful floats, so
|
||||
# compare the send buffers bitwise: -inf's high half is a NaN in fp16.
|
||||
assert torch.equal(actual.view(torch.int16), expected.view(torch.int16)), (
|
||||
"fused mask is not bit-exact with the eager mask"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("seed", range(8))
|
||||
def test_fused_mask_matches_eager_ragged(dtype, seed):
|
||||
"""Non-uniform per-request query lengths.
|
||||
|
||||
The uniform-stride cases above never exercise an unequal gap between
|
||||
consecutive ``query_start_loc`` entries, which is what the in-kernel
|
||||
boundary search has to get right. MTP with partial acceptance produces
|
||||
exactly this: some requests contribute one row, others several, in the
|
||||
same batch.
|
||||
"""
|
||||
torch.manual_seed(seed)
|
||||
device = "cuda"
|
||||
world_size, h_per_rank, head_dim = 8, 2, 512
|
||||
num_seqs = 17
|
||||
|
||||
query_lens = torch.randint(1, 5, (num_seqs,), dtype=torch.int32)
|
||||
query_start_loc = torch.zeros(num_seqs + 1, dtype=torch.int32)
|
||||
query_start_loc[1:] = torch.cumsum(query_lens, 0)
|
||||
# Mix real, empty, and (via padding rows) out-of-range shards.
|
||||
seq_lens = torch.randint(0, 2, (num_seqs,), dtype=torch.int32) * 16
|
||||
num_rows = int(query_start_loc[-1]) + 3
|
||||
|
||||
query_start_loc = query_start_loc.to(device)
|
||||
seq_lens = seq_lens.to(device)
|
||||
num_heads = world_size * h_per_rank
|
||||
out = torch.randn(num_rows, num_heads, head_dim, device=device, dtype=dtype)
|
||||
lse = torch.randn(num_rows, num_heads, device=device, dtype=torch.float32)
|
||||
|
||||
ref_lse = lse.clone()
|
||||
mask_dcp_empty_shards_(ref_lse, seq_lens, query_start_loc)
|
||||
expected = _pack(out, ref_lse, world_size, h_per_rank, head_dim, None, None)
|
||||
actual = _pack(
|
||||
out, lse.clone(), world_size, h_per_rank, head_dim, seq_lens, query_start_loc
|
||||
)
|
||||
|
||||
assert torch.equal(actual.view(torch.int16), expected.view(torch.int16)), (
|
||||
f"ragged query_start_loc mismatch: query_lens={query_lens.tolist()}, "
|
||||
f"seq_lens={seq_lens.tolist()}"
|
||||
)
|
||||
|
||||
|
||||
def test_mask_disabled_is_unmasked():
|
||||
"""Passing no seq_lens/query_start_loc must leave the LSE untouched."""
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
world_size, h_per_rank, head_dim = 8, 2, 512
|
||||
out = torch.randn(
|
||||
4, world_size * h_per_rank, head_dim, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
lse = torch.randn(4, world_size * h_per_rank, device=device, dtype=torch.float32)
|
||||
|
||||
packed = _pack(out, lse, world_size, h_per_rank, head_dim, None, None)
|
||||
assert torch.isfinite(packed.float()).all()
|
||||
@@ -45,13 +45,18 @@ if TYPE_CHECKING:
|
||||
# LSE/output combine
|
||||
|
||||
|
||||
def mask_dcp_empty_shards_(
|
||||
lse: torch.Tensor,
|
||||
def _validate_dcp_empty_shard_args(
|
||||
seq_lens: torch.Tensor | None,
|
||||
query_start_loc: torch.Tensor | None,
|
||||
) -> None:
|
||||
) -> bool:
|
||||
"""Validate the empty-shard mask inputs.
|
||||
|
||||
Returns True when masking is requested, False when it is disabled. Raises on
|
||||
an inconsistent pair. Performs no device work so it is safe to call from a
|
||||
CUDA-graph-captured region.
|
||||
"""
|
||||
if seq_lens is None and query_start_loc is None:
|
||||
return
|
||||
return False
|
||||
if seq_lens is None or query_start_loc is None:
|
||||
raise ValueError("seq_lens and query_start_loc must be provided together")
|
||||
if (
|
||||
@@ -60,6 +65,17 @@ def mask_dcp_empty_shards_(
|
||||
or query_start_loc.shape[0] != seq_lens.shape[0] + 1
|
||||
):
|
||||
raise ValueError("query_start_loc must contain one boundary per sequence")
|
||||
return True
|
||||
|
||||
|
||||
def mask_dcp_empty_shards_(
|
||||
lse: torch.Tensor,
|
||||
seq_lens: torch.Tensor | None,
|
||||
query_start_loc: torch.Tensor | None,
|
||||
) -> None:
|
||||
if not _validate_dcp_empty_shard_args(seq_lens, query_start_loc):
|
||||
return
|
||||
assert seq_lens is not None and query_start_loc is not None
|
||||
|
||||
row_indices = torch.arange(
|
||||
lse.shape[0], device=lse.device, dtype=query_start_loc.dtype
|
||||
@@ -603,6 +619,9 @@ def _dcp_a2a_pack_send_kernel(
|
||||
out_ptr,
|
||||
lse_ptr,
|
||||
send_ptr,
|
||||
seq_lens_ptr,
|
||||
query_start_loc_ptr,
|
||||
num_seqs,
|
||||
out_stride_B,
|
||||
out_stride_H,
|
||||
out_stride_D,
|
||||
@@ -616,11 +635,33 @@ def _dcp_a2a_pack_send_kernel(
|
||||
HEAD_DIM: tl.constexpr,
|
||||
H_PER_RANK: tl.constexpr,
|
||||
LSE_PACK_DIM: tl.constexpr,
|
||||
HAS_MASK: tl.constexpr,
|
||||
NUM_SEQS_POW2: tl.constexpr,
|
||||
):
|
||||
batch_idx = tl.program_id(0).to(tl.int64)
|
||||
local_head_idx = tl.program_id(1).to(tl.int64)
|
||||
d_offsets = tl.arange(0, HEAD_DIM)
|
||||
|
||||
# Empty-shard masking, fused in to avoid a separate per-layer pass over the
|
||||
# LSE. Rows that belong to a request with no local KV shard, and rows past
|
||||
# the end of the batch (CUDA-graph padding), must not contribute to the
|
||||
# combine, which the reduction achieves by giving them an LSE of -inf.
|
||||
row_is_empty = False
|
||||
if HAS_MASK:
|
||||
# searchsorted(query_start_loc[1:], batch_idx, right=True), i.e. the
|
||||
# number of request boundaries at or below this row.
|
||||
seq_offsets = tl.arange(0, NUM_SEQS_POW2)
|
||||
boundaries = tl.load(
|
||||
query_start_loc_ptr + 1 + seq_offsets,
|
||||
mask=seq_offsets < num_seqs,
|
||||
other=2**31 - 1,
|
||||
)
|
||||
seq_idx = tl.sum((boundaries <= batch_idx).to(tl.int32), axis=0)
|
||||
seq_idx = tl.minimum(seq_idx, num_seqs - 1)
|
||||
num_rows = tl.load(query_start_loc_ptr + num_seqs)
|
||||
seq_len = tl.load(seq_lens_ptr + seq_idx)
|
||||
row_is_empty = (batch_idx >= num_rows) | (seq_len == 0)
|
||||
|
||||
for rank_idx in tl.static_range(N):
|
||||
src_head_idx = rank_idx * H_PER_RANK + local_head_idx
|
||||
send_base = (
|
||||
@@ -642,6 +683,8 @@ def _dcp_a2a_pack_send_kernel(
|
||||
lse_val = tl.load(
|
||||
lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H
|
||||
).to(tl.float32)
|
||||
if HAS_MASK:
|
||||
lse_val = tl.where(row_is_empty, -float("inf"), lse_val)
|
||||
if LSE_PACK_DIM == 1:
|
||||
tl.store(
|
||||
send_ptr + send_base + HEAD_DIM * send_stride_D,
|
||||
@@ -797,12 +840,21 @@ def _dcp_a2a_pack_send(
|
||||
seq_lens: torch.Tensor | None = None,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc)
|
||||
# The empty-shard mask is fused into the pack kernel below rather than
|
||||
# applied as a separate pass: the mask is a pure function of seq_lens and
|
||||
# query_start_loc, which are identical for every MLA layer in a step, while
|
||||
# the eager form costs eight launch-bound kernels per layer.
|
||||
has_mask = _validate_dcp_empty_shard_args(seq_lens, query_start_loc)
|
||||
num_seqs = seq_lens.shape[0] if has_mask and seq_lens is not None else 0
|
||||
|
||||
grid = (cp_attn_out.shape[0], h_per_rank, 1)
|
||||
_dcp_a2a_pack_send_kernel[grid](
|
||||
cp_attn_out,
|
||||
cp_attn_lse,
|
||||
send_buffer,
|
||||
seq_lens,
|
||||
query_start_loc,
|
||||
num_seqs,
|
||||
cp_attn_out.stride(0),
|
||||
cp_attn_out.stride(1),
|
||||
cp_attn_out.stride(2),
|
||||
@@ -816,6 +868,11 @@ def _dcp_a2a_pack_send(
|
||||
HEAD_DIM=head_dim,
|
||||
H_PER_RANK=h_per_rank,
|
||||
LSE_PACK_DIM=lse_pack_dim,
|
||||
HAS_MASK=has_mask,
|
||||
# Only the arange bound is a constexpr: num_seqs itself is a runtime
|
||||
# argument so the kernel specializes on at most log2(max_num_seqs)
|
||||
# variants instead of one per distinct batch size.
|
||||
NUM_SEQS_POW2=triton.next_power_of_2(max(num_seqs, 1)),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user