[ROCm][DSV4.1][Perf] Stride the DSA decode candidate mask over the live context (#56628)
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,226 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm correctness tests for the strided DSA candidate mask.
|
||||
|
||||
``_apply_candidate_mask_strided`` is a decode-path variant of the shared
|
||||
``apply_candidate_mask``. It differs in two ways that need pinning down:
|
||||
|
||||
* it launches a fixed number of programs that stride the column axis, rather
|
||||
than one program per 1024-column tile, so at ``max_model_len`` widths the
|
||||
grid no longer scales with a workspace that is mostly padding;
|
||||
* it stops at each row's ``end`` instead of sanitizing the whole width.
|
||||
|
||||
The second is only sound because every consumer bounds its scan by the same
|
||||
ends -- ``top_k_per_row_decode`` takes ``rowEnd`` from ``seq_lens``. So the
|
||||
contract is agreement with the shared kernel over ``[0, end)`` and nothing
|
||||
beyond it, which is what these tests assert rather than comparing whole rows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
BLOCK_SIZE = 8
|
||||
TOPK_BLOCKS = 64
|
||||
# Above 128 tiles the grid saturates and programs start striding; below it the
|
||||
# min() clamp applies instead. Both regimes need covering, so the widths here
|
||||
# deliberately straddle 128 * 1024.
|
||||
NARROW = 8192
|
||||
WIDE = 262144
|
||||
|
||||
|
||||
def _inputs(rows, width, ends, starts, *, block_size, col_stride, dtype):
|
||||
"""Logits, bounds and candidate blocks shared by both kernels."""
|
||||
torch.manual_seed(0)
|
||||
# A column stride > 1 keeps the kernels honest about stride_col; the decode
|
||||
# caller slices a workspace, so contiguous rows are not guaranteed.
|
||||
logits = torch.randn(rows, width * col_stride, device="cuda")[:, ::col_stride]
|
||||
logits[0, : min(16, width)] = 0.0
|
||||
if rows > 2:
|
||||
logits[2, 5] = float("nan")
|
||||
row_ke = torch.tensor(ends, device="cuda", dtype=dtype)
|
||||
row_ks = torch.tensor(starts, device="cuda", dtype=dtype) if starts else None
|
||||
nblocks = (width + block_size - 1) // block_size
|
||||
candidates = torch.randint(
|
||||
0, nblocks, (rows, TOPK_BLOCKS), device="cuda", dtype=torch.int32
|
||||
)
|
||||
# -1 is the "no block" sentinel the selector emits for short rows.
|
||||
candidates[:, ::7] = -1
|
||||
return logits, row_ks, row_ke, candidates
|
||||
|
||||
|
||||
def _run_both(logits, row_ks, row_ke, candidates, block_size, row_repeat):
|
||||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
apply_candidate_mask,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
ref = logits.clone()
|
||||
got = logits.clone()
|
||||
apply_candidate_mask(ref, row_ks, row_ke, candidates, block_size, row_repeat)
|
||||
_apply_candidate_mask_strided(
|
||||
got, row_ks, row_ke, candidates, block_size, row_repeat
|
||||
)
|
||||
return ref, got
|
||||
|
||||
|
||||
def _assert_agrees_within_ends(ref, got, row_ke, row_repeat):
|
||||
"""Compare only the region a consumer can read."""
|
||||
for row in range(ref.shape[0]):
|
||||
end = int(row_ke[row // row_repeat])
|
||||
if end <= 0:
|
||||
continue
|
||||
torch.testing.assert_close(
|
||||
got[row, :end],
|
||||
ref[row, :end],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
equal_nan=True,
|
||||
msg=lambda m, r=row, e=end: f"row {r} differs within [0, {e}): {m}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [NARROW, WIDE])
|
||||
@pytest.mark.parametrize("col_stride", [1, 2])
|
||||
def test_matches_shared_kernel_ragged_ends(width, col_stride):
|
||||
"""Ends on and around tile boundaries, plus the empty and full rows."""
|
||||
ends = [0, 1, 1023, 1024, 1025, width - 1, width, width // 2]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
width,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=col_stride,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [8, 16])
|
||||
def test_matches_shared_kernel_with_starts(block_size):
|
||||
"""Non-zero starts: the decode caller passes zeros, prefill does not."""
|
||||
ends = [0, 64, 4096, 8192]
|
||||
starts = [0, 7, 1024, 3]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
NARROW,
|
||||
ends,
|
||||
starts,
|
||||
block_size=block_size,
|
||||
col_stride=1,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, block_size, 1)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, 1)
|
||||
|
||||
|
||||
def test_matches_shared_kernel_row_repeat():
|
||||
"""Speculative decode: next_n logit rows share one bound."""
|
||||
row_repeat = 3
|
||||
ends = [0, 1025, WIDE]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends) * row_repeat,
|
||||
WIDE,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, row_repeat)
|
||||
_assert_agrees_within_ends(ref, got, row_ke, row_repeat)
|
||||
|
||||
|
||||
def test_leaves_columns_past_end_untouched():
|
||||
"""The deliberate difference from the shared kernel, pinned explicitly.
|
||||
|
||||
Skipping the tail is where the speedup comes from, so a change that
|
||||
quietly restores full-width sanitizing should fail here rather than just
|
||||
get slower. Work stops at the tile boundary containing ``end``, not at
|
||||
``end`` itself, so that is the bound asserted.
|
||||
"""
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_MASK_TILE,
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
ends = [0, 1, 1025, WIDE // 2]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
len(ends),
|
||||
WIDE,
|
||||
ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
sentinel = 1.5
|
||||
logits.fill_(sentinel)
|
||||
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
for row, end in enumerate(ends):
|
||||
touched = min(-(-end // _MASK_TILE) * _MASK_TILE, WIDE)
|
||||
tail = logits[row, touched:]
|
||||
assert torch.equal(tail, torch.full_like(tail, sentinel)), (
|
||||
f"row {row} (end {end}) was written past column {touched}"
|
||||
)
|
||||
# And the bound is tight: a row that ends early must not have cost a
|
||||
# full-width pass, which is the whole point of the kernel.
|
||||
assert touched <= end + _MASK_TILE
|
||||
|
||||
|
||||
def test_cudagraph_replay_tracks_changing_ends():
|
||||
"""The grid is static so a FULL capture stays valid when ends change.
|
||||
|
||||
Capturing bakes the grid, so a data-dependent program count would freeze
|
||||
whatever lengths were resident at capture. Replaying against longer and
|
||||
shorter contexts than the captured ones is the property that matters.
|
||||
"""
|
||||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
apply_candidate_mask,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_apply_candidate_mask_strided,
|
||||
)
|
||||
|
||||
rows = 4
|
||||
captured_ends = [1024, 4096, 8192, 2048]
|
||||
logits, row_ks, row_ke, candidates = _inputs(
|
||||
rows,
|
||||
WIDE,
|
||||
captured_ends,
|
||||
None,
|
||||
block_size=BLOCK_SIZE,
|
||||
col_stride=1,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
base = logits.clone()
|
||||
|
||||
# Warm up outside the capture: Triton compiles on first call, and autotune
|
||||
# launches would otherwise happen inside the graph.
|
||||
_apply_candidate_mask_strided(
|
||||
logits.clone(), row_ks, row_ke, candidates, BLOCK_SIZE, 1
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
_apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
|
||||
for replay_ends in ([1, 131072, WIDE, 0], [WIDE, 1025, 3, 65536]):
|
||||
row_ke.copy_(torch.tensor(replay_ends, device="cuda", dtype=row_ke.dtype))
|
||||
logits.copy_(base)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
ref = base.clone()
|
||||
apply_candidate_mask(ref, row_ks, row_ke, candidates, BLOCK_SIZE, 1)
|
||||
_assert_agrees_within_ends(ref, logits, row_ke, 1)
|
||||
@@ -819,6 +819,123 @@ def rocm_fp8_mqa_logits(
|
||||
return fp8_mqa_logits_torch(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke)
|
||||
|
||||
|
||||
# Programs along the column axis of the decode mask grid. The shared kernel in
|
||||
# model_executor/kernels/attention/dsa/candidate_blocks.py launches one program
|
||||
# per 1024-column tile, which is 1024 of them per row at max_model_len 1048576
|
||||
# with only the first few doing work. This is a fixed count that strides
|
||||
# instead, measured on gfx950; it is not a portable choice, which is why this
|
||||
# variant lives here rather than replacing the shared one.
|
||||
_MASK_GRID_COLS = 128
|
||||
# Columns each program handles per iteration. Work stops at the tile boundary
|
||||
# containing a row's end rather than at the end itself; the overshoot is
|
||||
# masked the same way the shared kernel masks it, so it costs a tile and
|
||||
# changes nothing.
|
||||
_MASK_TILE = 1024
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["width", "nblocks"])
|
||||
def _mask_candidates_strided_kernel(
|
||||
logits,
|
||||
starts,
|
||||
ends,
|
||||
flags,
|
||||
stride_row,
|
||||
stride_col,
|
||||
stride_start,
|
||||
stride_end,
|
||||
width,
|
||||
nblocks,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_STARTS: tl.constexpr,
|
||||
ROW_REPEAT: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
|
||||
end = tl.load(ends + row // ROW_REPEAT * stride_end)
|
||||
edge = tl.load(flags + row * (nblocks + 1) + nblocks)
|
||||
step = tl.num_programs(1) * TILE
|
||||
tile_start = tl.program_id(1) * TILE
|
||||
# The shared kernel also sanitizes columns at or past `end`. Nothing reads
|
||||
# them: top_k_per_row_decode bounds its scan by the same row ends passed
|
||||
# here, and persistent_topk/cooperative_topk clamp by the same lengths.
|
||||
# Stopping at `end` is what makes the cost track the live context rather
|
||||
# than max_model_len.
|
||||
while tile_start < end:
|
||||
cols = tile_start + tl.arange(0, TILE)
|
||||
valid = (cols >= start) & (cols < end) & (cols < width)
|
||||
block = (cols - start) // BLOCK_SIZE
|
||||
keep = tl.load(flags + row * (nblocks + 1) + block, valid, other=0)
|
||||
keep = (keep != 0) | ((cols == width - 1) & (edge != 0))
|
||||
tl.store(
|
||||
logits + row * stride_row + cols * stride_col,
|
||||
-float("inf"),
|
||||
(cols < width) & ~(valid & keep),
|
||||
)
|
||||
tile_start += step
|
||||
|
||||
|
||||
def _apply_candidate_mask_strided(
|
||||
logits: torch.Tensor,
|
||||
row_ks: torch.Tensor | None,
|
||||
row_ke: torch.Tensor,
|
||||
candidate_blocks: torch.Tensor,
|
||||
block_size: int,
|
||||
row_repeat: int = 1,
|
||||
) -> None:
|
||||
"""ROCm decode variant of ``apply_candidate_mask``.
|
||||
|
||||
Same masking semantics over ``[0, end)``, but the grid is sized by a fixed
|
||||
program count rather than by the logits width. Only worth using where the
|
||||
width is the ``max_model_len`` workspace and the live context is far
|
||||
shorter, i.e. the paged decode path below; the prefill chunks pass
|
||||
chunk-sized logits and stay on the shared kernel.
|
||||
"""
|
||||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||||
_candidate_flags_kernel,
|
||||
)
|
||||
|
||||
rows, width = logits.shape
|
||||
if not rows or not width:
|
||||
return
|
||||
nblocks = triton.cdiv(width, block_size)
|
||||
flags = torch.empty((rows, nblocks + 1), device=logits.device, dtype=torch.uint8)
|
||||
start_stride = row_ks.stride(0) if row_ks is not None else 0
|
||||
_candidate_flags_kernel[(rows,)](
|
||||
candidate_blocks,
|
||||
row_ks,
|
||||
flags,
|
||||
*candidate_blocks.stride(),
|
||||
start_stride,
|
||||
width,
|
||||
nblocks,
|
||||
block_size,
|
||||
candidate_blocks.shape[1],
|
||||
row_ks is not None,
|
||||
row_repeat,
|
||||
)
|
||||
# Derived from width, which is a tensor shape, so the grid stays static and
|
||||
# a FULL cudagraph capture remains valid across replays; only the loop trip
|
||||
# count inside the kernel is data-dependent. The min keeps narrow widths
|
||||
# from launching programs that would only fall through.
|
||||
grid_cols = min(_MASK_GRID_COLS, triton.cdiv(width, _MASK_TILE))
|
||||
_mask_candidates_strided_kernel[(rows, grid_cols)](
|
||||
logits,
|
||||
row_ks,
|
||||
row_ke,
|
||||
flags,
|
||||
*logits.stride(),
|
||||
start_stride,
|
||||
row_ke.stride(0),
|
||||
width,
|
||||
nblocks,
|
||||
block_size,
|
||||
row_ks is not None,
|
||||
row_repeat,
|
||||
_MASK_TILE,
|
||||
)
|
||||
|
||||
|
||||
def _max_decode_logits_rows(num_batched_tokens: int) -> int:
|
||||
"""Upper bound on decode rows the paged-MQA logits buffer can ever hold.
|
||||
|
||||
@@ -1102,7 +1219,6 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
|
||||
if candidate_blocks is not None:
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||||
_apply_candidate_mask,
|
||||
_select_candidate_blocks,
|
||||
)
|
||||
|
||||
@@ -1123,7 +1239,7 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
decode_candidates,
|
||||
)
|
||||
else:
|
||||
_apply_candidate_mask(
|
||||
_apply_candidate_mask_strided(
|
||||
logits,
|
||||
row_starts,
|
||||
visible,
|
||||
|
||||
Reference in New Issue
Block a user