diff --git a/tests/kernels/attention/test_rocm_aiter_candidate_mask.py b/tests/kernels/attention/test_rocm_aiter_candidate_mask.py new file mode 100644 index 0000000000..85e742a096 --- /dev/null +++ b/tests/kernels/attention/test_rocm_aiter_candidate_mask.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm correctness tests for the strided DSA candidate mask. + +``_apply_candidate_mask_strided`` is a decode-path variant of the shared +``apply_candidate_mask``. It differs in two ways that need pinning down: + +* it launches a fixed number of programs that stride the column axis, rather + than one program per 1024-column tile, so at ``max_model_len`` widths the + grid no longer scales with a workspace that is mostly padding; +* it stops at each row's ``end`` instead of sanitizing the whole width. + +The second is only sound because every consumer bounds its scan by the same +ends -- ``top_k_per_row_decode`` takes ``rowEnd`` from ``seq_lens``. So the +contract is agreement with the shared kernel over ``[0, end)`` and nothing +beyond it, which is what these tests assert rather than comparing whole rows. +""" + +import pytest +import torch + +from vllm.platforms import current_platform + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-specific tests" +) + +BLOCK_SIZE = 8 +TOPK_BLOCKS = 64 +# Above 128 tiles the grid saturates and programs start striding; below it the +# min() clamp applies instead. Both regimes need covering, so the widths here +# deliberately straddle 128 * 1024. +NARROW = 8192 +WIDE = 262144 + + +def _inputs(rows, width, ends, starts, *, block_size, col_stride, dtype): + """Logits, bounds and candidate blocks shared by both kernels.""" + torch.manual_seed(0) + # A column stride > 1 keeps the kernels honest about stride_col; the decode + # caller slices a workspace, so contiguous rows are not guaranteed. + logits = torch.randn(rows, width * col_stride, device="cuda")[:, ::col_stride] + logits[0, : min(16, width)] = 0.0 + if rows > 2: + logits[2, 5] = float("nan") + row_ke = torch.tensor(ends, device="cuda", dtype=dtype) + row_ks = torch.tensor(starts, device="cuda", dtype=dtype) if starts else None + nblocks = (width + block_size - 1) // block_size + candidates = torch.randint( + 0, nblocks, (rows, TOPK_BLOCKS), device="cuda", dtype=torch.int32 + ) + # -1 is the "no block" sentinel the selector emits for short rows. + candidates[:, ::7] = -1 + return logits, row_ks, row_ke, candidates + + +def _run_both(logits, row_ks, row_ke, candidates, block_size, row_repeat): + from vllm.model_executor.kernels.attention.dsa.candidate_blocks import ( + apply_candidate_mask, + ) + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _apply_candidate_mask_strided, + ) + + ref = logits.clone() + got = logits.clone() + apply_candidate_mask(ref, row_ks, row_ke, candidates, block_size, row_repeat) + _apply_candidate_mask_strided( + got, row_ks, row_ke, candidates, block_size, row_repeat + ) + return ref, got + + +def _assert_agrees_within_ends(ref, got, row_ke, row_repeat): + """Compare only the region a consumer can read.""" + for row in range(ref.shape[0]): + end = int(row_ke[row // row_repeat]) + if end <= 0: + continue + torch.testing.assert_close( + got[row, :end], + ref[row, :end], + rtol=0, + atol=0, + equal_nan=True, + msg=lambda m, r=row, e=end: f"row {r} differs within [0, {e}): {m}", + ) + + +@pytest.mark.parametrize("width", [NARROW, WIDE]) +@pytest.mark.parametrize("col_stride", [1, 2]) +def test_matches_shared_kernel_ragged_ends(width, col_stride): + """Ends on and around tile boundaries, plus the empty and full rows.""" + ends = [0, 1, 1023, 1024, 1025, width - 1, width, width // 2] + logits, row_ks, row_ke, candidates = _inputs( + len(ends), + width, + ends, + None, + block_size=BLOCK_SIZE, + col_stride=col_stride, + dtype=torch.int32, + ) + ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1) + _assert_agrees_within_ends(ref, got, row_ke, 1) + + +@pytest.mark.parametrize("block_size", [8, 16]) +def test_matches_shared_kernel_with_starts(block_size): + """Non-zero starts: the decode caller passes zeros, prefill does not.""" + ends = [0, 64, 4096, 8192] + starts = [0, 7, 1024, 3] + logits, row_ks, row_ke, candidates = _inputs( + len(ends), + NARROW, + ends, + starts, + block_size=block_size, + col_stride=1, + dtype=torch.int64, + ) + ref, got = _run_both(logits, row_ks, row_ke, candidates, block_size, 1) + _assert_agrees_within_ends(ref, got, row_ke, 1) + + +def test_matches_shared_kernel_row_repeat(): + """Speculative decode: next_n logit rows share one bound.""" + row_repeat = 3 + ends = [0, 1025, WIDE] + logits, row_ks, row_ke, candidates = _inputs( + len(ends) * row_repeat, + WIDE, + ends, + None, + block_size=BLOCK_SIZE, + col_stride=1, + dtype=torch.int32, + ) + ref, got = _run_both(logits, row_ks, row_ke, candidates, BLOCK_SIZE, row_repeat) + _assert_agrees_within_ends(ref, got, row_ke, row_repeat) + + +def test_leaves_columns_past_end_untouched(): + """The deliberate difference from the shared kernel, pinned explicitly. + + Skipping the tail is where the speedup comes from, so a change that + quietly restores full-width sanitizing should fail here rather than just + get slower. Work stops at the tile boundary containing ``end``, not at + ``end`` itself, so that is the bound asserted. + """ + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _MASK_TILE, + _apply_candidate_mask_strided, + ) + + ends = [0, 1, 1025, WIDE // 2] + logits, row_ks, row_ke, candidates = _inputs( + len(ends), + WIDE, + ends, + None, + block_size=BLOCK_SIZE, + col_stride=1, + dtype=torch.int32, + ) + sentinel = 1.5 + logits.fill_(sentinel) + _apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1) + for row, end in enumerate(ends): + touched = min(-(-end // _MASK_TILE) * _MASK_TILE, WIDE) + tail = logits[row, touched:] + assert torch.equal(tail, torch.full_like(tail, sentinel)), ( + f"row {row} (end {end}) was written past column {touched}" + ) + # And the bound is tight: a row that ends early must not have cost a + # full-width pass, which is the whole point of the kernel. + assert touched <= end + _MASK_TILE + + +def test_cudagraph_replay_tracks_changing_ends(): + """The grid is static so a FULL capture stays valid when ends change. + + Capturing bakes the grid, so a data-dependent program count would freeze + whatever lengths were resident at capture. Replaying against longer and + shorter contexts than the captured ones is the property that matters. + """ + from vllm.model_executor.kernels.attention.dsa.candidate_blocks import ( + apply_candidate_mask, + ) + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _apply_candidate_mask_strided, + ) + + rows = 4 + captured_ends = [1024, 4096, 8192, 2048] + logits, row_ks, row_ke, candidates = _inputs( + rows, + WIDE, + captured_ends, + None, + block_size=BLOCK_SIZE, + col_stride=1, + dtype=torch.int32, + ) + base = logits.clone() + + # Warm up outside the capture: Triton compiles on first call, and autotune + # launches would otherwise happen inside the graph. + _apply_candidate_mask_strided( + logits.clone(), row_ks, row_ke, candidates, BLOCK_SIZE, 1 + ) + torch.accelerator.synchronize() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + _apply_candidate_mask_strided(logits, row_ks, row_ke, candidates, BLOCK_SIZE, 1) + + for replay_ends in ([1, 131072, WIDE, 0], [WIDE, 1025, 3, 65536]): + row_ke.copy_(torch.tensor(replay_ends, device="cuda", dtype=row_ke.dtype)) + logits.copy_(base) + graph.replay() + torch.accelerator.synchronize() + + ref = base.clone() + apply_candidate_mask(ref, row_ks, row_ke, candidates, BLOCK_SIZE, 1) + _assert_agrees_within_ends(ref, logits, row_ke, 1) diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 598baee0b0..44b6017a05 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -819,6 +819,123 @@ def rocm_fp8_mqa_logits( return fp8_mqa_logits_torch(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke) +# Programs along the column axis of the decode mask grid. The shared kernel in +# model_executor/kernels/attention/dsa/candidate_blocks.py launches one program +# per 1024-column tile, which is 1024 of them per row at max_model_len 1048576 +# with only the first few doing work. This is a fixed count that strides +# instead, measured on gfx950; it is not a portable choice, which is why this +# variant lives here rather than replacing the shared one. +_MASK_GRID_COLS = 128 +# Columns each program handles per iteration. Work stops at the tile boundary +# containing a row's end rather than at the end itself; the overshoot is +# masked the same way the shared kernel masks it, so it costs a tile and +# changes nothing. +_MASK_TILE = 1024 + + +@triton.jit(do_not_specialize=["width", "nblocks"]) +def _mask_candidates_strided_kernel( + logits, + starts, + ends, + flags, + stride_row, + stride_col, + stride_start, + stride_end, + width, + nblocks, + BLOCK_SIZE: tl.constexpr, + HAS_STARTS: tl.constexpr, + ROW_REPEAT: tl.constexpr, + TILE: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0 + end = tl.load(ends + row // ROW_REPEAT * stride_end) + edge = tl.load(flags + row * (nblocks + 1) + nblocks) + step = tl.num_programs(1) * TILE + tile_start = tl.program_id(1) * TILE + # The shared kernel also sanitizes columns at or past `end`. Nothing reads + # them: top_k_per_row_decode bounds its scan by the same row ends passed + # here, and persistent_topk/cooperative_topk clamp by the same lengths. + # Stopping at `end` is what makes the cost track the live context rather + # than max_model_len. + while tile_start < end: + cols = tile_start + tl.arange(0, TILE) + valid = (cols >= start) & (cols < end) & (cols < width) + block = (cols - start) // BLOCK_SIZE + keep = tl.load(flags + row * (nblocks + 1) + block, valid, other=0) + keep = (keep != 0) | ((cols == width - 1) & (edge != 0)) + tl.store( + logits + row * stride_row + cols * stride_col, + -float("inf"), + (cols < width) & ~(valid & keep), + ) + tile_start += step + + +def _apply_candidate_mask_strided( + logits: torch.Tensor, + row_ks: torch.Tensor | None, + row_ke: torch.Tensor, + candidate_blocks: torch.Tensor, + block_size: int, + row_repeat: int = 1, +) -> None: + """ROCm decode variant of ``apply_candidate_mask``. + + Same masking semantics over ``[0, end)``, but the grid is sized by a fixed + program count rather than by the logits width. Only worth using where the + width is the ``max_model_len`` workspace and the live context is far + shorter, i.e. the paged decode path below; the prefill chunks pass + chunk-sized logits and stay on the shared kernel. + """ + from vllm.model_executor.kernels.attention.dsa.candidate_blocks import ( + _candidate_flags_kernel, + ) + + rows, width = logits.shape + if not rows or not width: + return + nblocks = triton.cdiv(width, block_size) + flags = torch.empty((rows, nblocks + 1), device=logits.device, dtype=torch.uint8) + start_stride = row_ks.stride(0) if row_ks is not None else 0 + _candidate_flags_kernel[(rows,)]( + candidate_blocks, + row_ks, + flags, + *candidate_blocks.stride(), + start_stride, + width, + nblocks, + block_size, + candidate_blocks.shape[1], + row_ks is not None, + row_repeat, + ) + # Derived from width, which is a tensor shape, so the grid stays static and + # a FULL cudagraph capture remains valid across replays; only the loop trip + # count inside the kernel is data-dependent. The min keeps narrow widths + # from launching programs that would only fall through. + grid_cols = min(_MASK_GRID_COLS, triton.cdiv(width, _MASK_TILE)) + _mask_candidates_strided_kernel[(rows, grid_cols)]( + logits, + row_ks, + row_ke, + flags, + *logits.stride(), + start_stride, + row_ke.stride(0), + width, + nblocks, + block_size, + row_ks is not None, + row_repeat, + _MASK_TILE, + ) + + def _max_decode_logits_rows(num_batched_tokens: int) -> int: """Upper bound on decode rows the paged-MQA logits buffer can ever hold. @@ -1102,7 +1219,6 @@ def rocm_aiter_sparse_attn_indexer( if candidate_blocks is not None: from vllm.model_executor.layers.sparse_attn_indexer import ( - _apply_candidate_mask, _select_candidate_blocks, ) @@ -1123,7 +1239,7 @@ def rocm_aiter_sparse_attn_indexer( decode_candidates, ) else: - _apply_candidate_mask( + _apply_candidate_mask_strided( logits, row_starts, visible,