From 75fad5bbef7b2c0264c4b7ce6c4c10e32233d523 Mon Sep 17 00:00:00 2001 From: Mehmet Cagri Date: Mon, 28 Sep 2026 18:40:11 -0500 Subject: [PATCH] [Bugfix][ROCm] Drop -1 sentinels when building the ragged sparse-MLA indices (#58058) Signed-off-by: Mehmet Cagri Kaymak Co-authored-by: Claude Opus 5.5 --- .../attention/test_rocm_triton_attn_dsv4.py | 32 +++++ .../v1/attention/ops/rocm_aiter_mla_sparse.py | 119 ++++++++++++++---- 2 files changed, 124 insertions(+), 27 deletions(-) diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index bd41048d9b..387e690e1b 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -8,6 +8,7 @@ import pytest import torch from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_random_seed pytestmark = pytest.mark.skipif( not current_platform.is_rocm(), reason="Only used by ROCm" @@ -424,6 +425,37 @@ def test_compute_global_topk_ragged_indices_and_indptr() -> None: torch.testing.assert_close(actual_lens, expected_lens) +@pytest.mark.parametrize("width", [16, 640, 2176]) +@pytest.mark.parametrize("num_queries", [1, 70, 1000]) +@torch.inference_mode() +def test_build_ragged_indices_from_dense_drops_invalid_entries( + width: int, num_queries: int +) -> None: + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + build_ragged_indices_from_dense, + ) + + device = torch.device("cuda") + set_random_seed(width + num_queries) + num_rows = 5000 + indices = torch.randint(0, num_rows + 100, (num_queries, width)) + indices[torch.rand(num_queries, width) < 0.3] = -1 + lengths = torch.randint(0, width + 8, (num_queries,)) + + flat, indptr = build_ragged_indices_from_dense( + indices.to(device, torch.int32), + lengths.to(device, torch.int32), + num_rows=num_rows, + ) + + # Each row keeps its in-range entries within its length, in order. + expected = [ + [x for x in row[:n] if 0 <= x < num_rows] + for row, n in zip(indices.tolist(), lengths.tolist()) + ] + assert _rows_from_ragged(flat, indptr) == expected + + @torch.inference_mode() def test_combine_topk_swa_indices_adds_image_visibility() -> None: from vllm.models.deepseek_v4.amd.rocm import combine_topk_swa_indices diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index a3608a9c7e..005cbbb2ff 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -2017,7 +2017,48 @@ def _validate_dsv4_sparse_dims( @triton.jit -def _pack_dense_prefix_to_ragged_kernel( +def _count_valid_row_entries_kernel( + indices_ptr, + lengths_ptr, + counts_ptr, + indptr_ptr, + indices_stride0, + num_rows_limit, + num_queries, + row_width, + BLOCK_R: tl.constexpr, + BLOCK_W: tl.constexpr, +): + pid = tl.program_id(0) + # The scan fills indptr[1:], so write indptr[0] here and skip a zero fill. + if pid == 0: + tl.store(indptr_ptr, 0) + + rows = pid * BLOCK_R + tl.arange(0, BLOCK_R) + row_ok = rows < num_queries + row_len = tl.load(lengths_ptr + rows, mask=row_ok, other=0) + + acc = tl.zeros((BLOCK_R,), dtype=tl.int32) + for start in range(0, row_width, BLOCK_W): + cols = start + tl.arange(0, BLOCK_W) + in_row = cols[None, :] < row_width + vals = tl.load( + indices_ptr + rows[:, None] * indices_stride0 + cols[None, :], + mask=row_ok[:, None] & in_row, + other=-1, + ).to(tl.int32) + keep = ( + row_ok[:, None] & in_row & (cols[None, :] < row_len[:, None]) & (vals >= 0) + ) + if num_rows_limit >= 0: + keep = keep & (vals < num_rows_limit) + acc += tl.sum(keep.to(tl.int32), axis=1) + + tl.store(counts_ptr + rows, acc, mask=row_ok) + + +@triton.jit +def _compact_dense_row_to_ragged_kernel( indices_ptr, lengths_ptr, indptr_ptr, @@ -2025,28 +2066,25 @@ def _pack_dense_prefix_to_ragged_kernel( indices_stride0, num_rows_limit, row_width, - BLOCK_SIZE: tl.constexpr, + BLOCK_W: tl.constexpr, ): row_idx = tl.program_id(0) - block_idx = tl.program_id(1) - offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - row_len = tl.load(lengths_ptr + row_idx) - if block_idx * BLOCK_SIZE >= row_len: - return - - mask = offsets < row_len - safe_offsets = tl.where(offsets < row_width, offsets, 0) - vals = tl.load( - indices_ptr + row_idx * indices_stride0 + safe_offsets, - mask=mask & (offsets < row_width), - other=-1, - ).to(tl.int32) - if num_rows_limit >= 0: - vals = tl.where((vals >= 0) & (vals < num_rows_limit), vals, -1) - out_start = tl.load(indptr_ptr + row_idx) - tl.store(out_ptr + out_start + offsets, vals, mask=mask) + + written = 0 + for start in range(0, row_width, BLOCK_W): + cols = start + tl.arange(0, BLOCK_W) + in_row = cols < row_width + vals = tl.load( + indices_ptr + row_idx * indices_stride0 + cols, mask=in_row, other=-1 + ).to(tl.int32) + keep = in_row & (cols < row_len) & (vals >= 0) + if num_rows_limit >= 0: + keep = keep & (vals < num_rows_limit) + rank = tl.cumsum(keep.to(tl.int32), axis=0) - 1 + tl.store(out_ptr + out_start + written + rank, vals, mask=keep) + written += tl.sum(keep.to(tl.int32), axis=0) def build_ragged_indices_from_dense( @@ -2061,10 +2099,40 @@ def build_ragged_indices_from_dense( ) max_width = indices.shape[1] if indices.ndim == 2 else 0 - lengths = lengths.clamp(min=0, max=max_width).contiguous() + # Both kernels bound the row by row_width and row_len, so no clamp needed. + lengths = lengths.contiguous() - indptr = torch.zeros(indices.shape[0] + 1, dtype=torch.int32, device=indices.device) - torch.cumsum(lengths, dim=0, out=indptr[1:]) + # Dense rows reserve slots they may not fill, leaving -1 anywhere in the + # first `lengths` entries. Keep only the valid ones, since the consumers + # index the KV pool without checking the sign. + num_queries = indices.shape[0] + # max_width is whatever the caller packed (align(topk + window, 128) for the + # V4.1 indexer) and nothing caps it, so loop the row rather than size the + # block by it. + block = min(triton.next_power_of_2(max_width), 1024) if max_width > 0 else 1 + counts = torch.empty(num_queries, dtype=torch.int32, device=indices.device) + indptr = torch.empty(num_queries + 1, dtype=torch.int32, device=indices.device) + if num_queries > 0 and max_width > 0: + # A program per row only gets row_width elements, which is launch + # bound for narrow rows. Tile rows to fill the block. + block_r = max(1, 1024 // block) + _count_valid_row_entries_kernel[(triton.cdiv(num_queries, block_r),)]( + indices, + lengths, + counts, + indptr, + indices.stride(0), + int(num_rows), + num_queries, + max_width, + BLOCK_R=block_r, + BLOCK_W=block, + ) + else: + counts.zero_() + indptr.zero_() + + torch.cumsum(counts, dim=0, out=indptr[1:]) if indices.numel() == 0: flat = torch.empty(0, dtype=torch.int32, device=indices.device) @@ -2075,10 +2143,7 @@ def build_ragged_indices_from_dense( device=indices.device, ) if flat.numel() > 0: - block_size = 128 - _pack_dense_prefix_to_ragged_kernel[ - (indices.shape[0], triton.cdiv(max_width, block_size)) - ]( + _compact_dense_row_to_ragged_kernel[(indices.shape[0],)]( indices, lengths, indptr, @@ -2086,7 +2151,7 @@ def build_ragged_indices_from_dense( indices.stride(0), int(num_rows), max_width, - BLOCK_SIZE=block_size, + BLOCK_W=block, ) return flat, indptr