[Bugfix][ROCm] Drop -1 sentinels when building the ragged sparse-MLA indices (#58058)
Signed-off-by: Mehmet Cagri Kaymak <mehmet.kaymak@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9b29370524
commit
75fad5bbef
@@ -8,6 +8,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="Only used by ROCm"
|
||||
@@ -424,6 +425,37 @@ def test_compute_global_topk_ragged_indices_and_indptr() -> None:
|
||||
torch.testing.assert_close(actual_lens, expected_lens)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width", [16, 640, 2176])
|
||||
@pytest.mark.parametrize("num_queries", [1, 70, 1000])
|
||||
@torch.inference_mode()
|
||||
def test_build_ragged_indices_from_dense_drops_invalid_entries(
|
||||
width: int, num_queries: int
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
build_ragged_indices_from_dense,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(width + num_queries)
|
||||
num_rows = 5000
|
||||
indices = torch.randint(0, num_rows + 100, (num_queries, width))
|
||||
indices[torch.rand(num_queries, width) < 0.3] = -1
|
||||
lengths = torch.randint(0, width + 8, (num_queries,))
|
||||
|
||||
flat, indptr = build_ragged_indices_from_dense(
|
||||
indices.to(device, torch.int32),
|
||||
lengths.to(device, torch.int32),
|
||||
num_rows=num_rows,
|
||||
)
|
||||
|
||||
# Each row keeps its in-range entries within its length, in order.
|
||||
expected = [
|
||||
[x for x in row[:n] if 0 <= x < num_rows]
|
||||
for row, n in zip(indices.tolist(), lengths.tolist())
|
||||
]
|
||||
assert _rows_from_ragged(flat, indptr) == expected
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_combine_topk_swa_indices_adds_image_visibility() -> None:
|
||||
from vllm.models.deepseek_v4.amd.rocm import combine_topk_swa_indices
|
||||
|
||||
@@ -2017,7 +2017,48 @@ def _validate_dsv4_sparse_dims(
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_dense_prefix_to_ragged_kernel(
|
||||
def _count_valid_row_entries_kernel(
|
||||
indices_ptr,
|
||||
lengths_ptr,
|
||||
counts_ptr,
|
||||
indptr_ptr,
|
||||
indices_stride0,
|
||||
num_rows_limit,
|
||||
num_queries,
|
||||
row_width,
|
||||
BLOCK_R: tl.constexpr,
|
||||
BLOCK_W: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
# The scan fills indptr[1:], so write indptr[0] here and skip a zero fill.
|
||||
if pid == 0:
|
||||
tl.store(indptr_ptr, 0)
|
||||
|
||||
rows = pid * BLOCK_R + tl.arange(0, BLOCK_R)
|
||||
row_ok = rows < num_queries
|
||||
row_len = tl.load(lengths_ptr + rows, mask=row_ok, other=0)
|
||||
|
||||
acc = tl.zeros((BLOCK_R,), dtype=tl.int32)
|
||||
for start in range(0, row_width, BLOCK_W):
|
||||
cols = start + tl.arange(0, BLOCK_W)
|
||||
in_row = cols[None, :] < row_width
|
||||
vals = tl.load(
|
||||
indices_ptr + rows[:, None] * indices_stride0 + cols[None, :],
|
||||
mask=row_ok[:, None] & in_row,
|
||||
other=-1,
|
||||
).to(tl.int32)
|
||||
keep = (
|
||||
row_ok[:, None] & in_row & (cols[None, :] < row_len[:, None]) & (vals >= 0)
|
||||
)
|
||||
if num_rows_limit >= 0:
|
||||
keep = keep & (vals < num_rows_limit)
|
||||
acc += tl.sum(keep.to(tl.int32), axis=1)
|
||||
|
||||
tl.store(counts_ptr + rows, acc, mask=row_ok)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compact_dense_row_to_ragged_kernel(
|
||||
indices_ptr,
|
||||
lengths_ptr,
|
||||
indptr_ptr,
|
||||
@@ -2025,28 +2066,25 @@ def _pack_dense_prefix_to_ragged_kernel(
|
||||
indices_stride0,
|
||||
num_rows_limit,
|
||||
row_width,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_W: tl.constexpr,
|
||||
):
|
||||
row_idx = tl.program_id(0)
|
||||
block_idx = tl.program_id(1)
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
|
||||
row_len = tl.load(lengths_ptr + row_idx)
|
||||
if block_idx * BLOCK_SIZE >= row_len:
|
||||
return
|
||||
|
||||
mask = offsets < row_len
|
||||
safe_offsets = tl.where(offsets < row_width, offsets, 0)
|
||||
vals = tl.load(
|
||||
indices_ptr + row_idx * indices_stride0 + safe_offsets,
|
||||
mask=mask & (offsets < row_width),
|
||||
other=-1,
|
||||
).to(tl.int32)
|
||||
if num_rows_limit >= 0:
|
||||
vals = tl.where((vals >= 0) & (vals < num_rows_limit), vals, -1)
|
||||
|
||||
out_start = tl.load(indptr_ptr + row_idx)
|
||||
tl.store(out_ptr + out_start + offsets, vals, mask=mask)
|
||||
|
||||
written = 0
|
||||
for start in range(0, row_width, BLOCK_W):
|
||||
cols = start + tl.arange(0, BLOCK_W)
|
||||
in_row = cols < row_width
|
||||
vals = tl.load(
|
||||
indices_ptr + row_idx * indices_stride0 + cols, mask=in_row, other=-1
|
||||
).to(tl.int32)
|
||||
keep = in_row & (cols < row_len) & (vals >= 0)
|
||||
if num_rows_limit >= 0:
|
||||
keep = keep & (vals < num_rows_limit)
|
||||
rank = tl.cumsum(keep.to(tl.int32), axis=0) - 1
|
||||
tl.store(out_ptr + out_start + written + rank, vals, mask=keep)
|
||||
written += tl.sum(keep.to(tl.int32), axis=0)
|
||||
|
||||
|
||||
def build_ragged_indices_from_dense(
|
||||
@@ -2061,10 +2099,40 @@ def build_ragged_indices_from_dense(
|
||||
)
|
||||
|
||||
max_width = indices.shape[1] if indices.ndim == 2 else 0
|
||||
lengths = lengths.clamp(min=0, max=max_width).contiguous()
|
||||
# Both kernels bound the row by row_width and row_len, so no clamp needed.
|
||||
lengths = lengths.contiguous()
|
||||
|
||||
indptr = torch.zeros(indices.shape[0] + 1, dtype=torch.int32, device=indices.device)
|
||||
torch.cumsum(lengths, dim=0, out=indptr[1:])
|
||||
# Dense rows reserve slots they may not fill, leaving -1 anywhere in the
|
||||
# first `lengths` entries. Keep only the valid ones, since the consumers
|
||||
# index the KV pool without checking the sign.
|
||||
num_queries = indices.shape[0]
|
||||
# max_width is whatever the caller packed (align(topk + window, 128) for the
|
||||
# V4.1 indexer) and nothing caps it, so loop the row rather than size the
|
||||
# block by it.
|
||||
block = min(triton.next_power_of_2(max_width), 1024) if max_width > 0 else 1
|
||||
counts = torch.empty(num_queries, dtype=torch.int32, device=indices.device)
|
||||
indptr = torch.empty(num_queries + 1, dtype=torch.int32, device=indices.device)
|
||||
if num_queries > 0 and max_width > 0:
|
||||
# A program per row only gets row_width elements, which is launch
|
||||
# bound for narrow rows. Tile rows to fill the block.
|
||||
block_r = max(1, 1024 // block)
|
||||
_count_valid_row_entries_kernel[(triton.cdiv(num_queries, block_r),)](
|
||||
indices,
|
||||
lengths,
|
||||
counts,
|
||||
indptr,
|
||||
indices.stride(0),
|
||||
int(num_rows),
|
||||
num_queries,
|
||||
max_width,
|
||||
BLOCK_R=block_r,
|
||||
BLOCK_W=block,
|
||||
)
|
||||
else:
|
||||
counts.zero_()
|
||||
indptr.zero_()
|
||||
|
||||
torch.cumsum(counts, dim=0, out=indptr[1:])
|
||||
|
||||
if indices.numel() == 0:
|
||||
flat = torch.empty(0, dtype=torch.int32, device=indices.device)
|
||||
@@ -2075,10 +2143,7 @@ def build_ragged_indices_from_dense(
|
||||
device=indices.device,
|
||||
)
|
||||
if flat.numel() > 0:
|
||||
block_size = 128
|
||||
_pack_dense_prefix_to_ragged_kernel[
|
||||
(indices.shape[0], triton.cdiv(max_width, block_size))
|
||||
](
|
||||
_compact_dense_row_to_ragged_kernel[(indices.shape[0],)](
|
||||
indices,
|
||||
lengths,
|
||||
indptr,
|
||||
@@ -2086,7 +2151,7 @@ def build_ragged_indices_from_dense(
|
||||
indices.stride(0),
|
||||
int(num_rows),
|
||||
max_width,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_W=block,
|
||||
)
|
||||
|
||||
return flat, indptr
|
||||
|
||||
Reference in New Issue
Block a user