[Perf][Sparse MLA] Drop the atomic contention in the index remap (#50365)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-08-08 16:32:40 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 1c1077c6cc
commit 9e6be4a72b
3 changed files with 87 additions and 33 deletions
@@ -674,17 +674,20 @@ def test_dcp_filter_compacts_valid_slots_for_sparse_kernel():
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") @pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@pytest.mark.parametrize("interleave", [1, 2]) @pytest.mark.parametrize("interleave", [1, 2])
@pytest.mark.parametrize("dcp_rank", [0, 1]) @pytest.mark.parametrize("dcp_rank", [0, 1])
def test_dcp_filter_compaction_matches_reference(interleave: int, dcp_rank: int): # 384 is not a power of two, so it exercises the multi-tile atomic allocator
"""In-kernel compaction (atomic slot allocator across multiple column tiles) # rather than the single-tile path 1024 takes.
must, for every row, produce exactly the rank-owned physical slots packed @pytest.mark.parametrize("num_topk", [1024, 384])
into [0, valid_count) with -1 in the tail -- the same SET a reference filter def test_dcp_filter_compaction_matches_reference(
+ sort/gather produces. Uses wide rows (> BLOCK_N valid slots) so the interleave: int, dcp_rank: int, num_topk: int
cross-tile atomic allocation is exercised, with interior -1 gaps.""" ):
"""In-kernel compaction must, for every row, produce exactly the rank-owned
physical slots packed into [0, valid_count) with -1 in the tail -- the same
SET a reference filter + sort/gather produces. Uses wide rows (> BLOCK_N
valid slots), with interior -1 gaps."""
device = torch.device("cuda") device = torch.device("cuda")
torch.manual_seed(7) torch.manual_seed(7)
dcp_size = 2 dcp_size = 2
block_size = 8 block_size = 8
num_topk = 1024 # > BLOCK_N(128) -> multiple tiles per row
num_rows = 5 num_rows = 5
max_blocks = 64 max_blocks = 64
seq = max_blocks * block_size seq = max_blocks * block_size
@@ -698,7 +701,9 @@ def test_dcp_filter_compaction_matches_reference(interleave: int, dcp_rank: int)
(num_rows, num_topk), -1, dtype=torch.int32, device=device (num_rows, num_topk), -1, dtype=torch.int32, device=device
) )
for r in range(num_rows): for r in range(num_rows):
n_valid = int(torch.randint(200, 600, (1,)).item()) # Ids are distinct, so a row holds at most `seq` valid entries.
hi = min(num_topk * 3 // 4, seq)
n_valid = int(torch.randint(num_topk // 4, hi, (1,)).item())
perm = torch.randperm(seq, device=device)[:n_valid].to(torch.int32) perm = torch.randperm(seq, device=device)[:n_valid].to(torch.int32)
token_indices[r, :n_valid] = perm token_indices[r, :n_valid] = perm
+13 -9
View File
@@ -1250,32 +1250,36 @@ def test_split_indexer_prefill_chunks_single_request_overflow():
assert out == expected assert out == expected
def test_triton_convert_returns_valid_counts(): # 384 is not a power of two, so it counts via the tiled atomic accumulation
# rather than the single-tile path 128 takes.
@pytest.mark.parametrize("num_topk_tokens", [128, 384])
def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
"""Test that return_valid_counts correctly counts non-negative indices.""" """Test that return_valid_counts correctly counts non-negative indices."""
device = torch.device(DEVICE_TYPE) device = torch.device(DEVICE_TYPE)
num_tokens = 8 num_tokens = 8
num_requests = 2 num_requests = 2
max_blocks_per_req = 10 max_blocks_per_req = 10
block_size = 64 block_size = 64
num_topk_tokens = 128
req_id = torch.tensor([0, 0, 0, 0, 1, 1, 1, 1], dtype=torch.int32, device=device) req_id = torch.tensor([0, 0, 0, 0, 1, 1, 1, 1], dtype=torch.int32, device=device)
block_table = torch.arange( block_table = torch.arange(
num_requests * max_blocks_per_req, dtype=torch.int32, device=device num_requests * max_blocks_per_req, dtype=torch.int32, device=device
).view(num_requests, max_blocks_per_req) ).view(num_requests, max_blocks_per_req)
# Create token indices with varying numbers of valid entries # Create token indices with varying numbers of valid entries: half the row,
# Token 0: 64 valid, 64 invalid (-1) # a quarter of it, the whole row, then a single valid entry -- twice over.
# Token 1: 32 valid, 96 invalid
# Token 2: 128 valid (all)
# Token 3: 1 valid, 127 invalid
# etc.
token_indices = torch.full( token_indices = torch.full(
(num_tokens, num_topk_tokens), -1, dtype=torch.int32, device=device (num_tokens, num_topk_tokens), -1, dtype=torch.int32, device=device
) )
valid_counts_per_token = [
num_topk_tokens // 2,
num_topk_tokens // 4,
num_topk_tokens,
1,
] * 2
expected_valid = [] expected_valid = []
for i in range(num_tokens): for i in range(num_tokens):
num_valid = [64, 32, 128, 1, 64, 32, 128, 1][i] num_valid = valid_counts_per_token[i]
token_indices[i, :num_valid] = torch.arange( token_indices[i, :num_valid] = torch.arange(
num_valid, dtype=torch.int32, device=device num_valid, dtype=torch.int32, device=device
) % (block_size * max_blocks_per_req) ) % (block_size * max_blocks_per_req)
+61 -16
View File
@@ -23,6 +23,9 @@ def _convert_req_index_to_global_index_kernel(
BLOCK_N: tl.constexpr, # tile width along columns BLOCK_N: tl.constexpr, # tile width along columns
HAS_PREFILL: tl.constexpr, HAS_PREFILL: tl.constexpr,
COUNT_VALID: tl.constexpr, # whether to count valid indices COUNT_VALID: tl.constexpr, # whether to count valid indices
# BLOCK_N == NUM_TOPK_TOKENS: one program owns the row, so the valid count
# is an in-register reduction and needs no atomic.
SINGLE_TILE: tl.constexpr,
# When set, scatter valid slots to a contiguous prefix [0, valid_count) using # When set, scatter valid slots to a contiguous prefix [0, valid_count) using
# valid_count_ptr as an atomic slot allocator (DCP filtering leaves interior # valid_count_ptr as an atomic slot allocator (DCP filtering leaves interior
# -1 gaps; the trtllm-gen sparse kernel reads the first valid_count entries). # -1 gaps; the trtllm-gen sparse kernel reads the first valid_count entries).
@@ -99,10 +102,15 @@ def _convert_req_index_to_global_index_kernel(
# sum gives each valid lane a distinct local offset; one atomic add of the # sum gives each valid lane a distinct local offset; one atomic add of the
# tile's valid count reserves a contiguous base across racing tiles. The # tile's valid count reserves a contiguous base across racing tiles. The
# out buffer is pre-filled with -1, so unwritten tail slots stay -1. # out buffer is pre-filled with -1, so unwritten tail slots stay -1.
# With no racing tiles the base is 0 and the allocator becomes a store.
is_valid = (~is_invalid_tok).to(tl.int32) is_valid = (~is_invalid_tok).to(tl.int32)
local_offset = tl.cumsum(is_valid) - is_valid local_offset = tl.cumsum(is_valid) - is_valid
tile_valid_count = tl.sum(is_valid) tile_valid_count = tl.sum(is_valid)
base = tl.atomic_add(valid_count_ptr + token_id, tile_valid_count) if SINGLE_TILE:
base = 0
tl.store(valid_count_ptr + token_id, tile_valid_count)
else:
base = tl.atomic_add(valid_count_ptr + token_id, tile_valid_count)
dest = base + local_offset dest = base + local_offset
out_ptr_dest = out_ptr + token_id * out_stride0 + dest * out_stride1 out_ptr_dest = out_ptr + token_id * out_stride0 + dest * out_stride1
tl.store(out_ptr_dest, out_val, mask=is_valid == 1) tl.store(out_ptr_dest, out_val, mask=is_valid == 1)
@@ -111,10 +119,36 @@ def _convert_req_index_to_global_index_kernel(
out_ptr_ij = out_ptr + token_id * out_stride0 + indice_id * out_stride1 out_ptr_ij = out_ptr + token_id * out_stride0 + indice_id * out_stride1
tl.store(out_ptr_ij, out_val) tl.store(out_ptr_ij, out_val)
# Count valid indices in this tile and atomically add to row total # Accumulate the tile's valid count into the row total; a single tile's
# reduction *is* the total.
if COUNT_VALID: if COUNT_VALID:
tile_valid_count = tl.sum((~is_invalid_tok).to(tl.int32)) tile_valid_count = tl.sum((~is_invalid_tok).to(tl.int32))
tl.atomic_add(valid_count_ptr + token_id, tile_valid_count) if SINGLE_TILE:
tl.store(valid_count_ptr + token_id, tile_valid_count)
else:
tl.atomic_add(valid_count_ptr + token_id, tile_valid_count)
def _remap_tiling(
NUM_TOPK_TOKENS: int, BLOCK_N: int, count_valid: bool
) -> tuple[bool, int, int, int]:
"""Pick the column tiling for the index remap kernel.
Counting the valid slots per row is the only reason the column tiles have to
talk to each other, so when counting give one program the whole row: the
count becomes an in-register reduction plus a plain store, needing neither
atomics nor a zero-initialized counter. The row is one ``tl.arange``, so this
needs a power-of-two width; other top-k sizes stay tiled and atomic.
Returns:
(single_tile, block_n, tiles_per_row, num_warps)
"""
single_tile = (
count_valid and triton.next_power_of_2(NUM_TOPK_TOKENS) == NUM_TOPK_TOKENS
)
if single_tile:
return True, NUM_TOPK_TOKENS, 1, 8
return False, BLOCK_N, NUM_TOPK_TOKENS // BLOCK_N, 4
def triton_convert_req_index_to_global_index( def triton_convert_req_index_to_global_index(
@@ -172,7 +206,10 @@ def triton_convert_req_index_to_global_index(
num_tokens = req_id.shape[0] num_tokens = req_id.shape[0]
max_num_blocks_per_req = block_table.shape[1] max_num_blocks_per_req = block_table.shape[1]
tiles_per_row = NUM_TOPK_TOKENS // BLOCK_N
single_tile, block_n, tiles_per_row, num_warps = _remap_tiling(
NUM_TOPK_TOKENS, BLOCK_N, return_valid_counts
)
# Ensure contiguous tensors on the same device # Ensure contiguous tensors on the same device
req_id_c = req_id.contiguous() req_id_c = req_id.contiguous()
@@ -180,12 +217,11 @@ def triton_convert_req_index_to_global_index(
token_indices_c = token_indices.contiguous() token_indices_c = token_indices.contiguous()
out = torch.empty_like(token_indices_c) out = torch.empty_like(token_indices_c)
# Allocate valid count buffer if needed (must be zero-initialized for atomics)
valid_counts: torch.Tensor | None = None valid_counts: torch.Tensor | None = None
if return_valid_counts: if return_valid_counts:
valid_counts = torch.zeros( # Zero-init only matters for the atomic accumulation path.
num_tokens, dtype=torch.int32, device=token_indices.device alloc = torch.empty if single_tile else torch.zeros
) valid_counts = alloc(num_tokens, dtype=torch.int32, device=token_indices.device)
# Strides in elements # Strides in elements
bt_stride0, bt_stride1 = block_table_c.stride() bt_stride0, bt_stride1 = block_table_c.stride()
@@ -213,9 +249,10 @@ def triton_convert_req_index_to_global_index(
# shapes / constexprs # shapes / constexprs
max_num_blocks_per_req, max_num_blocks_per_req,
BLOCK_SIZE, BLOCK_SIZE,
BLOCK_N, block_n,
HAS_PREFILL_WORKSPACE, HAS_PREFILL_WORKSPACE,
return_valid_counts, return_valid_counts,
single_tile,
False, # COMPACT_TO_FRONT: keep input column == output column False, # COMPACT_TO_FRONT: keep input column == output column
# DCP disabled (no-op de-interleave) # DCP disabled (no-op de-interleave)
1, 1,
@@ -228,6 +265,7 @@ def triton_convert_req_index_to_global_index(
ti_stride1, ti_stride1,
out_stride0, out_stride0,
out_stride1, out_stride1,
num_warps=num_warps,
) )
if return_valid_counts: if return_valid_counts:
@@ -287,15 +325,20 @@ def triton_filter_and_convert_dcp_index(
num_tokens = req_id.shape[0] num_tokens = req_id.shape[0]
max_num_blocks_per_req = block_table.shape[1] max_num_blocks_per_req = block_table.shape[1]
tiles_per_row = NUM_TOPK_TOKENS // BLOCK_N
req_id_c = req_id.contiguous() req_id_c = req_id.contiguous()
block_table_c = block_table.contiguous() block_table_c = block_table.contiguous()
token_indices_c = token_indices.contiguous() token_indices_c = token_indices.contiguous()
# The compaction uses the valid-count buffer as an atomic slot allocator, so # The compaction uses the valid-count buffer as a slot allocator, so it
# it requires counting. Pre-fill out with -1 so the unwritten tail stays -1. # requires counting. Pre-fill out with -1 so the unwritten tail stays -1.
count_valid = return_valid_counts or compact_valid_to_front count_valid = return_valid_counts or compact_valid_to_front
# The compaction builds on the counting, so it shares the tiling.
single_tile, block_n, tiles_per_row, num_warps = _remap_tiling(
NUM_TOPK_TOKENS, BLOCK_N, count_valid
)
if compact_valid_to_front: if compact_valid_to_front:
out = torch.full_like(token_indices_c, -1) out = torch.full_like(token_indices_c, -1)
else: else:
@@ -303,9 +346,9 @@ def triton_filter_and_convert_dcp_index(
valid_counts: torch.Tensor | None = None valid_counts: torch.Tensor | None = None
if count_valid: if count_valid:
valid_counts = torch.zeros( # Zero-init only matters for the atomic accumulation path.
num_tokens, dtype=torch.int32, device=token_indices.device alloc = torch.empty if single_tile else torch.zeros
) valid_counts = alloc(num_tokens, dtype=torch.int32, device=token_indices.device)
bt_stride0, bt_stride1 = block_table_c.stride() bt_stride0, bt_stride1 = block_table_c.stride()
ti_stride0, ti_stride1 = token_indices_c.stride() ti_stride0, ti_stride1 = token_indices_c.stride()
@@ -322,9 +365,10 @@ def triton_filter_and_convert_dcp_index(
None, None,
max_num_blocks_per_req, max_num_blocks_per_req,
BLOCK_SIZE, BLOCK_SIZE,
BLOCK_N, block_n,
False, # HAS_PREFILL False, # HAS_PREFILL
count_valid, count_valid,
single_tile,
compact_valid_to_front, compact_valid_to_front,
dcp_size, dcp_size,
dcp_rank, dcp_rank,
@@ -335,6 +379,7 @@ def triton_filter_and_convert_dcp_index(
ti_stride1, ti_stride1,
out_stride0, out_stride0,
out_stride1, out_stride1,
num_warps=num_warps,
) )
if return_valid_counts: if return_valid_counts: