[Perf][Sparse MLA] Drop the atomic contention in the index remap (#50365)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-08-08 16:32:40 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 1c1077c6cc
commit 9e6be4a72b
3 changed files with 87 additions and 33 deletions
@@ -674,17 +674,20 @@ def test_dcp_filter_compacts_valid_slots_for_sparse_kernel():
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@pytest.mark.parametrize("interleave", [1, 2])
@pytest.mark.parametrize("dcp_rank", [0, 1])
def test_dcp_filter_compaction_matches_reference(interleave: int, dcp_rank: int):
"""In-kernel compaction (atomic slot allocator across multiple column tiles)
must, for every row, produce exactly the rank-owned physical slots packed
into [0, valid_count) with -1 in the tail -- the same SET a reference filter
+ sort/gather produces. Uses wide rows (> BLOCK_N valid slots) so the
cross-tile atomic allocation is exercised, with interior -1 gaps."""
# 384 is not a power of two, so it exercises the multi-tile atomic allocator
# rather than the single-tile path 1024 takes.
@pytest.mark.parametrize("num_topk", [1024, 384])
def test_dcp_filter_compaction_matches_reference(
interleave: int, dcp_rank: int, num_topk: int
):
"""In-kernel compaction must, for every row, produce exactly the rank-owned
physical slots packed into [0, valid_count) with -1 in the tail -- the same
SET a reference filter + sort/gather produces. Uses wide rows (> BLOCK_N
valid slots), with interior -1 gaps."""
device = torch.device("cuda")
torch.manual_seed(7)
dcp_size = 2
block_size = 8
num_topk = 1024 # > BLOCK_N(128) -> multiple tiles per row
num_rows = 5
max_blocks = 64
seq = max_blocks * block_size
@@ -698,7 +701,9 @@ def test_dcp_filter_compaction_matches_reference(interleave: int, dcp_rank: int)
(num_rows, num_topk), -1, dtype=torch.int32, device=device
)
for r in range(num_rows):
n_valid = int(torch.randint(200, 600, (1,)).item())
# Ids are distinct, so a row holds at most `seq` valid entries.
hi = min(num_topk * 3 // 4, seq)
n_valid = int(torch.randint(num_topk // 4, hi, (1,)).item())
perm = torch.randperm(seq, device=device)[:n_valid].to(torch.int32)
token_indices[r, :n_valid] = perm
+13 -9
View File
@@ -1250,32 +1250,36 @@ def test_split_indexer_prefill_chunks_single_request_overflow():
assert out == expected
def test_triton_convert_returns_valid_counts():
# 384 is not a power of two, so it counts via the tiled atomic accumulation
# rather than the single-tile path 128 takes.
@pytest.mark.parametrize("num_topk_tokens", [128, 384])
def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
"""Test that return_valid_counts correctly counts non-negative indices."""
device = torch.device(DEVICE_TYPE)
num_tokens = 8
num_requests = 2
max_blocks_per_req = 10
block_size = 64
num_topk_tokens = 128
req_id = torch.tensor([0, 0, 0, 0, 1, 1, 1, 1], dtype=torch.int32, device=device)
block_table = torch.arange(
num_requests * max_blocks_per_req, dtype=torch.int32, device=device
).view(num_requests, max_blocks_per_req)
# Create token indices with varying numbers of valid entries
# Token 0: 64 valid, 64 invalid (-1)
# Token 1: 32 valid, 96 invalid
# Token 2: 128 valid (all)
# Token 3: 1 valid, 127 invalid
# etc.
# Create token indices with varying numbers of valid entries: half the row,
# a quarter of it, the whole row, then a single valid entry -- twice over.
token_indices = torch.full(
(num_tokens, num_topk_tokens), -1, dtype=torch.int32, device=device
)
valid_counts_per_token = [
num_topk_tokens // 2,
num_topk_tokens // 4,
num_topk_tokens,
1,
] * 2
expected_valid = []
for i in range(num_tokens):
num_valid = [64, 32, 128, 1, 64, 32, 128, 1][i]
num_valid = valid_counts_per_token[i]
token_indices[i, :num_valid] = torch.arange(
num_valid, dtype=torch.int32, device=device
) % (block_size * max_blocks_per_req)