[Perf][Sparse MLA] Drop the atomic contention in the index remap (#50365)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1c1077c6cc
commit
9e6be4a72b
@@ -674,17 +674,20 @@ def test_dcp_filter_compacts_valid_slots_for_sparse_kernel():
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
@pytest.mark.parametrize("interleave", [1, 2])
|
||||
@pytest.mark.parametrize("dcp_rank", [0, 1])
|
||||
def test_dcp_filter_compaction_matches_reference(interleave: int, dcp_rank: int):
|
||||
"""In-kernel compaction (atomic slot allocator across multiple column tiles)
|
||||
must, for every row, produce exactly the rank-owned physical slots packed
|
||||
into [0, valid_count) with -1 in the tail -- the same SET a reference filter
|
||||
+ sort/gather produces. Uses wide rows (> BLOCK_N valid slots) so the
|
||||
cross-tile atomic allocation is exercised, with interior -1 gaps."""
|
||||
# 384 is not a power of two, so it exercises the multi-tile atomic allocator
|
||||
# rather than the single-tile path 1024 takes.
|
||||
@pytest.mark.parametrize("num_topk", [1024, 384])
|
||||
def test_dcp_filter_compaction_matches_reference(
|
||||
interleave: int, dcp_rank: int, num_topk: int
|
||||
):
|
||||
"""In-kernel compaction must, for every row, produce exactly the rank-owned
|
||||
physical slots packed into [0, valid_count) with -1 in the tail -- the same
|
||||
SET a reference filter + sort/gather produces. Uses wide rows (> BLOCK_N
|
||||
valid slots), with interior -1 gaps."""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(7)
|
||||
dcp_size = 2
|
||||
block_size = 8
|
||||
num_topk = 1024 # > BLOCK_N(128) -> multiple tiles per row
|
||||
num_rows = 5
|
||||
max_blocks = 64
|
||||
seq = max_blocks * block_size
|
||||
@@ -698,7 +701,9 @@ def test_dcp_filter_compaction_matches_reference(interleave: int, dcp_rank: int)
|
||||
(num_rows, num_topk), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
for r in range(num_rows):
|
||||
n_valid = int(torch.randint(200, 600, (1,)).item())
|
||||
# Ids are distinct, so a row holds at most `seq` valid entries.
|
||||
hi = min(num_topk * 3 // 4, seq)
|
||||
n_valid = int(torch.randint(num_topk // 4, hi, (1,)).item())
|
||||
perm = torch.randperm(seq, device=device)[:n_valid].to(torch.int32)
|
||||
token_indices[r, :n_valid] = perm
|
||||
|
||||
|
||||
@@ -1250,32 +1250,36 @@ def test_split_indexer_prefill_chunks_single_request_overflow():
|
||||
assert out == expected
|
||||
|
||||
|
||||
def test_triton_convert_returns_valid_counts():
|
||||
# 384 is not a power of two, so it counts via the tiled atomic accumulation
|
||||
# rather than the single-tile path 128 takes.
|
||||
@pytest.mark.parametrize("num_topk_tokens", [128, 384])
|
||||
def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
|
||||
"""Test that return_valid_counts correctly counts non-negative indices."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_tokens = 8
|
||||
num_requests = 2
|
||||
max_blocks_per_req = 10
|
||||
block_size = 64
|
||||
num_topk_tokens = 128
|
||||
|
||||
req_id = torch.tensor([0, 0, 0, 0, 1, 1, 1, 1], dtype=torch.int32, device=device)
|
||||
block_table = torch.arange(
|
||||
num_requests * max_blocks_per_req, dtype=torch.int32, device=device
|
||||
).view(num_requests, max_blocks_per_req)
|
||||
|
||||
# Create token indices with varying numbers of valid entries
|
||||
# Token 0: 64 valid, 64 invalid (-1)
|
||||
# Token 1: 32 valid, 96 invalid
|
||||
# Token 2: 128 valid (all)
|
||||
# Token 3: 1 valid, 127 invalid
|
||||
# etc.
|
||||
# Create token indices with varying numbers of valid entries: half the row,
|
||||
# a quarter of it, the whole row, then a single valid entry -- twice over.
|
||||
token_indices = torch.full(
|
||||
(num_tokens, num_topk_tokens), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
valid_counts_per_token = [
|
||||
num_topk_tokens // 2,
|
||||
num_topk_tokens // 4,
|
||||
num_topk_tokens,
|
||||
1,
|
||||
] * 2
|
||||
expected_valid = []
|
||||
for i in range(num_tokens):
|
||||
num_valid = [64, 32, 128, 1, 64, 32, 128, 1][i]
|
||||
num_valid = valid_counts_per_token[i]
|
||||
token_indices[i, :num_valid] = torch.arange(
|
||||
num_valid, dtype=torch.int32, device=device
|
||||
) % (block_size * max_blocks_per_req)
|
||||
|
||||
Reference in New Issue
Block a user