[Perf][GLM] Fuse the kpool tail slot mapping into one Triton kernel (#57534)

Signed-off-by: Jared Wen <jaredwen@inferact.ai>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jared Wen
2026-09-20 18:20:55 +08:00
committed by GitHub
co-authored by Claude Fable 5.1 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent db1bfdd4fb
commit 1b9fa3eaa8
2 changed files with 120 additions and 2 deletions
@@ -391,3 +391,66 @@ def test_interleaved_decode_pollution_legacy_vs_circular():
# The circular mapping keeps the rings isolated under interleaving.
torch.testing.assert_close(circular, ground_truth)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA device")
@pytest.mark.parametrize(
"per_req,num_actual,padded_len",
[
([list(range(10)), list(range(12))], 22, 22),
([list(range(10)), list(range(12))], 22, 30),
([[3, 4], [0], [7, 8, 9]], 6, 8),
([[5]], 1, 1),
],
)
def test_triton_mapping_matches_cpu(per_req, num_actual, padded_len):
"""The CUDA (Triton) path must match the CPU torch reference, including
tokens between the last request boundary and num_actual_tokens (mapped to
the last request) and untouched padding beyond num_actual."""
positions, qsl, slot_mapping, _, num_reqs = make_batch(
per_req, padded_len=padded_len
)
# Replace the all--1 placeholder slots with sentinel values to check the
# padding range is copied through untouched.
slot_mapping = torch.arange(padded_len, dtype=torch.int64) + 1000
bt = make_tail_block_table(list(range(5, 5 + num_reqs)))
ref = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
got = circular_tail_slots(
slot_mapping.cuda(),
bt.cuda(),
qsl.cuda().to(torch.int32),
positions.cuda(),
num_actual,
num_reqs,
)
torch.testing.assert_close(got.cpu(), ref)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a CUDA device")
def test_triton_mapping_reads_strided_block_table():
"""The kernel must address the block table through its real stride(0),
not a dense assumption (the #57477 class of bug): a tail block table that
is a column view of a wider allocation must still read column 0."""
per_req = [list(range(10)), list(range(12))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
own_blocks = [11, 22]
wide = torch.zeros(num_reqs, 8, dtype=torch.int32)
bt = wide[:, 1:4] # non-contiguous view, stride(0) == 8
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
ref = circular_tail_slots(
slot_mapping, bt.contiguous(), qsl, positions, num_actual, num_reqs
)
got = circular_tail_slots(
slot_mapping.cuda(),
bt.cuda(),
qsl.cuda().to(torch.int32),
positions.cuda(),
num_actual,
num_reqs,
)
torch.testing.assert_close(got.cpu(), ref)
for req, blk in enumerate(own_blocks):
start, end = int(qsl[req]), int(qsl[req + 1])
assert (got[start:end] // KPOOL == blk).all()