[Core] Triton kernel for small-batch top-p only masking (#54651)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nick Hill
2026-09-03 07:52:47 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent bb363db9a5
commit bf95f58d10
3 changed files with 627 additions and 13 deletions
+71
View File
@@ -771,6 +771,77 @@ class TestTritonTopkTopp:
if finite_in > 0:
assert kept > 0, f"Row {i}: no tokens kept"
def test_mixed_batch_grammar_rows_get_topp(self):
"""Grammar-masked rows in a mixed top-k batch must still get top-p.
Regression: with the split top-p pipeline active (batch <= 64), a
row with k < vocab but <= k finite logits was covered by neither the
monolithic kernel (standalone top-p disabled) nor the split pipeline
(skips rows with k < vocab), so its top-p was silently dropped.
"""
batch_size, vocab_size = 8, 32000
logits = torch.full((batch_size, vocab_size), float("-inf"))
# Rows 0-2: grammar rows (30 finite tokens); k=50 is a no-op for them.
for i in range(3):
idx = torch.randperm(vocab_size, generator=self.generator)[:30]
logits[i, idx] = torch.randn(30, generator=self.generator)
# Rows 3-4: p-only rows (k = vocab); rows 5-7: normal top-k+top-p.
logits[3:5] = torch.randn(2, vocab_size, generator=self.generator)
logits[5:] = torch.randn(3, vocab_size, generator=self.generator)
k = torch.tensor(
[50, 50, 50, vocab_size, vocab_size, 50, 50, 50], dtype=torch.int32
)
p = torch.tensor([0.5, 0.5, 0.5, 0.9, 0.9, 0.9, 0.9, 0.9])
self._compare_results(logits, k, p)
# Top-p must actually trim the grammar rows beyond the (no-op) top-k.
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
kept_topk_only = (
apply_top_k_top_p_triton(logits.clone(), k, None)[:3] != float("-inf")
).sum(-1)
kept_topk_topp = (
apply_top_k_top_p_triton(logits.clone(), k, p)[:3] != float("-inf")
).sum(-1)
assert (kept_topk_topp < kept_topk_only).all(), (
f"top-p did not trim grammar rows: {kept_topk_only.tolist()} -> "
f"{kept_topk_topp.tolist()}"
)
def test_topp_tie_break_deterministic(self):
"""Boundary-duplicate trimming must be deterministic across calls.
Regression: the split top-p mask kernel handed out the tie budget in
cross-program atomic order, so which copies of a tied boundary value
survived could vary between identical calls.
"""
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
batch_size, vocab_size = 4, 128256
# 1000 tied top tokens; the p=0.5 boundary lands inside the tie
# group (each tied token has mass e/(1000e+1000) ~ 0.073%).
logits = torch.full((batch_size, vocab_size), float("-inf"))
logits[:, :1000] = 1.0
logits[:, 1000:2000] = 0.0
p = torch.full((batch_size,), 0.5, dtype=torch.float32)
results = [apply_top_k_top_p_triton(logits.clone(), None, p) for _ in range(5)]
for r in results[1:]:
assert torch.equal(results[0], r), "non-deterministic tie break"
# The split pipeline now keeps the first `numkeep` copies in index
# order, exactly like the monolithic kernel and the PyTorch impl.
kept_idx = (results[0][0] != float("-inf")).nonzero().flatten()
assert kept_idx.min().item() == 0
assert kept_idx.max().item() == len(kept_idx) - 1
pytorch_kept = (
(apply_top_k_top_p_pytorch(logits.clone(), None, p)[0] != float("-inf"))
.sum()
.item()
)
assert len(kept_idx) == pytorch_kept
# =============================================================================
# FlashInfer top-k/top-p robustness tests