[Perf][Sampler] Speed up the small-batch split top-p pipeline (#61996)

Signed-off-by: Bowen Wang <bowen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
Bowen Wang
2026-10-16 21:22:11 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Nick Hill vllm-agent
parent 556ac15eb9
commit 7b6310c109
9 changed files with 148 additions and 24 deletions
+52 -2
View File
@@ -293,7 +293,7 @@ class TestTritonTopkTopp:
self._compare_results(logits, k, p=None)
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
def test_topp_only(self, batch_size: int, vocab_size: int):
"""Test top-p only (k=None)."""
@@ -307,7 +307,7 @@ class TestTritonTopkTopp:
self._compare_results(logits, k=None, p=p)
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
def test_topk_and_topp(self, batch_size: int, vocab_size: int):
"""Test combined top-k and top-p."""
@@ -328,6 +328,24 @@ class TestTritonTopkTopp:
self._compare_results(logits, k, p)
@pytest.mark.parametrize("batch_size", [8, 48, 64])
@pytest.mark.parametrize("vocab_size", [151936, 200064, 262144])
def test_topk_and_topp_large_vocab(self, batch_size: int, vocab_size: int):
"""Mixed batches at vocab sizes where the split top-p pipeline uses
its largest split counts (up to _SPLIT_MAX_SPLITS)."""
logits = torch.randn(
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
)
k = torch.randint(1, 100, (batch_size,), generator=self.generator)
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
# Rows cycle through top-k + top-p, top-p only, top-k only, neither.
k[1::4] = vocab_size
p[2::4] = 1.0
k[3::4] = vocab_size
p[3::4] = 1.0
self._compare_results(logits, k, p)
def test_both_disabled(self):
"""Test when both k and p are None (should be no-op)."""
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
@@ -788,6 +806,38 @@ class TestTritonTopkTopp:
f"{kept_topk_topp.tolist()}"
)
@pytest.mark.parametrize("batch_size", [8, 48])
def test_skip_split_without_topp_only_rows(
self, batch_size: int, monkeypatch: pytest.MonkeyPatch
):
"""has_topp_only_rows=False skips the split pipeline; the result must
not change when every top-p row also has a top-k, grammar rows
(k >= #finite logits) included."""
import vllm.v1.sample.ops.topk_topp_triton as tt
vocab_size = 32000
logits = torch.randn(
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
)
logits[1] = float("-inf")
idx = torch.randperm(vocab_size, generator=self.generator)[:30]
logits[1, idx] = torch.randn(30, generator=self.generator)
k = torch.randint(
50, 100, (batch_size,), generator=self.generator, dtype=torch.int32
)
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
k[::4] = vocab_size
p[::4] = 1.0
expected = tt.apply_top_k_top_p_triton(logits.clone(), k, p)
monkeypatch.setattr(
tt, "_apply_topp_split", lambda *args: pytest.fail("split launched")
)
actual = tt.apply_top_k_top_p_triton(
logits.clone(), k, p, has_topp_only_rows=False
)
assert torch.equal(actual, expected)
def test_topp_tie_break_deterministic(self):
"""Boundary-duplicate trimming must be deterministic across calls.