[Perf][Sampler] Speed up the small-batch split top-p pipeline (#61996)

Signed-off-by: Bowen Wang <bowen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
Bowen Wang
2026-10-16 21:22:11 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Nick Hill vllm-agent
parent 556ac15eb9
commit 7b6310c109
9 changed files with 148 additions and 24 deletions
+52 -2
View File
@@ -293,7 +293,7 @@ class TestTritonTopkTopp:
self._compare_results(logits, k, p=None)
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
def test_topp_only(self, batch_size: int, vocab_size: int):
"""Test top-p only (k=None)."""
@@ -307,7 +307,7 @@ class TestTritonTopkTopp:
self._compare_results(logits, k=None, p=p)
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
def test_topk_and_topp(self, batch_size: int, vocab_size: int):
"""Test combined top-k and top-p."""
@@ -328,6 +328,24 @@ class TestTritonTopkTopp:
self._compare_results(logits, k, p)
@pytest.mark.parametrize("batch_size", [8, 48, 64])
@pytest.mark.parametrize("vocab_size", [151936, 200064, 262144])
def test_topk_and_topp_large_vocab(self, batch_size: int, vocab_size: int):
"""Mixed batches at vocab sizes where the split top-p pipeline uses
its largest split counts (up to _SPLIT_MAX_SPLITS)."""
logits = torch.randn(
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
)
k = torch.randint(1, 100, (batch_size,), generator=self.generator)
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
# Rows cycle through top-k + top-p, top-p only, top-k only, neither.
k[1::4] = vocab_size
p[2::4] = 1.0
k[3::4] = vocab_size
p[3::4] = 1.0
self._compare_results(logits, k, p)
def test_both_disabled(self):
"""Test when both k and p are None (should be no-op)."""
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
@@ -788,6 +806,38 @@ class TestTritonTopkTopp:
f"{kept_topk_topp.tolist()}"
)
@pytest.mark.parametrize("batch_size", [8, 48])
def test_skip_split_without_topp_only_rows(
self, batch_size: int, monkeypatch: pytest.MonkeyPatch
):
"""has_topp_only_rows=False skips the split pipeline; the result must
not change when every top-p row also has a top-k, grammar rows
(k >= #finite logits) included."""
import vllm.v1.sample.ops.topk_topp_triton as tt
vocab_size = 32000
logits = torch.randn(
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
)
logits[1] = float("-inf")
idx = torch.randperm(vocab_size, generator=self.generator)[:30]
logits[1, idx] = torch.randn(30, generator=self.generator)
k = torch.randint(
50, 100, (batch_size,), generator=self.generator, dtype=torch.int32
)
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
k[::4] = vocab_size
p[::4] = 1.0
expected = tt.apply_top_k_top_p_triton(logits.clone(), k, p)
monkeypatch.setattr(
tt, "_apply_topp_split", lambda *args: pytest.fail("split launched")
)
actual = tt.apply_top_k_top_p_triton(
logits.clone(), k, p, has_topp_only_rows=False
)
assert torch.equal(actual, expected)
def test_topp_tie_break_deterministic(self):
"""Boundary-duplicate trimming must be deterministic across calls.
+24
View File
@@ -109,6 +109,30 @@ def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool):
assert sampler.needs_logits_processing[3] == admitted
def test_has_topp_only_rows():
"""A row is top-p-only when it has top-p and no effective top-k; top_k
of 0 (the default) or at least the logits width disables top-k."""
sampler = _make_sampler()
sampler.add_request(0, SamplingParams(top_k=10, top_p=0.9))
sampler.add_request(1, SamplingParams(top_k=10))
sampler.add_request(2, SamplingParams(top_p=0.9))
sampler.add_request(3, SamplingParams(top_k=VOCAB_SIZE + 1, top_p=0.9))
states = sampler.sampling_states
states.apply_staged_writes()
def has_topp_only_rows(idx: list[int], vocab_size: int = VOCAB_SIZE) -> bool:
idx_np = np.array(idx, dtype=np.int32)
idx_gpu = torch.from_numpy(idx_np).to(DEVICE)
return states.get_top_k_top_p(idx_gpu, idx_np, vocab_size)[2]
assert not has_topp_only_rows([0, 1])
assert has_topp_only_rows([0, 2])
assert has_topp_only_rows([3])
# The kernel compares top_k against the logits width, so a top_k at or
# above a narrower logits width counts as no top-k.
assert has_topp_only_rows([0], vocab_size=10)
def test_logits_processing_cache_only_checks_active_requests():
sampler = _make_sampler()
sampler.add_request(0, SamplingParams(temperature=0.0))
+7 -1
View File
@@ -368,6 +368,7 @@ def test_gpu_sampler_respects_mixed_request_watermarking(
None,
None,
False,
False,
)
assert torch.equal(sampled, torch.tensor([3, 7]))
@@ -383,7 +384,7 @@ def test_gpu_sampler_filters_top_k_top_p_before_watermarking(
# reference path; the assertion is about the call order, not the kernel.
monkeypatch.setattr(
"vllm.v1.watermarking.gpu_sampler.apply_top_k_top_p",
lambda logits, k, p: apply_top_k_top_p_pytorch(
lambda logits, k, p, **kwargs: apply_top_k_top_p_pytorch(
logits, k, p, allow_cpu_sync=True
),
)
@@ -415,6 +416,7 @@ def test_gpu_sampler_filters_top_k_top_p_before_watermarking(
torch.tensor([2]),
torch.tensor([0.8]),
False,
False,
)
assert watermarker.captured_logits is not None
@@ -482,6 +484,7 @@ def test_gpu_sampler_skips_watermarking_for_repeated_contexts(monkeypatch):
None,
None,
False,
False,
)
assert torch.equal(sampled, torch.tensor([3, 7]))
@@ -517,6 +520,7 @@ def test_gpu_sampler_can_disable_context_deduplication(monkeypatch):
None,
None,
False,
False,
)
assert torch.equal(sampled, torch.tensor([7, 7]))
@@ -956,6 +960,7 @@ def test_gpu_sampler_uses_fused_gumbel_for_repeated_contexts():
None,
None,
False,
False,
)
assert torch.equal(repeated, torch.tensor([True, False], device=device))
@@ -992,6 +997,7 @@ def test_gpu_sampler_skips_watermarking_for_greedy_batch(monkeypatch):
None,
None,
False,
False,
)
assert actual is expected