[Perf][Sampler] Speed up the small-batch split top-p pipeline (#61996)

Signed-off-by: Bowen Wang <bowen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
Bowen Wang
2026-10-16 21:22:11 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Nick Hill vllm-agent
parent 556ac15eb9
commit 7b6310c109
9 changed files with 148 additions and 24 deletions
+24
View File
@@ -109,6 +109,30 @@ def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool):
assert sampler.needs_logits_processing[3] == admitted
def test_has_topp_only_rows():
"""A row is top-p-only when it has top-p and no effective top-k; top_k
of 0 (the default) or at least the logits width disables top-k."""
sampler = _make_sampler()
sampler.add_request(0, SamplingParams(top_k=10, top_p=0.9))
sampler.add_request(1, SamplingParams(top_k=10))
sampler.add_request(2, SamplingParams(top_p=0.9))
sampler.add_request(3, SamplingParams(top_k=VOCAB_SIZE + 1, top_p=0.9))
states = sampler.sampling_states
states.apply_staged_writes()
def has_topp_only_rows(idx: list[int], vocab_size: int = VOCAB_SIZE) -> bool:
idx_np = np.array(idx, dtype=np.int32)
idx_gpu = torch.from_numpy(idx_np).to(DEVICE)
return states.get_top_k_top_p(idx_gpu, idx_np, vocab_size)[2]
assert not has_topp_only_rows([0, 1])
assert has_topp_only_rows([0, 2])
assert has_topp_only_rows([3])
# The kernel compares top_k against the logits width, so a top_k at or
# above a narrower logits width counts as no top-k.
assert has_topp_only_rows([0], vocab_size=10)
def test_logits_processing_cache_only_checks_active_requests():
sampler = _make_sampler()
sampler.add_request(0, SamplingParams(temperature=0.0))