[Perf][Sampler] Speed up the small-batch split top-p pipeline (#61996)
Signed-off-by: Bowen Wang <bowen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
Claude Opus 5.5
Nick Hill
vllm-agent
parent
556ac15eb9
commit
7b6310c109
@@ -293,7 +293,7 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k, p=None)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
|
||||
def test_topp_only(self, batch_size: int, vocab_size: int):
|
||||
"""Test top-p only (k=None)."""
|
||||
@@ -307,7 +307,7 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k=None, p=p)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
|
||||
def test_topk_and_topp(self, batch_size: int, vocab_size: int):
|
||||
"""Test combined top-k and top-p."""
|
||||
@@ -328,6 +328,24 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k, p)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [8, 48, 64])
|
||||
@pytest.mark.parametrize("vocab_size", [151936, 200064, 262144])
|
||||
def test_topk_and_topp_large_vocab(self, batch_size: int, vocab_size: int):
|
||||
"""Mixed batches at vocab sizes where the split top-p pipeline uses
|
||||
its largest split counts (up to _SPLIT_MAX_SPLITS)."""
|
||||
logits = torch.randn(
|
||||
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
|
||||
)
|
||||
k = torch.randint(1, 100, (batch_size,), generator=self.generator)
|
||||
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
|
||||
# Rows cycle through top-k + top-p, top-p only, top-k only, neither.
|
||||
k[1::4] = vocab_size
|
||||
p[2::4] = 1.0
|
||||
k[3::4] = vocab_size
|
||||
p[3::4] = 1.0
|
||||
|
||||
self._compare_results(logits, k, p)
|
||||
|
||||
def test_both_disabled(self):
|
||||
"""Test when both k and p are None (should be no-op)."""
|
||||
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
|
||||
@@ -788,6 +806,38 @@ class TestTritonTopkTopp:
|
||||
f"{kept_topk_topp.tolist()}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [8, 48])
|
||||
def test_skip_split_without_topp_only_rows(
|
||||
self, batch_size: int, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""has_topp_only_rows=False skips the split pipeline; the result must
|
||||
not change when every top-p row also has a top-k, grammar rows
|
||||
(k >= #finite logits) included."""
|
||||
import vllm.v1.sample.ops.topk_topp_triton as tt
|
||||
|
||||
vocab_size = 32000
|
||||
logits = torch.randn(
|
||||
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
|
||||
)
|
||||
logits[1] = float("-inf")
|
||||
idx = torch.randperm(vocab_size, generator=self.generator)[:30]
|
||||
logits[1, idx] = torch.randn(30, generator=self.generator)
|
||||
k = torch.randint(
|
||||
50, 100, (batch_size,), generator=self.generator, dtype=torch.int32
|
||||
)
|
||||
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
|
||||
k[::4] = vocab_size
|
||||
p[::4] = 1.0
|
||||
|
||||
expected = tt.apply_top_k_top_p_triton(logits.clone(), k, p)
|
||||
monkeypatch.setattr(
|
||||
tt, "_apply_topp_split", lambda *args: pytest.fail("split launched")
|
||||
)
|
||||
actual = tt.apply_top_k_top_p_triton(
|
||||
logits.clone(), k, p, has_topp_only_rows=False
|
||||
)
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
def test_topp_tie_break_deterministic(self):
|
||||
"""Boundary-duplicate trimming must be deterministic across calls.
|
||||
|
||||
|
||||
@@ -109,6 +109,30 @@ def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool):
|
||||
assert sampler.needs_logits_processing[3] == admitted
|
||||
|
||||
|
||||
def test_has_topp_only_rows():
|
||||
"""A row is top-p-only when it has top-p and no effective top-k; top_k
|
||||
of 0 (the default) or at least the logits width disables top-k."""
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(0, SamplingParams(top_k=10, top_p=0.9))
|
||||
sampler.add_request(1, SamplingParams(top_k=10))
|
||||
sampler.add_request(2, SamplingParams(top_p=0.9))
|
||||
sampler.add_request(3, SamplingParams(top_k=VOCAB_SIZE + 1, top_p=0.9))
|
||||
states = sampler.sampling_states
|
||||
states.apply_staged_writes()
|
||||
|
||||
def has_topp_only_rows(idx: list[int], vocab_size: int = VOCAB_SIZE) -> bool:
|
||||
idx_np = np.array(idx, dtype=np.int32)
|
||||
idx_gpu = torch.from_numpy(idx_np).to(DEVICE)
|
||||
return states.get_top_k_top_p(idx_gpu, idx_np, vocab_size)[2]
|
||||
|
||||
assert not has_topp_only_rows([0, 1])
|
||||
assert has_topp_only_rows([0, 2])
|
||||
assert has_topp_only_rows([3])
|
||||
# The kernel compares top_k against the logits width, so a top_k at or
|
||||
# above a narrower logits width counts as no top-k.
|
||||
assert has_topp_only_rows([0], vocab_size=10)
|
||||
|
||||
|
||||
def test_logits_processing_cache_only_checks_active_requests():
|
||||
sampler = _make_sampler()
|
||||
sampler.add_request(0, SamplingParams(temperature=0.0))
|
||||
|
||||
@@ -368,6 +368,7 @@ def test_gpu_sampler_respects_mixed_request_watermarking(
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([3, 7]))
|
||||
@@ -383,7 +384,7 @@ def test_gpu_sampler_filters_top_k_top_p_before_watermarking(
|
||||
# reference path; the assertion is about the call order, not the kernel.
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.watermarking.gpu_sampler.apply_top_k_top_p",
|
||||
lambda logits, k, p: apply_top_k_top_p_pytorch(
|
||||
lambda logits, k, p, **kwargs: apply_top_k_top_p_pytorch(
|
||||
logits, k, p, allow_cpu_sync=True
|
||||
),
|
||||
)
|
||||
@@ -415,6 +416,7 @@ def test_gpu_sampler_filters_top_k_top_p_before_watermarking(
|
||||
torch.tensor([2]),
|
||||
torch.tensor([0.8]),
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert watermarker.captured_logits is not None
|
||||
@@ -482,6 +484,7 @@ def test_gpu_sampler_skips_watermarking_for_repeated_contexts(monkeypatch):
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([3, 7]))
|
||||
@@ -517,6 +520,7 @@ def test_gpu_sampler_can_disable_context_deduplication(monkeypatch):
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(sampled, torch.tensor([7, 7]))
|
||||
@@ -956,6 +960,7 @@ def test_gpu_sampler_uses_fused_gumbel_for_repeated_contexts():
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert torch.equal(repeated, torch.tensor([True, False], device=device))
|
||||
@@ -992,6 +997,7 @@ def test_gpu_sampler_skips_watermarking_for_greedy_batch(monkeypatch):
|
||||
None,
|
||||
None,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
assert actual is expected
|
||||
|
||||
Reference in New Issue
Block a user