[Perf][Sampler] Speed up the small-batch split top-p pipeline (#61996)
Signed-off-by: Bowen Wang <bowen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
Claude Opus 5.5
Nick Hill
vllm-agent
parent
556ac15eb9
commit
7b6310c109
@@ -293,7 +293,7 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k, p=None)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
|
||||
def test_topp_only(self, batch_size: int, vocab_size: int):
|
||||
"""Test top-p only (k=None)."""
|
||||
@@ -307,7 +307,7 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k=None, p=p)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
|
||||
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
|
||||
def test_topk_and_topp(self, batch_size: int, vocab_size: int):
|
||||
"""Test combined top-k and top-p."""
|
||||
@@ -328,6 +328,24 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k, p)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [8, 48, 64])
|
||||
@pytest.mark.parametrize("vocab_size", [151936, 200064, 262144])
|
||||
def test_topk_and_topp_large_vocab(self, batch_size: int, vocab_size: int):
|
||||
"""Mixed batches at vocab sizes where the split top-p pipeline uses
|
||||
its largest split counts (up to _SPLIT_MAX_SPLITS)."""
|
||||
logits = torch.randn(
|
||||
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
|
||||
)
|
||||
k = torch.randint(1, 100, (batch_size,), generator=self.generator)
|
||||
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
|
||||
# Rows cycle through top-k + top-p, top-p only, top-k only, neither.
|
||||
k[1::4] = vocab_size
|
||||
p[2::4] = 1.0
|
||||
k[3::4] = vocab_size
|
||||
p[3::4] = 1.0
|
||||
|
||||
self._compare_results(logits, k, p)
|
||||
|
||||
def test_both_disabled(self):
|
||||
"""Test when both k and p are None (should be no-op)."""
|
||||
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
|
||||
@@ -788,6 +806,38 @@ class TestTritonTopkTopp:
|
||||
f"{kept_topk_topp.tolist()}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [8, 48])
|
||||
def test_skip_split_without_topp_only_rows(
|
||||
self, batch_size: int, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""has_topp_only_rows=False skips the split pipeline; the result must
|
||||
not change when every top-p row also has a top-k, grammar rows
|
||||
(k >= #finite logits) included."""
|
||||
import vllm.v1.sample.ops.topk_topp_triton as tt
|
||||
|
||||
vocab_size = 32000
|
||||
logits = torch.randn(
|
||||
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
|
||||
)
|
||||
logits[1] = float("-inf")
|
||||
idx = torch.randperm(vocab_size, generator=self.generator)[:30]
|
||||
logits[1, idx] = torch.randn(30, generator=self.generator)
|
||||
k = torch.randint(
|
||||
50, 100, (batch_size,), generator=self.generator, dtype=torch.int32
|
||||
)
|
||||
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
|
||||
k[::4] = vocab_size
|
||||
p[::4] = 1.0
|
||||
|
||||
expected = tt.apply_top_k_top_p_triton(logits.clone(), k, p)
|
||||
monkeypatch.setattr(
|
||||
tt, "_apply_topp_split", lambda *args: pytest.fail("split launched")
|
||||
)
|
||||
actual = tt.apply_top_k_top_p_triton(
|
||||
logits.clone(), k, p, has_topp_only_rows=False
|
||||
)
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
def test_topp_tie_break_deterministic(self):
|
||||
"""Boundary-duplicate trimming must be deterministic across calls.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user