[Perf][Model Runner V2] Compact sampling masks on GPU instead of unpacking the full-vocab bitmask on CPU (#54901)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: vx120 <893600387@qq.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: vx120 <893600387@qq.com>
This commit is contained in:
aoshen02
2026-09-03 19:03:36 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 vx120
parent d6bce42983
commit 560ef78bfe
10 changed files with 188 additions and 108 deletions
+30 -36
View File
@@ -3,6 +3,7 @@
from unittest import TestCase
import numpy as np
import pytest
import torch
from vllm.platforms import current_platform
@@ -61,23 +62,6 @@ def test_logprobs_tensors_tolists_with_tensor_boundaries():
assert sliced.sampled_token_ranks.tolist() == [3]
def test_sampling_mask_tensors_tolist():
tensors = SamplingMaskTensors(
packed_mask=torch.tensor(
[[0b00101000], [0b00000000], [0b10000000]],
dtype=torch.uint8,
),
counts=torch.tensor([2, 0, 1], dtype=torch.int32),
vocab_size=8,
)
result = tensors.tolists(np.array([1, 0, 1]))
assert result.token_ids.tolist() == [3, 5, 7]
assert result.offsets.tolist() == [0, 2, 3]
assert result.cu_num_generated_tokens == [0, 1, 1, 2]
def test_sampling_mask_lists_to_nested_list():
from vllm.v1.outputs import SamplingMaskLists
@@ -89,46 +73,56 @@ def test_sampling_mask_lists_to_nested_list():
nested = mask.to_nested_list()
assert nested == [[10, 11, 12], [20, 21]]
assert SamplingMaskLists(np.array([7, 9])).to_nested_list() == [[7, 9]]
def test_sampling_mask_tensors_from_logits():
@pytest.mark.parametrize("max_num_kept", [512, 20_001])
def test_sampling_mask_tensors_match_finite_support(max_num_kept):
"""Whatever its size (empty, one, around the compact width, beyond the
cap, the whole vocab), a sampled row's mask is exactly the ascending set
of its finite logits; rows that sampled nothing are empty."""
from vllm.v1.worker.gpu.sample.output import MAX_COMPACT_SUPPORT
vocab_size = 20_001
sizes = [0, 1, 7, 511, 512, 513, MAX_COMPACT_SUPPORT + 5, vocab_size, 40, 3]
gen = torch.Generator().manual_seed(0)
logits = torch.full((len(sizes), vocab_size), float("-inf"))
expected = []
for row, size in enumerate(sizes):
kept = torch.randperm(vocab_size, generator=gen)[:size].sort().values
logits[row, kept] = torch.randn(size, generator=gen)
expected.append(kept.tolist())
num_sampled_tokens = torch.ones(len(sizes), dtype=torch.int32)
num_sampled_tokens[[1, 7]] = 0
expected[1] = expected[7] = []
tensors = SamplingMaskTensors.from_logits(
logits=torch.tensor(
[
[1.0, float("-inf"), 2.0],
[3.0, 4.0, float("-inf")],
[float("-inf"), 5.0, 6.0],
],
device=DEVICE_TYPE,
),
num_sampled_tokens=torch.tensor([1, 0, 1], device=DEVICE_TYPE),
logits.to(DEVICE_TYPE), num_sampled_tokens.to(DEVICE_TYPE), max_num_kept
)
result = tensors.tolists(np.array([1, 0, 1]))
assert result.token_ids.tolist() == [0, 2, 1, 2]
assert result.offsets.tolist() == [0, 2, 4]
assert result.cu_num_generated_tokens == [0, 1, 1, 2]
assert tensors.token_ids.shape[1] == min(max_num_kept, MAX_COMPACT_SUPPORT)
assert tensors.to_cpu_nonblocking().tolists().to_nested_list() == expected
def test_sampling_mask_matches_processed_top_k_top_p_support():
def test_sampling_mask_preserves_top_k_boundary_ties():
processed_logits = apply_top_k_top_p(
logits=torch.tensor(
[[6.0, 5.0, 4.0, 4.0, 4.0, 2.0, 1.0, 0.0]], device=DEVICE_TYPE
),
k=torch.tensor([3], device=DEVICE_TYPE),
p=torch.tensor([0.9], device=DEVICE_TYPE),
p=None,
)
expected_token_ids = (
torch.isfinite(processed_logits[0]).nonzero().flatten().tolist()
)
assert 0 < len(expected_token_ids) < processed_logits.shape[1]
assert len(expected_token_ids) > 3
tensors = SamplingMaskTensors.from_logits(
processed_logits,
num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE),
max_num_kept=3,
)
result = tensors.tolists(np.array([1]))
result = tensors.tolists()
assert result.to_nested_list() == [expected_token_ids]