[Perf][Model Runner V2] Compact sampling masks on GPU instead of unpacking the full-vocab bitmask on CPU (#54901)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: vx120 <893600387@qq.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: vx120 <893600387@qq.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
vx120
parent
d6bce42983
commit
560ef78bfe
@@ -137,6 +137,7 @@ async def test_generate_sampling_mask(client):
|
||||
"top_p": 0.9,
|
||||
"ignore_eos": True,
|
||||
"seed": 0,
|
||||
"logprobs": top_k,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
@@ -150,11 +151,16 @@ async def test_generate_sampling_mask(client):
|
||||
assert len(token_ids) == len(sampling_mask)
|
||||
|
||||
vocab_size = get_vocab_size(MODEL_NAME)
|
||||
for token_id, support in zip(token_ids, sampling_mask):
|
||||
logprobs_content = choice["logprobs"]["content"]
|
||||
for token_id, support, entry in zip(token_ids, sampling_mask, logprobs_content):
|
||||
assert support
|
||||
assert len(support) == len(set(support))
|
||||
assert all(0 <= support_token_id < vocab_size for support_token_id in support)
|
||||
assert token_id in support
|
||||
# processed_logprobs: exactly the support carries finite probability.
|
||||
for top in entry["top_logprobs"]:
|
||||
in_support = int(top["token"].removeprefix("token_id:")) in support
|
||||
assert in_support == (top["logprob"] > -9999.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -753,6 +753,22 @@ def test_v1_model_runner_rejects_v2_only_features():
|
||||
VllmConfig._validate_v1_model_runner(config)
|
||||
|
||||
|
||||
def test_batch_sharded_sampling_rejects_return_sampling_mask():
|
||||
"""The batch-sharded gather drops sampling masks, so the combination must
|
||||
fail loudly instead of returning ``sampling_mask=None``."""
|
||||
config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
enable_batch_sharded_sampling=True, tensor_parallel_size=2
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=8),
|
||||
model_config=SimpleNamespace(max_logprobs=20, return_sampling_mask=True),
|
||||
speculative_config=None,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="sampling masks"):
|
||||
VllmConfig._validate_batch_sharded_sampling(config)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_with_hf_config_populates_missing_architectures_from_causal_lm_mapping(
|
||||
monkeypatch,
|
||||
|
||||
@@ -4,6 +4,7 @@ import dataclasses
|
||||
from concurrent.futures import Future
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -43,6 +44,7 @@ from vllm.v1.outputs import (
|
||||
ECConnectorOutput,
|
||||
KVConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
SamplingMaskLists,
|
||||
make_empty_encoder_model_runner_output,
|
||||
)
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
@@ -667,6 +669,51 @@ def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
|
||||
assert output2.num_scheduled_tokens[requests[2].request_id] == 800 - 224 - 224
|
||||
|
||||
|
||||
def test_update_from_output_routes_sampling_masks_by_request():
|
||||
"""Each request receives the sampler row at its own batch index."""
|
||||
scheduler = create_scheduler()
|
||||
scheduler.return_sampling_mask = True
|
||||
requests = create_requests(num_requests=3, max_tokens=10)
|
||||
for req in requests:
|
||||
req.num_computed_tokens = req.num_tokens
|
||||
scheduler.requests[req.request_id] = req
|
||||
scheduler.running.append(req)
|
||||
req.status = RequestStatus.RUNNING
|
||||
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
num_scheduled_tokens={req.request_id: 1 for req in requests},
|
||||
total_num_scheduled_tokens=3,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
model_output = ModelRunnerOutput(
|
||||
req_ids=[req.request_id for req in requests],
|
||||
req_id_to_index={req.request_id: i for i, req in enumerate(requests)},
|
||||
sampled_token_ids=[[1], [3], [4]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
sampling_masks=SamplingMaskLists(
|
||||
np.array([1, 2, 3, 4, 5, 6], dtype=np.int32), np.array([0, 2, 3, 6])
|
||||
),
|
||||
)
|
||||
|
||||
outputs = scheduler.update_from_output(scheduler_output, model_output)[0].outputs
|
||||
|
||||
assert [out.request_id for out in outputs] == [req.request_id for req in requests]
|
||||
assert [out.new_sampling_mask.token_ids.tolist() for out in outputs] == [
|
||||
[1, 2],
|
||||
[3],
|
||||
[4, 5, 6],
|
||||
]
|
||||
assert all(out.new_sampling_mask.offsets is None for out in outputs)
|
||||
|
||||
|
||||
def test_stop_via_update_from_output():
|
||||
"""Test stopping behavior through update_from_output"""
|
||||
scheduler = create_scheduler(num_speculative_tokens=1)
|
||||
|
||||
+30
-36
@@ -3,6 +3,7 @@
|
||||
from unittest import TestCase
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
@@ -61,23 +62,6 @@ def test_logprobs_tensors_tolists_with_tensor_boundaries():
|
||||
assert sliced.sampled_token_ranks.tolist() == [3]
|
||||
|
||||
|
||||
def test_sampling_mask_tensors_tolist():
|
||||
tensors = SamplingMaskTensors(
|
||||
packed_mask=torch.tensor(
|
||||
[[0b00101000], [0b00000000], [0b10000000]],
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
counts=torch.tensor([2, 0, 1], dtype=torch.int32),
|
||||
vocab_size=8,
|
||||
)
|
||||
|
||||
result = tensors.tolists(np.array([1, 0, 1]))
|
||||
|
||||
assert result.token_ids.tolist() == [3, 5, 7]
|
||||
assert result.offsets.tolist() == [0, 2, 3]
|
||||
assert result.cu_num_generated_tokens == [0, 1, 1, 2]
|
||||
|
||||
|
||||
def test_sampling_mask_lists_to_nested_list():
|
||||
from vllm.v1.outputs import SamplingMaskLists
|
||||
|
||||
@@ -89,46 +73,56 @@ def test_sampling_mask_lists_to_nested_list():
|
||||
nested = mask.to_nested_list()
|
||||
|
||||
assert nested == [[10, 11, 12], [20, 21]]
|
||||
assert SamplingMaskLists(np.array([7, 9])).to_nested_list() == [[7, 9]]
|
||||
|
||||
|
||||
def test_sampling_mask_tensors_from_logits():
|
||||
@pytest.mark.parametrize("max_num_kept", [512, 20_001])
|
||||
def test_sampling_mask_tensors_match_finite_support(max_num_kept):
|
||||
"""Whatever its size (empty, one, around the compact width, beyond the
|
||||
cap, the whole vocab), a sampled row's mask is exactly the ascending set
|
||||
of its finite logits; rows that sampled nothing are empty."""
|
||||
from vllm.v1.worker.gpu.sample.output import MAX_COMPACT_SUPPORT
|
||||
|
||||
vocab_size = 20_001
|
||||
sizes = [0, 1, 7, 511, 512, 513, MAX_COMPACT_SUPPORT + 5, vocab_size, 40, 3]
|
||||
gen = torch.Generator().manual_seed(0)
|
||||
logits = torch.full((len(sizes), vocab_size), float("-inf"))
|
||||
expected = []
|
||||
for row, size in enumerate(sizes):
|
||||
kept = torch.randperm(vocab_size, generator=gen)[:size].sort().values
|
||||
logits[row, kept] = torch.randn(size, generator=gen)
|
||||
expected.append(kept.tolist())
|
||||
num_sampled_tokens = torch.ones(len(sizes), dtype=torch.int32)
|
||||
num_sampled_tokens[[1, 7]] = 0
|
||||
expected[1] = expected[7] = []
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
logits=torch.tensor(
|
||||
[
|
||||
[1.0, float("-inf"), 2.0],
|
||||
[3.0, 4.0, float("-inf")],
|
||||
[float("-inf"), 5.0, 6.0],
|
||||
],
|
||||
device=DEVICE_TYPE,
|
||||
),
|
||||
num_sampled_tokens=torch.tensor([1, 0, 1], device=DEVICE_TYPE),
|
||||
logits.to(DEVICE_TYPE), num_sampled_tokens.to(DEVICE_TYPE), max_num_kept
|
||||
)
|
||||
|
||||
result = tensors.tolists(np.array([1, 0, 1]))
|
||||
|
||||
assert result.token_ids.tolist() == [0, 2, 1, 2]
|
||||
assert result.offsets.tolist() == [0, 2, 4]
|
||||
assert result.cu_num_generated_tokens == [0, 1, 1, 2]
|
||||
assert tensors.token_ids.shape[1] == min(max_num_kept, MAX_COMPACT_SUPPORT)
|
||||
assert tensors.to_cpu_nonblocking().tolists().to_nested_list() == expected
|
||||
|
||||
|
||||
def test_sampling_mask_matches_processed_top_k_top_p_support():
|
||||
def test_sampling_mask_preserves_top_k_boundary_ties():
|
||||
processed_logits = apply_top_k_top_p(
|
||||
logits=torch.tensor(
|
||||
[[6.0, 5.0, 4.0, 4.0, 4.0, 2.0, 1.0, 0.0]], device=DEVICE_TYPE
|
||||
),
|
||||
k=torch.tensor([3], device=DEVICE_TYPE),
|
||||
p=torch.tensor([0.9], device=DEVICE_TYPE),
|
||||
p=None,
|
||||
)
|
||||
expected_token_ids = (
|
||||
torch.isfinite(processed_logits[0]).nonzero().flatten().tolist()
|
||||
)
|
||||
assert 0 < len(expected_token_ids) < processed_logits.shape[1]
|
||||
assert len(expected_token_ids) > 3
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
processed_logits,
|
||||
num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE),
|
||||
max_num_kept=3,
|
||||
)
|
||||
result = tensors.tolists(np.array([1]))
|
||||
result = tensors.tolists()
|
||||
|
||||
assert result.to_nested_list() == [expected_token_ids]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user