[Feature][Spec Decode] Support sampling mask replay for MRV2 MTP (#59359)
Signed-off-by: Cuiping Cheng <chengcuipingswu@163.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Cuiping Cheng <chengcuipingswu@163.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Cuiping Cheng
aoshen02
Claude Opus 5.5
parent
37cecc8141
commit
70be6aae19
@@ -63,9 +63,12 @@ The mask is also available via the `/inference/v1/generate` HTTP endpoint:
|
||||
| `top_k > 0` | Bounds mask size; pure top-p can produce vocab-sized masks |
|
||||
| Model Runner V2 | Required by the async D2H copy pipeline |
|
||||
|
||||
Speculative decoding is supported; each emitted token gets the support of the
|
||||
target distribution at its position.
|
||||
|
||||
The engine rejects unsupported combinations at startup or request time:
|
||||
|
||||
- Speculative decoding
|
||||
- Speculative decoding with adaptive verification
|
||||
- Diffusion models
|
||||
- Custom logits processors (engine-level `--logits-processors`)
|
||||
|
||||
|
||||
@@ -147,6 +147,97 @@ def test_rocm_mm_prefix_lm_disables_chunked_mm_input(
|
||||
assert config.scheduler_config.disable_chunked_mm_input is expected
|
||||
|
||||
|
||||
def _sampling_replay_config(
|
||||
*,
|
||||
return_sampling_mask: bool = True,
|
||||
use_v2_model_runner: bool = True,
|
||||
speculative_method: str | None = None,
|
||||
rejection_sample_method: str = "standard",
|
||||
adaptive: bool = False,
|
||||
is_diffusion: bool = False,
|
||||
logits_processors: list[str] | None = None,
|
||||
logprobs_mode: str = "processed_logprobs",
|
||||
):
|
||||
speculative_config = None
|
||||
if speculative_method is not None:
|
||||
speculative_config = SimpleNamespace(
|
||||
method=speculative_method,
|
||||
enable_adaptive_verification=adaptive,
|
||||
rejection_sample_method=rejection_sample_method,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
return_sampling_mask=return_sampling_mask,
|
||||
is_diffusion=is_diffusion,
|
||||
logits_processors=logits_processors or [],
|
||||
logprobs_mode=logprobs_mode,
|
||||
),
|
||||
use_v2_model_runner=use_v2_model_runner,
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config", "message"),
|
||||
[
|
||||
(_sampling_replay_config(), None),
|
||||
*(
|
||||
(_sampling_replay_config(speculative_method=method), None)
|
||||
for method in (
|
||||
"mtp",
|
||||
"eagle",
|
||||
"eagle3",
|
||||
"dflash",
|
||||
"dspark",
|
||||
"draft_model",
|
||||
)
|
||||
),
|
||||
*(
|
||||
(
|
||||
_sampling_replay_config(
|
||||
speculative_method="mtp",
|
||||
rejection_sample_method=rejection_sample_method,
|
||||
),
|
||||
None,
|
||||
)
|
||||
for rejection_sample_method in ("standard", "block", "synthetic")
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(
|
||||
return_sampling_mask=False, speculative_method="dspark", adaptive=True
|
||||
),
|
||||
None,
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(speculative_method="dspark", adaptive=True),
|
||||
"requires fixed verification boundaries",
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(is_diffusion=True),
|
||||
"does not support diffusion models",
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(logits_processors=["custom"]),
|
||||
"does not support custom logits processors",
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(logprobs_mode="raw_logprobs"),
|
||||
"requires logprobs_mode='processed_logprobs'",
|
||||
),
|
||||
(
|
||||
_sampling_replay_config(use_v2_model_runner=False),
|
||||
"requires Model Runner V2",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sampling_replay_config(config, message):
|
||||
if message is None:
|
||||
VllmConfig._verify_sampling_replay_config(config)
|
||||
else:
|
||||
with pytest.raises(ValueError, match=message):
|
||||
VllmConfig._verify_sampling_replay_config(config)
|
||||
|
||||
|
||||
def test_kda_recoverssm_derivation_is_revalidated():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
|
||||
@@ -908,6 +908,49 @@ def test_update_from_output_routes_sampling_masks_by_request():
|
||||
assert all(out.new_sampling_mask.offsets is None for out in outputs)
|
||||
|
||||
|
||||
def test_update_from_output_routes_multi_position_sampling_masks():
|
||||
scheduler = create_scheduler()
|
||||
scheduler.return_sampling_mask = True
|
||||
requests = create_requests(num_requests=2, max_tokens=10)
|
||||
for req in requests:
|
||||
req.num_computed_tokens = req.num_tokens
|
||||
scheduler.requests[req.request_id] = req
|
||||
scheduler.running.append(req)
|
||||
req.status = RequestStatus.RUNNING
|
||||
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
num_scheduled_tokens={req.request_id: 1 for req in requests},
|
||||
total_num_scheduled_tokens=2,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
model_output = ModelRunnerOutput(
|
||||
req_ids=[req.request_id for req in requests],
|
||||
req_id_to_index={req.request_id: i for i, req in enumerate(requests)},
|
||||
sampled_token_ids=[[1, 2], [3, 4, 5]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
sampling_masks=SamplingMaskLists(
|
||||
token_ids=np.array([1, 6, 2, 3, 7, 8, 4, 5, 9], dtype=np.int32),
|
||||
offsets=np.array([0, 2, 3, 6, 7, 9]),
|
||||
cu_num_generated_tokens=[0, 2, 5],
|
||||
),
|
||||
)
|
||||
|
||||
outputs = scheduler.update_from_output(scheduler_output, model_output)[0].outputs
|
||||
|
||||
assert [out.new_sampling_mask.to_nested_list() for out in outputs] == [
|
||||
[[1, 6], [2]],
|
||||
[[3, 7, 8], [4], [5, 9]],
|
||||
]
|
||||
|
||||
|
||||
def test_stop_via_update_from_output():
|
||||
"""Test stopping behavior through update_from_output."""
|
||||
scheduler = create_scheduler(num_speculative_tokens=1)
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.v1.engine.output_processor import (
|
||||
RequestState,
|
||||
)
|
||||
from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats
|
||||
from vllm.v1.outputs import SamplingMaskLists
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flat_logprobs", [False, True])
|
||||
@@ -62,6 +63,31 @@ def test_delta_output_without_new_tokens_returns_empty_logprobs(
|
||||
assert len(output.logprobs) == 0
|
||||
|
||||
|
||||
def test_completion_output_preserves_each_sampling_mask_position() -> None:
|
||||
state = RequestState.__new__(RequestState)
|
||||
state.detokenizer = MagicMock()
|
||||
state.detokenizer.get_next_output_text.return_value = ""
|
||||
state.logprobs_processor = MagicMock()
|
||||
state.logprobs_processor.logprobs = None
|
||||
state.logprobs_processor.cumulative_logprob = None
|
||||
state.output_kind = RequestOutputKind.DELTA
|
||||
state.request_index = 0
|
||||
state.sampling_mask_chunks = [
|
||||
SamplingMaskLists(
|
||||
token_ids=np.array([10, 11, 20]),
|
||||
offsets=np.array([0, 2, 3]),
|
||||
),
|
||||
SamplingMaskLists(token_ids=np.array([30, 31, 32])),
|
||||
]
|
||||
state.routed_experts_chunks = []
|
||||
state.spec_decode_metrics = None
|
||||
|
||||
output = state._new_completion_output([1, 2, 3], FinishReason.LENGTH, None)
|
||||
|
||||
assert output.sampling_mask is not None
|
||||
assert output.sampling_mask.token_ids == [[10, 11], [20], [30, 31, 32]]
|
||||
|
||||
|
||||
def _ref_convert_id_to_token(
|
||||
tokenizer: TokenizerLike,
|
||||
token_id: int,
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.v1.outputs import (
|
||||
LogprobsLists,
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
SamplingMaskLists,
|
||||
)
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
|
||||
@@ -63,8 +64,6 @@ def test_logprobs_tensors_tolists_with_tensor_boundaries():
|
||||
|
||||
|
||||
def test_sampling_mask_lists_to_nested_list():
|
||||
from vllm.v1.outputs import SamplingMaskLists
|
||||
|
||||
mask = SamplingMaskLists(
|
||||
token_ids=np.array([10, 11, 12, 20, 21]),
|
||||
offsets=np.array([0, 3, 5]),
|
||||
@@ -76,6 +75,22 @@ def test_sampling_mask_lists_to_nested_list():
|
||||
assert SamplingMaskLists(np.array([7, 9])).to_nested_list() == [[7, 9]]
|
||||
|
||||
|
||||
def test_sampling_mask_lists_slices_multiple_positions_by_request():
|
||||
masks = SamplingMaskLists(
|
||||
token_ids=np.array([10, 11, 20, 30, 31, 32, 40]),
|
||||
offsets=np.array([0, 2, 3, 6, 7]),
|
||||
cu_num_generated_tokens=[0, 1, 3, 4],
|
||||
)
|
||||
|
||||
single = masks.slice_request(0, 1)
|
||||
multi = masks.slice_request(1, 2)
|
||||
|
||||
assert single.to_nested_list() == [[10, 11]]
|
||||
assert single.offsets is None
|
||||
assert multi.to_nested_list() == [[20], [30, 31, 32]]
|
||||
assert multi.cu_num_generated_tokens is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("max_num_kept", [512, 20_001])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_xpu(),
|
||||
@@ -102,7 +117,10 @@ def test_sampling_mask_tensors_match_finite_support(max_num_kept):
|
||||
expected[1] = expected[7] = []
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
logits.to(DEVICE_TYPE), num_sampled_tokens.to(DEVICE_TYPE), max_num_kept
|
||||
logits.to(DEVICE_TYPE),
|
||||
torch.arange(len(sizes) + 1, device=DEVICE_TYPE, dtype=torch.int32),
|
||||
num_sampled_tokens.to(DEVICE_TYPE),
|
||||
max_num_kept,
|
||||
)
|
||||
|
||||
assert tensors.token_ids.shape[1] == min(max_num_kept, MAX_COMPACT_SUPPORT)
|
||||
@@ -127,6 +145,7 @@ def test_sampling_mask_matches_processed_top_k_top_p_support():
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
processed_logits,
|
||||
cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
|
||||
num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE),
|
||||
max_num_kept=3,
|
||||
)
|
||||
@@ -135,6 +154,50 @@ def test_sampling_mask_matches_processed_top_k_top_p_support():
|
||||
assert result.to_nested_list() == [expected_token_ids]
|
||||
|
||||
|
||||
def test_sampling_mask_tensors_uses_request_boundaries():
|
||||
logits = torch.full((6, 8), -float("inf"), device=DEVICE_TYPE)
|
||||
logits[0, [1, 3]] = 0
|
||||
logits[2, [2, 4, 6]] = 0
|
||||
logits[5, [0, 7]] = 0
|
||||
counts = torch.tensor([1, 0, 1], device=DEVICE_TYPE, dtype=torch.int32)
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
logits,
|
||||
torch.tensor([0, 2, 5, 6], device=DEVICE_TYPE, dtype=torch.int32),
|
||||
counts,
|
||||
max_num_kept=3,
|
||||
)
|
||||
|
||||
assert tensors.tolists().to_nested_list() == [[1, 3], [], [0, 7]]
|
||||
|
||||
|
||||
def test_sampling_mask_tensors_multirow_request_layout():
|
||||
rows_per_request = 4
|
||||
counts = np.array([1, 2, 4, 0], dtype=np.int32)
|
||||
logits = torch.full((16, 16), -float("inf"), device=DEVICE_TYPE)
|
||||
expected = []
|
||||
for row in [0, 4, 5, 8, 9, 10, 11]:
|
||||
kept = torch.tensor(
|
||||
[row % 16, (row + 3) % 16, (row + 7) % 16], device=DEVICE_TYPE
|
||||
).unique(sorted=True)
|
||||
logits[row, kept] = 0
|
||||
expected.append(kept.tolist())
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
logits,
|
||||
torch.arange(0, 17, 4, device=DEVICE_TYPE, dtype=torch.int32),
|
||||
torch.from_numpy(counts).to(DEVICE_TYPE),
|
||||
max_num_kept=2,
|
||||
rows_per_request=rows_per_request,
|
||||
)
|
||||
result = tensors.to_cpu_nonblocking().tolists(counts)
|
||||
|
||||
assert tensors.token_ids.shape == (16, 2)
|
||||
assert tensors.rows_per_request == rows_per_request
|
||||
assert result.to_nested_list() == expected
|
||||
assert result.cu_num_generated_tokens == [0, 1, 3, 7, 7]
|
||||
|
||||
|
||||
def test_sampling_mask_preserves_top_k_boundary_ties():
|
||||
"""When the kept support is wider than `max_num_kept` (e.g. a top-k
|
||||
boundary tie keeps more than k logits), the mask must fall back to the
|
||||
@@ -147,6 +210,7 @@ def test_sampling_mask_preserves_top_k_boundary_ties():
|
||||
|
||||
tensors = SamplingMaskTensors.from_logits(
|
||||
processed_logits,
|
||||
cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
|
||||
num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE),
|
||||
max_num_kept=3,
|
||||
)
|
||||
|
||||
@@ -10,12 +10,53 @@ import torch
|
||||
|
||||
from vllm.config.model import PROCESSED_LOGPROBS_MODES, LogprobsMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
|
||||
RejectionSampler,
|
||||
_iter_request_chunks,
|
||||
)
|
||||
|
||||
|
||||
def _make_input_batch(
|
||||
cu_num_logits_np: np.ndarray,
|
||||
idx_mapping_np: np.ndarray,
|
||||
device: torch.device,
|
||||
) -> SimpleNamespace:
|
||||
"""Minimal InputBatch fields read by RejectionSampler._verify_in_chunks."""
|
||||
num_reqs = len(idx_mapping_np)
|
||||
num_logits_per_req = np.diff(cu_num_logits_np)
|
||||
return SimpleNamespace(
|
||||
num_reqs=num_reqs,
|
||||
cu_num_logits_np=cu_num_logits_np,
|
||||
cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device),
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
idx_mapping=torch.from_numpy(idx_mapping_np).to(device),
|
||||
expanded_idx_mapping=torch.from_numpy(
|
||||
np.repeat(idx_mapping_np, num_logits_per_req)
|
||||
).to(device),
|
||||
expanded_local_pos=torch.from_numpy(
|
||||
np.concatenate(
|
||||
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
|
||||
)
|
||||
).to(device),
|
||||
seq_lens_cpu_upper_bound=torch.full((num_reqs,), 64, dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def _make_rejection_sampler(
|
||||
sampler: SimpleNamespace, num_speculative_steps: int
|
||||
) -> RejectionSampler:
|
||||
"""RejectionSampler for standard, fixed-boundary, unwatermarked verification."""
|
||||
rejection_sampler = object.__new__(RejectionSampler)
|
||||
rejection_sampler.sampler = sampler
|
||||
rejection_sampler.num_speculative_steps = num_speculative_steps
|
||||
rejection_sampler.enable_adaptive_verification = False
|
||||
rejection_sampler.synthetic_conditional_rates = None
|
||||
rejection_sampler.use_block_verification = False
|
||||
rejection_sampler.watermark_key = None
|
||||
return rejection_sampler
|
||||
|
||||
|
||||
def test_iter_request_chunks_preserves_request_boundaries():
|
||||
cu_num_logits = np.array([0, 3, 4, 11, 13], dtype=np.int32)
|
||||
|
||||
@@ -33,28 +74,11 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
|
||||
cu_num_logits_np = np.array([0, 3, 4, 8, 10], dtype=np.int32)
|
||||
num_logits_per_req = np.diff(cu_num_logits_np)
|
||||
idx_mapping_np = np.array([7, 2, 9, 1], dtype=np.int32)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=4,
|
||||
cu_num_logits_np=cu_num_logits_np,
|
||||
cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device),
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
idx_mapping=torch.from_numpy(idx_mapping_np).to(device),
|
||||
expanded_idx_mapping=torch.from_numpy(
|
||||
np.repeat(idx_mapping_np, num_logits_per_req)
|
||||
).to(device),
|
||||
expanded_local_pos=torch.from_numpy(
|
||||
np.concatenate(
|
||||
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
|
||||
)
|
||||
).to(device),
|
||||
seq_lens_cpu_upper_bound=torch.from_numpy(
|
||||
np.array([10, 20, 30, 40], dtype=np.int32)
|
||||
),
|
||||
input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device)
|
||||
rejection_sampler = _make_rejection_sampler(
|
||||
SimpleNamespace(logprobs_mode=logprobs_mode, return_sampling_mask=False),
|
||||
num_speculative_steps=3,
|
||||
)
|
||||
rejection_sampler = object.__new__(RejectionSampler)
|
||||
rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode)
|
||||
rejection_sampler.num_speculative_steps = 3
|
||||
rejection_sampler.enable_adaptive_verification = False
|
||||
|
||||
def fake_verify(
|
||||
self,
|
||||
@@ -75,14 +99,16 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
|
||||
rejection_sampler._verify = MethodType(fake_verify, rejection_sampler)
|
||||
logits = torch.arange(170, dtype=torch.float32, device=device).view(10, 17)
|
||||
|
||||
sampled, num_sampled, chunked_logprobs = rejection_sampler._verify_in_chunks(
|
||||
logits,
|
||||
input_batch,
|
||||
draft_logits=None,
|
||||
draft_sampled=torch.arange(10, device=device),
|
||||
pos=torch.arange(10, device=device),
|
||||
max_chunk_logits=5,
|
||||
max_num_logprobs=2,
|
||||
sampled, num_sampled, chunked_logprobs, sampling_mask_tensors = (
|
||||
rejection_sampler._verify_in_chunks(
|
||||
logits,
|
||||
input_batch,
|
||||
draft_logits=None,
|
||||
draft_sampled=torch.arange(10, device=device),
|
||||
pos=torch.arange(10, device=device),
|
||||
max_chunk_logits=5,
|
||||
max_num_logprobs=2,
|
||||
)
|
||||
)
|
||||
score_logits = logits + 1 if logprobs_mode in PROCESSED_LOGPROBS_MODES else logits
|
||||
full_logprobs = rejection_sampler._get_logprobs_tensors(
|
||||
@@ -111,3 +137,155 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
|
||||
chunked_logprobs.cu_num_generated_tokens
|
||||
== full_logprobs.cu_num_generated_tokens
|
||||
)
|
||||
assert sampling_mask_tensors is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
def test_replay_on_off_preserves_rejection_sampling_and_rng(monkeypatch):
|
||||
device = torch.device("cuda")
|
||||
num_reqs = 3
|
||||
num_speculative_steps = 3
|
||||
rows_per_request = num_speculative_steps + 1
|
||||
vocab_size = 8
|
||||
cu_num_logits_np = np.arange(0, 13, rows_per_request, dtype=np.int32)
|
||||
idx_mapping_np = np.arange(num_reqs, dtype=np.int32)
|
||||
input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device)
|
||||
processed_logits = torch.zeros((12, vocab_size), device=device)
|
||||
processed_logits[0, 0] = -float("inf")
|
||||
processed_logits[rows_per_request + 1, 1] = -float("inf")
|
||||
draft_logits = torch.zeros(
|
||||
(num_reqs, num_speculative_steps, vocab_size), device=device
|
||||
)
|
||||
draft_sampled = torch.zeros(
|
||||
(num_reqs, rows_per_request), dtype=torch.int64, device=device
|
||||
)
|
||||
draft_sampled[:, 1:] = torch.tensor([0, 1, 2], device=device)
|
||||
draft_sampled = draft_sampled.flatten()
|
||||
sampler = SimpleNamespace(
|
||||
logprobs_mode="processed_logprobs",
|
||||
return_sampling_mask=False,
|
||||
sampling_states=SimpleNamespace(
|
||||
top_k=SimpleNamespace(np=np.full(num_reqs, vocab_size)),
|
||||
temperature=SimpleNamespace(gpu=torch.ones(num_reqs, device=device)),
|
||||
seeds=SimpleNamespace(
|
||||
gpu=torch.arange(num_reqs, dtype=torch.int64, device=device)
|
||||
),
|
||||
),
|
||||
use_fp64_gumbel=False,
|
||||
apply_sampling_params=lambda logits, *_: logits,
|
||||
)
|
||||
rejection_sampler = _make_rejection_sampler(sampler, num_speculative_steps)
|
||||
pos = torch.arange(12, dtype=torch.int32, device=device)
|
||||
|
||||
pack_calls: list[tuple[list[int], int, int]] = []
|
||||
pack = SamplingMaskTensors.from_logits.__func__
|
||||
|
||||
def track_pack(
|
||||
cls,
|
||||
logits,
|
||||
cu_num_logits,
|
||||
num_sampled_tokens,
|
||||
max_num_kept,
|
||||
rows_per_request=1,
|
||||
):
|
||||
pack_calls.append(
|
||||
(cu_num_logits.cpu().tolist(), max_num_kept, rows_per_request)
|
||||
)
|
||||
return pack(
|
||||
cls,
|
||||
logits,
|
||||
cu_num_logits,
|
||||
num_sampled_tokens,
|
||||
max_num_kept,
|
||||
rows_per_request,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
SamplingMaskTensors,
|
||||
"from_logits",
|
||||
classmethod(track_pack),
|
||||
)
|
||||
|
||||
rng_before = torch.cuda.get_rng_state()
|
||||
off_sampled, off_counts, off_logprobs, off_masks = (
|
||||
rejection_sampler._verify_in_chunks(
|
||||
processed_logits,
|
||||
input_batch,
|
||||
draft_logits,
|
||||
draft_sampled,
|
||||
pos,
|
||||
max_chunk_logits=8,
|
||||
max_num_logprobs=0,
|
||||
)
|
||||
)
|
||||
rng_after_off = torch.cuda.get_rng_state()
|
||||
assert pack_calls == []
|
||||
|
||||
sampler.return_sampling_mask = True
|
||||
on_sampled, on_counts, on_logprobs, on_masks = rejection_sampler._verify_in_chunks(
|
||||
processed_logits,
|
||||
input_batch,
|
||||
draft_logits,
|
||||
draft_sampled,
|
||||
pos,
|
||||
max_chunk_logits=8,
|
||||
max_num_logprobs=0,
|
||||
)
|
||||
rng_after_on = torch.cuda.get_rng_state()
|
||||
|
||||
assert pack_calls == [
|
||||
([0, 4, 8], vocab_size, rows_per_request),
|
||||
([0, 4], vocab_size, rows_per_request),
|
||||
]
|
||||
assert off_masks is None and on_masks is not None
|
||||
assert on_masks.rows_per_request == rows_per_request
|
||||
assert on_masks.token_ids.shape == (num_reqs * rows_per_request, vocab_size)
|
||||
assert on_masks.packed_mask.shape[0] == num_reqs * rows_per_request
|
||||
assert off_counts.tolist() == [1, 2, 4]
|
||||
assert torch.equal(off_counts, on_counts)
|
||||
counts = off_counts.cpu().tolist()
|
||||
for req_idx, count in enumerate(counts):
|
||||
assert torch.equal(off_sampled[req_idx, :count], on_sampled[req_idx, :count])
|
||||
|
||||
mapped_rows = [
|
||||
int(input_batch.cu_num_logits_np[req_idx]) + slot_idx
|
||||
for req_idx, count in enumerate(counts)
|
||||
for slot_idx in range(count)
|
||||
]
|
||||
assert off_logprobs is not None and on_logprobs is not None
|
||||
assert torch.equal(
|
||||
off_logprobs.logprob_token_ids[mapped_rows],
|
||||
on_logprobs.logprob_token_ids[mapped_rows],
|
||||
)
|
||||
assert torch.equal(
|
||||
off_logprobs.logprobs[mapped_rows], on_logprobs.logprobs[mapped_rows]
|
||||
)
|
||||
assert torch.equal(
|
||||
off_logprobs.selected_token_ranks[mapped_rows],
|
||||
on_logprobs.selected_token_ranks[mapped_rows],
|
||||
)
|
||||
assert torch.equal(rng_before, rng_after_off)
|
||||
assert torch.equal(rng_after_off, rng_after_on)
|
||||
|
||||
masks = (
|
||||
on_masks.to_cpu_nonblocking().tolists(on_counts.cpu().numpy()).to_nested_list()
|
||||
)
|
||||
expected_masks = [
|
||||
torch.isfinite(processed_logits[row]).nonzero().flatten().tolist()
|
||||
for row in mapped_rows
|
||||
]
|
||||
emitted = [
|
||||
int(on_sampled[req_idx, slot_idx])
|
||||
for req_idx, count in enumerate(on_counts.cpu().tolist())
|
||||
for slot_idx in range(count)
|
||||
]
|
||||
expected_logprobs = torch.stack(
|
||||
[
|
||||
torch.log_softmax(processed_logits[row], dim=-1)[token_id]
|
||||
for row, token_id in zip(mapped_rows, emitted)
|
||||
]
|
||||
)
|
||||
|
||||
assert masks == expected_masks
|
||||
assert all(token_id in mask for token_id, mask in zip(emitted, masks))
|
||||
assert torch.equal(on_logprobs.logprobs[mapped_rows, 0], expected_logprobs)
|
||||
|
||||
+8
-2
@@ -1265,9 +1265,15 @@ class VllmConfig:
|
||||
return
|
||||
if not self.use_v2_model_runner:
|
||||
raise ValueError("sampling distribution replay requires Model Runner V2")
|
||||
if self.speculative_config is not None:
|
||||
speculative_config = self.speculative_config
|
||||
if (
|
||||
speculative_config is not None
|
||||
and speculative_config.enable_adaptive_verification
|
||||
):
|
||||
raise ValueError(
|
||||
"sampling distribution replay does not support speculative decoding"
|
||||
"sampling distribution replay with speculative decoding "
|
||||
"requires fixed verification boundaries; disable adaptive "
|
||||
"verification"
|
||||
)
|
||||
if model_config.is_diffusion:
|
||||
raise ValueError(
|
||||
|
||||
@@ -441,7 +441,11 @@ class RequestState:
|
||||
sampling_mask = None
|
||||
if finished and self.sampling_mask_chunks:
|
||||
sampling_mask = SamplingMask(
|
||||
[chunk.token_ids.tolist() for chunk in self.sampling_mask_chunks]
|
||||
[
|
||||
position
|
||||
for chunk in self.sampling_mask_chunks
|
||||
for position in chunk.to_nested_list()
|
||||
]
|
||||
)
|
||||
|
||||
# Concatenate routed experts on finish
|
||||
|
||||
+10
-4
@@ -57,19 +57,25 @@ class LogprobsLists(NamedTuple):
|
||||
|
||||
|
||||
class SamplingMaskLists(NamedTuple):
|
||||
"""CSR sampling masks; a step slice holds one position (``offsets=None``)."""
|
||||
"""CSR sampling masks; a request slice may contain multiple positions."""
|
||||
|
||||
# [num_kept_tokens]
|
||||
token_ids: np.ndarray
|
||||
# [num_positions + 1], or None for a single position
|
||||
offsets: np.ndarray | None = None
|
||||
# Unused with one position per request; kept for the wire layout.
|
||||
# [num_requests + 1] for multi-position request batches.
|
||||
cu_num_generated_tokens: list[int] | None = None
|
||||
|
||||
def slice_request(self, req_idx: int, num_positions: int) -> "SamplingMaskLists":
|
||||
assert num_positions == 1 and self.offsets is not None
|
||||
assert self.offsets is not None
|
||||
cu = self.cu_num_generated_tokens
|
||||
start = req_idx if cu is None else cu[req_idx]
|
||||
end = start + num_positions
|
||||
lo, hi = self.offsets[start], self.offsets[end]
|
||||
if num_positions == 1:
|
||||
return SamplingMaskLists(self.token_ids[lo:hi])
|
||||
return SamplingMaskLists(
|
||||
self.token_ids[self.offsets[req_idx] : self.offsets[req_idx + 1]]
|
||||
self.token_ids[lo:hi], self.offsets[start : end + 1] - lo
|
||||
)
|
||||
|
||||
def to_nested_list(self) -> list[list[int]]:
|
||||
|
||||
@@ -185,7 +185,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
|
||||
|
||||
if self.sampling_mask_tensors is not None:
|
||||
self.model_runner_output.sampling_masks = (
|
||||
self.sampling_mask_tensors.tolists()
|
||||
self.sampling_mask_tensors.tolists(self.num_sampled_tokens_np)
|
||||
)
|
||||
|
||||
if self.num_nans is not None:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import NamedTuple
|
||||
|
||||
@@ -27,6 +28,7 @@ def _compact_sampling_mask_kernel(
|
||||
logits_ptr,
|
||||
logits_row_stride,
|
||||
logits_col_stride,
|
||||
cu_num_logits_ptr,
|
||||
num_sampled_tokens_ptr,
|
||||
token_ids_ptr,
|
||||
token_ids_row_stride,
|
||||
@@ -35,25 +37,32 @@ def _compact_sampling_mask_kernel(
|
||||
counts_ptr,
|
||||
vocab_size,
|
||||
max_num_kept,
|
||||
ROWS_PER_REQUEST: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
"""Per row: first ``max_num_kept`` finite-logit ids, the full count, the bitmask."""
|
||||
req_idx = tl.program_id(0)
|
||||
is_active = tl.load(num_sampled_tokens_ptr + req_idx) > 0
|
||||
output_row = tl.program_id(0)
|
||||
req_idx = output_row // ROWS_PER_REQUEST
|
||||
slot_idx = output_row % ROWS_PER_REQUEST
|
||||
if slot_idx >= tl.load(num_sampled_tokens_ptr + req_idx):
|
||||
tl.store(counts_ptr + output_row, 0)
|
||||
return
|
||||
source_row = tl.load(cu_num_logits_ptr + req_idx) + slot_idx
|
||||
|
||||
count = tl.zeros((), dtype=tl.int32)
|
||||
|
||||
for start_idx in range(0, vocab_size, BLOCK_SIZE):
|
||||
offsets = start_idx + tl.arange(0, BLOCK_SIZE)
|
||||
logits = tl.load(
|
||||
logits_ptr + req_idx * logits_row_stride + offsets * logits_col_stride,
|
||||
logits_ptr + source_row * logits_row_stride + offsets * logits_col_stride,
|
||||
mask=offsets < vocab_size,
|
||||
other=-float("inf"),
|
||||
)
|
||||
keep = (logits > -float("inf")) & (logits < float("inf")) & is_active
|
||||
keep = (logits > -float("inf")) & (logits < float("inf"))
|
||||
keep_i32 = keep.to(tl.int32)
|
||||
pos = count + tl.cumsum(keep_i32, axis=0) - keep_i32
|
||||
tl.store(
|
||||
token_ids_ptr + req_idx * token_ids_row_stride + pos,
|
||||
token_ids_ptr + output_row * token_ids_row_stride + pos,
|
||||
offsets.to(tl.int32),
|
||||
mask=keep & (pos < max_num_kept),
|
||||
)
|
||||
@@ -62,12 +71,12 @@ def _compact_sampling_mask_kernel(
|
||||
bits = tl.reshape(keep_i32, (BLOCK_SIZE // 8, 8)) << tl.arange(0, 8)[None, :]
|
||||
byte_offsets = start_idx // 8 + tl.arange(0, BLOCK_SIZE // 8)
|
||||
tl.store(
|
||||
packed_mask_ptr + req_idx * packed_mask_row_stride + byte_offsets,
|
||||
packed_mask_ptr + output_row * packed_mask_row_stride + byte_offsets,
|
||||
tl.sum(bits, axis=1).to(tl.uint8),
|
||||
mask=byte_offsets < tl.cdiv(vocab_size, 8),
|
||||
)
|
||||
|
||||
tl.store(counts_ptr + req_idx, count)
|
||||
tl.store(counts_ptr + output_row, count)
|
||||
|
||||
|
||||
# Bounds the [num_reqs, width] int32 buffer; wider rows use the bitmask instead.
|
||||
@@ -78,37 +87,46 @@ class SamplingMaskTensors(NamedTuple):
|
||||
"""Device-side masks pending async D2H: compact ids, plus the bitmask as
|
||||
the exact fallback for rows wider than ``max_num_kept``."""
|
||||
|
||||
# [num_requests, max_num_kept]
|
||||
# [num_requests * rows_per_request, max_num_kept]
|
||||
token_ids: torch.Tensor
|
||||
# [num_requests, ceil(vocab_size / 8)]
|
||||
# [num_requests * rows_per_request, ceil(vocab_size / 8)]
|
||||
packed_mask: torch.Tensor
|
||||
# [num_requests]
|
||||
# [num_requests * rows_per_request]
|
||||
counts: torch.Tensor
|
||||
vocab_size: int
|
||||
# Fixed request-major output slots. Ordinary sampling has one slot.
|
||||
rows_per_request: int = 1
|
||||
|
||||
@classmethod
|
||||
def from_logits(
|
||||
cls,
|
||||
logits: torch.Tensor,
|
||||
cu_num_logits: torch.Tensor,
|
||||
num_sampled_tokens: torch.Tensor,
|
||||
max_num_kept: int,
|
||||
rows_per_request: int = 1,
|
||||
) -> SamplingMaskTensors:
|
||||
"""Capture the finite-logit support of every row with a sampled token."""
|
||||
num_reqs, vocab_size = logits.shape
|
||||
"""Capture committed target supports in fixed request-major slots."""
|
||||
num_reqs = num_sampled_tokens.shape[0]
|
||||
vocab_size = logits.shape[1]
|
||||
max_num_kept = min(max_num_kept, vocab_size, MAX_COMPACT_SUPPORT)
|
||||
num_output_rows = num_reqs * rows_per_request
|
||||
device = logits.device
|
||||
|
||||
token_ids = torch.empty(
|
||||
(num_reqs, max_num_kept), dtype=torch.int32, device=device
|
||||
(num_output_rows, max_num_kept), dtype=torch.int32, device=device
|
||||
)
|
||||
packed_mask = torch.empty(
|
||||
(num_reqs, (vocab_size + 7) // 8), dtype=torch.uint8, device=device
|
||||
(num_output_rows, (vocab_size + 7) // 8),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
counts = torch.empty(num_reqs, dtype=torch.int32, device=device)
|
||||
_compact_sampling_mask_kernel[(num_reqs,)](
|
||||
counts = torch.empty(num_output_rows, dtype=torch.int32, device=device)
|
||||
_compact_sampling_mask_kernel[(num_output_rows,)](
|
||||
logits,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
cu_num_logits,
|
||||
num_sampled_tokens,
|
||||
token_ids,
|
||||
token_ids.stride(0),
|
||||
@@ -117,9 +135,23 @@ class SamplingMaskTensors(NamedTuple):
|
||||
counts,
|
||||
vocab_size,
|
||||
max_num_kept,
|
||||
ROWS_PER_REQUEST=rows_per_request,
|
||||
BLOCK_SIZE=8192,
|
||||
)
|
||||
return cls(token_ids, packed_mask, counts, vocab_size)
|
||||
return cls(token_ids, packed_mask, counts, vocab_size, rows_per_request)
|
||||
|
||||
@classmethod
|
||||
def cat(cls, chunks: Sequence[SamplingMaskTensors]) -> SamplingMaskTensors:
|
||||
"""Join request-contiguous chunks that share one output layout."""
|
||||
first = chunks[0]
|
||||
assert len({(c.vocab_size, c.rows_per_request) for c in chunks}) == 1
|
||||
return cls(
|
||||
torch.cat([chunk.token_ids for chunk in chunks]),
|
||||
torch.cat([chunk.packed_mask for chunk in chunks]),
|
||||
torch.cat([chunk.counts for chunk in chunks]),
|
||||
first.vocab_size,
|
||||
first.rows_per_request,
|
||||
)
|
||||
|
||||
def to_cpu_nonblocking(self) -> SamplingMaskTensors:
|
||||
if self.token_ids.device.type == "cpu":
|
||||
@@ -129,15 +161,31 @@ class SamplingMaskTensors(NamedTuple):
|
||||
self.packed_mask.to("cpu", non_blocking=True),
|
||||
self.counts.to("cpu", non_blocking=True),
|
||||
self.vocab_size,
|
||||
self.rows_per_request,
|
||||
)
|
||||
|
||||
def tolists(self) -> SamplingMaskLists:
|
||||
"""CSR over all requests; rows without a sampled token are empty."""
|
||||
def tolists(
|
||||
self, num_sampled_tokens: np.ndarray | None = None
|
||||
) -> SamplingMaskLists:
|
||||
"""Convert fixed output rows to the scheduler's CSR representation."""
|
||||
counts = self.counts.cpu().numpy()
|
||||
token_ids = self.token_ids.cpu().numpy()
|
||||
packed_mask = self.packed_mask.cpu().numpy()
|
||||
width = token_ids.shape[1]
|
||||
|
||||
cu_num_generated_tokens = None
|
||||
if self.rows_per_request == 1:
|
||||
sampled_rows = np.arange(len(counts))
|
||||
else:
|
||||
assert num_sampled_tokens is not None
|
||||
active_slots = (
|
||||
np.arange(self.rows_per_request)[None, :] < num_sampled_tokens[:, None]
|
||||
)
|
||||
sampled_rows = np.flatnonzero(active_slots)
|
||||
cu_num_generated_tokens = np.cumsum(
|
||||
np.concatenate(([0], num_sampled_tokens))
|
||||
).tolist()
|
||||
|
||||
def support(row: int) -> np.ndarray:
|
||||
if counts[row] <= width:
|
||||
return token_ids[row, : counts[row]]
|
||||
@@ -147,7 +195,8 @@ class SamplingMaskTensors(NamedTuple):
|
||||
)
|
||||
return np.flatnonzero(bits).astype(np.int32, copy=False)
|
||||
|
||||
supports = [support(row) for row in range(len(counts))]
|
||||
supports = [support(row) for row in sampled_rows]
|
||||
offsets = np.zeros(len(supports) + 1, dtype=np.int64)
|
||||
np.cumsum([len(s) for s in supports], out=offsets[1:])
|
||||
return SamplingMaskLists(np.concatenate(supports), offsets)
|
||||
flat = np.concatenate([np.empty(0, dtype=np.int32), *supports])
|
||||
return SamplingMaskLists(flat, offsets, cu_num_generated_tokens)
|
||||
|
||||
@@ -203,7 +203,10 @@ class Sampler:
|
||||
# Size by the validated top_k batch max; wider supports use the bitmask.
|
||||
max_num_kept = int(np.max(self.sampling_states.top_k.np[idx_mapping_np]))
|
||||
sampling_mask_tensors = SamplingMaskTensors.from_logits(
|
||||
processed_logits, num_sampled, max_num_kept
|
||||
processed_logits,
|
||||
input_batch.cu_num_logits,
|
||||
num_sampled,
|
||||
max_num_kept,
|
||||
)
|
||||
|
||||
# These are GPU tensors.
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.v1.worker.gpu.input_batch import (
|
||||
)
|
||||
from vllm.v1.worker.gpu.metrics.logits import get_num_nans
|
||||
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
from vllm.v1.worker.gpu.sample.states import NO_LOGPROBS
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
|
||||
@@ -296,7 +296,12 @@ class RejectionSampler:
|
||||
max_chunk_logits: int,
|
||||
max_num_logprobs: int,
|
||||
verify_draft_sampled: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]:
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
LogprobsTensors | None,
|
||||
SamplingMaskTensors | None,
|
||||
]:
|
||||
if verify_draft_sampled is None:
|
||||
verify_draft_sampled = draft_sampled
|
||||
cu_num_logits_np = input_batch.cu_num_logits_np
|
||||
@@ -316,6 +321,14 @@ class RejectionSampler:
|
||||
sampled_chunks: list[torch.Tensor] = []
|
||||
num_sampled_chunks: list[torch.Tensor] = []
|
||||
logprobs_chunks: list[LogprobsTensors] = []
|
||||
sampling_mask_chunks: list[SamplingMaskTensors] = []
|
||||
max_num_kept = None
|
||||
if self.sampler.return_sampling_mask:
|
||||
max_num_kept = int(
|
||||
np.max(
|
||||
self.sampler.sampling_states.top_k.np[input_batch.idx_mapping_np]
|
||||
)
|
||||
)
|
||||
|
||||
for start, end in request_chunks:
|
||||
lo = int(cu_num_logits_np[start])
|
||||
@@ -346,13 +359,31 @@ class RejectionSampler:
|
||||
)
|
||||
if chunk_logprobs is not None:
|
||||
logprobs_chunks.append(chunk_logprobs)
|
||||
if max_num_kept is not None:
|
||||
sampling_mask_chunks.append(
|
||||
SamplingMaskTensors.from_logits(
|
||||
processed_logits,
|
||||
chunk_cu_num_logits,
|
||||
num_sampled,
|
||||
max_num_kept,
|
||||
self.num_speculative_steps + 1,
|
||||
)
|
||||
)
|
||||
del processed_logits
|
||||
sampled_chunks.append(sampled)
|
||||
num_sampled_chunks.append(num_sampled)
|
||||
|
||||
if len(sampled_chunks) == 1:
|
||||
logprobs_tensors = logprobs_chunks[0] if logprobs_chunks else None
|
||||
return sampled_chunks[0], num_sampled_chunks[0], logprobs_tensors
|
||||
sampling_mask_tensors = (
|
||||
sampling_mask_chunks[0] if sampling_mask_chunks else None
|
||||
)
|
||||
return (
|
||||
sampled_chunks[0],
|
||||
num_sampled_chunks[0],
|
||||
logprobs_tensors,
|
||||
sampling_mask_tensors,
|
||||
)
|
||||
|
||||
logprobs_tensors = None
|
||||
if logprobs_chunks:
|
||||
@@ -366,7 +397,12 @@ class RejectionSampler:
|
||||
|
||||
sampled = torch.cat(sampled_chunks)
|
||||
num_sampled = torch.cat(num_sampled_chunks)
|
||||
return sampled, num_sampled, logprobs_tensors
|
||||
sampling_mask_tensors = (
|
||||
SamplingMaskTensors.cat(sampling_mask_chunks)
|
||||
if sampling_mask_chunks
|
||||
else None
|
||||
)
|
||||
return sampled, num_sampled, logprobs_tensors, sampling_mask_tensors
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -400,7 +436,12 @@ class RejectionSampler:
|
||||
input_batch.idx_mapping_np
|
||||
)
|
||||
chunk_logit_limit = get_max_chunk_logits(logits.shape[1])
|
||||
sampled, num_sampled, logprobs_tensors = self._verify_in_chunks(
|
||||
(
|
||||
sampled,
|
||||
num_sampled,
|
||||
logprobs_tensors,
|
||||
sampling_mask_tensors,
|
||||
) = self._verify_in_chunks(
|
||||
logits,
|
||||
input_batch,
|
||||
draft_logits,
|
||||
@@ -425,4 +466,5 @@ class RejectionSampler:
|
||||
num_nans=num_nans,
|
||||
num_sampled=num_sampled,
|
||||
num_rejected=num_rejected,
|
||||
sampling_mask_tensors=sampling_mask_tensors,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user