[Feature][Spec Decode] Support sampling mask replay for MRV2 MTP (#59359)

Signed-off-by: Cuiping Cheng <chengcuipingswu@163.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Cuiping Cheng <chengcuipingswu@163.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
vx120
2026-10-01 13:51:39 +08:00
committed by GitHub
co-authored by Cuiping Cheng aoshen02 Claude Opus 5.5
parent 37cecc8141
commit 70be6aae19
13 changed files with 583 additions and 68 deletions
+91
View File
@@ -147,6 +147,97 @@ def test_rocm_mm_prefix_lm_disables_chunked_mm_input(
assert config.scheduler_config.disable_chunked_mm_input is expected
def _sampling_replay_config(
*,
return_sampling_mask: bool = True,
use_v2_model_runner: bool = True,
speculative_method: str | None = None,
rejection_sample_method: str = "standard",
adaptive: bool = False,
is_diffusion: bool = False,
logits_processors: list[str] | None = None,
logprobs_mode: str = "processed_logprobs",
):
speculative_config = None
if speculative_method is not None:
speculative_config = SimpleNamespace(
method=speculative_method,
enable_adaptive_verification=adaptive,
rejection_sample_method=rejection_sample_method,
)
return SimpleNamespace(
model_config=SimpleNamespace(
return_sampling_mask=return_sampling_mask,
is_diffusion=is_diffusion,
logits_processors=logits_processors or [],
logprobs_mode=logprobs_mode,
),
use_v2_model_runner=use_v2_model_runner,
speculative_config=speculative_config,
)
@pytest.mark.parametrize(
("config", "message"),
[
(_sampling_replay_config(), None),
*(
(_sampling_replay_config(speculative_method=method), None)
for method in (
"mtp",
"eagle",
"eagle3",
"dflash",
"dspark",
"draft_model",
)
),
*(
(
_sampling_replay_config(
speculative_method="mtp",
rejection_sample_method=rejection_sample_method,
),
None,
)
for rejection_sample_method in ("standard", "block", "synthetic")
),
(
_sampling_replay_config(
return_sampling_mask=False, speculative_method="dspark", adaptive=True
),
None,
),
(
_sampling_replay_config(speculative_method="dspark", adaptive=True),
"requires fixed verification boundaries",
),
(
_sampling_replay_config(is_diffusion=True),
"does not support diffusion models",
),
(
_sampling_replay_config(logits_processors=["custom"]),
"does not support custom logits processors",
),
(
_sampling_replay_config(logprobs_mode="raw_logprobs"),
"requires logprobs_mode='processed_logprobs'",
),
(
_sampling_replay_config(use_v2_model_runner=False),
"requires Model Runner V2",
),
],
)
def test_sampling_replay_config(config, message):
if message is None:
VllmConfig._verify_sampling_replay_config(config)
else:
with pytest.raises(ValueError, match=message):
VllmConfig._verify_sampling_replay_config(config)
def test_kda_recoverssm_derivation_is_revalidated():
config = SimpleNamespace(
cache_config=SimpleNamespace(
+43
View File
@@ -908,6 +908,49 @@ def test_update_from_output_routes_sampling_masks_by_request():
assert all(out.new_sampling_mask.offsets is None for out in outputs)
def test_update_from_output_routes_multi_position_sampling_masks():
scheduler = create_scheduler()
scheduler.return_sampling_mask = True
requests = create_requests(num_requests=2, max_tokens=10)
for req in requests:
req.num_computed_tokens = req.num_tokens
scheduler.requests[req.request_id] = req
scheduler.running.append(req)
req.status = RequestStatus.RUNNING
scheduler_output = SchedulerOutput(
scheduled_new_reqs=[],
scheduled_cached_reqs=CachedRequestData.make_empty(),
num_scheduled_tokens={req.request_id: 1 for req in requests},
total_num_scheduled_tokens=2,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
model_output = ModelRunnerOutput(
req_ids=[req.request_id for req in requests],
req_id_to_index={req.request_id: i for i, req in enumerate(requests)},
sampled_token_ids=[[1, 2], [3, 4, 5]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
sampling_masks=SamplingMaskLists(
token_ids=np.array([1, 6, 2, 3, 7, 8, 4, 5, 9], dtype=np.int32),
offsets=np.array([0, 2, 3, 6, 7, 9]),
cu_num_generated_tokens=[0, 2, 5],
),
)
outputs = scheduler.update_from_output(scheduler_output, model_output)[0].outputs
assert [out.new_sampling_mask.to_nested_list() for out in outputs] == [
[[1, 6], [2]],
[[3, 7, 8], [4], [5, 9]],
]
def test_stop_via_update_from_output():
"""Test stopping behavior through update_from_output."""
scheduler = create_scheduler(num_speculative_tokens=1)
+26
View File
@@ -35,6 +35,7 @@ from vllm.v1.engine.output_processor import (
RequestState,
)
from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats
from vllm.v1.outputs import SamplingMaskLists
@pytest.mark.parametrize("flat_logprobs", [False, True])
@@ -62,6 +63,31 @@ def test_delta_output_without_new_tokens_returns_empty_logprobs(
assert len(output.logprobs) == 0
def test_completion_output_preserves_each_sampling_mask_position() -> None:
state = RequestState.__new__(RequestState)
state.detokenizer = MagicMock()
state.detokenizer.get_next_output_text.return_value = ""
state.logprobs_processor = MagicMock()
state.logprobs_processor.logprobs = None
state.logprobs_processor.cumulative_logprob = None
state.output_kind = RequestOutputKind.DELTA
state.request_index = 0
state.sampling_mask_chunks = [
SamplingMaskLists(
token_ids=np.array([10, 11, 20]),
offsets=np.array([0, 2, 3]),
),
SamplingMaskLists(token_ids=np.array([30, 31, 32])),
]
state.routed_experts_chunks = []
state.spec_decode_metrics = None
output = state._new_completion_output([1, 2, 3], FinishReason.LENGTH, None)
assert output.sampling_mask is not None
assert output.sampling_mask.token_ids == [[10, 11], [20], [30, 31, 32]]
def _ref_convert_id_to_token(
tokenizer: TokenizerLike,
token_id: int,
+67 -3
View File
@@ -13,6 +13,7 @@ from vllm.v1.outputs import (
LogprobsLists,
LogprobsTensors,
ModelRunnerOutput,
SamplingMaskLists,
)
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
@@ -63,8 +64,6 @@ def test_logprobs_tensors_tolists_with_tensor_boundaries():
def test_sampling_mask_lists_to_nested_list():
from vllm.v1.outputs import SamplingMaskLists
mask = SamplingMaskLists(
token_ids=np.array([10, 11, 12, 20, 21]),
offsets=np.array([0, 3, 5]),
@@ -76,6 +75,22 @@ def test_sampling_mask_lists_to_nested_list():
assert SamplingMaskLists(np.array([7, 9])).to_nested_list() == [[7, 9]]
def test_sampling_mask_lists_slices_multiple_positions_by_request():
masks = SamplingMaskLists(
token_ids=np.array([10, 11, 20, 30, 31, 32, 40]),
offsets=np.array([0, 2, 3, 6, 7]),
cu_num_generated_tokens=[0, 1, 3, 4],
)
single = masks.slice_request(0, 1)
multi = masks.slice_request(1, 2)
assert single.to_nested_list() == [[10, 11]]
assert single.offsets is None
assert multi.to_nested_list() == [[20], [30, 31, 32]]
assert multi.cu_num_generated_tokens is None
@pytest.mark.parametrize("max_num_kept", [512, 20_001])
@pytest.mark.skipif(
current_platform.is_xpu(),
@@ -102,7 +117,10 @@ def test_sampling_mask_tensors_match_finite_support(max_num_kept):
expected[1] = expected[7] = []
tensors = SamplingMaskTensors.from_logits(
logits.to(DEVICE_TYPE), num_sampled_tokens.to(DEVICE_TYPE), max_num_kept
logits.to(DEVICE_TYPE),
torch.arange(len(sizes) + 1, device=DEVICE_TYPE, dtype=torch.int32),
num_sampled_tokens.to(DEVICE_TYPE),
max_num_kept,
)
assert tensors.token_ids.shape[1] == min(max_num_kept, MAX_COMPACT_SUPPORT)
@@ -127,6 +145,7 @@ def test_sampling_mask_matches_processed_top_k_top_p_support():
tensors = SamplingMaskTensors.from_logits(
processed_logits,
cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE),
max_num_kept=3,
)
@@ -135,6 +154,50 @@ def test_sampling_mask_matches_processed_top_k_top_p_support():
assert result.to_nested_list() == [expected_token_ids]
def test_sampling_mask_tensors_uses_request_boundaries():
logits = torch.full((6, 8), -float("inf"), device=DEVICE_TYPE)
logits[0, [1, 3]] = 0
logits[2, [2, 4, 6]] = 0
logits[5, [0, 7]] = 0
counts = torch.tensor([1, 0, 1], device=DEVICE_TYPE, dtype=torch.int32)
tensors = SamplingMaskTensors.from_logits(
logits,
torch.tensor([0, 2, 5, 6], device=DEVICE_TYPE, dtype=torch.int32),
counts,
max_num_kept=3,
)
assert tensors.tolists().to_nested_list() == [[1, 3], [], [0, 7]]
def test_sampling_mask_tensors_multirow_request_layout():
rows_per_request = 4
counts = np.array([1, 2, 4, 0], dtype=np.int32)
logits = torch.full((16, 16), -float("inf"), device=DEVICE_TYPE)
expected = []
for row in [0, 4, 5, 8, 9, 10, 11]:
kept = torch.tensor(
[row % 16, (row + 3) % 16, (row + 7) % 16], device=DEVICE_TYPE
).unique(sorted=True)
logits[row, kept] = 0
expected.append(kept.tolist())
tensors = SamplingMaskTensors.from_logits(
logits,
torch.arange(0, 17, 4, device=DEVICE_TYPE, dtype=torch.int32),
torch.from_numpy(counts).to(DEVICE_TYPE),
max_num_kept=2,
rows_per_request=rows_per_request,
)
result = tensors.to_cpu_nonblocking().tolists(counts)
assert tensors.token_ids.shape == (16, 2)
assert tensors.rows_per_request == rows_per_request
assert result.to_nested_list() == expected
assert result.cu_num_generated_tokens == [0, 1, 3, 7, 7]
def test_sampling_mask_preserves_top_k_boundary_ties():
"""When the kept support is wider than `max_num_kept` (e.g. a top-k
boundary tie keeps more than k logits), the mask must fall back to the
@@ -147,6 +210,7 @@ def test_sampling_mask_preserves_top_k_boundary_ties():
tensors = SamplingMaskTensors.from_logits(
processed_logits,
cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE),
max_num_kept=3,
)
@@ -10,12 +10,53 @@ import torch
from vllm.config.model import PROCESSED_LOGPROBS_MODES, LogprobsMode
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
RejectionSampler,
_iter_request_chunks,
)
def _make_input_batch(
cu_num_logits_np: np.ndarray,
idx_mapping_np: np.ndarray,
device: torch.device,
) -> SimpleNamespace:
"""Minimal InputBatch fields read by RejectionSampler._verify_in_chunks."""
num_reqs = len(idx_mapping_np)
num_logits_per_req = np.diff(cu_num_logits_np)
return SimpleNamespace(
num_reqs=num_reqs,
cu_num_logits_np=cu_num_logits_np,
cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device),
idx_mapping_np=idx_mapping_np,
idx_mapping=torch.from_numpy(idx_mapping_np).to(device),
expanded_idx_mapping=torch.from_numpy(
np.repeat(idx_mapping_np, num_logits_per_req)
).to(device),
expanded_local_pos=torch.from_numpy(
np.concatenate(
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
)
).to(device),
seq_lens_cpu_upper_bound=torch.full((num_reqs,), 64, dtype=torch.int32),
)
def _make_rejection_sampler(
sampler: SimpleNamespace, num_speculative_steps: int
) -> RejectionSampler:
"""RejectionSampler for standard, fixed-boundary, unwatermarked verification."""
rejection_sampler = object.__new__(RejectionSampler)
rejection_sampler.sampler = sampler
rejection_sampler.num_speculative_steps = num_speculative_steps
rejection_sampler.enable_adaptive_verification = False
rejection_sampler.synthetic_conditional_rates = None
rejection_sampler.use_block_verification = False
rejection_sampler.watermark_key = None
return rejection_sampler
def test_iter_request_chunks_preserves_request_boundaries():
cu_num_logits = np.array([0, 3, 4, 11, 13], dtype=np.int32)
@@ -33,28 +74,11 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
cu_num_logits_np = np.array([0, 3, 4, 8, 10], dtype=np.int32)
num_logits_per_req = np.diff(cu_num_logits_np)
idx_mapping_np = np.array([7, 2, 9, 1], dtype=np.int32)
input_batch = SimpleNamespace(
num_reqs=4,
cu_num_logits_np=cu_num_logits_np,
cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device),
idx_mapping_np=idx_mapping_np,
idx_mapping=torch.from_numpy(idx_mapping_np).to(device),
expanded_idx_mapping=torch.from_numpy(
np.repeat(idx_mapping_np, num_logits_per_req)
).to(device),
expanded_local_pos=torch.from_numpy(
np.concatenate(
[np.arange(count, dtype=np.int32) for count in num_logits_per_req]
)
).to(device),
seq_lens_cpu_upper_bound=torch.from_numpy(
np.array([10, 20, 30, 40], dtype=np.int32)
),
input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device)
rejection_sampler = _make_rejection_sampler(
SimpleNamespace(logprobs_mode=logprobs_mode, return_sampling_mask=False),
num_speculative_steps=3,
)
rejection_sampler = object.__new__(RejectionSampler)
rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode)
rejection_sampler.num_speculative_steps = 3
rejection_sampler.enable_adaptive_verification = False
def fake_verify(
self,
@@ -75,14 +99,16 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
rejection_sampler._verify = MethodType(fake_verify, rejection_sampler)
logits = torch.arange(170, dtype=torch.float32, device=device).view(10, 17)
sampled, num_sampled, chunked_logprobs = rejection_sampler._verify_in_chunks(
logits,
input_batch,
draft_logits=None,
draft_sampled=torch.arange(10, device=device),
pos=torch.arange(10, device=device),
max_chunk_logits=5,
max_num_logprobs=2,
sampled, num_sampled, chunked_logprobs, sampling_mask_tensors = (
rejection_sampler._verify_in_chunks(
logits,
input_batch,
draft_logits=None,
draft_sampled=torch.arange(10, device=device),
pos=torch.arange(10, device=device),
max_chunk_logits=5,
max_num_logprobs=2,
)
)
score_logits = logits + 1 if logprobs_mode in PROCESSED_LOGPROBS_MODES else logits
full_logprobs = rejection_sampler._get_logprobs_tensors(
@@ -111,3 +137,155 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
chunked_logprobs.cu_num_generated_tokens
== full_logprobs.cu_num_generated_tokens
)
assert sampling_mask_tensors is None
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_replay_on_off_preserves_rejection_sampling_and_rng(monkeypatch):
device = torch.device("cuda")
num_reqs = 3
num_speculative_steps = 3
rows_per_request = num_speculative_steps + 1
vocab_size = 8
cu_num_logits_np = np.arange(0, 13, rows_per_request, dtype=np.int32)
idx_mapping_np = np.arange(num_reqs, dtype=np.int32)
input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device)
processed_logits = torch.zeros((12, vocab_size), device=device)
processed_logits[0, 0] = -float("inf")
processed_logits[rows_per_request + 1, 1] = -float("inf")
draft_logits = torch.zeros(
(num_reqs, num_speculative_steps, vocab_size), device=device
)
draft_sampled = torch.zeros(
(num_reqs, rows_per_request), dtype=torch.int64, device=device
)
draft_sampled[:, 1:] = torch.tensor([0, 1, 2], device=device)
draft_sampled = draft_sampled.flatten()
sampler = SimpleNamespace(
logprobs_mode="processed_logprobs",
return_sampling_mask=False,
sampling_states=SimpleNamespace(
top_k=SimpleNamespace(np=np.full(num_reqs, vocab_size)),
temperature=SimpleNamespace(gpu=torch.ones(num_reqs, device=device)),
seeds=SimpleNamespace(
gpu=torch.arange(num_reqs, dtype=torch.int64, device=device)
),
),
use_fp64_gumbel=False,
apply_sampling_params=lambda logits, *_: logits,
)
rejection_sampler = _make_rejection_sampler(sampler, num_speculative_steps)
pos = torch.arange(12, dtype=torch.int32, device=device)
pack_calls: list[tuple[list[int], int, int]] = []
pack = SamplingMaskTensors.from_logits.__func__
def track_pack(
cls,
logits,
cu_num_logits,
num_sampled_tokens,
max_num_kept,
rows_per_request=1,
):
pack_calls.append(
(cu_num_logits.cpu().tolist(), max_num_kept, rows_per_request)
)
return pack(
cls,
logits,
cu_num_logits,
num_sampled_tokens,
max_num_kept,
rows_per_request,
)
monkeypatch.setattr(
SamplingMaskTensors,
"from_logits",
classmethod(track_pack),
)
rng_before = torch.cuda.get_rng_state()
off_sampled, off_counts, off_logprobs, off_masks = (
rejection_sampler._verify_in_chunks(
processed_logits,
input_batch,
draft_logits,
draft_sampled,
pos,
max_chunk_logits=8,
max_num_logprobs=0,
)
)
rng_after_off = torch.cuda.get_rng_state()
assert pack_calls == []
sampler.return_sampling_mask = True
on_sampled, on_counts, on_logprobs, on_masks = rejection_sampler._verify_in_chunks(
processed_logits,
input_batch,
draft_logits,
draft_sampled,
pos,
max_chunk_logits=8,
max_num_logprobs=0,
)
rng_after_on = torch.cuda.get_rng_state()
assert pack_calls == [
([0, 4, 8], vocab_size, rows_per_request),
([0, 4], vocab_size, rows_per_request),
]
assert off_masks is None and on_masks is not None
assert on_masks.rows_per_request == rows_per_request
assert on_masks.token_ids.shape == (num_reqs * rows_per_request, vocab_size)
assert on_masks.packed_mask.shape[0] == num_reqs * rows_per_request
assert off_counts.tolist() == [1, 2, 4]
assert torch.equal(off_counts, on_counts)
counts = off_counts.cpu().tolist()
for req_idx, count in enumerate(counts):
assert torch.equal(off_sampled[req_idx, :count], on_sampled[req_idx, :count])
mapped_rows = [
int(input_batch.cu_num_logits_np[req_idx]) + slot_idx
for req_idx, count in enumerate(counts)
for slot_idx in range(count)
]
assert off_logprobs is not None and on_logprobs is not None
assert torch.equal(
off_logprobs.logprob_token_ids[mapped_rows],
on_logprobs.logprob_token_ids[mapped_rows],
)
assert torch.equal(
off_logprobs.logprobs[mapped_rows], on_logprobs.logprobs[mapped_rows]
)
assert torch.equal(
off_logprobs.selected_token_ranks[mapped_rows],
on_logprobs.selected_token_ranks[mapped_rows],
)
assert torch.equal(rng_before, rng_after_off)
assert torch.equal(rng_after_off, rng_after_on)
masks = (
on_masks.to_cpu_nonblocking().tolists(on_counts.cpu().numpy()).to_nested_list()
)
expected_masks = [
torch.isfinite(processed_logits[row]).nonzero().flatten().tolist()
for row in mapped_rows
]
emitted = [
int(on_sampled[req_idx, slot_idx])
for req_idx, count in enumerate(on_counts.cpu().tolist())
for slot_idx in range(count)
]
expected_logprobs = torch.stack(
[
torch.log_softmax(processed_logits[row], dim=-1)[token_id]
for row, token_id in zip(mapped_rows, emitted)
]
)
assert masks == expected_masks
assert all(token_id in mask for token_id, mask in zip(emitted, masks))
assert torch.equal(on_logprobs.logprobs[mapped_rows, 0], expected_logprobs)