From 70be6aae19587a251987c405b5bdf2d540da5991 Mon Sep 17 00:00:00 2001 From: vx120 <57470515+vx120@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:51:39 +0800 Subject: [PATCH] [Feature][Spec Decode] Support sampling mask replay for MRV2 MTP (#59359) Signed-off-by: Cuiping Cheng Signed-off-by: aoshen02 Co-authored-by: Cuiping Cheng Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- docs/training/sampling_mask.md | 5 +- tests/test_config.py | 91 +++++++ tests/v1/core/test_scheduler.py | 43 ++++ tests/v1/engine/test_output_processor.py | 26 ++ tests/v1/test_outputs.py | 70 +++++- .../test_gpu_rejection_sampler_chunking.py | 236 +++++++++++++++--- vllm/config/vllm.py | 10 +- vllm/v1/engine/output_processor.py | 6 +- vllm/v1/outputs.py | 14 +- vllm/v1/worker/gpu/async_utils.py | 2 +- vllm/v1/worker/gpu/sample/output.py | 91 +++++-- vllm/v1/worker/gpu/sample/sampler.py | 5 +- .../gpu/spec_decode/rejection_sampler.py | 52 +++- 13 files changed, 583 insertions(+), 68 deletions(-) diff --git a/docs/training/sampling_mask.md b/docs/training/sampling_mask.md index 67e0709388..c8425e0245 100644 --- a/docs/training/sampling_mask.md +++ b/docs/training/sampling_mask.md @@ -63,9 +63,12 @@ The mask is also available via the `/inference/v1/generate` HTTP endpoint: | `top_k > 0` | Bounds mask size; pure top-p can produce vocab-sized masks | | Model Runner V2 | Required by the async D2H copy pipeline | +Speculative decoding is supported; each emitted token gets the support of the +target distribution at its position. + The engine rejects unsupported combinations at startup or request time: -- Speculative decoding +- Speculative decoding with adaptive verification - Diffusion models - Custom logits processors (engine-level `--logits-processors`) diff --git a/tests/test_config.py b/tests/test_config.py index 712d01fd33..e7389d18b4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -147,6 +147,97 @@ def test_rocm_mm_prefix_lm_disables_chunked_mm_input( assert config.scheduler_config.disable_chunked_mm_input is expected +def _sampling_replay_config( + *, + return_sampling_mask: bool = True, + use_v2_model_runner: bool = True, + speculative_method: str | None = None, + rejection_sample_method: str = "standard", + adaptive: bool = False, + is_diffusion: bool = False, + logits_processors: list[str] | None = None, + logprobs_mode: str = "processed_logprobs", +): + speculative_config = None + if speculative_method is not None: + speculative_config = SimpleNamespace( + method=speculative_method, + enable_adaptive_verification=adaptive, + rejection_sample_method=rejection_sample_method, + ) + return SimpleNamespace( + model_config=SimpleNamespace( + return_sampling_mask=return_sampling_mask, + is_diffusion=is_diffusion, + logits_processors=logits_processors or [], + logprobs_mode=logprobs_mode, + ), + use_v2_model_runner=use_v2_model_runner, + speculative_config=speculative_config, + ) + + +@pytest.mark.parametrize( + ("config", "message"), + [ + (_sampling_replay_config(), None), + *( + (_sampling_replay_config(speculative_method=method), None) + for method in ( + "mtp", + "eagle", + "eagle3", + "dflash", + "dspark", + "draft_model", + ) + ), + *( + ( + _sampling_replay_config( + speculative_method="mtp", + rejection_sample_method=rejection_sample_method, + ), + None, + ) + for rejection_sample_method in ("standard", "block", "synthetic") + ), + ( + _sampling_replay_config( + return_sampling_mask=False, speculative_method="dspark", adaptive=True + ), + None, + ), + ( + _sampling_replay_config(speculative_method="dspark", adaptive=True), + "requires fixed verification boundaries", + ), + ( + _sampling_replay_config(is_diffusion=True), + "does not support diffusion models", + ), + ( + _sampling_replay_config(logits_processors=["custom"]), + "does not support custom logits processors", + ), + ( + _sampling_replay_config(logprobs_mode="raw_logprobs"), + "requires logprobs_mode='processed_logprobs'", + ), + ( + _sampling_replay_config(use_v2_model_runner=False), + "requires Model Runner V2", + ), + ], +) +def test_sampling_replay_config(config, message): + if message is None: + VllmConfig._verify_sampling_replay_config(config) + else: + with pytest.raises(ValueError, match=message): + VllmConfig._verify_sampling_replay_config(config) + + def test_kda_recoverssm_derivation_is_revalidated(): config = SimpleNamespace( cache_config=SimpleNamespace( diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index c8ac4dec64..722cbfb79e 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -908,6 +908,49 @@ def test_update_from_output_routes_sampling_masks_by_request(): assert all(out.new_sampling_mask.offsets is None for out in outputs) +def test_update_from_output_routes_multi_position_sampling_masks(): + scheduler = create_scheduler() + scheduler.return_sampling_mask = True + requests = create_requests(num_requests=2, max_tokens=10) + for req in requests: + req.num_computed_tokens = req.num_tokens + scheduler.requests[req.request_id] = req + scheduler.running.append(req) + req.status = RequestStatus.RUNNING + + scheduler_output = SchedulerOutput( + scheduled_new_reqs=[], + scheduled_cached_reqs=CachedRequestData.make_empty(), + num_scheduled_tokens={req.request_id: 1 for req in requests}, + total_num_scheduled_tokens=2, + scheduled_encoder_inputs={}, + scheduled_spec_decode_tokens={}, + num_common_prefix_blocks=[], + finished_req_ids=set(), + free_encoder_mm_hashes=[], + ) + model_output = ModelRunnerOutput( + req_ids=[req.request_id for req in requests], + req_id_to_index={req.request_id: i for i, req in enumerate(requests)}, + sampled_token_ids=[[1, 2], [3, 4, 5]], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + sampling_masks=SamplingMaskLists( + token_ids=np.array([1, 6, 2, 3, 7, 8, 4, 5, 9], dtype=np.int32), + offsets=np.array([0, 2, 3, 6, 7, 9]), + cu_num_generated_tokens=[0, 2, 5], + ), + ) + + outputs = scheduler.update_from_output(scheduler_output, model_output)[0].outputs + + assert [out.new_sampling_mask.to_nested_list() for out in outputs] == [ + [[1, 6], [2]], + [[3, 7, 8], [4], [5, 9]], + ] + + def test_stop_via_update_from_output(): """Test stopping behavior through update_from_output.""" scheduler = create_scheduler(num_speculative_tokens=1) diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 485187f222..53b8e0f552 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -35,6 +35,7 @@ from vllm.v1.engine.output_processor import ( RequestState, ) from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats +from vllm.v1.outputs import SamplingMaskLists @pytest.mark.parametrize("flat_logprobs", [False, True]) @@ -62,6 +63,31 @@ def test_delta_output_without_new_tokens_returns_empty_logprobs( assert len(output.logprobs) == 0 +def test_completion_output_preserves_each_sampling_mask_position() -> None: + state = RequestState.__new__(RequestState) + state.detokenizer = MagicMock() + state.detokenizer.get_next_output_text.return_value = "" + state.logprobs_processor = MagicMock() + state.logprobs_processor.logprobs = None + state.logprobs_processor.cumulative_logprob = None + state.output_kind = RequestOutputKind.DELTA + state.request_index = 0 + state.sampling_mask_chunks = [ + SamplingMaskLists( + token_ids=np.array([10, 11, 20]), + offsets=np.array([0, 2, 3]), + ), + SamplingMaskLists(token_ids=np.array([30, 31, 32])), + ] + state.routed_experts_chunks = [] + state.spec_decode_metrics = None + + output = state._new_completion_output([1, 2, 3], FinishReason.LENGTH, None) + + assert output.sampling_mask is not None + assert output.sampling_mask.token_ids == [[10, 11], [20], [30, 31, 32]] + + def _ref_convert_id_to_token( tokenizer: TokenizerLike, token_id: int, diff --git a/tests/v1/test_outputs.py b/tests/v1/test_outputs.py index 5d52c86374..471cac6f95 100644 --- a/tests/v1/test_outputs.py +++ b/tests/v1/test_outputs.py @@ -13,6 +13,7 @@ from vllm.v1.outputs import ( LogprobsLists, LogprobsTensors, ModelRunnerOutput, + SamplingMaskLists, ) from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors @@ -63,8 +64,6 @@ def test_logprobs_tensors_tolists_with_tensor_boundaries(): def test_sampling_mask_lists_to_nested_list(): - from vllm.v1.outputs import SamplingMaskLists - mask = SamplingMaskLists( token_ids=np.array([10, 11, 12, 20, 21]), offsets=np.array([0, 3, 5]), @@ -76,6 +75,22 @@ def test_sampling_mask_lists_to_nested_list(): assert SamplingMaskLists(np.array([7, 9])).to_nested_list() == [[7, 9]] +def test_sampling_mask_lists_slices_multiple_positions_by_request(): + masks = SamplingMaskLists( + token_ids=np.array([10, 11, 20, 30, 31, 32, 40]), + offsets=np.array([0, 2, 3, 6, 7]), + cu_num_generated_tokens=[0, 1, 3, 4], + ) + + single = masks.slice_request(0, 1) + multi = masks.slice_request(1, 2) + + assert single.to_nested_list() == [[10, 11]] + assert single.offsets is None + assert multi.to_nested_list() == [[20], [30, 31, 32]] + assert multi.cu_num_generated_tokens is None + + @pytest.mark.parametrize("max_num_kept", [512, 20_001]) @pytest.mark.skipif( current_platform.is_xpu(), @@ -102,7 +117,10 @@ def test_sampling_mask_tensors_match_finite_support(max_num_kept): expected[1] = expected[7] = [] tensors = SamplingMaskTensors.from_logits( - logits.to(DEVICE_TYPE), num_sampled_tokens.to(DEVICE_TYPE), max_num_kept + logits.to(DEVICE_TYPE), + torch.arange(len(sizes) + 1, device=DEVICE_TYPE, dtype=torch.int32), + num_sampled_tokens.to(DEVICE_TYPE), + max_num_kept, ) assert tensors.token_ids.shape[1] == min(max_num_kept, MAX_COMPACT_SUPPORT) @@ -127,6 +145,7 @@ def test_sampling_mask_matches_processed_top_k_top_p_support(): tensors = SamplingMaskTensors.from_logits( processed_logits, + cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE), num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE), max_num_kept=3, ) @@ -135,6 +154,50 @@ def test_sampling_mask_matches_processed_top_k_top_p_support(): assert result.to_nested_list() == [expected_token_ids] +def test_sampling_mask_tensors_uses_request_boundaries(): + logits = torch.full((6, 8), -float("inf"), device=DEVICE_TYPE) + logits[0, [1, 3]] = 0 + logits[2, [2, 4, 6]] = 0 + logits[5, [0, 7]] = 0 + counts = torch.tensor([1, 0, 1], device=DEVICE_TYPE, dtype=torch.int32) + + tensors = SamplingMaskTensors.from_logits( + logits, + torch.tensor([0, 2, 5, 6], device=DEVICE_TYPE, dtype=torch.int32), + counts, + max_num_kept=3, + ) + + assert tensors.tolists().to_nested_list() == [[1, 3], [], [0, 7]] + + +def test_sampling_mask_tensors_multirow_request_layout(): + rows_per_request = 4 + counts = np.array([1, 2, 4, 0], dtype=np.int32) + logits = torch.full((16, 16), -float("inf"), device=DEVICE_TYPE) + expected = [] + for row in [0, 4, 5, 8, 9, 10, 11]: + kept = torch.tensor( + [row % 16, (row + 3) % 16, (row + 7) % 16], device=DEVICE_TYPE + ).unique(sorted=True) + logits[row, kept] = 0 + expected.append(kept.tolist()) + + tensors = SamplingMaskTensors.from_logits( + logits, + torch.arange(0, 17, 4, device=DEVICE_TYPE, dtype=torch.int32), + torch.from_numpy(counts).to(DEVICE_TYPE), + max_num_kept=2, + rows_per_request=rows_per_request, + ) + result = tensors.to_cpu_nonblocking().tolists(counts) + + assert tensors.token_ids.shape == (16, 2) + assert tensors.rows_per_request == rows_per_request + assert result.to_nested_list() == expected + assert result.cu_num_generated_tokens == [0, 1, 3, 7, 7] + + def test_sampling_mask_preserves_top_k_boundary_ties(): """When the kept support is wider than `max_num_kept` (e.g. a top-k boundary tie keeps more than k logits), the mask must fall back to the @@ -147,6 +210,7 @@ def test_sampling_mask_preserves_top_k_boundary_ties(): tensors = SamplingMaskTensors.from_logits( processed_logits, + cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE), num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE), max_num_kept=3, ) diff --git a/tests/v1/worker/test_gpu_rejection_sampler_chunking.py b/tests/v1/worker/test_gpu_rejection_sampler_chunking.py index efd2b59d75..a6f6731884 100644 --- a/tests/v1/worker/test_gpu_rejection_sampler_chunking.py +++ b/tests/v1/worker/test_gpu_rejection_sampler_chunking.py @@ -10,12 +10,53 @@ import torch from vllm.config.model import PROCESSED_LOGPROBS_MODES, LogprobsMode from vllm.platforms import current_platform +from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors from vllm.v1.worker.gpu.spec_decode.rejection_sampler import ( RejectionSampler, _iter_request_chunks, ) +def _make_input_batch( + cu_num_logits_np: np.ndarray, + idx_mapping_np: np.ndarray, + device: torch.device, +) -> SimpleNamespace: + """Minimal InputBatch fields read by RejectionSampler._verify_in_chunks.""" + num_reqs = len(idx_mapping_np) + num_logits_per_req = np.diff(cu_num_logits_np) + return SimpleNamespace( + num_reqs=num_reqs, + cu_num_logits_np=cu_num_logits_np, + cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device), + idx_mapping_np=idx_mapping_np, + idx_mapping=torch.from_numpy(idx_mapping_np).to(device), + expanded_idx_mapping=torch.from_numpy( + np.repeat(idx_mapping_np, num_logits_per_req) + ).to(device), + expanded_local_pos=torch.from_numpy( + np.concatenate( + [np.arange(count, dtype=np.int32) for count in num_logits_per_req] + ) + ).to(device), + seq_lens_cpu_upper_bound=torch.full((num_reqs,), 64, dtype=torch.int32), + ) + + +def _make_rejection_sampler( + sampler: SimpleNamespace, num_speculative_steps: int +) -> RejectionSampler: + """RejectionSampler for standard, fixed-boundary, unwatermarked verification.""" + rejection_sampler = object.__new__(RejectionSampler) + rejection_sampler.sampler = sampler + rejection_sampler.num_speculative_steps = num_speculative_steps + rejection_sampler.enable_adaptive_verification = False + rejection_sampler.synthetic_conditional_rates = None + rejection_sampler.use_block_verification = False + rejection_sampler.watermark_key = None + return rejection_sampler + + def test_iter_request_chunks_preserves_request_boundaries(): cu_num_logits = np.array([0, 3, 4, 11, 13], dtype=np.int32) @@ -33,28 +74,11 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str): cu_num_logits_np = np.array([0, 3, 4, 8, 10], dtype=np.int32) num_logits_per_req = np.diff(cu_num_logits_np) idx_mapping_np = np.array([7, 2, 9, 1], dtype=np.int32) - input_batch = SimpleNamespace( - num_reqs=4, - cu_num_logits_np=cu_num_logits_np, - cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device), - idx_mapping_np=idx_mapping_np, - idx_mapping=torch.from_numpy(idx_mapping_np).to(device), - expanded_idx_mapping=torch.from_numpy( - np.repeat(idx_mapping_np, num_logits_per_req) - ).to(device), - expanded_local_pos=torch.from_numpy( - np.concatenate( - [np.arange(count, dtype=np.int32) for count in num_logits_per_req] - ) - ).to(device), - seq_lens_cpu_upper_bound=torch.from_numpy( - np.array([10, 20, 30, 40], dtype=np.int32) - ), + input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device) + rejection_sampler = _make_rejection_sampler( + SimpleNamespace(logprobs_mode=logprobs_mode, return_sampling_mask=False), + num_speculative_steps=3, ) - rejection_sampler = object.__new__(RejectionSampler) - rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode) - rejection_sampler.num_speculative_steps = 3 - rejection_sampler.enable_adaptive_verification = False def fake_verify( self, @@ -75,14 +99,16 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str): rejection_sampler._verify = MethodType(fake_verify, rejection_sampler) logits = torch.arange(170, dtype=torch.float32, device=device).view(10, 17) - sampled, num_sampled, chunked_logprobs = rejection_sampler._verify_in_chunks( - logits, - input_batch, - draft_logits=None, - draft_sampled=torch.arange(10, device=device), - pos=torch.arange(10, device=device), - max_chunk_logits=5, - max_num_logprobs=2, + sampled, num_sampled, chunked_logprobs, sampling_mask_tensors = ( + rejection_sampler._verify_in_chunks( + logits, + input_batch, + draft_logits=None, + draft_sampled=torch.arange(10, device=device), + pos=torch.arange(10, device=device), + max_chunk_logits=5, + max_num_logprobs=2, + ) ) score_logits = logits + 1 if logprobs_mode in PROCESSED_LOGPROBS_MODES else logits full_logprobs = rejection_sampler._get_logprobs_tensors( @@ -111,3 +137,155 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str): chunked_logprobs.cu_num_generated_tokens == full_logprobs.cu_num_generated_tokens ) + assert sampling_mask_tensors is None + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +def test_replay_on_off_preserves_rejection_sampling_and_rng(monkeypatch): + device = torch.device("cuda") + num_reqs = 3 + num_speculative_steps = 3 + rows_per_request = num_speculative_steps + 1 + vocab_size = 8 + cu_num_logits_np = np.arange(0, 13, rows_per_request, dtype=np.int32) + idx_mapping_np = np.arange(num_reqs, dtype=np.int32) + input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device) + processed_logits = torch.zeros((12, vocab_size), device=device) + processed_logits[0, 0] = -float("inf") + processed_logits[rows_per_request + 1, 1] = -float("inf") + draft_logits = torch.zeros( + (num_reqs, num_speculative_steps, vocab_size), device=device + ) + draft_sampled = torch.zeros( + (num_reqs, rows_per_request), dtype=torch.int64, device=device + ) + draft_sampled[:, 1:] = torch.tensor([0, 1, 2], device=device) + draft_sampled = draft_sampled.flatten() + sampler = SimpleNamespace( + logprobs_mode="processed_logprobs", + return_sampling_mask=False, + sampling_states=SimpleNamespace( + top_k=SimpleNamespace(np=np.full(num_reqs, vocab_size)), + temperature=SimpleNamespace(gpu=torch.ones(num_reqs, device=device)), + seeds=SimpleNamespace( + gpu=torch.arange(num_reqs, dtype=torch.int64, device=device) + ), + ), + use_fp64_gumbel=False, + apply_sampling_params=lambda logits, *_: logits, + ) + rejection_sampler = _make_rejection_sampler(sampler, num_speculative_steps) + pos = torch.arange(12, dtype=torch.int32, device=device) + + pack_calls: list[tuple[list[int], int, int]] = [] + pack = SamplingMaskTensors.from_logits.__func__ + + def track_pack( + cls, + logits, + cu_num_logits, + num_sampled_tokens, + max_num_kept, + rows_per_request=1, + ): + pack_calls.append( + (cu_num_logits.cpu().tolist(), max_num_kept, rows_per_request) + ) + return pack( + cls, + logits, + cu_num_logits, + num_sampled_tokens, + max_num_kept, + rows_per_request, + ) + + monkeypatch.setattr( + SamplingMaskTensors, + "from_logits", + classmethod(track_pack), + ) + + rng_before = torch.cuda.get_rng_state() + off_sampled, off_counts, off_logprobs, off_masks = ( + rejection_sampler._verify_in_chunks( + processed_logits, + input_batch, + draft_logits, + draft_sampled, + pos, + max_chunk_logits=8, + max_num_logprobs=0, + ) + ) + rng_after_off = torch.cuda.get_rng_state() + assert pack_calls == [] + + sampler.return_sampling_mask = True + on_sampled, on_counts, on_logprobs, on_masks = rejection_sampler._verify_in_chunks( + processed_logits, + input_batch, + draft_logits, + draft_sampled, + pos, + max_chunk_logits=8, + max_num_logprobs=0, + ) + rng_after_on = torch.cuda.get_rng_state() + + assert pack_calls == [ + ([0, 4, 8], vocab_size, rows_per_request), + ([0, 4], vocab_size, rows_per_request), + ] + assert off_masks is None and on_masks is not None + assert on_masks.rows_per_request == rows_per_request + assert on_masks.token_ids.shape == (num_reqs * rows_per_request, vocab_size) + assert on_masks.packed_mask.shape[0] == num_reqs * rows_per_request + assert off_counts.tolist() == [1, 2, 4] + assert torch.equal(off_counts, on_counts) + counts = off_counts.cpu().tolist() + for req_idx, count in enumerate(counts): + assert torch.equal(off_sampled[req_idx, :count], on_sampled[req_idx, :count]) + + mapped_rows = [ + int(input_batch.cu_num_logits_np[req_idx]) + slot_idx + for req_idx, count in enumerate(counts) + for slot_idx in range(count) + ] + assert off_logprobs is not None and on_logprobs is not None + assert torch.equal( + off_logprobs.logprob_token_ids[mapped_rows], + on_logprobs.logprob_token_ids[mapped_rows], + ) + assert torch.equal( + off_logprobs.logprobs[mapped_rows], on_logprobs.logprobs[mapped_rows] + ) + assert torch.equal( + off_logprobs.selected_token_ranks[mapped_rows], + on_logprobs.selected_token_ranks[mapped_rows], + ) + assert torch.equal(rng_before, rng_after_off) + assert torch.equal(rng_after_off, rng_after_on) + + masks = ( + on_masks.to_cpu_nonblocking().tolists(on_counts.cpu().numpy()).to_nested_list() + ) + expected_masks = [ + torch.isfinite(processed_logits[row]).nonzero().flatten().tolist() + for row in mapped_rows + ] + emitted = [ + int(on_sampled[req_idx, slot_idx]) + for req_idx, count in enumerate(on_counts.cpu().tolist()) + for slot_idx in range(count) + ] + expected_logprobs = torch.stack( + [ + torch.log_softmax(processed_logits[row], dim=-1)[token_id] + for row, token_id in zip(mapped_rows, emitted) + ] + ) + + assert masks == expected_masks + assert all(token_id in mask for token_id, mask in zip(emitted, masks)) + assert torch.equal(on_logprobs.logprobs[mapped_rows, 0], expected_logprobs) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 0fd536ff87..c25ed27a28 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1265,9 +1265,15 @@ class VllmConfig: return if not self.use_v2_model_runner: raise ValueError("sampling distribution replay requires Model Runner V2") - if self.speculative_config is not None: + speculative_config = self.speculative_config + if ( + speculative_config is not None + and speculative_config.enable_adaptive_verification + ): raise ValueError( - "sampling distribution replay does not support speculative decoding" + "sampling distribution replay with speculative decoding " + "requires fixed verification boundaries; disable adaptive " + "verification" ) if model_config.is_diffusion: raise ValueError( diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index 5ab5636f81..e291b18806 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -441,7 +441,11 @@ class RequestState: sampling_mask = None if finished and self.sampling_mask_chunks: sampling_mask = SamplingMask( - [chunk.token_ids.tolist() for chunk in self.sampling_mask_chunks] + [ + position + for chunk in self.sampling_mask_chunks + for position in chunk.to_nested_list() + ] ) # Concatenate routed experts on finish diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index 1f04c2ccf9..22169b1fda 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -57,19 +57,25 @@ class LogprobsLists(NamedTuple): class SamplingMaskLists(NamedTuple): - """CSR sampling masks; a step slice holds one position (``offsets=None``).""" + """CSR sampling masks; a request slice may contain multiple positions.""" # [num_kept_tokens] token_ids: np.ndarray # [num_positions + 1], or None for a single position offsets: np.ndarray | None = None - # Unused with one position per request; kept for the wire layout. + # [num_requests + 1] for multi-position request batches. cu_num_generated_tokens: list[int] | None = None def slice_request(self, req_idx: int, num_positions: int) -> "SamplingMaskLists": - assert num_positions == 1 and self.offsets is not None + assert self.offsets is not None + cu = self.cu_num_generated_tokens + start = req_idx if cu is None else cu[req_idx] + end = start + num_positions + lo, hi = self.offsets[start], self.offsets[end] + if num_positions == 1: + return SamplingMaskLists(self.token_ids[lo:hi]) return SamplingMaskLists( - self.token_ids[self.offsets[req_idx] : self.offsets[req_idx + 1]] + self.token_ids[lo:hi], self.offsets[start : end + 1] - lo ) def to_nested_list(self) -> list[list[int]]: diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index 3c58433318..647b0c297c 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -185,7 +185,7 @@ class AsyncOutput(AsyncModelRunnerOutput): if self.sampling_mask_tensors is not None: self.model_runner_output.sampling_masks = ( - self.sampling_mask_tensors.tolists() + self.sampling_mask_tensors.tolists(self.num_sampled_tokens_np) ) if self.num_nans is not None: diff --git a/vllm/v1/worker/gpu/sample/output.py b/vllm/v1/worker/gpu/sample/output.py index aedff5ce18..2b16883486 100644 --- a/vllm/v1/worker/gpu/sample/output.py +++ b/vllm/v1/worker/gpu/sample/output.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass from typing import NamedTuple @@ -27,6 +28,7 @@ def _compact_sampling_mask_kernel( logits_ptr, logits_row_stride, logits_col_stride, + cu_num_logits_ptr, num_sampled_tokens_ptr, token_ids_ptr, token_ids_row_stride, @@ -35,25 +37,32 @@ def _compact_sampling_mask_kernel( counts_ptr, vocab_size, max_num_kept, + ROWS_PER_REQUEST: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): """Per row: first ``max_num_kept`` finite-logit ids, the full count, the bitmask.""" - req_idx = tl.program_id(0) - is_active = tl.load(num_sampled_tokens_ptr + req_idx) > 0 + output_row = tl.program_id(0) + req_idx = output_row // ROWS_PER_REQUEST + slot_idx = output_row % ROWS_PER_REQUEST + if slot_idx >= tl.load(num_sampled_tokens_ptr + req_idx): + tl.store(counts_ptr + output_row, 0) + return + source_row = tl.load(cu_num_logits_ptr + req_idx) + slot_idx + count = tl.zeros((), dtype=tl.int32) for start_idx in range(0, vocab_size, BLOCK_SIZE): offsets = start_idx + tl.arange(0, BLOCK_SIZE) logits = tl.load( - logits_ptr + req_idx * logits_row_stride + offsets * logits_col_stride, + logits_ptr + source_row * logits_row_stride + offsets * logits_col_stride, mask=offsets < vocab_size, other=-float("inf"), ) - keep = (logits > -float("inf")) & (logits < float("inf")) & is_active + keep = (logits > -float("inf")) & (logits < float("inf")) keep_i32 = keep.to(tl.int32) pos = count + tl.cumsum(keep_i32, axis=0) - keep_i32 tl.store( - token_ids_ptr + req_idx * token_ids_row_stride + pos, + token_ids_ptr + output_row * token_ids_row_stride + pos, offsets.to(tl.int32), mask=keep & (pos < max_num_kept), ) @@ -62,12 +71,12 @@ def _compact_sampling_mask_kernel( bits = tl.reshape(keep_i32, (BLOCK_SIZE // 8, 8)) << tl.arange(0, 8)[None, :] byte_offsets = start_idx // 8 + tl.arange(0, BLOCK_SIZE // 8) tl.store( - packed_mask_ptr + req_idx * packed_mask_row_stride + byte_offsets, + packed_mask_ptr + output_row * packed_mask_row_stride + byte_offsets, tl.sum(bits, axis=1).to(tl.uint8), mask=byte_offsets < tl.cdiv(vocab_size, 8), ) - tl.store(counts_ptr + req_idx, count) + tl.store(counts_ptr + output_row, count) # Bounds the [num_reqs, width] int32 buffer; wider rows use the bitmask instead. @@ -78,37 +87,46 @@ class SamplingMaskTensors(NamedTuple): """Device-side masks pending async D2H: compact ids, plus the bitmask as the exact fallback for rows wider than ``max_num_kept``.""" - # [num_requests, max_num_kept] + # [num_requests * rows_per_request, max_num_kept] token_ids: torch.Tensor - # [num_requests, ceil(vocab_size / 8)] + # [num_requests * rows_per_request, ceil(vocab_size / 8)] packed_mask: torch.Tensor - # [num_requests] + # [num_requests * rows_per_request] counts: torch.Tensor vocab_size: int + # Fixed request-major output slots. Ordinary sampling has one slot. + rows_per_request: int = 1 @classmethod def from_logits( cls, logits: torch.Tensor, + cu_num_logits: torch.Tensor, num_sampled_tokens: torch.Tensor, max_num_kept: int, + rows_per_request: int = 1, ) -> SamplingMaskTensors: - """Capture the finite-logit support of every row with a sampled token.""" - num_reqs, vocab_size = logits.shape + """Capture committed target supports in fixed request-major slots.""" + num_reqs = num_sampled_tokens.shape[0] + vocab_size = logits.shape[1] max_num_kept = min(max_num_kept, vocab_size, MAX_COMPACT_SUPPORT) + num_output_rows = num_reqs * rows_per_request device = logits.device token_ids = torch.empty( - (num_reqs, max_num_kept), dtype=torch.int32, device=device + (num_output_rows, max_num_kept), dtype=torch.int32, device=device ) packed_mask = torch.empty( - (num_reqs, (vocab_size + 7) // 8), dtype=torch.uint8, device=device + (num_output_rows, (vocab_size + 7) // 8), + dtype=torch.uint8, + device=device, ) - counts = torch.empty(num_reqs, dtype=torch.int32, device=device) - _compact_sampling_mask_kernel[(num_reqs,)]( + counts = torch.empty(num_output_rows, dtype=torch.int32, device=device) + _compact_sampling_mask_kernel[(num_output_rows,)]( logits, logits.stride(0), logits.stride(1), + cu_num_logits, num_sampled_tokens, token_ids, token_ids.stride(0), @@ -117,9 +135,23 @@ class SamplingMaskTensors(NamedTuple): counts, vocab_size, max_num_kept, + ROWS_PER_REQUEST=rows_per_request, BLOCK_SIZE=8192, ) - return cls(token_ids, packed_mask, counts, vocab_size) + return cls(token_ids, packed_mask, counts, vocab_size, rows_per_request) + + @classmethod + def cat(cls, chunks: Sequence[SamplingMaskTensors]) -> SamplingMaskTensors: + """Join request-contiguous chunks that share one output layout.""" + first = chunks[0] + assert len({(c.vocab_size, c.rows_per_request) for c in chunks}) == 1 + return cls( + torch.cat([chunk.token_ids for chunk in chunks]), + torch.cat([chunk.packed_mask for chunk in chunks]), + torch.cat([chunk.counts for chunk in chunks]), + first.vocab_size, + first.rows_per_request, + ) def to_cpu_nonblocking(self) -> SamplingMaskTensors: if self.token_ids.device.type == "cpu": @@ -129,15 +161,31 @@ class SamplingMaskTensors(NamedTuple): self.packed_mask.to("cpu", non_blocking=True), self.counts.to("cpu", non_blocking=True), self.vocab_size, + self.rows_per_request, ) - def tolists(self) -> SamplingMaskLists: - """CSR over all requests; rows without a sampled token are empty.""" + def tolists( + self, num_sampled_tokens: np.ndarray | None = None + ) -> SamplingMaskLists: + """Convert fixed output rows to the scheduler's CSR representation.""" counts = self.counts.cpu().numpy() token_ids = self.token_ids.cpu().numpy() packed_mask = self.packed_mask.cpu().numpy() width = token_ids.shape[1] + cu_num_generated_tokens = None + if self.rows_per_request == 1: + sampled_rows = np.arange(len(counts)) + else: + assert num_sampled_tokens is not None + active_slots = ( + np.arange(self.rows_per_request)[None, :] < num_sampled_tokens[:, None] + ) + sampled_rows = np.flatnonzero(active_slots) + cu_num_generated_tokens = np.cumsum( + np.concatenate(([0], num_sampled_tokens)) + ).tolist() + def support(row: int) -> np.ndarray: if counts[row] <= width: return token_ids[row, : counts[row]] @@ -147,7 +195,8 @@ class SamplingMaskTensors(NamedTuple): ) return np.flatnonzero(bits).astype(np.int32, copy=False) - supports = [support(row) for row in range(len(counts))] + supports = [support(row) for row in sampled_rows] offsets = np.zeros(len(supports) + 1, dtype=np.int64) np.cumsum([len(s) for s in supports], out=offsets[1:]) - return SamplingMaskLists(np.concatenate(supports), offsets) + flat = np.concatenate([np.empty(0, dtype=np.int32), *supports]) + return SamplingMaskLists(flat, offsets, cu_num_generated_tokens) diff --git a/vllm/v1/worker/gpu/sample/sampler.py b/vllm/v1/worker/gpu/sample/sampler.py index 802adb6f64..7ffc5bf493 100644 --- a/vllm/v1/worker/gpu/sample/sampler.py +++ b/vllm/v1/worker/gpu/sample/sampler.py @@ -203,7 +203,10 @@ class Sampler: # Size by the validated top_k batch max; wider supports use the bitmask. max_num_kept = int(np.max(self.sampling_states.top_k.np[idx_mapping_np])) sampling_mask_tensors = SamplingMaskTensors.from_logits( - processed_logits, num_sampled, max_num_kept + processed_logits, + input_batch.cu_num_logits, + num_sampled, + max_num_kept, ) # These are GPU tensors. diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 1983126b77..b96007b6c1 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -18,7 +18,7 @@ from vllm.v1.worker.gpu.input_batch import ( ) from vllm.v1.worker.gpu.metrics.logits import get_num_nans from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores -from vllm.v1.worker.gpu.sample.output import SamplerOutput +from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors from vllm.v1.worker.gpu.sample.sampler import Sampler from vllm.v1.worker.gpu.sample.states import NO_LOGPROBS from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import ( @@ -296,7 +296,12 @@ class RejectionSampler: max_chunk_logits: int, max_num_logprobs: int, verify_draft_sampled: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]: + ) -> tuple[ + torch.Tensor, + torch.Tensor, + LogprobsTensors | None, + SamplingMaskTensors | None, + ]: if verify_draft_sampled is None: verify_draft_sampled = draft_sampled cu_num_logits_np = input_batch.cu_num_logits_np @@ -316,6 +321,14 @@ class RejectionSampler: sampled_chunks: list[torch.Tensor] = [] num_sampled_chunks: list[torch.Tensor] = [] logprobs_chunks: list[LogprobsTensors] = [] + sampling_mask_chunks: list[SamplingMaskTensors] = [] + max_num_kept = None + if self.sampler.return_sampling_mask: + max_num_kept = int( + np.max( + self.sampler.sampling_states.top_k.np[input_batch.idx_mapping_np] + ) + ) for start, end in request_chunks: lo = int(cu_num_logits_np[start]) @@ -346,13 +359,31 @@ class RejectionSampler: ) if chunk_logprobs is not None: logprobs_chunks.append(chunk_logprobs) + if max_num_kept is not None: + sampling_mask_chunks.append( + SamplingMaskTensors.from_logits( + processed_logits, + chunk_cu_num_logits, + num_sampled, + max_num_kept, + self.num_speculative_steps + 1, + ) + ) del processed_logits sampled_chunks.append(sampled) num_sampled_chunks.append(num_sampled) if len(sampled_chunks) == 1: logprobs_tensors = logprobs_chunks[0] if logprobs_chunks else None - return sampled_chunks[0], num_sampled_chunks[0], logprobs_tensors + sampling_mask_tensors = ( + sampling_mask_chunks[0] if sampling_mask_chunks else None + ) + return ( + sampled_chunks[0], + num_sampled_chunks[0], + logprobs_tensors, + sampling_mask_tensors, + ) logprobs_tensors = None if logprobs_chunks: @@ -366,7 +397,12 @@ class RejectionSampler: sampled = torch.cat(sampled_chunks) num_sampled = torch.cat(num_sampled_chunks) - return sampled, num_sampled, logprobs_tensors + sampling_mask_tensors = ( + SamplingMaskTensors.cat(sampling_mask_chunks) + if sampling_mask_chunks + else None + ) + return sampled, num_sampled, logprobs_tensors, sampling_mask_tensors def __call__( self, @@ -400,7 +436,12 @@ class RejectionSampler: input_batch.idx_mapping_np ) chunk_logit_limit = get_max_chunk_logits(logits.shape[1]) - sampled, num_sampled, logprobs_tensors = self._verify_in_chunks( + ( + sampled, + num_sampled, + logprobs_tensors, + sampling_mask_tensors, + ) = self._verify_in_chunks( logits, input_batch, draft_logits, @@ -425,4 +466,5 @@ class RejectionSampler: num_nans=num_nans, num_sampled=num_sampled, num_rejected=num_rejected, + sampling_mask_tensors=sampling_mask_tensors, )