From d5cadcee8641a9fcec15facb5a9157d157daa207 Mon Sep 17 00:00:00 2001 From: Naveenraj Kamalakannan Date: Tue, 25 Aug 2026 10:22:33 -0400 Subject: [PATCH] [Feature][DSpark]: Logprobs adaptive verification (#52242) Signed-off-by: Naveenraj Kamalakannan Co-authored-by: Benjamin Chislett --- .../adaptive_verification.md | 2 +- tests/v1/engine/test_output_processor.py | 2 +- tests/v1/test_outputs.py | 19 ++++++++++++++ .../test_gpu_rejection_sampler_chunking.py | 1 + vllm/sampling_params.py | 14 ----------- vllm/v1/engine/logprobs.py | 2 +- vllm/v1/outputs.py | 25 ++++++++++++++++--- vllm/v1/worker/gpu/sample/logprob.py | 6 +++-- .../gpu/spec_decode/rejection_sampler.py | 13 +++++++++- vllm/v1/worker/gpu_model_runner.py | 2 +- 10 files changed, 61 insertions(+), 25 deletions(-) diff --git a/docs/features/speculative_decoding/adaptive_verification.md b/docs/features/speculative_decoding/adaptive_verification.md index 58c9d57cee..e9314c0e09 100644 --- a/docs/features/speculative_decoding/adaptive_verification.md +++ b/docs/features/speculative_decoding/adaptive_verification.md @@ -36,7 +36,7 @@ Set `enable_adaptive_verification: false` to verify the full block for every req - The attention backend must tolerate device-decided query lengths, since the CPU lengths only bound them from above. Backends that plan off the CPU lengths are excluded by the attention selector, and rejected at startup for models that hard-wire their backend. - Full cudagraphs are required: step costs are profiled from captured graphs, so `--enforce-eager` is rejected at startup. -- Not supported with LoRA (the per-token LoRA mapping is built from CPU-side boundaries), pipeline parallelism (cost curves and confidences exist only on the last rank), or output logprobs (to be fixed). +- Not supported with LoRA (the per-token LoRA mapping is built from CPU-side boundaries) or pipeline parallelism (cost curves and confidences exist only on the last rank). ## Tuning the cost profile diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 51dbb9c989..fa63b52200 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -411,7 +411,7 @@ def _validate_logprobs( ref_prompt_logprob_toks, ref_prompt_logprob_vals, ref_prompt_token_ranks, - _, + *_, ) = ref_prompt_logprobs for idx, (prompt_token, pos_logprob_dict) in enumerate( zip(prompt_token_ids[1:], prompt_logprobs[1:]) diff --git a/tests/v1/test_outputs.py b/tests/v1/test_outputs.py index d93ae96e54..b6e120e896 100644 --- a/tests/v1/test_outputs.py +++ b/tests/v1/test_outputs.py @@ -42,6 +42,25 @@ def test_logprobs_tensors_cat(): assert LogprobsTensors.cat([first]) is first +def test_logprobs_tensors_tolists_with_tensor_boundaries(): + """Adaptive verification hands over the request boundaries as a tensor + (they only exist on device); tolists() must materialize it as a plain + list so slice_request splits requests correctly.""" + tensors = LogprobsTensors( + torch.tensor([[1, 2], [3, 4], [5, 6]]), + torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]), + torch.tensor([1, 2, 3]), + cu_num_generated_tokens_tensor=torch.tensor([0, 2, 3], dtype=torch.int32), + ) + + lists = tensors.to_cpu_nonblocking().tolists() + + assert lists.cu_num_generated_tokens == [0, 2, 3] + sliced = lists.slice_request(1, 1) + assert sliced.logprob_token_ids.tolist() == [[5, 6]] + assert sliced.sampled_token_ranks.tolist() == [3] + + def test_sampling_mask_tensors_tolist(): tensors = SamplingMaskTensors( packed_mask=torch.tensor( diff --git a/tests/v1/worker/test_gpu_rejection_sampler_chunking.py b/tests/v1/worker/test_gpu_rejection_sampler_chunking.py index 22cb71e67b..2e5adfabd1 100644 --- a/tests/v1/worker/test_gpu_rejection_sampler_chunking.py +++ b/tests/v1/worker/test_gpu_rejection_sampler_chunking.py @@ -51,6 +51,7 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str): rejection_sampler = object.__new__(RejectionSampler) rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode) rejection_sampler.num_speculative_steps = 3 + rejection_sampler.enable_adaptive_verification = False def fake_verify( self, diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index f0e325c255..ee50afb18a 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -974,20 +974,6 @@ class SamplingParams( if speculative_config is None: return - # Adaptive verification compacts logits after the forward pass, while - # compute_topk_scores uses the scheduled layout in cu_num_logits_np. - # TODO(lucas): lift this restriction. The true boundaries exist on device as - # cu_num_logits; cu_num_generated_tokens is only read host-side after - # the logprobs D2H, so it could ride along on that copy. - if ( - speculative_config.enable_adaptive_verification - and self.num_logprobs is not None - ): - raise ValueError( - "Output logprobs are not supported with DSpark confidence-based " - "verification." - ) - # Some sampling parameters are not yet compatible with spec decoding. if self.min_p > _SAMPLING_EPS or self.logit_bias: raise VLLMValidationError( diff --git a/vllm/v1/engine/logprobs.py b/vllm/v1/engine/logprobs.py index 74a45ab1e4..8eef567b07 100644 --- a/vllm/v1/engine/logprobs.py +++ b/vllm/v1/engine/logprobs.py @@ -134,7 +134,7 @@ class LogprobsProcessor: assert self.num_prompt_logprobs is not None assert self.prompt_logprobs is not None - token_ids, logprobs, ranks, _ = prompt_logprobs_tensors + token_ids, logprobs, ranks, *_ = prompt_logprobs_tensors # Recover shapes. num_prompt_tokens, num_logprobs = logprobs.shape diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index 80b909afbc..94f1776c05 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -98,27 +98,38 @@ class LogprobsTensors(NamedTuple): logprobs: torch.Tensor # [num_reqs x num_generated_tokens] selected_token_ranks: torch.Tensor - # [num_reqs] + # [num_reqs + 1] cu_num_generated_tokens: list[int] | None = None + # [num_reqs + 1]. Set instead of cu_num_generated_tokens when the + # boundaries only exist on device (adaptive verification); rides along on + # the async D2H copy. + cu_num_generated_tokens_tensor: torch.Tensor | None = None def tolists(self, cu_num_generated_tokens: list[int] | None = None): + if cu_num_generated_tokens is None: + if self.cu_num_generated_tokens_tensor is not None: + cu_num_generated_tokens = self.cu_num_generated_tokens_tensor.tolist() + else: + cu_num_generated_tokens = self.cu_num_generated_tokens return LogprobsLists( self.logprob_token_ids.cpu().numpy(), self.logprobs.cpu().numpy(), self.selected_token_ranks.cpu().numpy(), - cu_num_generated_tokens - if cu_num_generated_tokens is not None - else self.cu_num_generated_tokens, + cu_num_generated_tokens, ) def to_cpu_nonblocking(self) -> "LogprobsTensors": if self.logprob_token_ids.device.type == "cpu": return self + cu_tensor = self.cu_num_generated_tokens_tensor + if cu_tensor is not None: + cu_tensor = cu_tensor.to("cpu", non_blocking=True) return LogprobsTensors( self.logprob_token_ids.to("cpu", non_blocking=True), self.logprobs.to("cpu", non_blocking=True), self.selected_token_ranks.to("cpu", non_blocking=True), self.cu_num_generated_tokens, + cu_tensor, ) def filter(self, mask: torch.Tensor) -> "LogprobsTensors": @@ -126,6 +137,9 @@ class LogprobsTensors(NamedTuple): assert self.cu_num_generated_tokens is None, ( "filter can't be used with cu_num_generated_tokens" ) + assert self.cu_num_generated_tokens_tensor is None, ( + "filter can't be used with cu_num_generated_tokens_tensor" + ) return LogprobsTensors( self.logprob_token_ids[mask], self.logprobs[mask], @@ -147,6 +161,9 @@ class LogprobsTensors(NamedTuple): if cu_num_generated_tokens is None: return tensor return tensor._replace(cu_num_generated_tokens=cu_num_generated_tokens) + # The multi-chunk path rebuilds boundaries from the CPU layout, which + # the device-only boundaries of adaptive verification never use. + assert all(tensor.cu_num_generated_tokens_tensor is None for tensor in tensors) return LogprobsTensors( logprob_token_ids=torch.cat( [tensor.logprob_token_ids for tensor in tensors] diff --git a/vllm/v1/worker/gpu/sample/logprob.py b/vllm/v1/worker/gpu/sample/logprob.py index c0f31cdf9f..384fcdb9fd 100644 --- a/vllm/v1/worker/gpu/sample/logprob.py +++ b/vllm/v1/worker/gpu/sample/logprob.py @@ -110,7 +110,7 @@ def compute_topk_scores( logits: torch.Tensor, num_logprobs: int, sampled_token_ids: torch.Tensor, - cu_num_logits: list[int] | None = None, + cu_num_logits: list[int] | torch.Tensor | None = None, logprob_token_ids_state: "LogprobTokenIdsState | None" = None, expanded_idx_mapping: torch.Tensor | None = None, max_per_req_token_ids: int = 0, @@ -177,11 +177,13 @@ def compute_topk_scores( vocab_size, BLOCK_SIZE=8192, # type: ignore ) + is_tensor = isinstance(cu_num_logits, torch.Tensor) return LogprobsTensors( logprob_token_ids=logprob_token_ids, logprobs=scores, selected_token_ranks=token_ranks, - cu_num_generated_tokens=cu_num_logits, + cu_num_generated_tokens=None if is_tensor else cu_num_logits, + cu_num_generated_tokens_tensor=cu_num_logits if is_tensor else None, ) diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 8410b6ce87..a9027e0550 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -81,6 +81,7 @@ class RejectionSampler: ): self.sampler = sampler self.num_speculative_steps = spec_config.num_speculative_tokens + self.enable_adaptive_verification = spec_config.enable_adaptive_verification rejection_sample_method = spec_config.rejection_sample_method self.use_block_verification: bool = False self.synthetic_conditional_rates: torch.Tensor | None = None @@ -122,11 +123,20 @@ class RejectionSampler: num_warps=1, ) expanded_logits = num_logits != num_reqs + cu_num_generated_tokens: list[int] | torch.Tensor | None = None + if expanded_logits: + if self.enable_adaptive_verification: + # Adaptive verification keeps the true per-request boundaries + # on device only; cu_num_logits_np holds the pre-compacted + # layout. + cu_num_generated_tokens = cu_num_logits.clone() + else: + cu_num_generated_tokens = cu_num_logits_np.tolist() return compute_topk_scores( logits, max_num_logprobs, flat_sampled, - cu_num_logits_np.tolist() if expanded_logits else None, + cu_num_generated_tokens, logits_mode=self.sampler.logprobs_mode in ("raw_logits", "processed_logits"), ) @@ -191,6 +201,7 @@ class RejectionSampler: # guarantees the compacted batch always lands here. request_chunks: Iterable[tuple[int, int]] = ((0, num_reqs),) else: + assert not self.enable_adaptive_verification request_chunks = _iter_request_chunks(cu_num_logits_np, max_chunk_logits) sampled_chunks: list[torch.Tensor] = [] diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 93e19fca21..0daabc978d 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -5803,7 +5803,7 @@ class GPUModelRunner( scores = logits.to(torch.float32) else: scores = self.sampler.compute_logprobs(logits) - token_ids, logprobs, ranks, _ = self.sampler.gather_logprobs( + token_ids, logprobs, ranks, *_ = self.sampler.gather_logprobs( scores, num_prompt_logprobs, tgt_token_ids )