[Feature][DSpark]: Logprobs adaptive verification (#52242)

Signed-off-by: Naveenraj Kamalakannan <therealnaveenkamal@gmail.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Naveenraj Kamalakannan
2026-08-25 10:22:33 -04:00
committed by GitHub
co-authored by Benjamin Chislett
parent ca29cd48ba
commit d5cadcee86
10 changed files with 61 additions and 25 deletions
@@ -36,7 +36,7 @@ Set `enable_adaptive_verification: false` to verify the full block for every req
- The attention backend must tolerate device-decided query lengths, since the CPU lengths only bound them from above. Backends that plan off the CPU lengths are excluded by the attention selector, and rejected at startup for models that hard-wire their backend.
- Full cudagraphs are required: step costs are profiled from captured graphs, so `--enforce-eager` is rejected at startup.
- Not supported with LoRA (the per-token LoRA mapping is built from CPU-side boundaries), pipeline parallelism (cost curves and confidences exist only on the last rank), or output logprobs (to be fixed).
- Not supported with LoRA (the per-token LoRA mapping is built from CPU-side boundaries) or pipeline parallelism (cost curves and confidences exist only on the last rank).
## Tuning the cost profile
+1 -1
View File
@@ -411,7 +411,7 @@ def _validate_logprobs(
ref_prompt_logprob_toks,
ref_prompt_logprob_vals,
ref_prompt_token_ranks,
_,
*_,
) = ref_prompt_logprobs
for idx, (prompt_token, pos_logprob_dict) in enumerate(
zip(prompt_token_ids[1:], prompt_logprobs[1:])
+19
View File
@@ -42,6 +42,25 @@ def test_logprobs_tensors_cat():
assert LogprobsTensors.cat([first]) is first
def test_logprobs_tensors_tolists_with_tensor_boundaries():
"""Adaptive verification hands over the request boundaries as a tensor
(they only exist on device); tolists() must materialize it as a plain
list so slice_request splits requests correctly."""
tensors = LogprobsTensors(
torch.tensor([[1, 2], [3, 4], [5, 6]]),
torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]),
torch.tensor([1, 2, 3]),
cu_num_generated_tokens_tensor=torch.tensor([0, 2, 3], dtype=torch.int32),
)
lists = tensors.to_cpu_nonblocking().tolists()
assert lists.cu_num_generated_tokens == [0, 2, 3]
sliced = lists.slice_request(1, 1)
assert sliced.logprob_token_ids.tolist() == [[5, 6]]
assert sliced.sampled_token_ranks.tolist() == [3]
def test_sampling_mask_tensors_tolist():
tensors = SamplingMaskTensors(
packed_mask=torch.tensor(
@@ -51,6 +51,7 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
rejection_sampler = object.__new__(RejectionSampler)
rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode)
rejection_sampler.num_speculative_steps = 3
rejection_sampler.enable_adaptive_verification = False
def fake_verify(
self,
-14
View File
@@ -974,20 +974,6 @@ class SamplingParams(
if speculative_config is None:
return
# Adaptive verification compacts logits after the forward pass, while
# compute_topk_scores uses the scheduled layout in cu_num_logits_np.
# TODO(lucas): lift this restriction. The true boundaries exist on device as
# cu_num_logits; cu_num_generated_tokens is only read host-side after
# the logprobs D2H, so it could ride along on that copy.
if (
speculative_config.enable_adaptive_verification
and self.num_logprobs is not None
):
raise ValueError(
"Output logprobs are not supported with DSpark confidence-based "
"verification."
)
# Some sampling parameters are not yet compatible with spec decoding.
if self.min_p > _SAMPLING_EPS or self.logit_bias:
raise VLLMValidationError(
+1 -1
View File
@@ -134,7 +134,7 @@ class LogprobsProcessor:
assert self.num_prompt_logprobs is not None
assert self.prompt_logprobs is not None
token_ids, logprobs, ranks, _ = prompt_logprobs_tensors
token_ids, logprobs, ranks, *_ = prompt_logprobs_tensors
# Recover shapes.
num_prompt_tokens, num_logprobs = logprobs.shape
+21 -4
View File
@@ -98,27 +98,38 @@ class LogprobsTensors(NamedTuple):
logprobs: torch.Tensor
# [num_reqs x num_generated_tokens]
selected_token_ranks: torch.Tensor
# [num_reqs]
# [num_reqs + 1]
cu_num_generated_tokens: list[int] | None = None
# [num_reqs + 1]. Set instead of cu_num_generated_tokens when the
# boundaries only exist on device (adaptive verification); rides along on
# the async D2H copy.
cu_num_generated_tokens_tensor: torch.Tensor | None = None
def tolists(self, cu_num_generated_tokens: list[int] | None = None):
if cu_num_generated_tokens is None:
if self.cu_num_generated_tokens_tensor is not None:
cu_num_generated_tokens = self.cu_num_generated_tokens_tensor.tolist()
else:
cu_num_generated_tokens = self.cu_num_generated_tokens
return LogprobsLists(
self.logprob_token_ids.cpu().numpy(),
self.logprobs.cpu().numpy(),
self.selected_token_ranks.cpu().numpy(),
cu_num_generated_tokens
if cu_num_generated_tokens is not None
else self.cu_num_generated_tokens,
cu_num_generated_tokens,
)
def to_cpu_nonblocking(self) -> "LogprobsTensors":
if self.logprob_token_ids.device.type == "cpu":
return self
cu_tensor = self.cu_num_generated_tokens_tensor
if cu_tensor is not None:
cu_tensor = cu_tensor.to("cpu", non_blocking=True)
return LogprobsTensors(
self.logprob_token_ids.to("cpu", non_blocking=True),
self.logprobs.to("cpu", non_blocking=True),
self.selected_token_ranks.to("cpu", non_blocking=True),
self.cu_num_generated_tokens,
cu_tensor,
)
def filter(self, mask: torch.Tensor) -> "LogprobsTensors":
@@ -126,6 +137,9 @@ class LogprobsTensors(NamedTuple):
assert self.cu_num_generated_tokens is None, (
"filter can't be used with cu_num_generated_tokens"
)
assert self.cu_num_generated_tokens_tensor is None, (
"filter can't be used with cu_num_generated_tokens_tensor"
)
return LogprobsTensors(
self.logprob_token_ids[mask],
self.logprobs[mask],
@@ -147,6 +161,9 @@ class LogprobsTensors(NamedTuple):
if cu_num_generated_tokens is None:
return tensor
return tensor._replace(cu_num_generated_tokens=cu_num_generated_tokens)
# The multi-chunk path rebuilds boundaries from the CPU layout, which
# the device-only boundaries of adaptive verification never use.
assert all(tensor.cu_num_generated_tokens_tensor is None for tensor in tensors)
return LogprobsTensors(
logprob_token_ids=torch.cat(
[tensor.logprob_token_ids for tensor in tensors]
+4 -2
View File
@@ -110,7 +110,7 @@ def compute_topk_scores(
logits: torch.Tensor,
num_logprobs: int,
sampled_token_ids: torch.Tensor,
cu_num_logits: list[int] | None = None,
cu_num_logits: list[int] | torch.Tensor | None = None,
logprob_token_ids_state: "LogprobTokenIdsState | None" = None,
expanded_idx_mapping: torch.Tensor | None = None,
max_per_req_token_ids: int = 0,
@@ -177,11 +177,13 @@ def compute_topk_scores(
vocab_size,
BLOCK_SIZE=8192, # type: ignore
)
is_tensor = isinstance(cu_num_logits, torch.Tensor)
return LogprobsTensors(
logprob_token_ids=logprob_token_ids,
logprobs=scores,
selected_token_ranks=token_ranks,
cu_num_generated_tokens=cu_num_logits,
cu_num_generated_tokens=None if is_tensor else cu_num_logits,
cu_num_generated_tokens_tensor=cu_num_logits if is_tensor else None,
)
@@ -81,6 +81,7 @@ class RejectionSampler:
):
self.sampler = sampler
self.num_speculative_steps = spec_config.num_speculative_tokens
self.enable_adaptive_verification = spec_config.enable_adaptive_verification
rejection_sample_method = spec_config.rejection_sample_method
self.use_block_verification: bool = False
self.synthetic_conditional_rates: torch.Tensor | None = None
@@ -122,11 +123,20 @@ class RejectionSampler:
num_warps=1,
)
expanded_logits = num_logits != num_reqs
cu_num_generated_tokens: list[int] | torch.Tensor | None = None
if expanded_logits:
if self.enable_adaptive_verification:
# Adaptive verification keeps the true per-request boundaries
# on device only; cu_num_logits_np holds the pre-compacted
# layout.
cu_num_generated_tokens = cu_num_logits.clone()
else:
cu_num_generated_tokens = cu_num_logits_np.tolist()
return compute_topk_scores(
logits,
max_num_logprobs,
flat_sampled,
cu_num_logits_np.tolist() if expanded_logits else None,
cu_num_generated_tokens,
logits_mode=self.sampler.logprobs_mode
in ("raw_logits", "processed_logits"),
)
@@ -191,6 +201,7 @@ class RejectionSampler:
# guarantees the compacted batch always lands here.
request_chunks: Iterable[tuple[int, int]] = ((0, num_reqs),)
else:
assert not self.enable_adaptive_verification
request_chunks = _iter_request_chunks(cu_num_logits_np, max_chunk_logits)
sampled_chunks: list[torch.Tensor] = []
+1 -1
View File
@@ -5803,7 +5803,7 @@ class GPUModelRunner(
scores = logits.to(torch.float32)
else:
scores = self.sampler.compute_logprobs(logits)
token_ids, logprobs, ranks, _ = self.sampler.gather_logprobs(
token_ids, logprobs, ranks, *_ = self.sampler.gather_logprobs(
scores, num_prompt_logprobs, tgt_token_ids
)