[Feature][DSpark]: Logprobs adaptive verification (#52242)
Signed-off-by: Naveenraj Kamalakannan <therealnaveenkamal@gmail.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
co-authored by
Benjamin Chislett
parent
ca29cd48ba
commit
d5cadcee86
@@ -36,7 +36,7 @@ Set `enable_adaptive_verification: false` to verify the full block for every req
|
||||
|
||||
- The attention backend must tolerate device-decided query lengths, since the CPU lengths only bound them from above. Backends that plan off the CPU lengths are excluded by the attention selector, and rejected at startup for models that hard-wire their backend.
|
||||
- Full cudagraphs are required: step costs are profiled from captured graphs, so `--enforce-eager` is rejected at startup.
|
||||
- Not supported with LoRA (the per-token LoRA mapping is built from CPU-side boundaries), pipeline parallelism (cost curves and confidences exist only on the last rank), or output logprobs (to be fixed).
|
||||
- Not supported with LoRA (the per-token LoRA mapping is built from CPU-side boundaries) or pipeline parallelism (cost curves and confidences exist only on the last rank).
|
||||
|
||||
## Tuning the cost profile
|
||||
|
||||
|
||||
@@ -411,7 +411,7 @@ def _validate_logprobs(
|
||||
ref_prompt_logprob_toks,
|
||||
ref_prompt_logprob_vals,
|
||||
ref_prompt_token_ranks,
|
||||
_,
|
||||
*_,
|
||||
) = ref_prompt_logprobs
|
||||
for idx, (prompt_token, pos_logprob_dict) in enumerate(
|
||||
zip(prompt_token_ids[1:], prompt_logprobs[1:])
|
||||
|
||||
@@ -42,6 +42,25 @@ def test_logprobs_tensors_cat():
|
||||
assert LogprobsTensors.cat([first]) is first
|
||||
|
||||
|
||||
def test_logprobs_tensors_tolists_with_tensor_boundaries():
|
||||
"""Adaptive verification hands over the request boundaries as a tensor
|
||||
(they only exist on device); tolists() must materialize it as a plain
|
||||
list so slice_request splits requests correctly."""
|
||||
tensors = LogprobsTensors(
|
||||
torch.tensor([[1, 2], [3, 4], [5, 6]]),
|
||||
torch.tensor([[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]]),
|
||||
torch.tensor([1, 2, 3]),
|
||||
cu_num_generated_tokens_tensor=torch.tensor([0, 2, 3], dtype=torch.int32),
|
||||
)
|
||||
|
||||
lists = tensors.to_cpu_nonblocking().tolists()
|
||||
|
||||
assert lists.cu_num_generated_tokens == [0, 2, 3]
|
||||
sliced = lists.slice_request(1, 1)
|
||||
assert sliced.logprob_token_ids.tolist() == [[5, 6]]
|
||||
assert sliced.sampled_token_ranks.tolist() == [3]
|
||||
|
||||
|
||||
def test_sampling_mask_tensors_tolist():
|
||||
tensors = SamplingMaskTensors(
|
||||
packed_mask=torch.tensor(
|
||||
|
||||
@@ -51,6 +51,7 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str):
|
||||
rejection_sampler = object.__new__(RejectionSampler)
|
||||
rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode)
|
||||
rejection_sampler.num_speculative_steps = 3
|
||||
rejection_sampler.enable_adaptive_verification = False
|
||||
|
||||
def fake_verify(
|
||||
self,
|
||||
|
||||
@@ -974,20 +974,6 @@ class SamplingParams(
|
||||
if speculative_config is None:
|
||||
return
|
||||
|
||||
# Adaptive verification compacts logits after the forward pass, while
|
||||
# compute_topk_scores uses the scheduled layout in cu_num_logits_np.
|
||||
# TODO(lucas): lift this restriction. The true boundaries exist on device as
|
||||
# cu_num_logits; cu_num_generated_tokens is only read host-side after
|
||||
# the logprobs D2H, so it could ride along on that copy.
|
||||
if (
|
||||
speculative_config.enable_adaptive_verification
|
||||
and self.num_logprobs is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"Output logprobs are not supported with DSpark confidence-based "
|
||||
"verification."
|
||||
)
|
||||
|
||||
# Some sampling parameters are not yet compatible with spec decoding.
|
||||
if self.min_p > _SAMPLING_EPS or self.logit_bias:
|
||||
raise VLLMValidationError(
|
||||
|
||||
@@ -134,7 +134,7 @@ class LogprobsProcessor:
|
||||
assert self.num_prompt_logprobs is not None
|
||||
assert self.prompt_logprobs is not None
|
||||
|
||||
token_ids, logprobs, ranks, _ = prompt_logprobs_tensors
|
||||
token_ids, logprobs, ranks, *_ = prompt_logprobs_tensors
|
||||
|
||||
# Recover shapes.
|
||||
num_prompt_tokens, num_logprobs = logprobs.shape
|
||||
|
||||
+21
-4
@@ -98,27 +98,38 @@ class LogprobsTensors(NamedTuple):
|
||||
logprobs: torch.Tensor
|
||||
# [num_reqs x num_generated_tokens]
|
||||
selected_token_ranks: torch.Tensor
|
||||
# [num_reqs]
|
||||
# [num_reqs + 1]
|
||||
cu_num_generated_tokens: list[int] | None = None
|
||||
# [num_reqs + 1]. Set instead of cu_num_generated_tokens when the
|
||||
# boundaries only exist on device (adaptive verification); rides along on
|
||||
# the async D2H copy.
|
||||
cu_num_generated_tokens_tensor: torch.Tensor | None = None
|
||||
|
||||
def tolists(self, cu_num_generated_tokens: list[int] | None = None):
|
||||
if cu_num_generated_tokens is None:
|
||||
if self.cu_num_generated_tokens_tensor is not None:
|
||||
cu_num_generated_tokens = self.cu_num_generated_tokens_tensor.tolist()
|
||||
else:
|
||||
cu_num_generated_tokens = self.cu_num_generated_tokens
|
||||
return LogprobsLists(
|
||||
self.logprob_token_ids.cpu().numpy(),
|
||||
self.logprobs.cpu().numpy(),
|
||||
self.selected_token_ranks.cpu().numpy(),
|
||||
cu_num_generated_tokens
|
||||
if cu_num_generated_tokens is not None
|
||||
else self.cu_num_generated_tokens,
|
||||
cu_num_generated_tokens,
|
||||
)
|
||||
|
||||
def to_cpu_nonblocking(self) -> "LogprobsTensors":
|
||||
if self.logprob_token_ids.device.type == "cpu":
|
||||
return self
|
||||
cu_tensor = self.cu_num_generated_tokens_tensor
|
||||
if cu_tensor is not None:
|
||||
cu_tensor = cu_tensor.to("cpu", non_blocking=True)
|
||||
return LogprobsTensors(
|
||||
self.logprob_token_ids.to("cpu", non_blocking=True),
|
||||
self.logprobs.to("cpu", non_blocking=True),
|
||||
self.selected_token_ranks.to("cpu", non_blocking=True),
|
||||
self.cu_num_generated_tokens,
|
||||
cu_tensor,
|
||||
)
|
||||
|
||||
def filter(self, mask: torch.Tensor) -> "LogprobsTensors":
|
||||
@@ -126,6 +137,9 @@ class LogprobsTensors(NamedTuple):
|
||||
assert self.cu_num_generated_tokens is None, (
|
||||
"filter can't be used with cu_num_generated_tokens"
|
||||
)
|
||||
assert self.cu_num_generated_tokens_tensor is None, (
|
||||
"filter can't be used with cu_num_generated_tokens_tensor"
|
||||
)
|
||||
return LogprobsTensors(
|
||||
self.logprob_token_ids[mask],
|
||||
self.logprobs[mask],
|
||||
@@ -147,6 +161,9 @@ class LogprobsTensors(NamedTuple):
|
||||
if cu_num_generated_tokens is None:
|
||||
return tensor
|
||||
return tensor._replace(cu_num_generated_tokens=cu_num_generated_tokens)
|
||||
# The multi-chunk path rebuilds boundaries from the CPU layout, which
|
||||
# the device-only boundaries of adaptive verification never use.
|
||||
assert all(tensor.cu_num_generated_tokens_tensor is None for tensor in tensors)
|
||||
return LogprobsTensors(
|
||||
logprob_token_ids=torch.cat(
|
||||
[tensor.logprob_token_ids for tensor in tensors]
|
||||
|
||||
@@ -110,7 +110,7 @@ def compute_topk_scores(
|
||||
logits: torch.Tensor,
|
||||
num_logprobs: int,
|
||||
sampled_token_ids: torch.Tensor,
|
||||
cu_num_logits: list[int] | None = None,
|
||||
cu_num_logits: list[int] | torch.Tensor | None = None,
|
||||
logprob_token_ids_state: "LogprobTokenIdsState | None" = None,
|
||||
expanded_idx_mapping: torch.Tensor | None = None,
|
||||
max_per_req_token_ids: int = 0,
|
||||
@@ -177,11 +177,13 @@ def compute_topk_scores(
|
||||
vocab_size,
|
||||
BLOCK_SIZE=8192, # type: ignore
|
||||
)
|
||||
is_tensor = isinstance(cu_num_logits, torch.Tensor)
|
||||
return LogprobsTensors(
|
||||
logprob_token_ids=logprob_token_ids,
|
||||
logprobs=scores,
|
||||
selected_token_ranks=token_ranks,
|
||||
cu_num_generated_tokens=cu_num_logits,
|
||||
cu_num_generated_tokens=None if is_tensor else cu_num_logits,
|
||||
cu_num_generated_tokens_tensor=cu_num_logits if is_tensor else None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -81,6 +81,7 @@ class RejectionSampler:
|
||||
):
|
||||
self.sampler = sampler
|
||||
self.num_speculative_steps = spec_config.num_speculative_tokens
|
||||
self.enable_adaptive_verification = spec_config.enable_adaptive_verification
|
||||
rejection_sample_method = spec_config.rejection_sample_method
|
||||
self.use_block_verification: bool = False
|
||||
self.synthetic_conditional_rates: torch.Tensor | None = None
|
||||
@@ -122,11 +123,20 @@ class RejectionSampler:
|
||||
num_warps=1,
|
||||
)
|
||||
expanded_logits = num_logits != num_reqs
|
||||
cu_num_generated_tokens: list[int] | torch.Tensor | None = None
|
||||
if expanded_logits:
|
||||
if self.enable_adaptive_verification:
|
||||
# Adaptive verification keeps the true per-request boundaries
|
||||
# on device only; cu_num_logits_np holds the pre-compacted
|
||||
# layout.
|
||||
cu_num_generated_tokens = cu_num_logits.clone()
|
||||
else:
|
||||
cu_num_generated_tokens = cu_num_logits_np.tolist()
|
||||
return compute_topk_scores(
|
||||
logits,
|
||||
max_num_logprobs,
|
||||
flat_sampled,
|
||||
cu_num_logits_np.tolist() if expanded_logits else None,
|
||||
cu_num_generated_tokens,
|
||||
logits_mode=self.sampler.logprobs_mode
|
||||
in ("raw_logits", "processed_logits"),
|
||||
)
|
||||
@@ -191,6 +201,7 @@ class RejectionSampler:
|
||||
# guarantees the compacted batch always lands here.
|
||||
request_chunks: Iterable[tuple[int, int]] = ((0, num_reqs),)
|
||||
else:
|
||||
assert not self.enable_adaptive_verification
|
||||
request_chunks = _iter_request_chunks(cu_num_logits_np, max_chunk_logits)
|
||||
|
||||
sampled_chunks: list[torch.Tensor] = []
|
||||
|
||||
@@ -5803,7 +5803,7 @@ class GPUModelRunner(
|
||||
scores = logits.to(torch.float32)
|
||||
else:
|
||||
scores = self.sampler.compute_logprobs(logits)
|
||||
token_ids, logprobs, ranks, _ = self.sampler.gather_logprobs(
|
||||
token_ids, logprobs, ranks, *_ = self.sampler.gather_logprobs(
|
||||
scores, num_prompt_logprobs, tgt_token_ids
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user