[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -0,0 +1,477 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Adaptive verification for DSpark speculative decoding."""
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Iterator
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.utils import CpuGpuBuffer
|
||||
from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
get_query_lens_mismatch_unsupported_backend,
|
||||
)
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_PROFILE_REPLAYS = 5
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
def _assign_draft_token_budget(
|
||||
confidence_probs: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
capacities: torch.Tensor,
|
||||
draft_budget: int,
|
||||
num_steps: int,
|
||||
) -> None:
|
||||
"""Admit the globally best `draft_budget` draft slots, in place.
|
||||
|
||||
Every (request, step) slot is scored by its survival probability, the running
|
||||
product of that request's per-position confidences, and the highest scores win.
|
||||
Survival only decreases along a request, so a global top-k always admits
|
||||
continuously along steps with a request.
|
||||
|
||||
`capacities` enters holding each request's scheduled draft count (which bounds its
|
||||
eligible slots) and leaves holding the admitted count. The caller only calls this
|
||||
when `draft_budget < capacities.sum()`, so every winner is a real draft slot.
|
||||
"""
|
||||
survival = confidence_probs[idx_mapping].cumprod(dim=1)
|
||||
steps = torch.arange(num_steps, device=survival.device)
|
||||
# Out-of-range slots score -inf so they never outrank a real draft.
|
||||
out_of_range = steps[None, :] >= capacities[:, None]
|
||||
survival = survival.masked_fill(out_of_range, -float("inf"))
|
||||
flat = survival.flatten()
|
||||
winners = flat.topk(draft_budget).indices
|
||||
admitted = torch.zeros_like(flat, dtype=torch.bool).index_fill_(0, winners, True)
|
||||
torch.sum(admitted.view_as(survival), dim=1, dtype=capacities.dtype, out=capacities)
|
||||
|
||||
|
||||
_assign_draft_token_budget_compiled = torch.compile(
|
||||
_assign_draft_token_budget, dynamic=True
|
||||
)
|
||||
|
||||
|
||||
def build_cost_tables_from_curves(
|
||||
draft_curve: list[tuple[int, float]],
|
||||
verify_curve: list[tuple[int, float]],
|
||||
max_num_reqs: int,
|
||||
max_batch_tokens: int,
|
||||
cudagraph_limit: int = 0,
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Build cost tables: graph-padded below the capture limit, smooth above.
|
||||
|
||||
Args:
|
||||
cudagraph_limit: Largest cudagraph-captured size. At or below it,
|
||||
execution pads up to the next captured size, so cost is a step
|
||||
function. Above it there is no padding, so cost is continuous.
|
||||
"""
|
||||
|
||||
def build_table(limit: int, curve: list[tuple[int, float]]) -> np.ndarray:
|
||||
xs, ys = np.asarray(curve, dtype=np.float64).T
|
||||
ys = np.maximum.accumulate(ys)
|
||||
values = np.arange(limit + 1)
|
||||
# Execution pads to the next captured size, so cost is a step
|
||||
# function of the padded size: smooth interpolation would invent
|
||||
# marginal per-token costs that don't exist within a pad bucket.
|
||||
idx = np.searchsorted(xs, values, side="left")
|
||||
result = ys[np.minimum(idx, len(xs) - 1)]
|
||||
# Past the capture limit nothing pads, so cost really is continuous in
|
||||
# size; snapping to the next profiled point overestimates badly when
|
||||
# the profiled points are far apart. Interpolate only between points
|
||||
# that are themselves past the limit: crossing the limit loses
|
||||
# cudagraphs entirely, which is a genuine discontinuity, so the first
|
||||
# point above it must not be blended with the last one below.
|
||||
if cudagraph_limit:
|
||||
smooth = values > cudagraph_limit
|
||||
above = xs > cudagraph_limit
|
||||
if smooth.any() and above.any():
|
||||
result[smooth] = np.interp(values[smooth], xs[above], ys[above])
|
||||
if len(xs) > 1:
|
||||
after = values > xs[-1]
|
||||
slope = (ys[-1] - ys[-2]) / (xs[-1] - xs[-2])
|
||||
result[after] = ys[-1] + slope * (values[after] - xs[-1])
|
||||
return result
|
||||
|
||||
draft_table = np.maximum(build_table(max_num_reqs, draft_curve), 0.0)
|
||||
verify_table = np.maximum(build_table(max_batch_tokens, verify_curve), 1e-6)
|
||||
return draft_table, verify_table
|
||||
|
||||
|
||||
class AdaptiveVerificationManager:
|
||||
def __init__(
|
||||
self,
|
||||
req_states: "RequestState",
|
||||
query_start_loc: torch.Tensor,
|
||||
num_bonus_tokens: int,
|
||||
max_total_logits: int,
|
||||
):
|
||||
self.req_states = req_states
|
||||
self.num_speculative_steps = req_states.num_speculative_steps
|
||||
device = req_states.device
|
||||
self._copy_stream = torch.cuda.Stream(device)
|
||||
|
||||
self.num_bonus_tokens = num_bonus_tokens
|
||||
# Rejection sampling verifies logits in one contiguous chunk; the
|
||||
# chunked path indexes by scheduled (untrimmed) offsets and cannot
|
||||
# address the compacted layout, so the budget must fit one chunk.
|
||||
self._max_total_logits = max_total_logits
|
||||
self.query_start_loc = query_start_loc
|
||||
self.cost_tables: tuple[np.ndarray, np.ndarray] | None = None
|
||||
# Largest cudagraph-captured token count; above it nothing pads.
|
||||
self._cudagraph_limit = 0
|
||||
self._batch_budget: tuple[dict[str, int], dict[str, int], int] | None = None
|
||||
max_num_reqs = req_states.max_num_reqs
|
||||
# Current per-slot confidences
|
||||
self._confidence_probs = torch.empty(
|
||||
(max_num_reqs, self.num_speculative_steps),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self._batch_draft_capacity = torch.empty(
|
||||
max_num_reqs, dtype=torch.int32, device=device
|
||||
)
|
||||
self._num_non_draft_tokens = torch.empty_like(query_start_loc[:-1])
|
||||
self._cu_num_logits = torch.empty_like(query_start_loc)
|
||||
|
||||
# Two D2H slots preserve stale inputs for budget selection.
|
||||
self._stale_confidences = [
|
||||
CpuGpuBuffer(
|
||||
max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
self._copy_events = [torch.cuda.Event(blocking=True) for _ in range(2)]
|
||||
self._pending_resets: list[int] = []
|
||||
self._stale_idx = 0
|
||||
for slot in self._stale_confidences:
|
||||
slot.np.fill(1.0)
|
||||
|
||||
def add_request(self, req_idx: int) -> None:
|
||||
self._stale_confidences[self._stale_idx].np[req_idx].fill(1.0)
|
||||
self._pending_resets.append(req_idx)
|
||||
self._confidence_probs[req_idx].fill_(1.0)
|
||||
|
||||
def batches_to_profile(self, capture_sizes: list[int]) -> Iterator[dict[str, int]]:
|
||||
"""Dummy-run kwargs whose step timings seed the cost tables.
|
||||
|
||||
Run these inside StepTimingCollector.collect(), then hand the block's
|
||||
timings to set_initial_cost_curves."""
|
||||
max_num_tokens = self.req_states.max_num_batched_tokens
|
||||
size = self._cudagraph_limit = capture_sizes[-1] if capture_sizes else 0
|
||||
# Also profile beyond the capture limit: real steps run there
|
||||
# (piecewise/eager) and linear extrapolation badly underestimates
|
||||
# them. These runs double as JIT warmup for the piecewise shapes.
|
||||
tail_sizes: set[int] = set()
|
||||
if size:
|
||||
tail_sizes.add(min(size + size // 2, max_num_tokens))
|
||||
while size < max_num_tokens:
|
||||
size = min(size * 2, max_num_tokens)
|
||||
tail_sizes.add(size)
|
||||
tail_sizes -= set(capture_sizes)
|
||||
for num_tokens in capture_sizes + sorted(tail_sizes):
|
||||
for _ in range(_PROFILE_REPLAYS):
|
||||
yield {
|
||||
"num_tokens": num_tokens,
|
||||
"context_len": envs.VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN,
|
||||
}
|
||||
|
||||
def set_initial_cost_curves(self, samples: list[StepTimingSample]) -> None:
|
||||
def median_curve(
|
||||
points: Iterable[tuple[int, float]],
|
||||
) -> list[tuple[int, float]]:
|
||||
grouped: defaultdict[int, list[float]] = defaultdict(list)
|
||||
for key, value in points:
|
||||
grouped[key].append(value)
|
||||
return [(k, float(np.median(v))) for k, v in sorted(grouped.items())]
|
||||
|
||||
# Draft curve: eager-target steps inflate drafter timings (the CPU is
|
||||
# still launching kernels, opening gaps between the drafter's events),
|
||||
# and request counts — unlike token counts — collide across execution
|
||||
# modes, so only graph-replay samples may price the draft curve.
|
||||
draft_curve = median_curve(
|
||||
(s.num_reqs, s.drafter_ms) for s in samples if s.full_cudagraph
|
||||
)
|
||||
verify_curve = median_curve(
|
||||
(s.num_target_tokens, s.forward_ms) for s in samples
|
||||
)
|
||||
self.set_cost_curves(draft_curve, verify_curve)
|
||||
|
||||
def set_cost_curves(
|
||||
self,
|
||||
draft_curve: list[tuple[int, float]],
|
||||
verify_curve: list[tuple[int, float]],
|
||||
) -> None:
|
||||
draft_curve, verify_curve = get_tp_group().broadcast_object(
|
||||
(draft_curve, verify_curve), src=0
|
||||
)
|
||||
if not draft_curve or not verify_curve:
|
||||
raise RuntimeError(
|
||||
"Adaptive verification could not profile step costs. Pass "
|
||||
"`enable_adaptive_verification=false` in the speculative config to "
|
||||
"verify a fixed number of drafts instead."
|
||||
)
|
||||
self.cost_tables = build_cost_tables_from_curves(
|
||||
draft_curve,
|
||||
verify_curve,
|
||||
self.req_states.max_num_reqs,
|
||||
self.req_states.max_num_batched_tokens,
|
||||
self._cudagraph_limit,
|
||||
)
|
||||
logger.debug("DSpark cost tables: %s", self.cost_tables)
|
||||
|
||||
def record_confidences(
|
||||
self,
|
||||
confidence_probs: torch.Tensor,
|
||||
input_batch: "InputBatch",
|
||||
) -> None:
|
||||
"""Publish this step's raw confidences for the ranking kernel and start
|
||||
copying them to the CPU, where a later step's budget reads them."""
|
||||
num_reqs = input_batch.num_reqs
|
||||
ready_idx = self._stale_idx ^ 1
|
||||
with gpu_sync_allowed():
|
||||
self._copy_events[ready_idx].synchronize()
|
||||
if self._pending_resets:
|
||||
self._stale_confidences[ready_idx].np[self._pending_resets] = 1.0
|
||||
self._pending_resets.clear()
|
||||
# Last step's copy has landed: budgets read it, this step overwrites the
|
||||
# slot they were reading before.
|
||||
self._stale_idx, write_idx = ready_idx, self._stale_idx
|
||||
|
||||
self._confidence_probs[input_batch.idx_mapping] = confidence_probs[:num_reqs]
|
||||
write_slot = self._stale_confidences[write_idx]
|
||||
write_slot.gpu.copy_(self._confidence_probs)
|
||||
|
||||
current_stream = torch.cuda.current_stream(self.req_states.device)
|
||||
self._copy_stream.wait_stream(current_stream)
|
||||
with stream(self._copy_stream, current_stream):
|
||||
write_slot.copy_to_cpu()
|
||||
self._copy_events[write_idx].record()
|
||||
|
||||
def get_num_tokens(
|
||||
self,
|
||||
num_tokens_per_req: dict[str, int],
|
||||
draft_tokens: dict[str, list[int]],
|
||||
) -> int:
|
||||
"""Token count once the draft budget is trimmed to fit.
|
||||
|
||||
Stashes the chosen budget in ``_batch_budget`` for the compaction and
|
||||
reallocation that follow in the same step.
|
||||
"""
|
||||
assert self.cost_tables is not None
|
||||
req_ids = list(num_tokens_per_req)
|
||||
num_reqs = len(req_ids)
|
||||
scheduled_tokens = np.fromiter(
|
||||
num_tokens_per_req.values(), dtype=np.int32, count=num_reqs
|
||||
)
|
||||
scheduled_drafts = np.fromiter(
|
||||
(len(draft_tokens.get(req_id, ())) for req_id in req_ids),
|
||||
dtype=np.int32,
|
||||
count=num_reqs,
|
||||
)
|
||||
num_non_draft_tokens = scheduled_tokens - scheduled_drafts
|
||||
slots = np.fromiter(
|
||||
(self.req_states.req_id_to_index[req_id] for req_id in req_ids),
|
||||
dtype=np.int32,
|
||||
count=len(req_ids),
|
||||
)
|
||||
stale_confidences = self._stale_confidences[self._stale_idx].np[slots]
|
||||
survival_probability = np.cumprod(stale_confidences.astype(np.float64), axis=1)
|
||||
steps = np.arange(self.num_speculative_steps)
|
||||
valid = steps[None, :] < scheduled_drafts[:, None]
|
||||
scores = np.sort(survival_probability[valid])[::-1]
|
||||
num_non_draft_tokens_total = int(num_non_draft_tokens.sum())
|
||||
max_draft_budget = min(
|
||||
int(scheduled_drafts.sum()),
|
||||
max(0, self._max_total_logits - num_reqs * self.num_bonus_tokens),
|
||||
)
|
||||
scores = scores[:max_draft_budget]
|
||||
draft_cost_ms, verify_cost_ms = self.cost_tables
|
||||
num_sampling_requests = np.count_nonzero(
|
||||
self.req_states.num_computed_tokens_np[slots] + num_non_draft_tokens
|
||||
>= self.req_states.prefill_len.np[slots]
|
||||
)
|
||||
num_tokens_to_estimated_accepted_tokens = np.concatenate(
|
||||
([num_sampling_requests], num_sampling_requests + np.cumsum(scores))
|
||||
)
|
||||
costs = (
|
||||
draft_cost_ms[len(req_ids)]
|
||||
+ verify_cost_ms[
|
||||
num_non_draft_tokens_total : num_non_draft_tokens_total
|
||||
+ max_draft_budget
|
||||
+ 1
|
||||
]
|
||||
)
|
||||
num_drafts_per_req = {
|
||||
req_id: int(num_drafts)
|
||||
for req_id, num_drafts in zip(req_ids, scheduled_drafts, strict=True)
|
||||
}
|
||||
num_non_draft_tokens_per_req = {
|
||||
req_id: int(num_tokens)
|
||||
for req_id, num_tokens in zip(req_ids, num_non_draft_tokens, strict=True)
|
||||
}
|
||||
draft_budget = int(np.argmax(num_tokens_to_estimated_accepted_tokens / costs))
|
||||
self._batch_budget = (
|
||||
num_drafts_per_req,
|
||||
num_non_draft_tokens_per_req,
|
||||
draft_budget,
|
||||
)
|
||||
return sum(num_non_draft_tokens_per_req.values()) + draft_budget
|
||||
|
||||
def compact_batch(
|
||||
self,
|
||||
num_draft_tokens_per_req: np.ndarray,
|
||||
num_scheduled_tokens: np.ndarray,
|
||||
cu_num_logits_np: np.ndarray,
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Compact the CPU batch to the chosen draft budget.
|
||||
|
||||
Returns the compacted per-request token counts and the CPU cu_num_logits_np.
|
||||
If the draft budget is 0, we can know cu_num_logits_np exactly, otherwise
|
||||
its unchanged/an-upper-bound.
|
||||
"""
|
||||
batch_budget = self._batch_budget
|
||||
assert batch_budget is not None
|
||||
_, _, draft_budget = batch_budget
|
||||
num_drafts = int(num_draft_tokens_per_req.sum())
|
||||
if draft_budget == num_drafts:
|
||||
return num_scheduled_tokens, cu_num_logits_np
|
||||
|
||||
num_non_draft_tokens = num_scheduled_tokens - num_draft_tokens_per_req
|
||||
if draft_budget == 0:
|
||||
# The draft budget is 0, so we can know cu_num_logits_np exactly. This helps
|
||||
# when we would exceed the sampler logit chunk size.
|
||||
num_reqs = num_scheduled_tokens.shape[0]
|
||||
cu_num_logits_np = (
|
||||
np.arange(num_reqs + 1, dtype=cu_num_logits_np.dtype)
|
||||
* self.num_bonus_tokens
|
||||
)
|
||||
return num_non_draft_tokens, cu_num_logits_np
|
||||
|
||||
is_verification_request = num_draft_tokens_per_req > 0
|
||||
num_verification_reqs = int(is_verification_request.sum())
|
||||
# sort_batch_req_ids keeps verification requests at the front.
|
||||
assert np.all(is_verification_request[:num_verification_reqs])
|
||||
# for the CPU side buffer we distribute draft tokens evenly
|
||||
draft_lens_cpu = np.zeros_like(num_non_draft_tokens)
|
||||
draft_lens_cpu[:num_verification_reqs] = draft_budget // num_verification_reqs
|
||||
draft_lens_cpu[: draft_budget % num_verification_reqs] += 1
|
||||
return num_non_draft_tokens + draft_lens_cpu, cu_num_logits_np
|
||||
|
||||
def reallocate_drafts(
|
||||
self, req_ids: list[str], idx_mapping: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, int]:
|
||||
batch_budget, self._batch_budget = self._batch_budget, None
|
||||
assert batch_budget is not None
|
||||
num_drafts_per_req, num_non_draft_tokens_per_req, draft_budget = batch_budget
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
scheduled_drafts = np.fromiter(
|
||||
(num_drafts_per_req[req_id] for req_id in req_ids),
|
||||
dtype=np.int32,
|
||||
count=num_reqs,
|
||||
)
|
||||
num_non_draft_tokens = np.fromiter(
|
||||
(num_non_draft_tokens_per_req[req_id] for req_id in req_ids),
|
||||
dtype=np.int32,
|
||||
count=num_reqs,
|
||||
)
|
||||
num_tokens = int(num_non_draft_tokens.sum()) + draft_budget
|
||||
|
||||
# Rank draft slots by survival probability and admit the best prefix.
|
||||
# capacities enters holding each request's valid draft count (the kernel
|
||||
# uses it to bound eligible slots) and leaves holding the admitted count.
|
||||
capacities = self._batch_draft_capacity[:num_reqs]
|
||||
if draft_budget == 0:
|
||||
capacities.zero_()
|
||||
else:
|
||||
async_copy_to_gpu(scheduled_drafts, out=capacities)
|
||||
if draft_budget < int(scheduled_drafts.sum()):
|
||||
_assign_draft_token_budget_compiled(
|
||||
self._confidence_probs,
|
||||
idx_mapping,
|
||||
capacities,
|
||||
draft_budget,
|
||||
self.num_speculative_steps,
|
||||
)
|
||||
|
||||
num_non_draft_tokens_gpu = self._num_non_draft_tokens[:num_reqs]
|
||||
async_copy_to_gpu(
|
||||
num_non_draft_tokens,
|
||||
out=num_non_draft_tokens_gpu,
|
||||
)
|
||||
self._cu_num_logits[:1].zero_()
|
||||
torch.cumsum(
|
||||
capacities + self.num_bonus_tokens,
|
||||
dim=0,
|
||||
out=self._cu_num_logits[1 : num_reqs + 1],
|
||||
)
|
||||
self.query_start_loc[:1].zero_()
|
||||
torch.cumsum(
|
||||
capacities + num_non_draft_tokens_gpu,
|
||||
dim=0,
|
||||
out=self.query_start_loc[1 : num_reqs + 1],
|
||||
)
|
||||
self.query_start_loc[num_reqs + 1 :].fill_(num_tokens)
|
||||
return (
|
||||
self._cu_num_logits[: num_reqs + 1],
|
||||
self.query_start_loc,
|
||||
draft_budget,
|
||||
)
|
||||
|
||||
|
||||
def maybe_create_adaptive_verification_manager(
|
||||
*,
|
||||
enable_adaptive_verification: bool,
|
||||
attn_groups: list[list["AttentionGroup"]],
|
||||
attn_cg_support: "AttentionCGSupportInfo",
|
||||
req_states: "RequestState",
|
||||
query_start_loc: torch.Tensor,
|
||||
num_bonus_tokens: int,
|
||||
max_total_logits: int,
|
||||
) -> AdaptiveVerificationManager | None:
|
||||
if not enable_adaptive_verification:
|
||||
return None
|
||||
|
||||
# The selector rejects unsupported backends, but models that
|
||||
# hard-wire theirs (e.g. DeepSeek-V4) never go through it.
|
||||
backend = get_query_lens_mismatch_unsupported_backend(attn_groups)
|
||||
if backend is not None:
|
||||
raise ValueError(
|
||||
"Adaptive verification trims verification requests on device, which"
|
||||
f" the {backend} attention backend does not support. Pass "
|
||||
"enable_adaptive_verification=false in the speculative config, or "
|
||||
"use a backend that does."
|
||||
)
|
||||
|
||||
if attn_cg_support.min_cg_support != AttentionCGSupport.ALWAYS:
|
||||
raise ValueError(
|
||||
"Adaptive verification captures varlen decode cudagraphs, so every"
|
||||
" attention builder must report AttentionCGSupport.ALWAYS, but "
|
||||
f"{attn_cg_support.min_cg_attn_backend} reports "
|
||||
f"{attn_cg_support.min_cg_support}. Pass "
|
||||
"enable_adaptive_verification=false in the speculative config, or "
|
||||
"use a backend that does."
|
||||
)
|
||||
|
||||
return AdaptiveVerificationManager(
|
||||
req_states,
|
||||
query_start_loc,
|
||||
num_bonus_tokens,
|
||||
max_total_logits=max_total_logits,
|
||||
)
|
||||
@@ -135,11 +135,10 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
|
||||
def capture(self) -> None:
|
||||
logger.info("Capturing model for %s speculator...", self._speculator_name)
|
||||
# Reset sampling indices to zero to prevent stale values from prior
|
||||
# dummy runs from being baked into the captured graph.
|
||||
# Padded sample rows must not scatter into a live request during capture.
|
||||
self.sample_indices.zero_()
|
||||
self.sample_pos.zero_()
|
||||
self.sample_idx_mapping.zero_()
|
||||
self.sample_idx_mapping.fill_(-1)
|
||||
assert self.query_cudagraph_manager is not None
|
||||
self.query_cudagraph_manager.capture(
|
||||
self._generate_draft,
|
||||
@@ -589,7 +588,10 @@ def _prepare_dflash_inputs_kernel(
|
||||
# seq_lens is the absolute sequence length the draft attention
|
||||
# reads up to (context + query), not just the count of accepted
|
||||
# tokens this step.
|
||||
tl.store(out_seq_lens_ptr + req_idx, last_valid_pos + 1 + num_query_per_req)
|
||||
tl.store(
|
||||
out_seq_lens_ptr + req_idx,
|
||||
tl.minimum(last_valid_pos + 1 + num_query_per_req, max_model_len),
|
||||
)
|
||||
# Copy sampling state.
|
||||
tl.store(
|
||||
out_temperature_ptr + req_state_idx,
|
||||
|
||||
@@ -29,10 +29,13 @@ import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.utils import load_dspark_model
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DSparkSpeculator(DFlashSpeculator):
|
||||
_speculator_name = "DSpark"
|
||||
@@ -76,6 +79,13 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
self.draft_model_config.hf_config, "dspark_draft_topk", None
|
||||
)
|
||||
|
||||
self.draft_token_confidence_probs = torch.empty_like(
|
||||
self.draft_tokens, dtype=torch.float32
|
||||
)
|
||||
self.enable_adaptive_verification = (
|
||||
self.speculative_config.enable_adaptive_verification
|
||||
)
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
target_model: torch.nn.Module,
|
||||
@@ -98,6 +108,13 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
dtype=self.draft_logits.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
if self.enable_adaptive_verification and model.model.confidence_head is None:
|
||||
raise ValueError(
|
||||
"Adaptive verification needs a DSpark checkpoint with a confidence "
|
||||
"head, and this one has none. Pass "
|
||||
"enable_adaptive_verification=false in the speculative config to verify"
|
||||
" a fixed number of drafts instead."
|
||||
)
|
||||
return model
|
||||
|
||||
def _sample_logits(
|
||||
@@ -149,6 +166,7 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
|
||||
idx_map = self.sample_idx_mapping[:num_sample].view(num_reqs, n_spec)
|
||||
sample_pos = self.sample_pos[:num_sample].view(num_reqs, n_spec)
|
||||
confidence_markov_embeds = []
|
||||
|
||||
# Anchor (bonus) token per request = the input id at query offset 0,
|
||||
# read via the precomputed persistent index (fixed buffer for capture).
|
||||
@@ -157,6 +175,8 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
for i in range(n_spec):
|
||||
# Sequential stage: Markov bias from the previously sampled token.
|
||||
markov_embed = self.model.markov_embed(prev)
|
||||
if self.enable_adaptive_verification:
|
||||
confidence_markov_embeds.append(markov_embed)
|
||||
bias = self.model.markov_bias(markov_embed)
|
||||
logits_i = base_logits[:, i] + bias
|
||||
draft_sampled_i = self._sample_logits(
|
||||
@@ -165,6 +185,15 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
self.draft_tokens[:num_reqs, i] = draft_sampled_i
|
||||
prev = draft_sampled_i
|
||||
|
||||
if self.enable_adaptive_verification:
|
||||
confidence = self.model.compute_confidence(
|
||||
sample_hidden,
|
||||
torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1),
|
||||
)
|
||||
self.draft_token_confidence_probs[:num_reqs] = confidence.view(
|
||||
num_reqs, n_spec
|
||||
)
|
||||
|
||||
def _sample_sequential_topk(self, num_reqs: int, head_hidden: torch.Tensor) -> None:
|
||||
"""Apply the sequential Markov head only to top-k base-logit candidates.
|
||||
|
||||
@@ -186,10 +215,13 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
base_logits.fill_(float("-inf"))
|
||||
idx_map = self.sample_idx_mapping[:num_sample].view(num_reqs, n_spec)
|
||||
sample_pos = self.sample_pos[:num_sample].view(num_reqs, n_spec)
|
||||
confidence_markov_embeds = []
|
||||
prev = self.input_buffers.input_ids[self._anchor_idx[:num_reqs]]
|
||||
|
||||
for i in range(n_spec):
|
||||
markov_embed = self.model.markov_embed(prev)
|
||||
if self.enable_adaptive_verification:
|
||||
confidence_markov_embeds.append(markov_embed)
|
||||
logits_i = self.model.apply_markov_bias_gathered(
|
||||
markov_embed,
|
||||
base_logits[:, i],
|
||||
@@ -202,6 +234,15 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
self.draft_tokens[:num_reqs, i] = draft_sampled_i
|
||||
prev = draft_sampled_i
|
||||
|
||||
if self.enable_adaptive_verification:
|
||||
confidence = self.model.compute_confidence(
|
||||
sample_hidden,
|
||||
torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1),
|
||||
)
|
||||
self.draft_token_confidence_probs[:num_reqs] = confidence.view(
|
||||
num_reqs, n_spec
|
||||
)
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
num_reqs: int,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Iterable, Iterator
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -32,6 +32,11 @@ MAX_CHUNK_BYTES = 2**30 # 1GB
|
||||
_FP32_BYTES = 4
|
||||
|
||||
|
||||
def get_max_chunk_logits(vocab_size: int) -> int:
|
||||
"""Largest number of logits rows one verification chunk may hold."""
|
||||
return max(1, MAX_CHUNK_BYTES // (vocab_size * _FP32_BYTES))
|
||||
|
||||
|
||||
def _iter_request_chunks(
|
||||
cu_num_logits: np.ndarray, max_chunk_logits: int
|
||||
) -> Iterator[tuple[int, int]]:
|
||||
@@ -177,11 +182,22 @@ class RejectionSampler:
|
||||
) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]:
|
||||
cu_num_logits_np = input_batch.cu_num_logits_np
|
||||
use_processed_logits = self.sampler.logprobs_mode in PROCESSED_LOGPROBS_MODES
|
||||
num_reqs = input_batch.num_reqs
|
||||
|
||||
if logits.shape[0] <= max_chunk_logits:
|
||||
# One chunk covers the batch. Adaptive verification compacts the logits
|
||||
# without updating cu_num_logits_np (it keeps the pre-compacted layout),
|
||||
# so the stale sums must not pick chunk boundaries; its budget cap
|
||||
# guarantees the compacted batch always lands here.
|
||||
request_chunks: Iterable[tuple[int, int]] = ((0, num_reqs),)
|
||||
else:
|
||||
request_chunks = _iter_request_chunks(cu_num_logits_np, max_chunk_logits)
|
||||
|
||||
sampled_chunks: list[torch.Tensor] = []
|
||||
num_sampled_chunks: list[torch.Tensor] = []
|
||||
logprobs_chunks: list[LogprobsTensors] = []
|
||||
|
||||
for start, end in _iter_request_chunks(cu_num_logits_np, max_chunk_logits):
|
||||
for start, end in request_chunks:
|
||||
lo = int(cu_num_logits_np[start])
|
||||
hi = int(cu_num_logits_np[end])
|
||||
chunk_cu_num_logits_np = cu_num_logits_np[start : end + 1] - lo
|
||||
@@ -246,14 +262,14 @@ class RejectionSampler:
|
||||
max_num_logprobs = self.sampler.sampling_states.max_num_logprobs(
|
||||
input_batch.idx_mapping_np
|
||||
)
|
||||
max_chunk_logits = max(1, MAX_CHUNK_BYTES // (logits.shape[1] * _FP32_BYTES))
|
||||
chunk_logit_limit = get_max_chunk_logits(logits.shape[1])
|
||||
sampled, num_sampled, logprobs_tensors = self._verify_in_chunks(
|
||||
logits,
|
||||
input_batch,
|
||||
draft_logits,
|
||||
draft_sampled,
|
||||
pos,
|
||||
max_chunk_logits,
|
||||
chunk_logit_limit,
|
||||
max_num_logprobs,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user