[Model Runner V2] Acceptance estimation for adaptive verification (#52228)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-09-14 15:58:48 -07:00
committed by GitHub
parent 8e08cef46e
commit 3bb0a03f35
9 changed files with 900 additions and 28 deletions
@@ -0,0 +1,294 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""``predict`` writes one probability into two index spaces at once.
Adaptive verification reads the estimate in this step's batch order, while the
next step's refit reads it in persistent request-state slots. A mix-up between
the two is silent -- every value stays a plausible probability, it just belongs
to another request -- so it is asserted here rather than left to acceptance
metrics to hint at.
"""
import pytest
import torch
from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import (
OnlineAcceptanceEstimator,
)
pytest.importorskip("triton")
if not torch.cuda.is_available():
pytest.skip("CUDA required for acceptance estimator tests", allow_module_level=True)
MAX_NUM_REQS = 16
NUM_STEPS = 4
VOCAB_SIZE = 2048
def _make_estimator(device: torch.device) -> OnlineAcceptanceEstimator:
estimator = OnlineAcceptanceEstimator(MAX_NUM_REQS, NUM_STEPS, device)
# Distinct per-position intercepts, so reading the wrong one shows up.
estimator.slope.fill_(0.7)
estimator.intercepts[:] = torch.tensor([1.5, 1.0, 0.5, 0.0], device=device)
return estimator
def _expected_probs(
estimator: OnlineAcceptanceEstimator,
logits: torch.Tensor,
steps: torch.Tensor,
temperature: torch.Tensor,
) -> torch.Tensor:
"""logit(q_max) by a different route than the kernel takes.
The kernel shifts by the max and drops the unit term; here the argmax is
masked out and the complement is reduced with logsumexp, so agreement is
evidence about the value rather than about a shared formula.
"""
temp = torch.where(temperature > 0, temperature, torch.ones_like(temperature))
scaled = logits.double() / temp[:, None].double()
top = scaled.max(dim=-1)
rest = scaled.scatter(1, top.indices[:, None], float("-inf"))
feature = (top.values - torch.logsumexp(rest, dim=-1)).clamp(-40.0, 40.0)
weight, bias = estimator.slope, estimator.intercepts
return torch.sigmoid(weight.double() * feature + bias[steps].double()).float()
@pytest.mark.parametrize("tokens_per_req", [1, NUM_STEPS])
def test_predict_agrees_across_slot_and_batch_order(tokens_per_req: int):
"""One row per request (per-step drafting) and a whole block at once.
Which one it is is inferred from whether the draft step is per-row, so the
two cases differ only in how ``draft_step`` is shaped.
"""
device = torch.device("cuda")
torch.manual_seed(0)
estimator = _make_estimator(device)
num_reqs = 5
# Batch position i holds a shuffled persistent slot, as a served batch would.
slots = torch.randperm(MAX_NUM_REQS, device=device)[:num_reqs].to(torch.int32)
if tokens_per_req == 1:
idx_mapping = slots
draft_step = torch.tensor(2, device=device, dtype=torch.int32)
steps = draft_step.expand(num_reqs)
else:
idx_mapping = slots.repeat_interleave(NUM_STEPS)
draft_step = torch.arange(NUM_STEPS, device=device, dtype=torch.int32).repeat(
num_reqs
)
steps = draft_step
batch_rows = torch.arange(num_reqs, device=device).repeat_interleave(tokens_per_req)
logits = torch.randn(idx_mapping.shape[0], VOCAB_SIZE, device=device) * 4.0
confidence_probs = torch.zeros(MAX_NUM_REQS, NUM_STEPS, device=device)
temperature = torch.rand(MAX_NUM_REQS, device=device) + 0.5
estimator.predict(logits, idx_mapping, draft_step, confidence_probs, temperature)
expected = _expected_probs(
estimator, logits, steps.long(), temperature[idx_mapping.long()]
)
torch.testing.assert_close(
estimator.predictions[idx_mapping.long(), steps.long()],
expected,
atol=2e-3,
rtol=2e-3,
)
torch.testing.assert_close(
confidence_probs[batch_rows, steps.long()], expected, atol=2e-3, rtol=2e-3
)
def test_predict_skips_cudagraph_padded_rows():
"""Padded rows carry slot -1 and must not land on a live request."""
device = torch.device("cuda")
torch.manual_seed(0)
estimator = _make_estimator(device)
num_reqs, num_padded = 3, 6
slots = torch.tensor([7, 2, 11], device=device, dtype=torch.int32)
idx_mapping = torch.cat(
[slots, torch.full((num_padded,), -1, device=device, dtype=torch.int32)]
)
draft_step = torch.tensor(1, device=device, dtype=torch.int32)
logits = torch.randn(num_reqs + num_padded, VOCAB_SIZE, device=device) * 4.0
confidence_probs = torch.zeros(MAX_NUM_REQS, NUM_STEPS, device=device)
temperature = torch.rand(MAX_NUM_REQS, device=device) + 0.5
estimator.predict(logits, idx_mapping, draft_step, confidence_probs, temperature)
live = (slots.long(), torch.full((num_reqs,), 1, device=device))
expected = _expected_probs(
estimator, logits[:num_reqs], live[1], temperature[slots.long()]
)
torch.testing.assert_close(
estimator.predictions[live], expected, atol=2e-3, rtol=2e-3
)
# Nothing outside the live (request, position) cells was touched.
untouched = torch.ones(MAX_NUM_REQS, NUM_STEPS, dtype=torch.bool, device=device)
untouched[live] = False
assert not estimator.predictions[untouched].any()
assert not confidence_probs[num_reqs:].any()
# A refit takes one undamped Newton step, so it needs a realistic number of
# observations per round to be well conditioned -- a handful of requests can
# send the step far past the optimum. This is the batch a served step sees.
REFIT_NUM_REQS = 512
def _drive_round(estimator, features, num_accepted, slots):
"""Feed one step's graded drafts through predict's buffers into `step`."""
estimator.features[slots.long()] = features
estimator.predictions[slots.long()] = torch.sigmoid(
estimator.slope * features + estimator.intercepts
)
estimator.step(
slots,
(num_accepted + 1).to(torch.int32),
(features.shape[1] - num_accepted).to(torch.int32),
)
def test_refit_recovers_a_shared_slope_and_per_position_intercepts():
"""The fit must find the logistic that generated the observations.
One slope is shared across draft positions and each keeps its own intercept,
so the arrowhead solve is only correct if it recovers both from data whose
positions differ in level but not in slope.
"""
device = torch.device("cuda")
torch.manual_seed(0)
true_slope = 0.6
true_bias = torch.tensor([1.4, 0.9, 0.4, -0.1], device=device)
estimator = OnlineAcceptanceEstimator(REFIT_NUM_REQS, NUM_STEPS, device)
estimator.REFIT_INTERVAL = 5
slots = torch.arange(REFIT_NUM_REQS, device=device, dtype=torch.int32)
for _ in range(60):
features = torch.rand(REFIT_NUM_REQS, NUM_STEPS, device=device) * 8.0
accepted = torch.rand_like(features) < torch.sigmoid(
true_slope * features + true_bias
)
# Verification stops at the first rejection, so a position's label is
# conditional on every shallower one having been accepted.
num_accepted = (~accepted).float().argmax(dim=1)
num_accepted[accepted.all(dim=1)] = NUM_STEPS
_drive_round(estimator, features, num_accepted, slots)
slope, bias = estimator.slope, estimator.intercepts
torch.testing.assert_close(
slope[0], torch.tensor(true_slope, device=device), atol=0.1, rtol=0
)
torch.testing.assert_close(bias, true_bias, atol=0.3, rtol=0)
def test_refit_survives_a_position_with_no_observations():
"""A draft position nobody reached must not stall the shared slope.
Its statistics stay zero while the others accumulate, which is the degenerate
case the arrowhead denominator has to tolerate.
"""
device = torch.device("cuda")
torch.manual_seed(0)
estimator = OnlineAcceptanceEstimator(REFIT_NUM_REQS, NUM_STEPS, device)
estimator.REFIT_INTERVAL = 5
slots = torch.arange(REFIT_NUM_REQS, device=device, dtype=torch.int32)
before_slope = estimator.slope.clone()
before_intercepts = estimator.intercepts.clone()
# Every request is rejected at position 1, so positions 2+ are never graded.
num_accepted = torch.ones(REFIT_NUM_REQS, device=device)
for _ in range(20):
features = torch.rand(REFIT_NUM_REQS, NUM_STEPS, device=device) * 8.0
_drive_round(estimator, features, num_accepted, slots)
assert estimator.slope.isfinite().all()
assert estimator.intercepts.isfinite().all()
assert not torch.equal(estimator.slope, before_slope), "slope never moved"
# Positions 2 and 3 were never observed, so their intercepts must be untouched.
torch.testing.assert_close(
estimator.intercepts[2:], before_intercepts[2:], rtol=0, atol=0
)
def test_step_damping_scales_with_the_evidence():
"""A round moves a coefficient in proportion to the samples behind it.
The Newton step itself barely changes with how much data a round carries --
curvature and score both scale with it -- so scaling a round's statistics
isolates the damping factor n / (n + DAMPING_OBSERVATIONS). The slope/
intercept coupling is zeroed so the two parameters can be checked apart.
"""
device = torch.device("cuda")
damping = OnlineAcceptanceEstimator.DAMPING_OBSERVATIONS
per_position_n = 20.0
def refit_with(scale):
estimator = OnlineAcceptanceEstimator(MAX_NUM_REQS, NUM_STEPS, device)
estimator.info[:, 1] = 0.0 # sum(w*margin): no slope/intercept coupling
estimator.info[:, 2] = 1.0 * scale # sum(w)
estimator.grad[:, 1] = 0.5 * scale # sum(y - p)
# The slope's information and score are accumulated as per-position
# partials that the refit sums, so spread each total over the positions.
estimator.info[:, 0] = 30.0 * scale / NUM_STEPS # sum(w*margin^2)
estimator.grad[:, 0] = 2.0 * scale / NUM_STEPS # sum((y - p)*margin)
estimator.counts[:] = per_position_n * scale
estimator._steps_since_refit = estimator.REFIT_INTERVAL - 1
before_slope = estimator.slope.clone()
before_intercepts = estimator.intercepts.clone()
empty = torch.zeros(0, dtype=torch.int32, device=device)
estimator.step(empty, empty, empty)
return (
(estimator.slope - before_slope)[0].item(),
(estimator.intercepts - before_intercepts)[0].item(),
)
for scale in (1.0, 10.0, 100.0):
total_n = per_position_n * scale * NUM_STEPS
slope_delta, bias_delta = refit_with(scale)
assert slope_delta == pytest.approx(
(2.0 / 30.0) * total_n / (total_n + damping), rel=1e-3
)
assert bias_delta == pytest.approx(
0.5 * (per_position_n * scale) / (per_position_n * scale + damping),
rel=1e-3,
)
# A round with no observations at all must leave the coefficients alone.
assert refit_with(0.0) == (0.0, 0.0)
def test_feature_ignores_which_token_was_drawn():
"""The score must depend on the draft distribution, not on the draw.
This is what keeps rejection sampling lossless: if the score read the drawn
token, trimming a draft would condition on the token being trimmed and tilt
the emitted distribution toward the drafter's confident modes. ``predict``
takes no token argument, and this pins that -- two rows with identical
logits must score identically no matter what was sampled from them.
"""
device = torch.device("cuda")
torch.manual_seed(0)
estimator = _make_estimator(device)
row = torch.randn(VOCAB_SIZE, device=device) * 4.0
logits = row.expand(2, VOCAB_SIZE).contiguous()
idx_mapping = torch.tensor([0, 1], device=device, dtype=torch.int32)
draft_step = torch.tensor(0, device=device, dtype=torch.int32)
temperature = torch.ones(MAX_NUM_REQS, device=device)
confidence_probs = torch.zeros(MAX_NUM_REQS, NUM_STEPS, device=device)
estimator.predict(logits, idx_mapping, draft_step, confidence_probs, temperature)
assert estimator.features[0, 0] == estimator.features[1, 0]
# And the value is the top token's log-odds, not the top-2 margin.
scaled = row.double()
top = scaled.max(dim=-1)
rest = scaled.scatter(0, top.indices[None], float("-inf"))
torch.testing.assert_close(
estimator.features[0, 0].double(),
(top.values - torch.logsumexp(rest, dim=-1)).clamp(-40.0, 40.0),
atol=2e-3,
rtol=2e-3,
)
@@ -123,6 +123,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected)
draft_model_config=draft_model_config,
use_local_argmax_reduction=False,
draft_sample_method="greedy",
enable_adaptive_verification=False,
)
vllm_config = SimpleNamespace(
speculative_config=speculative_config,
@@ -161,6 +162,7 @@ def test_mm_support_configured_after_model_load(monkeypatch):
speculator.dtype = torch.float32
speculator.draft_model_config = draft_model_config
speculator.supports_mm_inputs = False
speculator.use_acceptance_estimator = False
checked_configs = []
@@ -195,6 +197,7 @@ def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):
speculator = object.__new__(_TestSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.use_acceptance_estimator = False
speculator.vllm_config = SimpleNamespace(model_config=object())
speculator.max_num_tokens = 4
speculator.hidden_size = 3
@@ -219,6 +222,7 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
speculator = object.__new__(_TestSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.use_acceptance_estimator = False
speculator.vllm_config = SimpleNamespace(model_config=object())
draft_model = _TextOnlyDraftModel()
speculator.test_draft_model = draft_model
@@ -251,6 +255,7 @@ def test_multi_module_mm_support_configured_after_model_load(monkeypatch):
speculator = object.__new__(MultiModuleMTPSpeculator)
speculator.supports_mm_inputs = False
speculator.inputs_embeds = None
speculator.use_acceptance_estimator = False
speculator.cached_draft_input_embeds = None
speculator.vllm_config = SimpleNamespace(model_config=object())
speculator.max_num_tokens = 4
@@ -60,6 +60,7 @@ def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch):
layer_name = "cache_only_layers.2"
speculator = object.__new__(ExtractHiddenStatesSpeculator)
speculator.vllm_config = cast(Any, SimpleNamespace())
speculator.acceptance_estimator = None
speculator.num_hidden_states = 2
speculator.hidden_states = torch.zeros(4, 2, 3)
speculator.draft_attn_layer_names = {layer_name}
@@ -111,6 +112,7 @@ def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch):
def test_propose_requires_aux_hidden_states():
speculator = object.__new__(ExtractHiddenStatesSpeculator)
speculator.acceptance_estimator = None
speculator.num_hidden_states = 2
input_batch = cast(
Any, SimpleNamespace(idx_mapping=torch.tensor([0], dtype=torch.int32))
+10 -2
View File
@@ -1561,8 +1561,16 @@ class SpeculativeConfig:
self.index_share_for_mtp_iteration
)
if self.method != "dspark" and self.enable_adaptive_verification:
raise ValueError("Adaptive verification only supported with DSpark")
if (
self.method != "dspark"
and self.enable_adaptive_verification
and self.use_local_argmax_reduction
):
raise ValueError(
"Adaptive verification estimates per-position acceptance from "
"the draft logits, which use_local_argmax_reduction never "
"materializes. Disable one of them."
)
return self
+4
View File
@@ -2120,6 +2120,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.speculator is not None:
assert self.sampler is not None
if isinstance(self.speculator, DraftModelSpeculator):
self.speculator.observe_verification(
input_batch.idx_mapping, num_sampled, num_rejected
)
# Let the target override the hidden state fed to the drafter
# (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The
# target returns a persistent buffer sized at max_num_batched_tokens;
@@ -0,0 +1,501 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
_compute_max_and_sumexp,
)
# The feature is a log-odds, log(q / (1 - q)), which a row carrying all of its
# mass on one token sends to infinity. Clamp it symmetrically.
_MAX_LOG_ODDS = 40.0
# Cold-start coefficients: the median slope and intercept fitted online across
# DeepSeek-V4-Flash, Muse-Glimmer-30B, MiMo-V2.5-Pro, Inkling and Kimi-K2.5.
# Log-loss is flat near the optimum, so one starting point serves all of them
# and a real fit replaces it within a refit or two.
_INIT_SLOPE = 0.5
_INIT_BIAS = -0.2
@triton.jit
def _accumulate_kernel(
info_ptr,
grad_ptr,
idx_mapping_ptr,
num_sampled_ptr,
num_rejected_ptr,
features_ptr,
features_stride,
pred_ptr,
pred_stride,
counts_ptr,
num_reqs,
BLOCK_R: tl.constexpr,
):
step = tl.program_id(0).to(tl.int64)
req_block = tl.arange(0, BLOCK_R)
req_mask = req_block < num_reqs
req_state_block = tl.load(idx_mapping_ptr + req_block, mask=req_mask, other=0).to(
tl.int64
)
num_sampled = tl.load(num_sampled_ptr + req_block, mask=req_mask, other=0).to(
tl.int64
)
# Subtract bonus token.
num_accepted = tl.maximum(num_sampled - 1, 0)
num_rejected = tl.load(num_rejected_ptr + req_block, mask=req_mask, other=0).to(
tl.int64
)
num_admitted = num_accepted + num_rejected
feature = tl.load(
features_ptr + req_state_block * features_stride + step,
mask=req_mask,
other=0.0,
).to(tl.float32)
pred = tl.load(
pred_ptr + req_state_block * pred_stride + step,
mask=req_mask,
other=0.0,
).to(tl.float32)
# The accepted tokens and the first rejected draft token (if one exists) are
# "observed" and given labels.
observed = req_mask & (step <= num_accepted) & (step < num_admitted)
label = tl.where(step < num_accepted, 1.0, 0.0)
w = tl.where(observed, pred * (1.0 - pred), 0.0)
resid = tl.where(observed, label - pred, 0.0)
# This block's contribution to the normal equations for design row
# x = [feature, eₖ]: the XᵀWX entries aₖ, bₖ, cₖ and the Xᵀr entries g₀ₖ, g₁ₖ.
# aₖ and g₀ₖ are partials of scalars shared by every position, which
# _refit_kernel sums.
a_k = tl.sum(w * feature * feature, axis=0)
b_k = tl.sum(w * feature, axis=0)
c_k = tl.sum(w, axis=0)
g0_k = tl.sum(resid * feature, axis=0)
g1_k = tl.sum(resid, axis=0)
count = tl.sum(tl.where(observed, 1.0, 0.0), axis=0)
tl.store(info_ptr + step * 3 + 0, tl.load(info_ptr + step * 3 + 0) + a_k)
tl.store(info_ptr + step * 3 + 1, tl.load(info_ptr + step * 3 + 1) + b_k)
tl.store(info_ptr + step * 3 + 2, tl.load(info_ptr + step * 3 + 2) + c_k)
tl.store(grad_ptr + step * 2 + 0, tl.load(grad_ptr + step * 2 + 0) + g0_k)
tl.store(grad_ptr + step * 2 + 1, tl.load(grad_ptr + step * 2 + 1) + g1_k)
tl.store(counts_ptr + step, tl.load(counts_ptr + step) + count)
@triton.jit
def _refit_kernel(
slope_ptr,
intercepts_ptr,
info_ptr,
info_row_stride,
grad_ptr,
grad_row_stride,
counts_ptr,
NUM_SPECULATIVE_STEPS: tl.constexpr,
L2: tl.constexpr,
DAMPING: tl.constexpr,
BLOCK: tl.constexpr,
):
k = tl.arange(0, BLOCK)
mask = k < NUM_SPECULATIVE_STEPS
# Load the stats accumulated over this round to solve (XᵀWX + λI) Δθ = Xᵀr.
a_k = tl.load(info_ptr + k * info_row_stride + 0, mask=mask, other=0.0)
b_k = tl.load(info_ptr + k * info_row_stride + 1, mask=mask, other=0.0)
c_k = tl.load(info_ptr + k * info_row_stride + 2, mask=mask, other=0.0) + L2
a = tl.sum(a_k, axis=0) + L2
g0_k = tl.load(grad_ptr + k * grad_row_stride + 0, mask=mask, other=0.0)
g1_k = tl.load(grad_ptr + k * grad_row_stride + 1, mask=mask, other=0.0)
g0 = tl.sum(g0_k, axis=0)
n = tl.load(counts_ptr + k, mask=mask, other=0.0)
# Load the current shared slope and per-position intercepts.
w = tl.load(slope_ptr)
bias = tl.load(intercepts_ptr + k, mask=mask, other=0.0)
# Solve for the shared slope delta, Δω, in the following arrowhead system:
# ⎡ a+λ b₀ b₁ … bₙ₋₁ ⎤ ⎡ Δω ⎤ ⎡ g₀ ⎤
# ⎢ b₀ c₀+λ ⎥ ⎢ Δβ₀ ⎥ ⎢ g₁₀ ⎥
# ⎢ b₁ c₁+λ ⎥ ⎢ Δβ₁ ⎥ = ⎢ g₁₁ ⎥
# ⎢ ⋮ ⋱ ⎥ ⎢ ⋮ ⎥ ⎢ ⋮ ⎥
# ⎣ bₙ₋₁ cₙ₋₁+λ ⎦ ⎣ Δβₙ₋₁ ⎦ ⎣ g₁ₙ₋₁ ⎦
# XᵀWX + λI Δθ Xᵀ(Y-P)
shrink = b_k * b_k / c_k
coupling = b_k * g1_k / c_k
step_w = (g0 - tl.sum(coupling, axis=0)) / (a - tl.sum(shrink, axis=0))
# The slope learns from every position, so it is damped by the round's
# total sample count.
total_n = tl.sum(n, axis=0)
step_w *= total_n / (total_n + DAMPING)
# Mask out NaNs and steps that would drive the slope negative.
step_w = tl.where((step_w == step_w) & (w + step_w >= 0.0), step_w, 0.0)
# Update the shared slope.
new_w = w + step_w
# Solve for the per-position intercept deltas, Δβₖ, in the same system.
step_bias = (g1_k - b_k * step_w) / c_k
# Damp by the position's sample count.
step_bias *= n / (n + DAMPING)
# Mask out NaNs.
step_bias = tl.where(step_bias == step_bias, step_bias, 0.0)
# Update the per-position intercepts.
new_bias = bias + step_bias
tl.store(slope_ptr, new_w)
tl.store(intercepts_ptr + k, new_bias, mask=mask)
# Start the round clean by clearing the per-round statistics.
tl.store(info_ptr + k * info_row_stride + 0, 0.0, mask=mask)
tl.store(info_ptr + k * info_row_stride + 1, 0.0, mask=mask)
tl.store(info_ptr + k * info_row_stride + 2, 0.0, mask=mask)
tl.store(grad_ptr + k * grad_row_stride + 0, 0.0, mask=mask)
tl.store(grad_ptr + k * grad_row_stride + 1, 0.0, mask=mask)
tl.store(counts_ptr + k, 0.0, mask=mask)
@triton.jit
def _local_max_sumexp_kernel(
local_max_ptr,
local_max_stride,
local_sumexp_ptr,
local_sumexp_stride,
logits_ptr,
logits_stride,
idx_mapping_ptr,
idx_mapping_stride,
temperature_ptr,
num_tokens,
vocab_size,
BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0).to(tl.int64)
if token_idx >= num_tokens:
return
block_idx = tl.program_id(1)
req_state_idx = tl.load(idx_mapping_ptr + token_idx * idx_mapping_stride).to(
tl.int64
)
if req_state_idx < 0:
# Cudagraph-padded requests carry -1. Write an empty partial so the
# buffer holds nothing uninitialized; _predict_kernel skips these rows.
tl.store(
local_max_ptr + token_idx * local_max_stride + block_idx, float("-inf")
)
tl.store(local_sumexp_ptr + token_idx * local_sumexp_stride + block_idx, 0.0)
return
# Draft logits are pre-temperature, and acceptance is decided on the scaled
# distribution the drafter actually sampled from, so scale first. A request
# with temp == 0 drafts greedily, so leave its logits alone rather than dividing
# by zero, matching what the sampler does.
temp = tl.load(temperature_ptr + req_state_idx).to(tl.float32)
temp = tl.where(temp > 0.0, temp, 1.0)
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < vocab_size
logits = (
tl.load(
logits_ptr + token_idx * logits_stride + offsets,
mask=mask,
other=float("-inf"),
).to(tl.float32)
/ temp
)
block_max, block_sumexp = _compute_max_and_sumexp(logits)
tl.store(local_max_ptr + token_idx * local_max_stride + block_idx, block_max)
tl.store(
local_sumexp_ptr + token_idx * local_sumexp_stride + block_idx, block_sumexp
)
@triton.jit
def _predict_kernel(
features_ptr,
features_stride,
pred_ptr,
pred_stride,
conf_ptr,
conf_stride,
slope_ptr,
intercepts_ptr,
local_max_ptr,
local_max_stride,
local_sumexp_ptr,
local_sumexp_stride,
idx_mapping_ptr,
idx_mapping_stride,
step_ptr,
num_tokens,
vocab_num_blocks,
per_token_step: tl.constexpr,
NUM_SPECULATIVE_STEPS: tl.constexpr,
MAX_LOG_ODDS: tl.constexpr,
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
):
token_idx = tl.program_id(0).to(tl.int64)
if token_idx >= num_tokens:
return
req_state_idx = tl.load(idx_mapping_ptr + token_idx * idx_mapping_stride).to(
tl.int64
)
if req_state_idx < 0:
# Cudagraph-padded requests carry -1. Skip them so that they don't
# scatter garbage over a live request's features.
return
# A per-row step means the speculator sampled the whole block in one pass, so
# each request owns NUM_SPECULATIVE_STEPS consecutive rows; a step shared by
# every row means one row per request. Batch position, unlike req_state_idx,
# is only meaningful for the step that computed it, and is what adaptive
# verification indexes by.
if per_token_step:
step = tl.load(step_ptr + token_idx).to(tl.int64)
batch_idx = token_idx // NUM_SPECULATIVE_STEPS
else:
step = tl.load(step_ptr).to(tl.int64)
batch_idx = token_idx
blocks = tl.arange(0, PADDED_VOCAB_NUM_BLOCKS)
blocks_mask = blocks < vocab_num_blocks
maxes = tl.load(
local_max_ptr + token_idx * local_max_stride + blocks,
mask=blocks_mask,
other=float("-inf"),
)
sumexps = tl.load(
local_sumexp_ptr + token_idx * local_sumexp_stride + blocks,
mask=blocks_mask,
other=0.0,
)
# The feature, logit(q(m*)) for the row's max draft logit, m*, is:
# q(x) = e^(x - m*) / Σ_z e^(z - m*)
# logit(q(m*)) = log(q(m*) / (1 - q(m*)))
# = m* - log(Σ_{z≠m*} e^(z - m*)).
# It scores the draft distribution, not the token drawn from it, so it is
# a function of the prefix alone. Trimming a draft therefore never
# conditions on the token being trimmed, and rejection sampling stays
# lossless. Scoring the drawn token instead would select on the proposal
# and bias the emitted distribution toward the drafter's confident modes.
# Rescale each block's partial to the global max and sum. The most likely
# token contributes exactly e^(m - m) = 1, so removing it leaves the sum
# over every other token.
global_max = tl.max(maxes, axis=0)
total_sumexp = tl.sum(
tl.where(blocks_mask, sumexps * tl.exp(maxes - global_max), 0.0), axis=0
)
complement_sumexp = tl.maximum(total_sumexp - 1.0, 0.0)
# Shifted by m*, the numerator is e^(m* - m*) = 1, so the feature is just the
# negated complement log-sum-exponential. Clamp to the min/max log-odds.
feature = -tl.log(complement_sumexp)
feature = tl.minimum(tl.maximum(feature, -MAX_LOG_ODDS), MAX_LOG_ODDS)
tl.store(features_ptr + req_state_idx * features_stride + step, feature)
# Predict the acceptance probability from the feature and current coefficients.
weight = tl.load(slope_ptr)
bias = tl.load(intercepts_ptr + step)
prob = tl.sigmoid(weight * feature + bias)
tl.store(pred_ptr + req_state_idx * pred_stride + step, prob)
tl.store(conf_ptr + batch_idx * conf_stride + step, prob)
class OnlineAcceptanceEstimator:
"""Predicts per-position acceptance, and calibrates itself while serving.
Lifecycle per step, driven by ``DraftModelSpeculator``:
1. ``step`` folds the previous step's drafts, now graded by the target,
into the IRLS accumulators, and periodically solves for new coefficients,
independently but identically on every rank.
2. ``predict`` runs inside the captured draft graph, turning this step's
draft logits into acceptance probabilities for adaptive verification.
Trimming starts immediately: survival is a running product of sigmoids, so it
decreases with draft position whatever the coefficients are, and an unfitted
estimator degrades to uniform-depth truncation rather than to anything harmful.
"""
# Refit every this many steps, accumulating samples in between.
REFIT_INTERVAL = 100
# Newton steps are damped by n / (n + DAMPING_OBSERVATIONS), so a round
# carrying this many observations moves a parameter half of a full step. It
# replaces a hard minimum-sample gate: a data-poor position keeps learning,
# just slowly, instead of freezing until it crosses a threshold. Lower values
# track a drifting workload faster, higher ones are steadier on thin data.
DAMPING_OBSERVATIONS = 50.0
# Ridge on the Newton solve.
L2 = 1e-3
def __init__(
self,
max_num_reqs: int,
num_speculative_steps: int,
device: torch.device,
):
self.num_speculative_steps = num_speculative_steps
self.device = device
self._steps_since_refit = 0
self._refits = 0
# The learned coefficients used for predicting acceptance.
self.slope = torch.full((1,), _INIT_SLOPE, dtype=torch.float32, device=device)
self.intercepts = torch.full(
(num_speculative_steps,), _INIT_BIAS, dtype=torch.float32, device=device
)
# Holds logit(max q), which is used as the feature for the logistic.
# Stored in stable slots keyed by persistent request-state index.
self.features = torch.zeros(
max_num_reqs, num_speculative_steps, dtype=torch.float32, device=device
)
# Holds the acceptance predictions made at draft time, computed as:
# p = σ(ω * f + βₖ)
# where f is feature (logit(max q)) at draft position k, ω is the learned
# shared slope, and βₖ is the learned per-position intercept. Stored in
# stable slots keyed by the persistent request-state index.
self.predictions = torch.zeros(
max_num_reqs, num_speculative_steps, dtype=torch.float32, device=device
)
# Per-round Newton-IRLS statistics, cleared after each refit. For observation
# i the design row is xᵢ = [fᵢ, eₖ], the IRLS weight is wᵢ = pᵢ(1-pᵢ), and the
# residual is yᵢ - pᵢ, where yᵢ is the label (1 for accepted, 0 for rejected).
# XᵀWX is then the negative Hessian of the Bernoulli log-likelihood loss and
# Xᵀ(Y-P) is its gradient, so the Newton step for θ = [ω, β₀ … βₙ₋₁] is the
# the solution of (XᵀWX + λI) Δθ = Xᵀ(Y-P).
#
# Columns [aₖ, bₖ, cₖ] are each position's contribution to the XᵀWX arrowhead:
# ⎡ a b₀ b₁ … bₙ₋₁ ⎤
# ⎢ b₀ c₀ ⎥
# ⎢ b₁ c₁ ⎥
# ⎢ ⋮ ⋱ ⎥
# ⎣ bₙ₋₁ cₙ₋₁ ⎦
# bₖ and cₖ are per-position, but a = Σₖ aₖ is one scalar shared by all of
# them.
self.info = torch.zeros(
num_speculative_steps, 3, dtype=torch.float32, device=device
)
# Columns [g₀ₖ, g₁ₖ] are each position's contribution to the Xᵀ(Y-P) vector:
# ⎡ g₀ ⎤
# ⎢ g₁₀ ⎥
# ⎢ g₁₁ ⎥
# ⎢ ⋮ ⎥
# ⎣ g₁ₙ₋₁ ⎦
# As above, g₁ₖ is per-position and g₀ = Σₖ g₀ₖ is shared.
self.grad = torch.zeros(
num_speculative_steps, 2, dtype=torch.float32, device=device
)
# Number of graded drafts per-position in the current round, zeroed after
# each refit. _refit_kernel uses this to damp coefficient updates by:
# n / (n + DAMPING_OBSERVATIONS)
self.counts = torch.zeros(
num_speculative_steps, dtype=torch.float32, device=device
)
def step(
self,
idx_mapping: torch.Tensor,
num_sampled: torch.Tensor,
num_rejected: torch.Tensor,
) -> None:
# Accumulate the previous step's graded drafts into the IRLS statistics.
num_reqs = idx_mapping.shape[0]
_accumulate_kernel[(self.num_speculative_steps,)](
self.info,
self.grad,
idx_mapping,
num_sampled,
num_rejected,
self.features,
self.features.stride(0),
self.predictions,
self.predictions.stride(0),
self.counts,
num_reqs,
BLOCK_R=triton.next_power_of_2(max(num_reqs, 1)),
)
self._steps_since_refit += 1
if self._steps_since_refit < self.REFIT_INTERVAL:
return
self._steps_since_refit = 0
# Fit the coefficients to the accumulated statistics gathered over the
# course of the last REFIT_INTERVAL steps.
_refit_kernel[(1,)](
self.slope,
self.intercepts,
self.info,
self.info.stride(0),
self.grad,
self.grad.stride(0),
self.counts,
NUM_SPECULATIVE_STEPS=self.num_speculative_steps,
L2=self.L2,
DAMPING=self.DAMPING_OBSERVATIONS,
BLOCK=triton.next_power_of_2(self.num_speculative_steps),
)
self._refits += 1
def predict(
self,
logits: torch.Tensor,
idx_mapping: torch.Tensor,
draft_step: torch.Tensor,
confidence_probs: torch.Tensor,
temperature: torch.Tensor,
) -> None:
num_tokens, vocab_size = logits.shape
VOCAB_BLOCK_SIZE = 4096
num_blocks = triton.cdiv(vocab_size, VOCAB_BLOCK_SIZE)
local_max = torch.empty(
num_tokens, num_blocks, dtype=torch.float32, device=self.device
)
local_sumexp = torch.empty_like(local_max)
_local_max_sumexp_kernel[(num_tokens, num_blocks)](
local_max,
local_max.stride(0),
local_sumexp,
local_sumexp.stride(0),
logits,
logits.stride(0),
idx_mapping,
idx_mapping.stride(0),
temperature,
num_tokens,
vocab_size,
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
)
_predict_kernel[(num_tokens,)](
self.features,
self.features.stride(0),
self.predictions,
self.predictions.stride(0),
confidence_probs,
confidence_probs.stride(0),
self.slope,
self.intercepts,
local_max,
local_max.stride(0),
local_sumexp,
local_sumexp.stride(0),
idx_mapping,
idx_mapping.stride(0),
draft_step,
num_tokens,
num_blocks,
per_token_step=draft_step.dim() > 0,
NUM_SPECULATIVE_STEPS=self.num_speculative_steps,
MAX_LOG_ODDS=_MAX_LOG_ODDS,
PADDED_VOCAB_NUM_BLOCKS=triton.next_power_of_2(num_blocks),
)
@@ -213,5 +213,11 @@ class DFlash2Speculator(DFlashSpeculator):
anchor_token_ids,
)
self._sample_path(candidate_ids, scores, num_reqs)
if self.enable_adaptive_verification:
self._maybe_predict_acceptance(
self._selector_scores[:num_reqs].flatten(0, 1),
self.sample_idx_mapping[:num_sample],
self.sample_col[:num_sample],
)
if self.draft_logits is not None:
self._cache_draft_logits(candidate_ids, num_sample)
@@ -79,12 +79,7 @@ class DSparkSpeculator(DFlashSpeculator):
self.draft_model_config.hf_config, "dspark_draft_topk", None
)
self.draft_token_confidence_probs = torch.empty_like(
self.draft_tokens, dtype=torch.float32
)
self.enable_adaptive_verification = (
self.speculative_config.enable_adaptive_verification
)
self.use_confidence_head: bool = False
def load_draft_model(
self,
@@ -108,13 +103,14 @@ class DSparkSpeculator(DFlashSpeculator):
dtype=self.draft_logits.dtype,
device=self.device,
)
if self.enable_adaptive_verification and model.model.confidence_head is None:
raise ValueError(
"Adaptive verification needs a DSpark checkpoint with a confidence "
"head, and this one has none. Pass "
"enable_adaptive_verification=false in the speculative config to verify"
" a fixed number of drafts instead."
)
self.use_confidence_head = (
self.enable_adaptive_verification
and model.model.confidence_head is not None
)
if self.use_confidence_head:
# The acceptance estimator is not needed when a trained confidence head
# is available.
self.use_acceptance_estimator = False
return model
def _sample_logits(
@@ -124,8 +120,10 @@ class DSparkSpeculator(DFlashSpeculator):
sample_pos: torch.Tensor,
step: int,
) -> torch.Tensor:
self._maybe_predict_acceptance(logits, idx_map, self._step_cols[step])
if self.draft_logits is None:
return self.model.map_draft_to_target(logits.argmax(dim=-1))
draft_ids = logits.argmax(dim=-1)
return self.model.map_draft_to_target(draft_ids)
# Probabilistic sampling and rejection operate in target-vocabulary
# space. A reduced draft vocabulary is scattered into its target rows.
@@ -184,7 +182,7 @@ class DSparkSpeculator(DFlashSpeculator):
for i in range(n_spec):
# Sequential stage: Markov bias from the previously sampled token.
markov_embed = self.model.markov_embed(prev)
if self.enable_adaptive_verification:
if self.use_confidence_head:
confidence_markov_embeds.append(markov_embed)
bias = self.model.markov_bias(markov_embed)
logits_i = base_logits[:, i] + bias
@@ -194,7 +192,7 @@ class DSparkSpeculator(DFlashSpeculator):
self.draft_tokens[:num_reqs, i] = draft_sampled_i
prev = draft_sampled_i
if self.enable_adaptive_verification:
if self.use_confidence_head:
confidence = self.model.compute_confidence(
sample_hidden,
torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1),
@@ -229,7 +227,7 @@ class DSparkSpeculator(DFlashSpeculator):
for i in range(n_spec):
markov_embed = self.model.markov_embed(prev)
if self.enable_adaptive_verification:
if self.use_confidence_head:
confidence_markov_embeds.append(markov_embed)
logits_i = self.model.apply_markov_bias_gathered(
markov_embed,
@@ -243,7 +241,7 @@ class DSparkSpeculator(DFlashSpeculator):
self.draft_tokens[:num_reqs, i] = draft_sampled_i
prev = draft_sampled_i
if self.enable_adaptive_verification:
if self.use_confidence_head:
confidence = self.model.compute_confidence(
sample_hidden,
torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1),
+62 -8
View File
@@ -33,6 +33,9 @@ from vllm.v1.worker.gpu.dp_utils import DPSyncState
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import (
OnlineAcceptanceEstimator,
)
from vllm.v1.worker.utils import AttentionGroup
if TYPE_CHECKING:
@@ -158,6 +161,15 @@ class DraftModelSpeculator(BaseSpeculator):
self.arange_np = np.arange(self.max_num_reqs + 1, dtype=np.int32)
self.draft_is_prefilling = torch.zeros(self.max_num_reqs, dtype=torch.bool)
self.draft_token_confidence_probs = torch.empty_like(
self.draft_tokens, dtype=torch.float32
)
self.enable_adaptive_verification = (
self.speculative_config.enable_adaptive_verification
)
self.use_acceptance_estimator = self.enable_adaptive_verification
self.acceptance_estimator: OnlineAcceptanceEstimator | None = None
self.draft_logits: torch.Tensor | None = None
if self.speculative_config.draft_sample_method == "probabilistic":
# Pre-temperature logits, cached from the previous decode step.
@@ -227,6 +239,20 @@ class DraftModelSpeculator(BaseSpeculator):
type(self.model).__name__,
)
if self.use_acceptance_estimator:
if self.use_local_argmax_reduction:
raise ValueError(
"Adaptive verification without a confidence head estimates "
"per-position acceptance from the draft logits, which "
"use_local_argmax_reduction never materializes. Disable one "
"of them."
)
self.acceptance_estimator = OnlineAcceptanceEstimator(
self.max_num_reqs,
self.num_speculative_steps,
self.device,
)
def set_eplb_state(self, eplb_state: EplbState) -> None:
"""Inject EPLB state after construction."""
self.eplb_state = eplb_state
@@ -371,12 +397,6 @@ class DraftModelSpeculator(BaseSpeculator):
"(communication: O(2*tp_size) vs O(vocab_size))."
)
def _greedy_sample_draft(self, hidden_states: torch.Tensor) -> torch.Tensor:
if self.use_local_argmax_reduction:
return self.model.get_top_tokens(hidden_states)
logits = self.model.compute_logits(hidden_states)
return logits.argmax(dim=-1)
def sample_draft(
self,
hidden_states: torch.Tensor,
@@ -408,8 +428,42 @@ class DraftModelSpeculator(BaseSpeculator):
idx_mapping,
temperature,
)
return sampled
return self._greedy_sample_draft(hidden_states)
elif self.use_local_argmax_reduction:
return self.model.get_top_tokens(hidden_states)
else:
logits = self.model.compute_logits(hidden_states)
sampled = logits.argmax(dim=-1)
self._maybe_predict_acceptance(logits, idx_mapping, draft_step)
return sampled
def _maybe_predict_acceptance(
self,
logits: torch.Tensor,
idx_mapping: torch.Tensor,
draft_step: torch.Tensor,
) -> None:
if self.acceptance_estimator is not None:
self.acceptance_estimator.predict(
logits,
idx_mapping,
draft_step,
self.draft_token_confidence_probs,
self.temperature,
)
def observe_verification(
self,
idx_mapping: torch.Tensor,
num_sampled: torch.Tensor,
num_rejected: torch.Tensor,
) -> None:
"""Fold the target's verdict on the last drafts into the estimator.
Must run before the next `propose`, which overwrites the per-slot
features the verdict grades.
"""
if self.acceptance_estimator is not None:
self.acceptance_estimator.step(idx_mapping, num_sampled, num_rejected)
def prepare_watermarking(
self, contexts: torch.Tensor, watermarking: torch.Tensor