From 3bb0a03f35269185b6358753e64932bb946cc531 Mon Sep 17 00:00:00 2001 From: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Date: Mon, 14 Sep 2026 15:58:48 -0700 Subject: [PATCH] [Model Runner V2] Acceptance estimation for adaptive verification (#52228) Signed-off-by: Giancarlo Delfin --- .../spec_decode/test_acceptance_estimator.py | 294 ++++++++++ .../test_gpu_autoregressive_speculator.py | 5 + ...st_gpu_extract_hidden_states_speculator.py | 2 + vllm/config/speculative.py | 12 +- vllm/v1/worker/gpu/model_runner.py | 4 + .../gpu/spec_decode/acceptance_estimator.py | 501 ++++++++++++++++++ .../gpu/spec_decode/dflash2/speculator.py | 6 + .../gpu/spec_decode/dspark/speculator.py | 34 +- vllm/v1/worker/gpu/spec_decode/speculator.py | 70 ++- 9 files changed, 900 insertions(+), 28 deletions(-) create mode 100644 tests/v1/spec_decode/test_acceptance_estimator.py create mode 100644 vllm/v1/worker/gpu/spec_decode/acceptance_estimator.py diff --git a/tests/v1/spec_decode/test_acceptance_estimator.py b/tests/v1/spec_decode/test_acceptance_estimator.py new file mode 100644 index 0000000000..a1f3da0940 --- /dev/null +++ b/tests/v1/spec_decode/test_acceptance_estimator.py @@ -0,0 +1,294 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""``predict`` writes one probability into two index spaces at once. + +Adaptive verification reads the estimate in this step's batch order, while the +next step's refit reads it in persistent request-state slots. A mix-up between +the two is silent -- every value stays a plausible probability, it just belongs +to another request -- so it is asserted here rather than left to acceptance +metrics to hint at. +""" + +import pytest +import torch + +from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import ( + OnlineAcceptanceEstimator, +) + +pytest.importorskip("triton") +if not torch.cuda.is_available(): + pytest.skip("CUDA required for acceptance estimator tests", allow_module_level=True) + +MAX_NUM_REQS = 16 +NUM_STEPS = 4 +VOCAB_SIZE = 2048 + + +def _make_estimator(device: torch.device) -> OnlineAcceptanceEstimator: + estimator = OnlineAcceptanceEstimator(MAX_NUM_REQS, NUM_STEPS, device) + # Distinct per-position intercepts, so reading the wrong one shows up. + estimator.slope.fill_(0.7) + estimator.intercepts[:] = torch.tensor([1.5, 1.0, 0.5, 0.0], device=device) + return estimator + + +def _expected_probs( + estimator: OnlineAcceptanceEstimator, + logits: torch.Tensor, + steps: torch.Tensor, + temperature: torch.Tensor, +) -> torch.Tensor: + """logit(q_max) by a different route than the kernel takes. + + The kernel shifts by the max and drops the unit term; here the argmax is + masked out and the complement is reduced with logsumexp, so agreement is + evidence about the value rather than about a shared formula. + """ + temp = torch.where(temperature > 0, temperature, torch.ones_like(temperature)) + scaled = logits.double() / temp[:, None].double() + top = scaled.max(dim=-1) + rest = scaled.scatter(1, top.indices[:, None], float("-inf")) + feature = (top.values - torch.logsumexp(rest, dim=-1)).clamp(-40.0, 40.0) + weight, bias = estimator.slope, estimator.intercepts + return torch.sigmoid(weight.double() * feature + bias[steps].double()).float() + + +@pytest.mark.parametrize("tokens_per_req", [1, NUM_STEPS]) +def test_predict_agrees_across_slot_and_batch_order(tokens_per_req: int): + """One row per request (per-step drafting) and a whole block at once. + + Which one it is is inferred from whether the draft step is per-row, so the + two cases differ only in how ``draft_step`` is shaped. + """ + device = torch.device("cuda") + torch.manual_seed(0) + estimator = _make_estimator(device) + num_reqs = 5 + + # Batch position i holds a shuffled persistent slot, as a served batch would. + slots = torch.randperm(MAX_NUM_REQS, device=device)[:num_reqs].to(torch.int32) + if tokens_per_req == 1: + idx_mapping = slots + draft_step = torch.tensor(2, device=device, dtype=torch.int32) + steps = draft_step.expand(num_reqs) + else: + idx_mapping = slots.repeat_interleave(NUM_STEPS) + draft_step = torch.arange(NUM_STEPS, device=device, dtype=torch.int32).repeat( + num_reqs + ) + steps = draft_step + batch_rows = torch.arange(num_reqs, device=device).repeat_interleave(tokens_per_req) + logits = torch.randn(idx_mapping.shape[0], VOCAB_SIZE, device=device) * 4.0 + + confidence_probs = torch.zeros(MAX_NUM_REQS, NUM_STEPS, device=device) + temperature = torch.rand(MAX_NUM_REQS, device=device) + 0.5 + estimator.predict(logits, idx_mapping, draft_step, confidence_probs, temperature) + + expected = _expected_probs( + estimator, logits, steps.long(), temperature[idx_mapping.long()] + ) + torch.testing.assert_close( + estimator.predictions[idx_mapping.long(), steps.long()], + expected, + atol=2e-3, + rtol=2e-3, + ) + torch.testing.assert_close( + confidence_probs[batch_rows, steps.long()], expected, atol=2e-3, rtol=2e-3 + ) + + +def test_predict_skips_cudagraph_padded_rows(): + """Padded rows carry slot -1 and must not land on a live request.""" + device = torch.device("cuda") + torch.manual_seed(0) + estimator = _make_estimator(device) + num_reqs, num_padded = 3, 6 + + slots = torch.tensor([7, 2, 11], device=device, dtype=torch.int32) + idx_mapping = torch.cat( + [slots, torch.full((num_padded,), -1, device=device, dtype=torch.int32)] + ) + draft_step = torch.tensor(1, device=device, dtype=torch.int32) + logits = torch.randn(num_reqs + num_padded, VOCAB_SIZE, device=device) * 4.0 + + confidence_probs = torch.zeros(MAX_NUM_REQS, NUM_STEPS, device=device) + temperature = torch.rand(MAX_NUM_REQS, device=device) + 0.5 + estimator.predict(logits, idx_mapping, draft_step, confidence_probs, temperature) + + live = (slots.long(), torch.full((num_reqs,), 1, device=device)) + expected = _expected_probs( + estimator, logits[:num_reqs], live[1], temperature[slots.long()] + ) + torch.testing.assert_close( + estimator.predictions[live], expected, atol=2e-3, rtol=2e-3 + ) + # Nothing outside the live (request, position) cells was touched. + untouched = torch.ones(MAX_NUM_REQS, NUM_STEPS, dtype=torch.bool, device=device) + untouched[live] = False + assert not estimator.predictions[untouched].any() + assert not confidence_probs[num_reqs:].any() + + +# A refit takes one undamped Newton step, so it needs a realistic number of +# observations per round to be well conditioned -- a handful of requests can +# send the step far past the optimum. This is the batch a served step sees. +REFIT_NUM_REQS = 512 + + +def _drive_round(estimator, features, num_accepted, slots): + """Feed one step's graded drafts through predict's buffers into `step`.""" + estimator.features[slots.long()] = features + estimator.predictions[slots.long()] = torch.sigmoid( + estimator.slope * features + estimator.intercepts + ) + estimator.step( + slots, + (num_accepted + 1).to(torch.int32), + (features.shape[1] - num_accepted).to(torch.int32), + ) + + +def test_refit_recovers_a_shared_slope_and_per_position_intercepts(): + """The fit must find the logistic that generated the observations. + + One slope is shared across draft positions and each keeps its own intercept, + so the arrowhead solve is only correct if it recovers both from data whose + positions differ in level but not in slope. + """ + device = torch.device("cuda") + torch.manual_seed(0) + true_slope = 0.6 + true_bias = torch.tensor([1.4, 0.9, 0.4, -0.1], device=device) + + estimator = OnlineAcceptanceEstimator(REFIT_NUM_REQS, NUM_STEPS, device) + estimator.REFIT_INTERVAL = 5 + slots = torch.arange(REFIT_NUM_REQS, device=device, dtype=torch.int32) + for _ in range(60): + features = torch.rand(REFIT_NUM_REQS, NUM_STEPS, device=device) * 8.0 + accepted = torch.rand_like(features) < torch.sigmoid( + true_slope * features + true_bias + ) + # Verification stops at the first rejection, so a position's label is + # conditional on every shallower one having been accepted. + num_accepted = (~accepted).float().argmax(dim=1) + num_accepted[accepted.all(dim=1)] = NUM_STEPS + _drive_round(estimator, features, num_accepted, slots) + + slope, bias = estimator.slope, estimator.intercepts + torch.testing.assert_close( + slope[0], torch.tensor(true_slope, device=device), atol=0.1, rtol=0 + ) + torch.testing.assert_close(bias, true_bias, atol=0.3, rtol=0) + + +def test_refit_survives_a_position_with_no_observations(): + """A draft position nobody reached must not stall the shared slope. + + Its statistics stay zero while the others accumulate, which is the degenerate + case the arrowhead denominator has to tolerate. + """ + device = torch.device("cuda") + torch.manual_seed(0) + estimator = OnlineAcceptanceEstimator(REFIT_NUM_REQS, NUM_STEPS, device) + estimator.REFIT_INTERVAL = 5 + slots = torch.arange(REFIT_NUM_REQS, device=device, dtype=torch.int32) + before_slope = estimator.slope.clone() + before_intercepts = estimator.intercepts.clone() + + # Every request is rejected at position 1, so positions 2+ are never graded. + num_accepted = torch.ones(REFIT_NUM_REQS, device=device) + for _ in range(20): + features = torch.rand(REFIT_NUM_REQS, NUM_STEPS, device=device) * 8.0 + _drive_round(estimator, features, num_accepted, slots) + + assert estimator.slope.isfinite().all() + assert estimator.intercepts.isfinite().all() + assert not torch.equal(estimator.slope, before_slope), "slope never moved" + # Positions 2 and 3 were never observed, so their intercepts must be untouched. + torch.testing.assert_close( + estimator.intercepts[2:], before_intercepts[2:], rtol=0, atol=0 + ) + + +def test_step_damping_scales_with_the_evidence(): + """A round moves a coefficient in proportion to the samples behind it. + + The Newton step itself barely changes with how much data a round carries -- + curvature and score both scale with it -- so scaling a round's statistics + isolates the damping factor n / (n + DAMPING_OBSERVATIONS). The slope/ + intercept coupling is zeroed so the two parameters can be checked apart. + """ + device = torch.device("cuda") + damping = OnlineAcceptanceEstimator.DAMPING_OBSERVATIONS + per_position_n = 20.0 + + def refit_with(scale): + estimator = OnlineAcceptanceEstimator(MAX_NUM_REQS, NUM_STEPS, device) + estimator.info[:, 1] = 0.0 # sum(w*margin): no slope/intercept coupling + estimator.info[:, 2] = 1.0 * scale # sum(w) + estimator.grad[:, 1] = 0.5 * scale # sum(y - p) + # The slope's information and score are accumulated as per-position + # partials that the refit sums, so spread each total over the positions. + estimator.info[:, 0] = 30.0 * scale / NUM_STEPS # sum(w*margin^2) + estimator.grad[:, 0] = 2.0 * scale / NUM_STEPS # sum((y - p)*margin) + estimator.counts[:] = per_position_n * scale + estimator._steps_since_refit = estimator.REFIT_INTERVAL - 1 + before_slope = estimator.slope.clone() + before_intercepts = estimator.intercepts.clone() + empty = torch.zeros(0, dtype=torch.int32, device=device) + estimator.step(empty, empty, empty) + return ( + (estimator.slope - before_slope)[0].item(), + (estimator.intercepts - before_intercepts)[0].item(), + ) + + for scale in (1.0, 10.0, 100.0): + total_n = per_position_n * scale * NUM_STEPS + slope_delta, bias_delta = refit_with(scale) + assert slope_delta == pytest.approx( + (2.0 / 30.0) * total_n / (total_n + damping), rel=1e-3 + ) + assert bias_delta == pytest.approx( + 0.5 * (per_position_n * scale) / (per_position_n * scale + damping), + rel=1e-3, + ) + + # A round with no observations at all must leave the coefficients alone. + assert refit_with(0.0) == (0.0, 0.0) + + +def test_feature_ignores_which_token_was_drawn(): + """The score must depend on the draft distribution, not on the draw. + + This is what keeps rejection sampling lossless: if the score read the drawn + token, trimming a draft would condition on the token being trimmed and tilt + the emitted distribution toward the drafter's confident modes. ``predict`` + takes no token argument, and this pins that -- two rows with identical + logits must score identically no matter what was sampled from them. + """ + device = torch.device("cuda") + torch.manual_seed(0) + estimator = _make_estimator(device) + + row = torch.randn(VOCAB_SIZE, device=device) * 4.0 + logits = row.expand(2, VOCAB_SIZE).contiguous() + idx_mapping = torch.tensor([0, 1], device=device, dtype=torch.int32) + draft_step = torch.tensor(0, device=device, dtype=torch.int32) + temperature = torch.ones(MAX_NUM_REQS, device=device) + confidence_probs = torch.zeros(MAX_NUM_REQS, NUM_STEPS, device=device) + + estimator.predict(logits, idx_mapping, draft_step, confidence_probs, temperature) + assert estimator.features[0, 0] == estimator.features[1, 0] + + # And the value is the top token's log-odds, not the top-2 margin. + scaled = row.double() + top = scaled.max(dim=-1) + rest = scaled.scatter(0, top.indices[None], float("-inf")) + torch.testing.assert_close( + estimator.features[0, 0].double(), + (top.values - torch.logsumexp(rest, dim=-1)).clamp(-40.0, 40.0), + atol=2e-3, + rtol=2e-3, + ) diff --git a/tests/v1/worker/test_gpu_autoregressive_speculator.py b/tests/v1/worker/test_gpu_autoregressive_speculator.py index 920c9a25f0..b9b3570b8a 100644 --- a/tests/v1/worker/test_gpu_autoregressive_speculator.py +++ b/tests/v1/worker/test_gpu_autoregressive_speculator.py @@ -123,6 +123,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected) draft_model_config=draft_model_config, use_local_argmax_reduction=False, draft_sample_method="greedy", + enable_adaptive_verification=False, ) vllm_config = SimpleNamespace( speculative_config=speculative_config, @@ -161,6 +162,7 @@ def test_mm_support_configured_after_model_load(monkeypatch): speculator.dtype = torch.float32 speculator.draft_model_config = draft_model_config speculator.supports_mm_inputs = False + speculator.use_acceptance_estimator = False checked_configs = [] @@ -195,6 +197,7 @@ def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch): speculator = object.__new__(_TestSpeculator) speculator.supports_mm_inputs = False speculator.inputs_embeds = None + speculator.use_acceptance_estimator = False speculator.vllm_config = SimpleNamespace(model_config=object()) speculator.max_num_tokens = 4 speculator.hidden_size = 3 @@ -219,6 +222,7 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch): speculator = object.__new__(_TestSpeculator) speculator.supports_mm_inputs = False speculator.inputs_embeds = None + speculator.use_acceptance_estimator = False speculator.vllm_config = SimpleNamespace(model_config=object()) draft_model = _TextOnlyDraftModel() speculator.test_draft_model = draft_model @@ -251,6 +255,7 @@ def test_multi_module_mm_support_configured_after_model_load(monkeypatch): speculator = object.__new__(MultiModuleMTPSpeculator) speculator.supports_mm_inputs = False speculator.inputs_embeds = None + speculator.use_acceptance_estimator = False speculator.cached_draft_input_embeds = None speculator.vllm_config = SimpleNamespace(model_config=object()) speculator.max_num_tokens = 4 diff --git a/tests/v1/worker/test_gpu_extract_hidden_states_speculator.py b/tests/v1/worker/test_gpu_extract_hidden_states_speculator.py index d1aa0798fe..74464f542f 100644 --- a/tests/v1/worker/test_gpu_extract_hidden_states_speculator.py +++ b/tests/v1/worker/test_gpu_extract_hidden_states_speculator.py @@ -60,6 +60,7 @@ def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch): layer_name = "cache_only_layers.2" speculator = object.__new__(ExtractHiddenStatesSpeculator) speculator.vllm_config = cast(Any, SimpleNamespace()) + speculator.acceptance_estimator = None speculator.num_hidden_states = 2 speculator.hidden_states = torch.zeros(4, 2, 3) speculator.draft_attn_layer_names = {layer_name} @@ -111,6 +112,7 @@ def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch): def test_propose_requires_aux_hidden_states(): speculator = object.__new__(ExtractHiddenStatesSpeculator) + speculator.acceptance_estimator = None speculator.num_hidden_states = 2 input_batch = cast( Any, SimpleNamespace(idx_mapping=torch.tensor([0], dtype=torch.int32)) diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 1861266820..2396b26cb0 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -1561,8 +1561,16 @@ class SpeculativeConfig: self.index_share_for_mtp_iteration ) - if self.method != "dspark" and self.enable_adaptive_verification: - raise ValueError("Adaptive verification only supported with DSpark") + if ( + self.method != "dspark" + and self.enable_adaptive_verification + and self.use_local_argmax_reduction + ): + raise ValueError( + "Adaptive verification estimates per-position acceptance from " + "the draft logits, which use_local_argmax_reduction never " + "materializes. Disable one of them." + ) return self diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 4b1520f276..761893303a 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -2120,6 +2120,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): if self.speculator is not None: assert self.sampler is not None + if isinstance(self.speculator, DraftModelSpeculator): + self.speculator.observe_verification( + input_batch.idx_mapping, num_sampled, num_rejected + ) # Let the target override the hidden state fed to the drafter # (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The # target returns a persistent buffer sized at max_num_batched_tokens; diff --git a/vllm/v1/worker/gpu/spec_decode/acceptance_estimator.py b/vllm/v1/worker/gpu/spec_decode/acceptance_estimator.py new file mode 100644 index 0000000000..46065d0546 --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/acceptance_estimator.py @@ -0,0 +1,501 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.triton_utils import tl, triton +from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import ( + _compute_max_and_sumexp, +) + +# The feature is a log-odds, log(q / (1 - q)), which a row carrying all of its +# mass on one token sends to infinity. Clamp it symmetrically. +_MAX_LOG_ODDS = 40.0 + +# Cold-start coefficients: the median slope and intercept fitted online across +# DeepSeek-V4-Flash, Muse-Glimmer-30B, MiMo-V2.5-Pro, Inkling and Kimi-K2.5. +# Log-loss is flat near the optimum, so one starting point serves all of them +# and a real fit replaces it within a refit or two. +_INIT_SLOPE = 0.5 +_INIT_BIAS = -0.2 + + +@triton.jit +def _accumulate_kernel( + info_ptr, + grad_ptr, + idx_mapping_ptr, + num_sampled_ptr, + num_rejected_ptr, + features_ptr, + features_stride, + pred_ptr, + pred_stride, + counts_ptr, + num_reqs, + BLOCK_R: tl.constexpr, +): + step = tl.program_id(0).to(tl.int64) + req_block = tl.arange(0, BLOCK_R) + req_mask = req_block < num_reqs + req_state_block = tl.load(idx_mapping_ptr + req_block, mask=req_mask, other=0).to( + tl.int64 + ) + + num_sampled = tl.load(num_sampled_ptr + req_block, mask=req_mask, other=0).to( + tl.int64 + ) + # Subtract bonus token. + num_accepted = tl.maximum(num_sampled - 1, 0) + num_rejected = tl.load(num_rejected_ptr + req_block, mask=req_mask, other=0).to( + tl.int64 + ) + num_admitted = num_accepted + num_rejected + + feature = tl.load( + features_ptr + req_state_block * features_stride + step, + mask=req_mask, + other=0.0, + ).to(tl.float32) + pred = tl.load( + pred_ptr + req_state_block * pred_stride + step, + mask=req_mask, + other=0.0, + ).to(tl.float32) + + # The accepted tokens and the first rejected draft token (if one exists) are + # "observed" and given labels. + observed = req_mask & (step <= num_accepted) & (step < num_admitted) + label = tl.where(step < num_accepted, 1.0, 0.0) + w = tl.where(observed, pred * (1.0 - pred), 0.0) + resid = tl.where(observed, label - pred, 0.0) + + # This block's contribution to the normal equations for design row + # x = [feature, eₖ]: the XᵀWX entries aₖ, bₖ, cₖ and the Xᵀr entries g₀ₖ, g₁ₖ. + # aₖ and g₀ₖ are partials of scalars shared by every position, which + # _refit_kernel sums. + a_k = tl.sum(w * feature * feature, axis=0) + b_k = tl.sum(w * feature, axis=0) + c_k = tl.sum(w, axis=0) + g0_k = tl.sum(resid * feature, axis=0) + g1_k = tl.sum(resid, axis=0) + count = tl.sum(tl.where(observed, 1.0, 0.0), axis=0) + tl.store(info_ptr + step * 3 + 0, tl.load(info_ptr + step * 3 + 0) + a_k) + tl.store(info_ptr + step * 3 + 1, tl.load(info_ptr + step * 3 + 1) + b_k) + tl.store(info_ptr + step * 3 + 2, tl.load(info_ptr + step * 3 + 2) + c_k) + tl.store(grad_ptr + step * 2 + 0, tl.load(grad_ptr + step * 2 + 0) + g0_k) + tl.store(grad_ptr + step * 2 + 1, tl.load(grad_ptr + step * 2 + 1) + g1_k) + tl.store(counts_ptr + step, tl.load(counts_ptr + step) + count) + + +@triton.jit +def _refit_kernel( + slope_ptr, + intercepts_ptr, + info_ptr, + info_row_stride, + grad_ptr, + grad_row_stride, + counts_ptr, + NUM_SPECULATIVE_STEPS: tl.constexpr, + L2: tl.constexpr, + DAMPING: tl.constexpr, + BLOCK: tl.constexpr, +): + k = tl.arange(0, BLOCK) + mask = k < NUM_SPECULATIVE_STEPS + + # Load the stats accumulated over this round to solve (XᵀWX + λI) Δθ = Xᵀr. + a_k = tl.load(info_ptr + k * info_row_stride + 0, mask=mask, other=0.0) + b_k = tl.load(info_ptr + k * info_row_stride + 1, mask=mask, other=0.0) + c_k = tl.load(info_ptr + k * info_row_stride + 2, mask=mask, other=0.0) + L2 + a = tl.sum(a_k, axis=0) + L2 + g0_k = tl.load(grad_ptr + k * grad_row_stride + 0, mask=mask, other=0.0) + g1_k = tl.load(grad_ptr + k * grad_row_stride + 1, mask=mask, other=0.0) + g0 = tl.sum(g0_k, axis=0) + n = tl.load(counts_ptr + k, mask=mask, other=0.0) + + # Load the current shared slope and per-position intercepts. + w = tl.load(slope_ptr) + bias = tl.load(intercepts_ptr + k, mask=mask, other=0.0) + + # Solve for the shared slope delta, Δω, in the following arrowhead system: + # ⎡ a+λ b₀ b₁ … bₙ₋₁ ⎤ ⎡ Δω ⎤ ⎡ g₀ ⎤ + # ⎢ b₀ c₀+λ ⎥ ⎢ Δβ₀ ⎥ ⎢ g₁₀ ⎥ + # ⎢ b₁ c₁+λ ⎥ ⎢ Δβ₁ ⎥ = ⎢ g₁₁ ⎥ + # ⎢ ⋮ ⋱ ⎥ ⎢ ⋮ ⎥ ⎢ ⋮ ⎥ + # ⎣ bₙ₋₁ cₙ₋₁+λ ⎦ ⎣ Δβₙ₋₁ ⎦ ⎣ g₁ₙ₋₁ ⎦ + # XᵀWX + λI Δθ Xᵀ(Y-P) + shrink = b_k * b_k / c_k + coupling = b_k * g1_k / c_k + step_w = (g0 - tl.sum(coupling, axis=0)) / (a - tl.sum(shrink, axis=0)) + + # The slope learns from every position, so it is damped by the round's + # total sample count. + total_n = tl.sum(n, axis=0) + step_w *= total_n / (total_n + DAMPING) + # Mask out NaNs and steps that would drive the slope negative. + step_w = tl.where((step_w == step_w) & (w + step_w >= 0.0), step_w, 0.0) + # Update the shared slope. + new_w = w + step_w + + # Solve for the per-position intercept deltas, Δβₖ, in the same system. + step_bias = (g1_k - b_k * step_w) / c_k + # Damp by the position's sample count. + step_bias *= n / (n + DAMPING) + # Mask out NaNs. + step_bias = tl.where(step_bias == step_bias, step_bias, 0.0) + # Update the per-position intercepts. + new_bias = bias + step_bias + + tl.store(slope_ptr, new_w) + tl.store(intercepts_ptr + k, new_bias, mask=mask) + + # Start the round clean by clearing the per-round statistics. + tl.store(info_ptr + k * info_row_stride + 0, 0.0, mask=mask) + tl.store(info_ptr + k * info_row_stride + 1, 0.0, mask=mask) + tl.store(info_ptr + k * info_row_stride + 2, 0.0, mask=mask) + tl.store(grad_ptr + k * grad_row_stride + 0, 0.0, mask=mask) + tl.store(grad_ptr + k * grad_row_stride + 1, 0.0, mask=mask) + tl.store(counts_ptr + k, 0.0, mask=mask) + + +@triton.jit +def _local_max_sumexp_kernel( + local_max_ptr, + local_max_stride, + local_sumexp_ptr, + local_sumexp_stride, + logits_ptr, + logits_stride, + idx_mapping_ptr, + idx_mapping_stride, + temperature_ptr, + num_tokens, + vocab_size, + BLOCK_SIZE: tl.constexpr, +): + token_idx = tl.program_id(0).to(tl.int64) + if token_idx >= num_tokens: + return + + block_idx = tl.program_id(1) + req_state_idx = tl.load(idx_mapping_ptr + token_idx * idx_mapping_stride).to( + tl.int64 + ) + if req_state_idx < 0: + # Cudagraph-padded requests carry -1. Write an empty partial so the + # buffer holds nothing uninitialized; _predict_kernel skips these rows. + tl.store( + local_max_ptr + token_idx * local_max_stride + block_idx, float("-inf") + ) + tl.store(local_sumexp_ptr + token_idx * local_sumexp_stride + block_idx, 0.0) + return + + # Draft logits are pre-temperature, and acceptance is decided on the scaled + # distribution the drafter actually sampled from, so scale first. A request + # with temp == 0 drafts greedily, so leave its logits alone rather than dividing + # by zero, matching what the sampler does. + temp = tl.load(temperature_ptr + req_state_idx).to(tl.float32) + temp = tl.where(temp > 0.0, temp, 1.0) + + offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = offsets < vocab_size + logits = ( + tl.load( + logits_ptr + token_idx * logits_stride + offsets, + mask=mask, + other=float("-inf"), + ).to(tl.float32) + / temp + ) + block_max, block_sumexp = _compute_max_and_sumexp(logits) + tl.store(local_max_ptr + token_idx * local_max_stride + block_idx, block_max) + tl.store( + local_sumexp_ptr + token_idx * local_sumexp_stride + block_idx, block_sumexp + ) + + +@triton.jit +def _predict_kernel( + features_ptr, + features_stride, + pred_ptr, + pred_stride, + conf_ptr, + conf_stride, + slope_ptr, + intercepts_ptr, + local_max_ptr, + local_max_stride, + local_sumexp_ptr, + local_sumexp_stride, + idx_mapping_ptr, + idx_mapping_stride, + step_ptr, + num_tokens, + vocab_num_blocks, + per_token_step: tl.constexpr, + NUM_SPECULATIVE_STEPS: tl.constexpr, + MAX_LOG_ODDS: tl.constexpr, + PADDED_VOCAB_NUM_BLOCKS: tl.constexpr, +): + token_idx = tl.program_id(0).to(tl.int64) + if token_idx >= num_tokens: + return + + req_state_idx = tl.load(idx_mapping_ptr + token_idx * idx_mapping_stride).to( + tl.int64 + ) + if req_state_idx < 0: + # Cudagraph-padded requests carry -1. Skip them so that they don't + # scatter garbage over a live request's features. + return + + # A per-row step means the speculator sampled the whole block in one pass, so + # each request owns NUM_SPECULATIVE_STEPS consecutive rows; a step shared by + # every row means one row per request. Batch position, unlike req_state_idx, + # is only meaningful for the step that computed it, and is what adaptive + # verification indexes by. + if per_token_step: + step = tl.load(step_ptr + token_idx).to(tl.int64) + batch_idx = token_idx // NUM_SPECULATIVE_STEPS + else: + step = tl.load(step_ptr).to(tl.int64) + batch_idx = token_idx + + blocks = tl.arange(0, PADDED_VOCAB_NUM_BLOCKS) + blocks_mask = blocks < vocab_num_blocks + maxes = tl.load( + local_max_ptr + token_idx * local_max_stride + blocks, + mask=blocks_mask, + other=float("-inf"), + ) + sumexps = tl.load( + local_sumexp_ptr + token_idx * local_sumexp_stride + blocks, + mask=blocks_mask, + other=0.0, + ) + + # The feature, logit(q(m*)) for the row's max draft logit, m*, is: + # q(x) = e^(x - m*) / Σ_z e^(z - m*) + # logit(q(m*)) = log(q(m*) / (1 - q(m*))) + # = m* - log(Σ_{z≠m*} e^(z - m*)). + # It scores the draft distribution, not the token drawn from it, so it is + # a function of the prefix alone. Trimming a draft therefore never + # conditions on the token being trimmed, and rejection sampling stays + # lossless. Scoring the drawn token instead would select on the proposal + # and bias the emitted distribution toward the drafter's confident modes. + + # Rescale each block's partial to the global max and sum. The most likely + # token contributes exactly e^(m - m) = 1, so removing it leaves the sum + # over every other token. + global_max = tl.max(maxes, axis=0) + total_sumexp = tl.sum( + tl.where(blocks_mask, sumexps * tl.exp(maxes - global_max), 0.0), axis=0 + ) + complement_sumexp = tl.maximum(total_sumexp - 1.0, 0.0) + + # Shifted by m*, the numerator is e^(m* - m*) = 1, so the feature is just the + # negated complement log-sum-exponential. Clamp to the min/max log-odds. + feature = -tl.log(complement_sumexp) + feature = tl.minimum(tl.maximum(feature, -MAX_LOG_ODDS), MAX_LOG_ODDS) + tl.store(features_ptr + req_state_idx * features_stride + step, feature) + + # Predict the acceptance probability from the feature and current coefficients. + weight = tl.load(slope_ptr) + bias = tl.load(intercepts_ptr + step) + prob = tl.sigmoid(weight * feature + bias) + tl.store(pred_ptr + req_state_idx * pred_stride + step, prob) + tl.store(conf_ptr + batch_idx * conf_stride + step, prob) + + +class OnlineAcceptanceEstimator: + """Predicts per-position acceptance, and calibrates itself while serving. + + Lifecycle per step, driven by ``DraftModelSpeculator``: + + 1. ``step`` folds the previous step's drafts, now graded by the target, + into the IRLS accumulators, and periodically solves for new coefficients, + independently but identically on every rank. + 2. ``predict`` runs inside the captured draft graph, turning this step's + draft logits into acceptance probabilities for adaptive verification. + + Trimming starts immediately: survival is a running product of sigmoids, so it + decreases with draft position whatever the coefficients are, and an unfitted + estimator degrades to uniform-depth truncation rather than to anything harmful. + """ + + # Refit every this many steps, accumulating samples in between. + REFIT_INTERVAL = 100 + # Newton steps are damped by n / (n + DAMPING_OBSERVATIONS), so a round + # carrying this many observations moves a parameter half of a full step. It + # replaces a hard minimum-sample gate: a data-poor position keeps learning, + # just slowly, instead of freezing until it crosses a threshold. Lower values + # track a drifting workload faster, higher ones are steadier on thin data. + DAMPING_OBSERVATIONS = 50.0 + # Ridge on the Newton solve. + L2 = 1e-3 + + def __init__( + self, + max_num_reqs: int, + num_speculative_steps: int, + device: torch.device, + ): + self.num_speculative_steps = num_speculative_steps + self.device = device + self._steps_since_refit = 0 + self._refits = 0 + + # The learned coefficients used for predicting acceptance. + self.slope = torch.full((1,), _INIT_SLOPE, dtype=torch.float32, device=device) + self.intercepts = torch.full( + (num_speculative_steps,), _INIT_BIAS, dtype=torch.float32, device=device + ) + # Holds logit(max q), which is used as the feature for the logistic. + # Stored in stable slots keyed by persistent request-state index. + self.features = torch.zeros( + max_num_reqs, num_speculative_steps, dtype=torch.float32, device=device + ) + # Holds the acceptance predictions made at draft time, computed as: + # p = σ(ω * f + βₖ) + # where f is feature (logit(max q)) at draft position k, ω is the learned + # shared slope, and βₖ is the learned per-position intercept. Stored in + # stable slots keyed by the persistent request-state index. + self.predictions = torch.zeros( + max_num_reqs, num_speculative_steps, dtype=torch.float32, device=device + ) + + # Per-round Newton-IRLS statistics, cleared after each refit. For observation + # i the design row is xᵢ = [fᵢ, eₖ], the IRLS weight is wᵢ = pᵢ(1-pᵢ), and the + # residual is yᵢ - pᵢ, where yᵢ is the label (1 for accepted, 0 for rejected). + # XᵀWX is then the negative Hessian of the Bernoulli log-likelihood loss and + # Xᵀ(Y-P) is its gradient, so the Newton step for θ = [ω, β₀ … βₙ₋₁] is the + # the solution of (XᵀWX + λI) Δθ = Xᵀ(Y-P). + # + # Columns [aₖ, bₖ, cₖ] are each position's contribution to the XᵀWX arrowhead: + # ⎡ a b₀ b₁ … bₙ₋₁ ⎤ + # ⎢ b₀ c₀ ⎥ + # ⎢ b₁ c₁ ⎥ + # ⎢ ⋮ ⋱ ⎥ + # ⎣ bₙ₋₁ cₙ₋₁ ⎦ + # bₖ and cₖ are per-position, but a = Σₖ aₖ is one scalar shared by all of + # them. + self.info = torch.zeros( + num_speculative_steps, 3, dtype=torch.float32, device=device + ) + # Columns [g₀ₖ, g₁ₖ] are each position's contribution to the Xᵀ(Y-P) vector: + # ⎡ g₀ ⎤ + # ⎢ g₁₀ ⎥ + # ⎢ g₁₁ ⎥ + # ⎢ ⋮ ⎥ + # ⎣ g₁ₙ₋₁ ⎦ + # As above, g₁ₖ is per-position and g₀ = Σₖ g₀ₖ is shared. + self.grad = torch.zeros( + num_speculative_steps, 2, dtype=torch.float32, device=device + ) + # Number of graded drafts per-position in the current round, zeroed after + # each refit. _refit_kernel uses this to damp coefficient updates by: + # n / (n + DAMPING_OBSERVATIONS) + self.counts = torch.zeros( + num_speculative_steps, dtype=torch.float32, device=device + ) + + def step( + self, + idx_mapping: torch.Tensor, + num_sampled: torch.Tensor, + num_rejected: torch.Tensor, + ) -> None: + # Accumulate the previous step's graded drafts into the IRLS statistics. + num_reqs = idx_mapping.shape[0] + _accumulate_kernel[(self.num_speculative_steps,)]( + self.info, + self.grad, + idx_mapping, + num_sampled, + num_rejected, + self.features, + self.features.stride(0), + self.predictions, + self.predictions.stride(0), + self.counts, + num_reqs, + BLOCK_R=triton.next_power_of_2(max(num_reqs, 1)), + ) + + self._steps_since_refit += 1 + if self._steps_since_refit < self.REFIT_INTERVAL: + return + self._steps_since_refit = 0 + + # Fit the coefficients to the accumulated statistics gathered over the + # course of the last REFIT_INTERVAL steps. + _refit_kernel[(1,)]( + self.slope, + self.intercepts, + self.info, + self.info.stride(0), + self.grad, + self.grad.stride(0), + self.counts, + NUM_SPECULATIVE_STEPS=self.num_speculative_steps, + L2=self.L2, + DAMPING=self.DAMPING_OBSERVATIONS, + BLOCK=triton.next_power_of_2(self.num_speculative_steps), + ) + self._refits += 1 + + def predict( + self, + logits: torch.Tensor, + idx_mapping: torch.Tensor, + draft_step: torch.Tensor, + confidence_probs: torch.Tensor, + temperature: torch.Tensor, + ) -> None: + num_tokens, vocab_size = logits.shape + VOCAB_BLOCK_SIZE = 4096 + num_blocks = triton.cdiv(vocab_size, VOCAB_BLOCK_SIZE) + local_max = torch.empty( + num_tokens, num_blocks, dtype=torch.float32, device=self.device + ) + local_sumexp = torch.empty_like(local_max) + _local_max_sumexp_kernel[(num_tokens, num_blocks)]( + local_max, + local_max.stride(0), + local_sumexp, + local_sumexp.stride(0), + logits, + logits.stride(0), + idx_mapping, + idx_mapping.stride(0), + temperature, + num_tokens, + vocab_size, + BLOCK_SIZE=VOCAB_BLOCK_SIZE, + ) + _predict_kernel[(num_tokens,)]( + self.features, + self.features.stride(0), + self.predictions, + self.predictions.stride(0), + confidence_probs, + confidence_probs.stride(0), + self.slope, + self.intercepts, + local_max, + local_max.stride(0), + local_sumexp, + local_sumexp.stride(0), + idx_mapping, + idx_mapping.stride(0), + draft_step, + num_tokens, + num_blocks, + per_token_step=draft_step.dim() > 0, + NUM_SPECULATIVE_STEPS=self.num_speculative_steps, + MAX_LOG_ODDS=_MAX_LOG_ODDS, + PADDED_VOCAB_NUM_BLOCKS=triton.next_power_of_2(num_blocks), + ) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py index 1cd19acdab..7db5284497 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py @@ -213,5 +213,11 @@ class DFlash2Speculator(DFlashSpeculator): anchor_token_ids, ) self._sample_path(candidate_ids, scores, num_reqs) + if self.enable_adaptive_verification: + self._maybe_predict_acceptance( + self._selector_scores[:num_reqs].flatten(0, 1), + self.sample_idx_mapping[:num_sample], + self.sample_col[:num_sample], + ) if self.draft_logits is not None: self._cache_draft_logits(candidate_ids, num_sample) diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index 036c3014f3..411aa7f041 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -79,12 +79,7 @@ class DSparkSpeculator(DFlashSpeculator): self.draft_model_config.hf_config, "dspark_draft_topk", None ) - self.draft_token_confidence_probs = torch.empty_like( - self.draft_tokens, dtype=torch.float32 - ) - self.enable_adaptive_verification = ( - self.speculative_config.enable_adaptive_verification - ) + self.use_confidence_head: bool = False def load_draft_model( self, @@ -108,13 +103,14 @@ class DSparkSpeculator(DFlashSpeculator): dtype=self.draft_logits.dtype, device=self.device, ) - if self.enable_adaptive_verification and model.model.confidence_head is None: - raise ValueError( - "Adaptive verification needs a DSpark checkpoint with a confidence " - "head, and this one has none. Pass " - "enable_adaptive_verification=false in the speculative config to verify" - " a fixed number of drafts instead." - ) + self.use_confidence_head = ( + self.enable_adaptive_verification + and model.model.confidence_head is not None + ) + if self.use_confidence_head: + # The acceptance estimator is not needed when a trained confidence head + # is available. + self.use_acceptance_estimator = False return model def _sample_logits( @@ -124,8 +120,10 @@ class DSparkSpeculator(DFlashSpeculator): sample_pos: torch.Tensor, step: int, ) -> torch.Tensor: + self._maybe_predict_acceptance(logits, idx_map, self._step_cols[step]) if self.draft_logits is None: - return self.model.map_draft_to_target(logits.argmax(dim=-1)) + draft_ids = logits.argmax(dim=-1) + return self.model.map_draft_to_target(draft_ids) # Probabilistic sampling and rejection operate in target-vocabulary # space. A reduced draft vocabulary is scattered into its target rows. @@ -184,7 +182,7 @@ class DSparkSpeculator(DFlashSpeculator): for i in range(n_spec): # Sequential stage: Markov bias from the previously sampled token. markov_embed = self.model.markov_embed(prev) - if self.enable_adaptive_verification: + if self.use_confidence_head: confidence_markov_embeds.append(markov_embed) bias = self.model.markov_bias(markov_embed) logits_i = base_logits[:, i] + bias @@ -194,7 +192,7 @@ class DSparkSpeculator(DFlashSpeculator): self.draft_tokens[:num_reqs, i] = draft_sampled_i prev = draft_sampled_i - if self.enable_adaptive_verification: + if self.use_confidence_head: confidence = self.model.compute_confidence( sample_hidden, torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1), @@ -229,7 +227,7 @@ class DSparkSpeculator(DFlashSpeculator): for i in range(n_spec): markov_embed = self.model.markov_embed(prev) - if self.enable_adaptive_verification: + if self.use_confidence_head: confidence_markov_embeds.append(markov_embed) logits_i = self.model.apply_markov_bias_gathered( markov_embed, @@ -243,7 +241,7 @@ class DSparkSpeculator(DFlashSpeculator): self.draft_tokens[:num_reqs, i] = draft_sampled_i prev = draft_sampled_i - if self.enable_adaptive_verification: + if self.use_confidence_head: confidence = self.model.compute_confidence( sample_hidden, torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1), diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index f670441219..299cac6416 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -33,6 +33,9 @@ from vllm.v1.worker.gpu.dp_utils import DPSyncState from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample +from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import ( + OnlineAcceptanceEstimator, +) from vllm.v1.worker.utils import AttentionGroup if TYPE_CHECKING: @@ -158,6 +161,15 @@ class DraftModelSpeculator(BaseSpeculator): self.arange_np = np.arange(self.max_num_reqs + 1, dtype=np.int32) self.draft_is_prefilling = torch.zeros(self.max_num_reqs, dtype=torch.bool) + self.draft_token_confidence_probs = torch.empty_like( + self.draft_tokens, dtype=torch.float32 + ) + self.enable_adaptive_verification = ( + self.speculative_config.enable_adaptive_verification + ) + self.use_acceptance_estimator = self.enable_adaptive_verification + self.acceptance_estimator: OnlineAcceptanceEstimator | None = None + self.draft_logits: torch.Tensor | None = None if self.speculative_config.draft_sample_method == "probabilistic": # Pre-temperature logits, cached from the previous decode step. @@ -227,6 +239,20 @@ class DraftModelSpeculator(BaseSpeculator): type(self.model).__name__, ) + if self.use_acceptance_estimator: + if self.use_local_argmax_reduction: + raise ValueError( + "Adaptive verification without a confidence head estimates " + "per-position acceptance from the draft logits, which " + "use_local_argmax_reduction never materializes. Disable one " + "of them." + ) + self.acceptance_estimator = OnlineAcceptanceEstimator( + self.max_num_reqs, + self.num_speculative_steps, + self.device, + ) + def set_eplb_state(self, eplb_state: EplbState) -> None: """Inject EPLB state after construction.""" self.eplb_state = eplb_state @@ -371,12 +397,6 @@ class DraftModelSpeculator(BaseSpeculator): "(communication: O(2*tp_size) vs O(vocab_size))." ) - def _greedy_sample_draft(self, hidden_states: torch.Tensor) -> torch.Tensor: - if self.use_local_argmax_reduction: - return self.model.get_top_tokens(hidden_states) - logits = self.model.compute_logits(hidden_states) - return logits.argmax(dim=-1) - def sample_draft( self, hidden_states: torch.Tensor, @@ -408,8 +428,42 @@ class DraftModelSpeculator(BaseSpeculator): idx_mapping, temperature, ) - return sampled - return self._greedy_sample_draft(hidden_states) + elif self.use_local_argmax_reduction: + return self.model.get_top_tokens(hidden_states) + else: + logits = self.model.compute_logits(hidden_states) + sampled = logits.argmax(dim=-1) + self._maybe_predict_acceptance(logits, idx_mapping, draft_step) + return sampled + + def _maybe_predict_acceptance( + self, + logits: torch.Tensor, + idx_mapping: torch.Tensor, + draft_step: torch.Tensor, + ) -> None: + if self.acceptance_estimator is not None: + self.acceptance_estimator.predict( + logits, + idx_mapping, + draft_step, + self.draft_token_confidence_probs, + self.temperature, + ) + + def observe_verification( + self, + idx_mapping: torch.Tensor, + num_sampled: torch.Tensor, + num_rejected: torch.Tensor, + ) -> None: + """Fold the target's verdict on the last drafts into the estimator. + + Must run before the next `propose`, which overwrites the per-slot + features the verdict grades. + """ + if self.acceptance_estimator is not None: + self.acceptance_estimator.step(idx_mapping, num_sampled, num_rejected) def prepare_watermarking( self, contexts: torch.Tensor, watermarking: torch.Tensor