[Model Runner V2] Acceptance estimation for adaptive verification (#52228)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -0,0 +1,294 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""``predict`` writes one probability into two index spaces at once.
|
||||
|
||||
Adaptive verification reads the estimate in this step's batch order, while the
|
||||
next step's refit reads it in persistent request-state slots. A mix-up between
|
||||
the two is silent -- every value stays a plausible probability, it just belongs
|
||||
to another request -- so it is asserted here rather than left to acceptance
|
||||
metrics to hint at.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import (
|
||||
OnlineAcceptanceEstimator,
|
||||
)
|
||||
|
||||
pytest.importorskip("triton")
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required for acceptance estimator tests", allow_module_level=True)
|
||||
|
||||
MAX_NUM_REQS = 16
|
||||
NUM_STEPS = 4
|
||||
VOCAB_SIZE = 2048
|
||||
|
||||
|
||||
def _make_estimator(device: torch.device) -> OnlineAcceptanceEstimator:
|
||||
estimator = OnlineAcceptanceEstimator(MAX_NUM_REQS, NUM_STEPS, device)
|
||||
# Distinct per-position intercepts, so reading the wrong one shows up.
|
||||
estimator.slope.fill_(0.7)
|
||||
estimator.intercepts[:] = torch.tensor([1.5, 1.0, 0.5, 0.0], device=device)
|
||||
return estimator
|
||||
|
||||
|
||||
def _expected_probs(
|
||||
estimator: OnlineAcceptanceEstimator,
|
||||
logits: torch.Tensor,
|
||||
steps: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""logit(q_max) by a different route than the kernel takes.
|
||||
|
||||
The kernel shifts by the max and drops the unit term; here the argmax is
|
||||
masked out and the complement is reduced with logsumexp, so agreement is
|
||||
evidence about the value rather than about a shared formula.
|
||||
"""
|
||||
temp = torch.where(temperature > 0, temperature, torch.ones_like(temperature))
|
||||
scaled = logits.double() / temp[:, None].double()
|
||||
top = scaled.max(dim=-1)
|
||||
rest = scaled.scatter(1, top.indices[:, None], float("-inf"))
|
||||
feature = (top.values - torch.logsumexp(rest, dim=-1)).clamp(-40.0, 40.0)
|
||||
weight, bias = estimator.slope, estimator.intercepts
|
||||
return torch.sigmoid(weight.double() * feature + bias[steps].double()).float()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokens_per_req", [1, NUM_STEPS])
|
||||
def test_predict_agrees_across_slot_and_batch_order(tokens_per_req: int):
|
||||
"""One row per request (per-step drafting) and a whole block at once.
|
||||
|
||||
Which one it is is inferred from whether the draft step is per-row, so the
|
||||
two cases differ only in how ``draft_step`` is shaped.
|
||||
"""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
estimator = _make_estimator(device)
|
||||
num_reqs = 5
|
||||
|
||||
# Batch position i holds a shuffled persistent slot, as a served batch would.
|
||||
slots = torch.randperm(MAX_NUM_REQS, device=device)[:num_reqs].to(torch.int32)
|
||||
if tokens_per_req == 1:
|
||||
idx_mapping = slots
|
||||
draft_step = torch.tensor(2, device=device, dtype=torch.int32)
|
||||
steps = draft_step.expand(num_reqs)
|
||||
else:
|
||||
idx_mapping = slots.repeat_interleave(NUM_STEPS)
|
||||
draft_step = torch.arange(NUM_STEPS, device=device, dtype=torch.int32).repeat(
|
||||
num_reqs
|
||||
)
|
||||
steps = draft_step
|
||||
batch_rows = torch.arange(num_reqs, device=device).repeat_interleave(tokens_per_req)
|
||||
logits = torch.randn(idx_mapping.shape[0], VOCAB_SIZE, device=device) * 4.0
|
||||
|
||||
confidence_probs = torch.zeros(MAX_NUM_REQS, NUM_STEPS, device=device)
|
||||
temperature = torch.rand(MAX_NUM_REQS, device=device) + 0.5
|
||||
estimator.predict(logits, idx_mapping, draft_step, confidence_probs, temperature)
|
||||
|
||||
expected = _expected_probs(
|
||||
estimator, logits, steps.long(), temperature[idx_mapping.long()]
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
estimator.predictions[idx_mapping.long(), steps.long()],
|
||||
expected,
|
||||
atol=2e-3,
|
||||
rtol=2e-3,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
confidence_probs[batch_rows, steps.long()], expected, atol=2e-3, rtol=2e-3
|
||||
)
|
||||
|
||||
|
||||
def test_predict_skips_cudagraph_padded_rows():
|
||||
"""Padded rows carry slot -1 and must not land on a live request."""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
estimator = _make_estimator(device)
|
||||
num_reqs, num_padded = 3, 6
|
||||
|
||||
slots = torch.tensor([7, 2, 11], device=device, dtype=torch.int32)
|
||||
idx_mapping = torch.cat(
|
||||
[slots, torch.full((num_padded,), -1, device=device, dtype=torch.int32)]
|
||||
)
|
||||
draft_step = torch.tensor(1, device=device, dtype=torch.int32)
|
||||
logits = torch.randn(num_reqs + num_padded, VOCAB_SIZE, device=device) * 4.0
|
||||
|
||||
confidence_probs = torch.zeros(MAX_NUM_REQS, NUM_STEPS, device=device)
|
||||
temperature = torch.rand(MAX_NUM_REQS, device=device) + 0.5
|
||||
estimator.predict(logits, idx_mapping, draft_step, confidence_probs, temperature)
|
||||
|
||||
live = (slots.long(), torch.full((num_reqs,), 1, device=device))
|
||||
expected = _expected_probs(
|
||||
estimator, logits[:num_reqs], live[1], temperature[slots.long()]
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
estimator.predictions[live], expected, atol=2e-3, rtol=2e-3
|
||||
)
|
||||
# Nothing outside the live (request, position) cells was touched.
|
||||
untouched = torch.ones(MAX_NUM_REQS, NUM_STEPS, dtype=torch.bool, device=device)
|
||||
untouched[live] = False
|
||||
assert not estimator.predictions[untouched].any()
|
||||
assert not confidence_probs[num_reqs:].any()
|
||||
|
||||
|
||||
# A refit takes one undamped Newton step, so it needs a realistic number of
|
||||
# observations per round to be well conditioned -- a handful of requests can
|
||||
# send the step far past the optimum. This is the batch a served step sees.
|
||||
REFIT_NUM_REQS = 512
|
||||
|
||||
|
||||
def _drive_round(estimator, features, num_accepted, slots):
|
||||
"""Feed one step's graded drafts through predict's buffers into `step`."""
|
||||
estimator.features[slots.long()] = features
|
||||
estimator.predictions[slots.long()] = torch.sigmoid(
|
||||
estimator.slope * features + estimator.intercepts
|
||||
)
|
||||
estimator.step(
|
||||
slots,
|
||||
(num_accepted + 1).to(torch.int32),
|
||||
(features.shape[1] - num_accepted).to(torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def test_refit_recovers_a_shared_slope_and_per_position_intercepts():
|
||||
"""The fit must find the logistic that generated the observations.
|
||||
|
||||
One slope is shared across draft positions and each keeps its own intercept,
|
||||
so the arrowhead solve is only correct if it recovers both from data whose
|
||||
positions differ in level but not in slope.
|
||||
"""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
true_slope = 0.6
|
||||
true_bias = torch.tensor([1.4, 0.9, 0.4, -0.1], device=device)
|
||||
|
||||
estimator = OnlineAcceptanceEstimator(REFIT_NUM_REQS, NUM_STEPS, device)
|
||||
estimator.REFIT_INTERVAL = 5
|
||||
slots = torch.arange(REFIT_NUM_REQS, device=device, dtype=torch.int32)
|
||||
for _ in range(60):
|
||||
features = torch.rand(REFIT_NUM_REQS, NUM_STEPS, device=device) * 8.0
|
||||
accepted = torch.rand_like(features) < torch.sigmoid(
|
||||
true_slope * features + true_bias
|
||||
)
|
||||
# Verification stops at the first rejection, so a position's label is
|
||||
# conditional on every shallower one having been accepted.
|
||||
num_accepted = (~accepted).float().argmax(dim=1)
|
||||
num_accepted[accepted.all(dim=1)] = NUM_STEPS
|
||||
_drive_round(estimator, features, num_accepted, slots)
|
||||
|
||||
slope, bias = estimator.slope, estimator.intercepts
|
||||
torch.testing.assert_close(
|
||||
slope[0], torch.tensor(true_slope, device=device), atol=0.1, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(bias, true_bias, atol=0.3, rtol=0)
|
||||
|
||||
|
||||
def test_refit_survives_a_position_with_no_observations():
|
||||
"""A draft position nobody reached must not stall the shared slope.
|
||||
|
||||
Its statistics stay zero while the others accumulate, which is the degenerate
|
||||
case the arrowhead denominator has to tolerate.
|
||||
"""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
estimator = OnlineAcceptanceEstimator(REFIT_NUM_REQS, NUM_STEPS, device)
|
||||
estimator.REFIT_INTERVAL = 5
|
||||
slots = torch.arange(REFIT_NUM_REQS, device=device, dtype=torch.int32)
|
||||
before_slope = estimator.slope.clone()
|
||||
before_intercepts = estimator.intercepts.clone()
|
||||
|
||||
# Every request is rejected at position 1, so positions 2+ are never graded.
|
||||
num_accepted = torch.ones(REFIT_NUM_REQS, device=device)
|
||||
for _ in range(20):
|
||||
features = torch.rand(REFIT_NUM_REQS, NUM_STEPS, device=device) * 8.0
|
||||
_drive_round(estimator, features, num_accepted, slots)
|
||||
|
||||
assert estimator.slope.isfinite().all()
|
||||
assert estimator.intercepts.isfinite().all()
|
||||
assert not torch.equal(estimator.slope, before_slope), "slope never moved"
|
||||
# Positions 2 and 3 were never observed, so their intercepts must be untouched.
|
||||
torch.testing.assert_close(
|
||||
estimator.intercepts[2:], before_intercepts[2:], rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
def test_step_damping_scales_with_the_evidence():
|
||||
"""A round moves a coefficient in proportion to the samples behind it.
|
||||
|
||||
The Newton step itself barely changes with how much data a round carries --
|
||||
curvature and score both scale with it -- so scaling a round's statistics
|
||||
isolates the damping factor n / (n + DAMPING_OBSERVATIONS). The slope/
|
||||
intercept coupling is zeroed so the two parameters can be checked apart.
|
||||
"""
|
||||
device = torch.device("cuda")
|
||||
damping = OnlineAcceptanceEstimator.DAMPING_OBSERVATIONS
|
||||
per_position_n = 20.0
|
||||
|
||||
def refit_with(scale):
|
||||
estimator = OnlineAcceptanceEstimator(MAX_NUM_REQS, NUM_STEPS, device)
|
||||
estimator.info[:, 1] = 0.0 # sum(w*margin): no slope/intercept coupling
|
||||
estimator.info[:, 2] = 1.0 * scale # sum(w)
|
||||
estimator.grad[:, 1] = 0.5 * scale # sum(y - p)
|
||||
# The slope's information and score are accumulated as per-position
|
||||
# partials that the refit sums, so spread each total over the positions.
|
||||
estimator.info[:, 0] = 30.0 * scale / NUM_STEPS # sum(w*margin^2)
|
||||
estimator.grad[:, 0] = 2.0 * scale / NUM_STEPS # sum((y - p)*margin)
|
||||
estimator.counts[:] = per_position_n * scale
|
||||
estimator._steps_since_refit = estimator.REFIT_INTERVAL - 1
|
||||
before_slope = estimator.slope.clone()
|
||||
before_intercepts = estimator.intercepts.clone()
|
||||
empty = torch.zeros(0, dtype=torch.int32, device=device)
|
||||
estimator.step(empty, empty, empty)
|
||||
return (
|
||||
(estimator.slope - before_slope)[0].item(),
|
||||
(estimator.intercepts - before_intercepts)[0].item(),
|
||||
)
|
||||
|
||||
for scale in (1.0, 10.0, 100.0):
|
||||
total_n = per_position_n * scale * NUM_STEPS
|
||||
slope_delta, bias_delta = refit_with(scale)
|
||||
assert slope_delta == pytest.approx(
|
||||
(2.0 / 30.0) * total_n / (total_n + damping), rel=1e-3
|
||||
)
|
||||
assert bias_delta == pytest.approx(
|
||||
0.5 * (per_position_n * scale) / (per_position_n * scale + damping),
|
||||
rel=1e-3,
|
||||
)
|
||||
|
||||
# A round with no observations at all must leave the coefficients alone.
|
||||
assert refit_with(0.0) == (0.0, 0.0)
|
||||
|
||||
|
||||
def test_feature_ignores_which_token_was_drawn():
|
||||
"""The score must depend on the draft distribution, not on the draw.
|
||||
|
||||
This is what keeps rejection sampling lossless: if the score read the drawn
|
||||
token, trimming a draft would condition on the token being trimmed and tilt
|
||||
the emitted distribution toward the drafter's confident modes. ``predict``
|
||||
takes no token argument, and this pins that -- two rows with identical
|
||||
logits must score identically no matter what was sampled from them.
|
||||
"""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
estimator = _make_estimator(device)
|
||||
|
||||
row = torch.randn(VOCAB_SIZE, device=device) * 4.0
|
||||
logits = row.expand(2, VOCAB_SIZE).contiguous()
|
||||
idx_mapping = torch.tensor([0, 1], device=device, dtype=torch.int32)
|
||||
draft_step = torch.tensor(0, device=device, dtype=torch.int32)
|
||||
temperature = torch.ones(MAX_NUM_REQS, device=device)
|
||||
confidence_probs = torch.zeros(MAX_NUM_REQS, NUM_STEPS, device=device)
|
||||
|
||||
estimator.predict(logits, idx_mapping, draft_step, confidence_probs, temperature)
|
||||
assert estimator.features[0, 0] == estimator.features[1, 0]
|
||||
|
||||
# And the value is the top token's log-odds, not the top-2 margin.
|
||||
scaled = row.double()
|
||||
top = scaled.max(dim=-1)
|
||||
rest = scaled.scatter(0, top.indices[None], float("-inf"))
|
||||
torch.testing.assert_close(
|
||||
estimator.features[0, 0].double(),
|
||||
(top.values - torch.logsumexp(rest, dim=-1)).clamp(-40.0, 40.0),
|
||||
atol=2e-3,
|
||||
rtol=2e-3,
|
||||
)
|
||||
@@ -123,6 +123,7 @@ def test_speculator_uses_draft_model_hidden_size(monkeypatch, hc_mult, expected)
|
||||
draft_model_config=draft_model_config,
|
||||
use_local_argmax_reduction=False,
|
||||
draft_sample_method="greedy",
|
||||
enable_adaptive_verification=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=speculative_config,
|
||||
@@ -161,6 +162,7 @@ def test_mm_support_configured_after_model_load(monkeypatch):
|
||||
speculator.dtype = torch.float32
|
||||
speculator.draft_model_config = draft_model_config
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.use_acceptance_estimator = False
|
||||
|
||||
checked_configs = []
|
||||
|
||||
@@ -195,6 +197,7 @@ def test_load_model_keeps_mm_support_for_capable_drafter(monkeypatch):
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.use_acceptance_estimator = False
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
speculator.max_num_tokens = 4
|
||||
speculator.hidden_size = 3
|
||||
@@ -219,6 +222,7 @@ def test_load_model_disables_mm_support_for_text_only_drafter(monkeypatch):
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.use_acceptance_estimator = False
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
draft_model = _TextOnlyDraftModel()
|
||||
speculator.test_draft_model = draft_model
|
||||
@@ -251,6 +255,7 @@ def test_multi_module_mm_support_configured_after_model_load(monkeypatch):
|
||||
speculator = object.__new__(MultiModuleMTPSpeculator)
|
||||
speculator.supports_mm_inputs = False
|
||||
speculator.inputs_embeds = None
|
||||
speculator.use_acceptance_estimator = False
|
||||
speculator.cached_draft_input_embeds = None
|
||||
speculator.vllm_config = SimpleNamespace(model_config=object())
|
||||
speculator.max_num_tokens = 4
|
||||
|
||||
@@ -60,6 +60,7 @@ def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch):
|
||||
layer_name = "cache_only_layers.2"
|
||||
speculator = object.__new__(ExtractHiddenStatesSpeculator)
|
||||
speculator.vllm_config = cast(Any, SimpleNamespace())
|
||||
speculator.acceptance_estimator = None
|
||||
speculator.num_hidden_states = 2
|
||||
speculator.hidden_states = torch.zeros(4, 2, 3)
|
||||
speculator.draft_attn_layer_names = {layer_name}
|
||||
@@ -111,6 +112,7 @@ def test_propose_caches_hidden_states_and_returns_sampled_tokens(monkeypatch):
|
||||
|
||||
def test_propose_requires_aux_hidden_states():
|
||||
speculator = object.__new__(ExtractHiddenStatesSpeculator)
|
||||
speculator.acceptance_estimator = None
|
||||
speculator.num_hidden_states = 2
|
||||
input_batch = cast(
|
||||
Any, SimpleNamespace(idx_mapping=torch.tensor([0], dtype=torch.int32))
|
||||
|
||||
@@ -1561,8 +1561,16 @@ class SpeculativeConfig:
|
||||
self.index_share_for_mtp_iteration
|
||||
)
|
||||
|
||||
if self.method != "dspark" and self.enable_adaptive_verification:
|
||||
raise ValueError("Adaptive verification only supported with DSpark")
|
||||
if (
|
||||
self.method != "dspark"
|
||||
and self.enable_adaptive_verification
|
||||
and self.use_local_argmax_reduction
|
||||
):
|
||||
raise ValueError(
|
||||
"Adaptive verification estimates per-position acceptance from "
|
||||
"the draft logits, which use_local_argmax_reduction never "
|
||||
"materializes. Disable one of them."
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
|
||||
@@ -2120,6 +2120,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
if self.speculator is not None:
|
||||
assert self.sampler is not None
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
self.speculator.observe_verification(
|
||||
input_batch.idx_mapping, num_sampled, num_rejected
|
||||
)
|
||||
# Let the target override the hidden state fed to the drafter
|
||||
# (e.g. DeepSeek V4 MTP needs the pre-hc_head residual). The
|
||||
# target returns a persistent buffer sized at max_num_batched_tokens;
|
||||
|
||||
@@ -0,0 +1,501 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import (
|
||||
_compute_max_and_sumexp,
|
||||
)
|
||||
|
||||
# The feature is a log-odds, log(q / (1 - q)), which a row carrying all of its
|
||||
# mass on one token sends to infinity. Clamp it symmetrically.
|
||||
_MAX_LOG_ODDS = 40.0
|
||||
|
||||
# Cold-start coefficients: the median slope and intercept fitted online across
|
||||
# DeepSeek-V4-Flash, Muse-Glimmer-30B, MiMo-V2.5-Pro, Inkling and Kimi-K2.5.
|
||||
# Log-loss is flat near the optimum, so one starting point serves all of them
|
||||
# and a real fit replaces it within a refit or two.
|
||||
_INIT_SLOPE = 0.5
|
||||
_INIT_BIAS = -0.2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _accumulate_kernel(
|
||||
info_ptr,
|
||||
grad_ptr,
|
||||
idx_mapping_ptr,
|
||||
num_sampled_ptr,
|
||||
num_rejected_ptr,
|
||||
features_ptr,
|
||||
features_stride,
|
||||
pred_ptr,
|
||||
pred_stride,
|
||||
counts_ptr,
|
||||
num_reqs,
|
||||
BLOCK_R: tl.constexpr,
|
||||
):
|
||||
step = tl.program_id(0).to(tl.int64)
|
||||
req_block = tl.arange(0, BLOCK_R)
|
||||
req_mask = req_block < num_reqs
|
||||
req_state_block = tl.load(idx_mapping_ptr + req_block, mask=req_mask, other=0).to(
|
||||
tl.int64
|
||||
)
|
||||
|
||||
num_sampled = tl.load(num_sampled_ptr + req_block, mask=req_mask, other=0).to(
|
||||
tl.int64
|
||||
)
|
||||
# Subtract bonus token.
|
||||
num_accepted = tl.maximum(num_sampled - 1, 0)
|
||||
num_rejected = tl.load(num_rejected_ptr + req_block, mask=req_mask, other=0).to(
|
||||
tl.int64
|
||||
)
|
||||
num_admitted = num_accepted + num_rejected
|
||||
|
||||
feature = tl.load(
|
||||
features_ptr + req_state_block * features_stride + step,
|
||||
mask=req_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
pred = tl.load(
|
||||
pred_ptr + req_state_block * pred_stride + step,
|
||||
mask=req_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
# The accepted tokens and the first rejected draft token (if one exists) are
|
||||
# "observed" and given labels.
|
||||
observed = req_mask & (step <= num_accepted) & (step < num_admitted)
|
||||
label = tl.where(step < num_accepted, 1.0, 0.0)
|
||||
w = tl.where(observed, pred * (1.0 - pred), 0.0)
|
||||
resid = tl.where(observed, label - pred, 0.0)
|
||||
|
||||
# This block's contribution to the normal equations for design row
|
||||
# x = [feature, eₖ]: the XᵀWX entries aₖ, bₖ, cₖ and the Xᵀr entries g₀ₖ, g₁ₖ.
|
||||
# aₖ and g₀ₖ are partials of scalars shared by every position, which
|
||||
# _refit_kernel sums.
|
||||
a_k = tl.sum(w * feature * feature, axis=0)
|
||||
b_k = tl.sum(w * feature, axis=0)
|
||||
c_k = tl.sum(w, axis=0)
|
||||
g0_k = tl.sum(resid * feature, axis=0)
|
||||
g1_k = tl.sum(resid, axis=0)
|
||||
count = tl.sum(tl.where(observed, 1.0, 0.0), axis=0)
|
||||
tl.store(info_ptr + step * 3 + 0, tl.load(info_ptr + step * 3 + 0) + a_k)
|
||||
tl.store(info_ptr + step * 3 + 1, tl.load(info_ptr + step * 3 + 1) + b_k)
|
||||
tl.store(info_ptr + step * 3 + 2, tl.load(info_ptr + step * 3 + 2) + c_k)
|
||||
tl.store(grad_ptr + step * 2 + 0, tl.load(grad_ptr + step * 2 + 0) + g0_k)
|
||||
tl.store(grad_ptr + step * 2 + 1, tl.load(grad_ptr + step * 2 + 1) + g1_k)
|
||||
tl.store(counts_ptr + step, tl.load(counts_ptr + step) + count)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _refit_kernel(
|
||||
slope_ptr,
|
||||
intercepts_ptr,
|
||||
info_ptr,
|
||||
info_row_stride,
|
||||
grad_ptr,
|
||||
grad_row_stride,
|
||||
counts_ptr,
|
||||
NUM_SPECULATIVE_STEPS: tl.constexpr,
|
||||
L2: tl.constexpr,
|
||||
DAMPING: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
k = tl.arange(0, BLOCK)
|
||||
mask = k < NUM_SPECULATIVE_STEPS
|
||||
|
||||
# Load the stats accumulated over this round to solve (XᵀWX + λI) Δθ = Xᵀr.
|
||||
a_k = tl.load(info_ptr + k * info_row_stride + 0, mask=mask, other=0.0)
|
||||
b_k = tl.load(info_ptr + k * info_row_stride + 1, mask=mask, other=0.0)
|
||||
c_k = tl.load(info_ptr + k * info_row_stride + 2, mask=mask, other=0.0) + L2
|
||||
a = tl.sum(a_k, axis=0) + L2
|
||||
g0_k = tl.load(grad_ptr + k * grad_row_stride + 0, mask=mask, other=0.0)
|
||||
g1_k = tl.load(grad_ptr + k * grad_row_stride + 1, mask=mask, other=0.0)
|
||||
g0 = tl.sum(g0_k, axis=0)
|
||||
n = tl.load(counts_ptr + k, mask=mask, other=0.0)
|
||||
|
||||
# Load the current shared slope and per-position intercepts.
|
||||
w = tl.load(slope_ptr)
|
||||
bias = tl.load(intercepts_ptr + k, mask=mask, other=0.0)
|
||||
|
||||
# Solve for the shared slope delta, Δω, in the following arrowhead system:
|
||||
# ⎡ a+λ b₀ b₁ … bₙ₋₁ ⎤ ⎡ Δω ⎤ ⎡ g₀ ⎤
|
||||
# ⎢ b₀ c₀+λ ⎥ ⎢ Δβ₀ ⎥ ⎢ g₁₀ ⎥
|
||||
# ⎢ b₁ c₁+λ ⎥ ⎢ Δβ₁ ⎥ = ⎢ g₁₁ ⎥
|
||||
# ⎢ ⋮ ⋱ ⎥ ⎢ ⋮ ⎥ ⎢ ⋮ ⎥
|
||||
# ⎣ bₙ₋₁ cₙ₋₁+λ ⎦ ⎣ Δβₙ₋₁ ⎦ ⎣ g₁ₙ₋₁ ⎦
|
||||
# XᵀWX + λI Δθ Xᵀ(Y-P)
|
||||
shrink = b_k * b_k / c_k
|
||||
coupling = b_k * g1_k / c_k
|
||||
step_w = (g0 - tl.sum(coupling, axis=0)) / (a - tl.sum(shrink, axis=0))
|
||||
|
||||
# The slope learns from every position, so it is damped by the round's
|
||||
# total sample count.
|
||||
total_n = tl.sum(n, axis=0)
|
||||
step_w *= total_n / (total_n + DAMPING)
|
||||
# Mask out NaNs and steps that would drive the slope negative.
|
||||
step_w = tl.where((step_w == step_w) & (w + step_w >= 0.0), step_w, 0.0)
|
||||
# Update the shared slope.
|
||||
new_w = w + step_w
|
||||
|
||||
# Solve for the per-position intercept deltas, Δβₖ, in the same system.
|
||||
step_bias = (g1_k - b_k * step_w) / c_k
|
||||
# Damp by the position's sample count.
|
||||
step_bias *= n / (n + DAMPING)
|
||||
# Mask out NaNs.
|
||||
step_bias = tl.where(step_bias == step_bias, step_bias, 0.0)
|
||||
# Update the per-position intercepts.
|
||||
new_bias = bias + step_bias
|
||||
|
||||
tl.store(slope_ptr, new_w)
|
||||
tl.store(intercepts_ptr + k, new_bias, mask=mask)
|
||||
|
||||
# Start the round clean by clearing the per-round statistics.
|
||||
tl.store(info_ptr + k * info_row_stride + 0, 0.0, mask=mask)
|
||||
tl.store(info_ptr + k * info_row_stride + 1, 0.0, mask=mask)
|
||||
tl.store(info_ptr + k * info_row_stride + 2, 0.0, mask=mask)
|
||||
tl.store(grad_ptr + k * grad_row_stride + 0, 0.0, mask=mask)
|
||||
tl.store(grad_ptr + k * grad_row_stride + 1, 0.0, mask=mask)
|
||||
tl.store(counts_ptr + k, 0.0, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _local_max_sumexp_kernel(
|
||||
local_max_ptr,
|
||||
local_max_stride,
|
||||
local_sumexp_ptr,
|
||||
local_sumexp_stride,
|
||||
logits_ptr,
|
||||
logits_stride,
|
||||
idx_mapping_ptr,
|
||||
idx_mapping_stride,
|
||||
temperature_ptr,
|
||||
num_tokens,
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
if token_idx >= num_tokens:
|
||||
return
|
||||
|
||||
block_idx = tl.program_id(1)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + token_idx * idx_mapping_stride).to(
|
||||
tl.int64
|
||||
)
|
||||
if req_state_idx < 0:
|
||||
# Cudagraph-padded requests carry -1. Write an empty partial so the
|
||||
# buffer holds nothing uninitialized; _predict_kernel skips these rows.
|
||||
tl.store(
|
||||
local_max_ptr + token_idx * local_max_stride + block_idx, float("-inf")
|
||||
)
|
||||
tl.store(local_sumexp_ptr + token_idx * local_sumexp_stride + block_idx, 0.0)
|
||||
return
|
||||
|
||||
# Draft logits are pre-temperature, and acceptance is decided on the scaled
|
||||
# distribution the drafter actually sampled from, so scale first. A request
|
||||
# with temp == 0 drafts greedily, so leave its logits alone rather than dividing
|
||||
# by zero, matching what the sampler does.
|
||||
temp = tl.load(temperature_ptr + req_state_idx).to(tl.float32)
|
||||
temp = tl.where(temp > 0.0, temp, 1.0)
|
||||
|
||||
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < vocab_size
|
||||
logits = (
|
||||
tl.load(
|
||||
logits_ptr + token_idx * logits_stride + offsets,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
/ temp
|
||||
)
|
||||
block_max, block_sumexp = _compute_max_and_sumexp(logits)
|
||||
tl.store(local_max_ptr + token_idx * local_max_stride + block_idx, block_max)
|
||||
tl.store(
|
||||
local_sumexp_ptr + token_idx * local_sumexp_stride + block_idx, block_sumexp
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _predict_kernel(
|
||||
features_ptr,
|
||||
features_stride,
|
||||
pred_ptr,
|
||||
pred_stride,
|
||||
conf_ptr,
|
||||
conf_stride,
|
||||
slope_ptr,
|
||||
intercepts_ptr,
|
||||
local_max_ptr,
|
||||
local_max_stride,
|
||||
local_sumexp_ptr,
|
||||
local_sumexp_stride,
|
||||
idx_mapping_ptr,
|
||||
idx_mapping_stride,
|
||||
step_ptr,
|
||||
num_tokens,
|
||||
vocab_num_blocks,
|
||||
per_token_step: tl.constexpr,
|
||||
NUM_SPECULATIVE_STEPS: tl.constexpr,
|
||||
MAX_LOG_ODDS: tl.constexpr,
|
||||
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
if token_idx >= num_tokens:
|
||||
return
|
||||
|
||||
req_state_idx = tl.load(idx_mapping_ptr + token_idx * idx_mapping_stride).to(
|
||||
tl.int64
|
||||
)
|
||||
if req_state_idx < 0:
|
||||
# Cudagraph-padded requests carry -1. Skip them so that they don't
|
||||
# scatter garbage over a live request's features.
|
||||
return
|
||||
|
||||
# A per-row step means the speculator sampled the whole block in one pass, so
|
||||
# each request owns NUM_SPECULATIVE_STEPS consecutive rows; a step shared by
|
||||
# every row means one row per request. Batch position, unlike req_state_idx,
|
||||
# is only meaningful for the step that computed it, and is what adaptive
|
||||
# verification indexes by.
|
||||
if per_token_step:
|
||||
step = tl.load(step_ptr + token_idx).to(tl.int64)
|
||||
batch_idx = token_idx // NUM_SPECULATIVE_STEPS
|
||||
else:
|
||||
step = tl.load(step_ptr).to(tl.int64)
|
||||
batch_idx = token_idx
|
||||
|
||||
blocks = tl.arange(0, PADDED_VOCAB_NUM_BLOCKS)
|
||||
blocks_mask = blocks < vocab_num_blocks
|
||||
maxes = tl.load(
|
||||
local_max_ptr + token_idx * local_max_stride + blocks,
|
||||
mask=blocks_mask,
|
||||
other=float("-inf"),
|
||||
)
|
||||
sumexps = tl.load(
|
||||
local_sumexp_ptr + token_idx * local_sumexp_stride + blocks,
|
||||
mask=blocks_mask,
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# The feature, logit(q(m*)) for the row's max draft logit, m*, is:
|
||||
# q(x) = e^(x - m*) / Σ_z e^(z - m*)
|
||||
# logit(q(m*)) = log(q(m*) / (1 - q(m*)))
|
||||
# = m* - log(Σ_{z≠m*} e^(z - m*)).
|
||||
# It scores the draft distribution, not the token drawn from it, so it is
|
||||
# a function of the prefix alone. Trimming a draft therefore never
|
||||
# conditions on the token being trimmed, and rejection sampling stays
|
||||
# lossless. Scoring the drawn token instead would select on the proposal
|
||||
# and bias the emitted distribution toward the drafter's confident modes.
|
||||
|
||||
# Rescale each block's partial to the global max and sum. The most likely
|
||||
# token contributes exactly e^(m - m) = 1, so removing it leaves the sum
|
||||
# over every other token.
|
||||
global_max = tl.max(maxes, axis=0)
|
||||
total_sumexp = tl.sum(
|
||||
tl.where(blocks_mask, sumexps * tl.exp(maxes - global_max), 0.0), axis=0
|
||||
)
|
||||
complement_sumexp = tl.maximum(total_sumexp - 1.0, 0.0)
|
||||
|
||||
# Shifted by m*, the numerator is e^(m* - m*) = 1, so the feature is just the
|
||||
# negated complement log-sum-exponential. Clamp to the min/max log-odds.
|
||||
feature = -tl.log(complement_sumexp)
|
||||
feature = tl.minimum(tl.maximum(feature, -MAX_LOG_ODDS), MAX_LOG_ODDS)
|
||||
tl.store(features_ptr + req_state_idx * features_stride + step, feature)
|
||||
|
||||
# Predict the acceptance probability from the feature and current coefficients.
|
||||
weight = tl.load(slope_ptr)
|
||||
bias = tl.load(intercepts_ptr + step)
|
||||
prob = tl.sigmoid(weight * feature + bias)
|
||||
tl.store(pred_ptr + req_state_idx * pred_stride + step, prob)
|
||||
tl.store(conf_ptr + batch_idx * conf_stride + step, prob)
|
||||
|
||||
|
||||
class OnlineAcceptanceEstimator:
|
||||
"""Predicts per-position acceptance, and calibrates itself while serving.
|
||||
|
||||
Lifecycle per step, driven by ``DraftModelSpeculator``:
|
||||
|
||||
1. ``step`` folds the previous step's drafts, now graded by the target,
|
||||
into the IRLS accumulators, and periodically solves for new coefficients,
|
||||
independently but identically on every rank.
|
||||
2. ``predict`` runs inside the captured draft graph, turning this step's
|
||||
draft logits into acceptance probabilities for adaptive verification.
|
||||
|
||||
Trimming starts immediately: survival is a running product of sigmoids, so it
|
||||
decreases with draft position whatever the coefficients are, and an unfitted
|
||||
estimator degrades to uniform-depth truncation rather than to anything harmful.
|
||||
"""
|
||||
|
||||
# Refit every this many steps, accumulating samples in between.
|
||||
REFIT_INTERVAL = 100
|
||||
# Newton steps are damped by n / (n + DAMPING_OBSERVATIONS), so a round
|
||||
# carrying this many observations moves a parameter half of a full step. It
|
||||
# replaces a hard minimum-sample gate: a data-poor position keeps learning,
|
||||
# just slowly, instead of freezing until it crosses a threshold. Lower values
|
||||
# track a drifting workload faster, higher ones are steadier on thin data.
|
||||
DAMPING_OBSERVATIONS = 50.0
|
||||
# Ridge on the Newton solve.
|
||||
L2 = 1e-3
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_num_reqs: int,
|
||||
num_speculative_steps: int,
|
||||
device: torch.device,
|
||||
):
|
||||
self.num_speculative_steps = num_speculative_steps
|
||||
self.device = device
|
||||
self._steps_since_refit = 0
|
||||
self._refits = 0
|
||||
|
||||
# The learned coefficients used for predicting acceptance.
|
||||
self.slope = torch.full((1,), _INIT_SLOPE, dtype=torch.float32, device=device)
|
||||
self.intercepts = torch.full(
|
||||
(num_speculative_steps,), _INIT_BIAS, dtype=torch.float32, device=device
|
||||
)
|
||||
# Holds logit(max q), which is used as the feature for the logistic.
|
||||
# Stored in stable slots keyed by persistent request-state index.
|
||||
self.features = torch.zeros(
|
||||
max_num_reqs, num_speculative_steps, dtype=torch.float32, device=device
|
||||
)
|
||||
# Holds the acceptance predictions made at draft time, computed as:
|
||||
# p = σ(ω * f + βₖ)
|
||||
# where f is feature (logit(max q)) at draft position k, ω is the learned
|
||||
# shared slope, and βₖ is the learned per-position intercept. Stored in
|
||||
# stable slots keyed by the persistent request-state index.
|
||||
self.predictions = torch.zeros(
|
||||
max_num_reqs, num_speculative_steps, dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
# Per-round Newton-IRLS statistics, cleared after each refit. For observation
|
||||
# i the design row is xᵢ = [fᵢ, eₖ], the IRLS weight is wᵢ = pᵢ(1-pᵢ), and the
|
||||
# residual is yᵢ - pᵢ, where yᵢ is the label (1 for accepted, 0 for rejected).
|
||||
# XᵀWX is then the negative Hessian of the Bernoulli log-likelihood loss and
|
||||
# Xᵀ(Y-P) is its gradient, so the Newton step for θ = [ω, β₀ … βₙ₋₁] is the
|
||||
# the solution of (XᵀWX + λI) Δθ = Xᵀ(Y-P).
|
||||
#
|
||||
# Columns [aₖ, bₖ, cₖ] are each position's contribution to the XᵀWX arrowhead:
|
||||
# ⎡ a b₀ b₁ … bₙ₋₁ ⎤
|
||||
# ⎢ b₀ c₀ ⎥
|
||||
# ⎢ b₁ c₁ ⎥
|
||||
# ⎢ ⋮ ⋱ ⎥
|
||||
# ⎣ bₙ₋₁ cₙ₋₁ ⎦
|
||||
# bₖ and cₖ are per-position, but a = Σₖ aₖ is one scalar shared by all of
|
||||
# them.
|
||||
self.info = torch.zeros(
|
||||
num_speculative_steps, 3, dtype=torch.float32, device=device
|
||||
)
|
||||
# Columns [g₀ₖ, g₁ₖ] are each position's contribution to the Xᵀ(Y-P) vector:
|
||||
# ⎡ g₀ ⎤
|
||||
# ⎢ g₁₀ ⎥
|
||||
# ⎢ g₁₁ ⎥
|
||||
# ⎢ ⋮ ⎥
|
||||
# ⎣ g₁ₙ₋₁ ⎦
|
||||
# As above, g₁ₖ is per-position and g₀ = Σₖ g₀ₖ is shared.
|
||||
self.grad = torch.zeros(
|
||||
num_speculative_steps, 2, dtype=torch.float32, device=device
|
||||
)
|
||||
# Number of graded drafts per-position in the current round, zeroed after
|
||||
# each refit. _refit_kernel uses this to damp coefficient updates by:
|
||||
# n / (n + DAMPING_OBSERVATIONS)
|
||||
self.counts = torch.zeros(
|
||||
num_speculative_steps, dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
def step(
|
||||
self,
|
||||
idx_mapping: torch.Tensor,
|
||||
num_sampled: torch.Tensor,
|
||||
num_rejected: torch.Tensor,
|
||||
) -> None:
|
||||
# Accumulate the previous step's graded drafts into the IRLS statistics.
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
_accumulate_kernel[(self.num_speculative_steps,)](
|
||||
self.info,
|
||||
self.grad,
|
||||
idx_mapping,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
self.features,
|
||||
self.features.stride(0),
|
||||
self.predictions,
|
||||
self.predictions.stride(0),
|
||||
self.counts,
|
||||
num_reqs,
|
||||
BLOCK_R=triton.next_power_of_2(max(num_reqs, 1)),
|
||||
)
|
||||
|
||||
self._steps_since_refit += 1
|
||||
if self._steps_since_refit < self.REFIT_INTERVAL:
|
||||
return
|
||||
self._steps_since_refit = 0
|
||||
|
||||
# Fit the coefficients to the accumulated statistics gathered over the
|
||||
# course of the last REFIT_INTERVAL steps.
|
||||
_refit_kernel[(1,)](
|
||||
self.slope,
|
||||
self.intercepts,
|
||||
self.info,
|
||||
self.info.stride(0),
|
||||
self.grad,
|
||||
self.grad.stride(0),
|
||||
self.counts,
|
||||
NUM_SPECULATIVE_STEPS=self.num_speculative_steps,
|
||||
L2=self.L2,
|
||||
DAMPING=self.DAMPING_OBSERVATIONS,
|
||||
BLOCK=triton.next_power_of_2(self.num_speculative_steps),
|
||||
)
|
||||
self._refits += 1
|
||||
|
||||
def predict(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
draft_step: torch.Tensor,
|
||||
confidence_probs: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
) -> None:
|
||||
num_tokens, vocab_size = logits.shape
|
||||
VOCAB_BLOCK_SIZE = 4096
|
||||
num_blocks = triton.cdiv(vocab_size, VOCAB_BLOCK_SIZE)
|
||||
local_max = torch.empty(
|
||||
num_tokens, num_blocks, dtype=torch.float32, device=self.device
|
||||
)
|
||||
local_sumexp = torch.empty_like(local_max)
|
||||
_local_max_sumexp_kernel[(num_tokens, num_blocks)](
|
||||
local_max,
|
||||
local_max.stride(0),
|
||||
local_sumexp,
|
||||
local_sumexp.stride(0),
|
||||
logits,
|
||||
logits.stride(0),
|
||||
idx_mapping,
|
||||
idx_mapping.stride(0),
|
||||
temperature,
|
||||
num_tokens,
|
||||
vocab_size,
|
||||
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
|
||||
)
|
||||
_predict_kernel[(num_tokens,)](
|
||||
self.features,
|
||||
self.features.stride(0),
|
||||
self.predictions,
|
||||
self.predictions.stride(0),
|
||||
confidence_probs,
|
||||
confidence_probs.stride(0),
|
||||
self.slope,
|
||||
self.intercepts,
|
||||
local_max,
|
||||
local_max.stride(0),
|
||||
local_sumexp,
|
||||
local_sumexp.stride(0),
|
||||
idx_mapping,
|
||||
idx_mapping.stride(0),
|
||||
draft_step,
|
||||
num_tokens,
|
||||
num_blocks,
|
||||
per_token_step=draft_step.dim() > 0,
|
||||
NUM_SPECULATIVE_STEPS=self.num_speculative_steps,
|
||||
MAX_LOG_ODDS=_MAX_LOG_ODDS,
|
||||
PADDED_VOCAB_NUM_BLOCKS=triton.next_power_of_2(num_blocks),
|
||||
)
|
||||
@@ -213,5 +213,11 @@ class DFlash2Speculator(DFlashSpeculator):
|
||||
anchor_token_ids,
|
||||
)
|
||||
self._sample_path(candidate_ids, scores, num_reqs)
|
||||
if self.enable_adaptive_verification:
|
||||
self._maybe_predict_acceptance(
|
||||
self._selector_scores[:num_reqs].flatten(0, 1),
|
||||
self.sample_idx_mapping[:num_sample],
|
||||
self.sample_col[:num_sample],
|
||||
)
|
||||
if self.draft_logits is not None:
|
||||
self._cache_draft_logits(candidate_ids, num_sample)
|
||||
|
||||
@@ -79,12 +79,7 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
self.draft_model_config.hf_config, "dspark_draft_topk", None
|
||||
)
|
||||
|
||||
self.draft_token_confidence_probs = torch.empty_like(
|
||||
self.draft_tokens, dtype=torch.float32
|
||||
)
|
||||
self.enable_adaptive_verification = (
|
||||
self.speculative_config.enable_adaptive_verification
|
||||
)
|
||||
self.use_confidence_head: bool = False
|
||||
|
||||
def load_draft_model(
|
||||
self,
|
||||
@@ -108,13 +103,14 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
dtype=self.draft_logits.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
if self.enable_adaptive_verification and model.model.confidence_head is None:
|
||||
raise ValueError(
|
||||
"Adaptive verification needs a DSpark checkpoint with a confidence "
|
||||
"head, and this one has none. Pass "
|
||||
"enable_adaptive_verification=false in the speculative config to verify"
|
||||
" a fixed number of drafts instead."
|
||||
)
|
||||
self.use_confidence_head = (
|
||||
self.enable_adaptive_verification
|
||||
and model.model.confidence_head is not None
|
||||
)
|
||||
if self.use_confidence_head:
|
||||
# The acceptance estimator is not needed when a trained confidence head
|
||||
# is available.
|
||||
self.use_acceptance_estimator = False
|
||||
return model
|
||||
|
||||
def _sample_logits(
|
||||
@@ -124,8 +120,10 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
sample_pos: torch.Tensor,
|
||||
step: int,
|
||||
) -> torch.Tensor:
|
||||
self._maybe_predict_acceptance(logits, idx_map, self._step_cols[step])
|
||||
if self.draft_logits is None:
|
||||
return self.model.map_draft_to_target(logits.argmax(dim=-1))
|
||||
draft_ids = logits.argmax(dim=-1)
|
||||
return self.model.map_draft_to_target(draft_ids)
|
||||
|
||||
# Probabilistic sampling and rejection operate in target-vocabulary
|
||||
# space. A reduced draft vocabulary is scattered into its target rows.
|
||||
@@ -184,7 +182,7 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
for i in range(n_spec):
|
||||
# Sequential stage: Markov bias from the previously sampled token.
|
||||
markov_embed = self.model.markov_embed(prev)
|
||||
if self.enable_adaptive_verification:
|
||||
if self.use_confidence_head:
|
||||
confidence_markov_embeds.append(markov_embed)
|
||||
bias = self.model.markov_bias(markov_embed)
|
||||
logits_i = base_logits[:, i] + bias
|
||||
@@ -194,7 +192,7 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
self.draft_tokens[:num_reqs, i] = draft_sampled_i
|
||||
prev = draft_sampled_i
|
||||
|
||||
if self.enable_adaptive_verification:
|
||||
if self.use_confidence_head:
|
||||
confidence = self.model.compute_confidence(
|
||||
sample_hidden,
|
||||
torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1),
|
||||
@@ -229,7 +227,7 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
|
||||
for i in range(n_spec):
|
||||
markov_embed = self.model.markov_embed(prev)
|
||||
if self.enable_adaptive_verification:
|
||||
if self.use_confidence_head:
|
||||
confidence_markov_embeds.append(markov_embed)
|
||||
logits_i = self.model.apply_markov_bias_gathered(
|
||||
markov_embed,
|
||||
@@ -243,7 +241,7 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
self.draft_tokens[:num_reqs, i] = draft_sampled_i
|
||||
prev = draft_sampled_i
|
||||
|
||||
if self.enable_adaptive_verification:
|
||||
if self.use_confidence_head:
|
||||
confidence = self.model.compute_confidence(
|
||||
sample_hidden,
|
||||
torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1),
|
||||
|
||||
@@ -33,6 +33,9 @@ from vllm.v1.worker.gpu.dp_utils import DPSyncState
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import (
|
||||
OnlineAcceptanceEstimator,
|
||||
)
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -158,6 +161,15 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
self.arange_np = np.arange(self.max_num_reqs + 1, dtype=np.int32)
|
||||
self.draft_is_prefilling = torch.zeros(self.max_num_reqs, dtype=torch.bool)
|
||||
|
||||
self.draft_token_confidence_probs = torch.empty_like(
|
||||
self.draft_tokens, dtype=torch.float32
|
||||
)
|
||||
self.enable_adaptive_verification = (
|
||||
self.speculative_config.enable_adaptive_verification
|
||||
)
|
||||
self.use_acceptance_estimator = self.enable_adaptive_verification
|
||||
self.acceptance_estimator: OnlineAcceptanceEstimator | None = None
|
||||
|
||||
self.draft_logits: torch.Tensor | None = None
|
||||
if self.speculative_config.draft_sample_method == "probabilistic":
|
||||
# Pre-temperature logits, cached from the previous decode step.
|
||||
@@ -227,6 +239,20 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
type(self.model).__name__,
|
||||
)
|
||||
|
||||
if self.use_acceptance_estimator:
|
||||
if self.use_local_argmax_reduction:
|
||||
raise ValueError(
|
||||
"Adaptive verification without a confidence head estimates "
|
||||
"per-position acceptance from the draft logits, which "
|
||||
"use_local_argmax_reduction never materializes. Disable one "
|
||||
"of them."
|
||||
)
|
||||
self.acceptance_estimator = OnlineAcceptanceEstimator(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
self.device,
|
||||
)
|
||||
|
||||
def set_eplb_state(self, eplb_state: EplbState) -> None:
|
||||
"""Inject EPLB state after construction."""
|
||||
self.eplb_state = eplb_state
|
||||
@@ -371,12 +397,6 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
"(communication: O(2*tp_size) vs O(vocab_size))."
|
||||
)
|
||||
|
||||
def _greedy_sample_draft(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
if self.use_local_argmax_reduction:
|
||||
return self.model.get_top_tokens(hidden_states)
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
return logits.argmax(dim=-1)
|
||||
|
||||
def sample_draft(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -408,8 +428,42 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
idx_mapping,
|
||||
temperature,
|
||||
)
|
||||
return sampled
|
||||
return self._greedy_sample_draft(hidden_states)
|
||||
elif self.use_local_argmax_reduction:
|
||||
return self.model.get_top_tokens(hidden_states)
|
||||
else:
|
||||
logits = self.model.compute_logits(hidden_states)
|
||||
sampled = logits.argmax(dim=-1)
|
||||
self._maybe_predict_acceptance(logits, idx_mapping, draft_step)
|
||||
return sampled
|
||||
|
||||
def _maybe_predict_acceptance(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
draft_step: torch.Tensor,
|
||||
) -> None:
|
||||
if self.acceptance_estimator is not None:
|
||||
self.acceptance_estimator.predict(
|
||||
logits,
|
||||
idx_mapping,
|
||||
draft_step,
|
||||
self.draft_token_confidence_probs,
|
||||
self.temperature,
|
||||
)
|
||||
|
||||
def observe_verification(
|
||||
self,
|
||||
idx_mapping: torch.Tensor,
|
||||
num_sampled: torch.Tensor,
|
||||
num_rejected: torch.Tensor,
|
||||
) -> None:
|
||||
"""Fold the target's verdict on the last drafts into the estimator.
|
||||
|
||||
Must run before the next `propose`, which overwrites the per-slot
|
||||
features the verdict grades.
|
||||
"""
|
||||
if self.acceptance_estimator is not None:
|
||||
self.acceptance_estimator.step(idx_mapping, num_sampled, num_rejected)
|
||||
|
||||
def prepare_watermarking(
|
||||
self, contexts: torch.Tensor, watermarking: torch.Tensor
|
||||
|
||||
Reference in New Issue
Block a user