[Model Runner V2][Spec Decode] Implement block verification for rejection sampling (#46781)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-06-30 08:07:24 -07:00
committed by GitHub
parent 00ebf19cca
commit db808b3961
4 changed files with 641 additions and 85 deletions
@@ -309,3 +309,106 @@ def test_placeholder_draft_token_rejected():
assert torch.equal(num_sampled, torch.ones_like(num_sampled))
recovered = sampled[:, 0]
assert (recovered >= 0).all() and (recovered < VOCAB_SIZE).all()
@pytest.mark.parametrize(
"num_speculative_steps,temperature",
[
(1, 0.6),
(3, 0.6),
(1, 1.0),
(3, 1.0),
(5, 1.0),
],
)
@pytest.mark.parametrize("has_draft_logits", [True, False])
def test_block_verification_rejection_sample(
num_speculative_steps: int, temperature: float, has_draft_logits: bool
):
"""
Verify that block verification (Sun et al.) preserves the target
distribution at every accepted position, for both the full draft-logits
case and the one-hot (no draft logits) case. Block verification changes
*which* prefix is accepted, but the marginal of every output position must
still match the target distribution p(x).
"""
torch.manual_seed(42)
device = "cuda"
num_trials = 10 * VOCAB_SIZE
target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
if temperature > 0:
target_logits_1d /= temperature
draft_logits_1d /= temperature
inputs = _build_rejection_sample_inputs(
target_logits_1d,
draft_logits_1d,
num_speculative_steps,
temperature=temperature,
num_trials=num_trials,
)
if not has_draft_logits:
inputs["draft_logits"] = None
sampled, num_sampled = rejection_sample(
**inputs,
num_speculative_steps=num_speculative_steps,
use_block_verification=True,
)
target_probs = torch.softmax(target_logits_1d, dim=0)
for pos in range(num_speculative_steps + 1):
accepted_mask = num_sampled >= pos + 1
_assert_distribution_match(
sampled[accepted_mask, pos], target_probs, device, label=f"position {pos}"
)
@pytest.mark.parametrize("num_speculative_steps", [3, 5])
def test_block_verification_accepts_at_least_as_many(num_speculative_steps: int):
"""
Block verification is designed to accept at least as long a prefix as
token verification in expectation. Verify the mean accepted length is no
worse than the standard method on the same inputs.
"""
torch.manual_seed(0)
device = "cuda"
num_trials = 20 * VOCAB_SIZE
temperature = 1.0
target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
# A draft close to the target gives block verification room to recover
# prefixes that token verification would have truncated.
draft_logits_1d = target_logits_1d + 0.5 * torch.randn(
VOCAB_SIZE, device=device, dtype=torch.float32
)
inputs = _build_rejection_sample_inputs(
target_logits_1d,
draft_logits_1d,
num_speculative_steps,
temperature=temperature,
num_trials=num_trials,
)
_, num_sampled_standard = rejection_sample(
**inputs, num_speculative_steps=num_speculative_steps
)
_, num_sampled_block = rejection_sample(
**inputs,
num_speculative_steps=num_speculative_steps,
use_block_verification=True,
)
mean_standard = (num_sampled_standard - 1).float().mean().item()
mean_block = (num_sampled_block - 1).float().mean().item()
# Allow a small slack for sampling noise.
assert mean_block >= mean_standard - 1e-2, (
f"Block verification mean accepted length {mean_block:.4f} is worse "
f"than standard {mean_standard:.4f}."
)
+4 -2
View File
@@ -67,7 +67,7 @@ SpeculativeMethod = Literal[
EagleModelTypes,
NgramGPUTypes,
]
RejectionSampleMethod = Literal["standard", "synthetic"]
RejectionSampleMethod = Literal["standard", "synthetic", "block"]
DraftSampleMethod = Literal["greedy", "probabilistic"]
@@ -201,7 +201,9 @@ class SpeculativeConfig:
"""The rejection sampling method to use. 'standard' uses probabilistic
rejection sampling (with or without cached draft logits, controlled by
draft_sample_method). 'synthetic' accepts draft tokens with a decaying
probability calibrated to synthetic_acceptance_rate."""
probability calibrated to synthetic_acceptance_rate. 'block' uses block
verification (Sun et al.), which jointly verifies the draft tokens as a
block instead of one at a time."""
synthetic_acceptance_rates: list[float] | None = None
"""Per-position *unconditional* acceptance rates for synthetic rejection
@@ -49,9 +49,10 @@ class RejectionSampler:
):
self.sampler = sampler
self.num_speculative_steps = spec_config.num_speculative_tokens
self.rejection_sample_method = spec_config.rejection_sample_method
rejection_sample_method = spec_config.rejection_sample_method
self.use_block_verification: bool = False
self.synthetic_conditional_rates: torch.Tensor | None = None
if self.rejection_sample_method == "synthetic":
if rejection_sample_method == "synthetic":
assert spec_config.synthetic_acceptance_rates is not None
self.synthetic_conditional_rates = torch.tensor(
unconditional_to_conditional_rates(
@@ -60,6 +61,8 @@ class RejectionSampler:
dtype=torch.float32,
device=device,
)
elif rejection_sample_method == "block":
self.use_block_verification = True
def _get_logprobs_tensors(
self,
@@ -129,6 +132,7 @@ class RejectionSampler:
self.num_speculative_steps,
self.synthetic_conditional_rates,
use_fp64=self.sampler.use_fp64_gumbel,
use_block_verification=self.use_block_verification,
)
logprobs_tensors = self._get_logprobs_tensors(
input_batch,
@@ -7,18 +7,18 @@ from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32
@triton.jit
def _compute_block_max_and_sumexp(logits):
block_max = tl.max(logits, axis=0)
block_sumexp = tl.where(
block_max > float("-inf"),
tl.sum(tl.exp(logits - block_max)),
def _compute_max_and_sumexp(logits):
max = tl.max(logits, axis=0)
sumexp = tl.where(
max > float("-inf"),
tl.sum(tl.exp(logits - max)),
0.0,
)
return block_max, block_sumexp
return max, sumexp
@triton.jit
def _compute_global_lse(
def _compute_global_logsumexp(
local_max_ptr,
local_max_stride,
local_sumexp_ptr,
@@ -45,7 +45,146 @@ def _compute_global_lse(
@triton.jit
def _compute_block_stats_kernel(
def _compute_global_residual_mass(
local_residual_mass_ptr,
local_residual_mass_stride,
prefix_joint_ratio,
target_logits_ptr,
target_logits_stride,
target_local_max_ptr,
target_local_max_stride,
target_local_sumexp_ptr,
target_local_sumexp_stride,
draft_sampled_ptr,
logit_idx,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
HAS_DRAFT_LOGITS: tl.constexpr,
):
if HAS_DRAFT_LOGITS:
blocks = tl.arange(0, PADDED_VOCAB_NUM_BLOCKS)
mask = blocks < vocab_num_blocks
partials = tl.load(
local_residual_mass_ptr + logit_idx * local_residual_mass_stride + blocks,
mask=mask,
other=0.0,
)
return tl.sum(partials, axis=0)
else:
# One-hot draft. M_s is a point mass at this draft token
# so the residual mass reduces to the closed form:
# p * (1 - M_b(draft_token)).
draft_token = tl.load(draft_sampled_ptr + logit_idx + 1).to(tl.int64)
target_lse = _compute_global_logsumexp(
target_local_max_ptr,
target_local_max_stride,
target_local_sumexp_ptr,
target_local_sumexp_stride,
logit_idx,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
)
target_logit = tl.load(
target_logits_ptr + logit_idx * target_logits_stride + draft_token,
).to(tl.float32)
m_b = tl.exp(target_logit - target_lse)
return prefix_joint_ratio * (1.0 - m_b)
@triton.jit
def _compute_global_target_argmax(
target_local_max_ptr,
target_local_max_stride,
target_local_argmax_ptr,
target_local_argmax_stride,
logit_idx,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
):
blocks = tl.arange(0, PADDED_VOCAB_NUM_BLOCKS)
blocks_mask = blocks < vocab_num_blocks
local_max = tl.load(
target_local_max_ptr + logit_idx * target_local_max_stride + blocks,
mask=blocks_mask,
other=float("-inf"),
)
max_block_idx = tl.argmax(local_max, axis=0)
return tl.load(
target_local_argmax_ptr + logit_idx * target_local_argmax_stride + max_block_idx
).to(tl.int64)
@triton.jit
def _compute_global_logprobs_and_logsumexp(
token,
mask,
logit_idx,
req_state_idx,
draft_step,
# [num_logits, V]
target_logits_ptr,
target_logits_stride,
# [num_logits, num_blocks]
target_local_max_ptr,
target_local_max_stride,
target_local_sumexp_ptr,
target_local_sumexp_stride,
# [max_num_reqs, num_speculative_steps, V]
draft_logits_ptr,
draft_logits_stride_0,
draft_logits_stride_1,
# [num_logits, num_blocks]
draft_local_max_ptr,
draft_local_max_stride,
draft_local_sumexp_ptr,
draft_local_sumexp_stride,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
HAS_DRAFT_LOGITS: tl.constexpr,
):
target_logit = tl.load(
target_logits_ptr + logit_idx * target_logits_stride + token,
mask=mask,
other=float("-inf"),
).to(tl.float32)
target_lse = _compute_global_logsumexp(
target_local_max_ptr,
target_local_max_stride,
target_local_sumexp_ptr,
target_local_sumexp_stride,
logit_idx,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
)
target_log_prob = target_logit - target_lse
if HAS_DRAFT_LOGITS:
draft_logit = tl.load(
draft_logits_ptr
+ req_state_idx * draft_logits_stride_0
+ draft_step * draft_logits_stride_1
+ token,
mask=mask,
other=float("-inf"),
).to(tl.float32)
draft_lse = _compute_global_logsumexp(
draft_local_max_ptr,
draft_local_max_stride,
draft_local_sumexp_ptr,
draft_local_sumexp_stride,
logit_idx,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
)
draft_log_prob = draft_logit - draft_lse
else:
# One-hot draft: q(token) = 1, log_q = 0.
draft_log_prob = 0.0
draft_lse = 0.0
return target_log_prob, draft_log_prob, target_lse, draft_lse
@triton.jit
def _compute_local_logits_stats_kernel(
# [num_logits, num_blocks]
target_local_argmax_ptr,
target_local_argmax_stride,
@@ -119,7 +258,7 @@ def _compute_block_stats_kernel(
mask=mask,
other=float("-inf"),
).to(tl.float32)
target_max, target_sumexp = _compute_block_max_and_sumexp(target_logits)
target_max, target_sumexp = _compute_max_and_sumexp(target_logits)
tl.store(
target_local_max_ptr + logit_idx * target_local_max_stride + block_idx,
target_max,
@@ -140,7 +279,7 @@ def _compute_block_stats_kernel(
mask=mask,
other=float("-inf"),
).to(tl.float32)
draft_max, draft_sumexp = _compute_block_max_and_sumexp(draft_logits)
draft_max, draft_sumexp = _compute_max_and_sumexp(draft_logits)
tl.store(
draft_local_max_ptr + logit_idx * draft_local_max_stride + block_idx,
draft_max,
@@ -153,6 +292,170 @@ def _compute_block_stats_kernel(
)
@triton.jit
def _compute_cumulative_log_p_kernel(
# [num_logits]
cumulative_log_p_ptr,
# [num_logits, V]
target_logits_ptr,
target_logits_stride,
# [num_logits, num_blocks]
target_local_max_ptr,
target_local_max_stride,
# [num_logits, num_blocks]
target_local_sumexp_ptr,
target_local_sumexp_stride,
# [num_logits]
draft_sampled_ptr,
# [max_num_reqs, num_speculative_steps, V]
draft_logits_ptr,
draft_logits_stride_0,
draft_logits_stride_1,
# [num_logits, num_blocks]
draft_local_max_ptr,
draft_local_max_stride,
# [num_logits, num_blocks]
draft_local_sumexp_ptr,
draft_local_sumexp_stride,
# [num_reqs + 1]
cu_num_logits_ptr,
# [num_reqs]
idx_mapping_ptr,
# [max_num_reqs]
temp_ptr,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
HAS_DRAFT_LOGITS: tl.constexpr,
):
req_idx = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + req_idx)
start_idx = tl.load(cu_num_logits_ptr + req_idx)
end_idx = tl.load(cu_num_logits_ptr + req_idx + 1)
num_draft_tokens = end_idx - start_idx - 1
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
if temp == 0.0:
return
log_p = 0.0
for step in range(num_draft_tokens):
logit_idx = start_idx + step
draft_token = tl.load(draft_sampled_ptr + logit_idx + 1).to(tl.int64)
target_logprob, draft_logprob, _, _ = _compute_global_logprobs_and_logsumexp(
draft_token,
True, # mask
logit_idx,
req_state_idx,
step,
target_logits_ptr,
target_logits_stride,
target_local_max_ptr,
target_local_max_stride,
target_local_sumexp_ptr,
target_local_sumexp_stride,
draft_logits_ptr,
draft_logits_stride_0,
draft_logits_stride_1,
draft_local_max_ptr,
draft_local_max_stride,
draft_local_sumexp_ptr,
draft_local_sumexp_stride,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
HAS_DRAFT_LOGITS,
)
log_p = tl.minimum(log_p + (target_logprob - draft_logprob), 0.0)
tl.store(cumulative_log_p_ptr + logit_idx, log_p)
@triton.jit
def _compute_local_residual_mass_kernel(
# [num_logits, num_blocks]
local_residual_mass_ptr,
local_residual_mass_stride,
# [num_logits]
cumulative_log_p_ptr,
# [num_logits, V]
target_logits_ptr,
target_logits_stride,
# [num_logits, num_blocks]
target_local_max_ptr,
target_local_max_stride,
# [num_logits, num_blocks]
target_local_sumexp_ptr,
target_local_sumexp_stride,
# [max_num_reqs, num_speculative_steps, V]
draft_logits_ptr,
draft_logits_stride_0,
draft_logits_stride_1,
# [num_logits, num_blocks]
draft_local_max_ptr,
draft_local_max_stride,
# [num_logits, num_blocks]
draft_local_sumexp_ptr,
draft_local_sumexp_stride,
# [num_logits]
expanded_idx_mapping_ptr,
# [num_logits]
expanded_local_pos_ptr,
# [max_num_reqs]
temp_ptr,
vocab_size,
num_speculative_steps,
vocab_num_blocks,
BLOCK_SIZE: tl.constexpr,
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
):
logit_idx = tl.program_id(0)
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
if draft_step_idx == 0 or draft_step_idx >= num_speculative_steps:
# The acceptance threshold, h, looks one position ahead and sums
# over: max(p_i * M_b(x|x_{<i}) - M_s(x|x_{<i}), 0). Tokens at the
# first and last (bonus) positions aren't needed for this computation.
return
req_state_idx = tl.load(expanded_idx_mapping_ptr + logit_idx)
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
if temp == 0.0:
return
block_idx = tl.program_id(1)
block_offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = block_offsets < vocab_size
target_log_probs, draft_log_probs, _, _ = _compute_global_logprobs_and_logsumexp(
block_offsets,
mask,
logit_idx,
req_state_idx,
draft_step_idx,
target_logits_ptr,
target_logits_stride,
target_local_max_ptr,
target_local_max_stride,
target_local_sumexp_ptr,
target_local_sumexp_stride,
draft_logits_ptr,
draft_logits_stride_0,
draft_logits_stride_1,
draft_local_max_ptr,
draft_local_max_stride,
draft_local_sumexp_ptr,
draft_local_sumexp_stride,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
True, # HAS_DRAFT_LOGITS
)
# Compute the residual mass: max(p_i * M_b(x|x_{<i}) - M_s(x|x_{<i}), 0)
p = tl.exp(tl.load(cumulative_log_p_ptr + logit_idx - 1).to(tl.float32))
m_b = tl.exp(target_log_probs)
m_s = tl.exp(draft_log_probs)
partial = tl.sum(tl.maximum(p * m_b - m_s, 0.0), axis=0)
tl.store(
local_residual_mass_ptr + logit_idx * local_residual_mass_stride + block_idx,
partial,
)
@triton.jit
def _rejection_kernel(
# [num_reqs, num_speculative_steps + 1]
@@ -200,50 +503,78 @@ def _rejection_kernel(
pos_ptr,
# [num_speculative_steps]
synthetic_conditional_rates_ptr,
# [num_logits]
cumulative_log_p_ptr,
# [num_logits, num_blocks]
local_residual_mass_ptr,
local_residual_mass_stride,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
HAS_DRAFT_LOGITS: tl.constexpr,
SYNTHETIC_MODE: tl.constexpr,
USE_BLOCK_VERIFICATION: tl.constexpr,
):
req_idx = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + req_idx).to(tl.int64)
start_idx = tl.load(cu_num_logits_ptr + req_idx).to(tl.int64)
end_idx = tl.load(cu_num_logits_ptr + req_idx + 1)
num_tokens = end_idx - start_idx
num_draft_tokens = end_idx - start_idx - 1
seed = tl.load(seed_ptr + req_state_idx)
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
is_greedy = temp == 0.0
rejected_step = 0
accepted_length = tl.zeros((), tl.int64)
target_lse = 0.0
draft_lse = 0.0
accepted = True
for i in range(num_tokens - 1):
if accepted:
logit_idx = start_idx + i
draft_sampled = tl.load(draft_sampled_ptr + logit_idx + 1).to(tl.int64)
if temp == 0.0:
for i in range(num_draft_tokens):
logit_idx = start_idx + i
draft_sampled = tl.load(draft_sampled_ptr + logit_idx + 1).to(tl.int64)
pos = tl.load(pos_ptr + logit_idx)
u = tl_rand32(seed, pos, includes_zero=False)
if USE_BLOCK_VERIFICATION and not is_greedy:
# Block verification (Sun et al., 2024): https://arxiv.org/abs/2403.10444
prefix_joint_ratio = tl.exp(
tl.load(cumulative_log_p_ptr + logit_idx).to(tl.float32)
)
if i < num_draft_tokens - 1:
residual_mass = _compute_global_residual_mass(
local_residual_mass_ptr,
local_residual_mass_stride,
prefix_joint_ratio,
target_logits_ptr,
target_logits_stride,
target_local_max_ptr,
target_local_max_stride,
target_local_sumexp_ptr,
target_local_sumexp_stride,
draft_sampled_ptr,
logit_idx + 1,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
HAS_DRAFT_LOGITS,
)
denom = residual_mass + 1.0 - prefix_joint_ratio
h = tl.where(denom > 0.0, residual_mass / denom, 1.0)
else:
h = prefix_joint_ratio
accepted_length = tl.where(u <= h, i + 1, accepted_length)
tl.store(sampled_ptr + req_idx * sampled_stride + i, draft_sampled)
elif accepted:
if is_greedy:
# Greedy sampling. Accept IFF draft matches target argmax.
# NOTE: Target argmax is stored directly so that resampling
# can be skipped upon rejection.
target_blocks = tl.arange(0, PADDED_VOCAB_NUM_BLOCKS)
target_blocks_mask = target_blocks < vocab_num_blocks
target_local_max = tl.load(
target_local_max_ptr
+ logit_idx * target_local_max_stride
+ target_blocks,
mask=target_blocks_mask,
other=float("-inf"),
target_argmax = _compute_global_target_argmax(
target_local_max_ptr,
target_local_max_stride,
target_local_argmax_ptr,
target_local_argmax_stride,
logit_idx,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
)
max_target_block_idx = tl.argmax(target_local_max, axis=0)
target_argmax = tl.load(
target_local_argmax_ptr
+ logit_idx * target_local_argmax_stride
+ max_target_block_idx
).to(tl.int64)
if SYNTHETIC_MODE:
pos = tl.load(pos_ptr + logit_idx)
u = tl_rand32(seed, pos, includes_zero=False)
rate = tl.load(synthetic_conditional_rates_ptr + i)
# -1 is used for padded draft token ids that should be rejected.
accepted &= (u < rate) & (draft_sampled >= 0)
@@ -254,57 +585,70 @@ def _rejection_kernel(
draft_sampled if accepted else target_argmax,
)
else:
# Speculative decoding (Leviathan et al., 2023): https://arxiv.org/abs/2211.17192
# -1 is used for padded draft token ids that should be rejected.
is_valid_draft = draft_sampled >= 0
# Avoid possible OOB ptr access.
draft_sampled = tl.maximum(0, draft_sampled)
target_logit = tl.load(
target_logits_ptr + logit_idx * target_logits_stride + draft_sampled
).to(tl.float32)
target_lse = _compute_global_lse(
target_local_max_ptr,
target_local_max_stride,
target_local_sumexp_ptr,
target_local_sumexp_stride,
logit_idx,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
)
target_log_prob = target_logit - target_lse
pos = tl.load(pos_ptr + logit_idx)
u = tl_rand32(seed, pos, includes_zero=False)
if HAS_DRAFT_LOGITS:
draft_logit = tl.load(
draft_logits_ptr
+ req_state_idx * draft_logits_stride_0
+ i * draft_logits_stride_1
+ draft_sampled
).to(tl.float32)
draft_lse = _compute_global_lse(
target_logprob, draft_logprob, target_lse, draft_lse = (
_compute_global_logprobs_and_logsumexp(
draft_sampled,
True, # mask
logit_idx,
req_state_idx,
i,
target_logits_ptr,
target_logits_stride,
target_local_max_ptr,
target_local_max_stride,
target_local_sumexp_ptr,
target_local_sumexp_stride,
draft_logits_ptr,
draft_logits_stride_0,
draft_logits_stride_1,
draft_local_max_ptr,
draft_local_max_stride,
draft_local_sumexp_ptr,
draft_local_sumexp_stride,
logit_idx,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
HAS_DRAFT_LOGITS,
)
draft_log_prob = draft_logit - draft_lse
else:
# One-hot draft: q(draft_token) = 1, log_q = 0.
draft_log_prob = 0
)
if SYNTHETIC_MODE:
rate = tl.load(synthetic_conditional_rates_ptr + i)
accepted &= u < rate
else:
# Probability ratio test: p(x) > u * q(x)
# Equivalent log form: log_p(x) > log(u) + log_q(x)
accepted &= target_log_prob > tl.log(u) + draft_log_prob
accepted &= target_logprob > tl.log(u) + draft_logprob
accepted &= is_valid_draft
tl.store(sampled_ptr + req_idx * sampled_stride + i, draft_sampled)
rejected_step += accepted
tl.store(rejected_steps_ptr + req_idx, rejected_step)
accepted_length += accepted
tl.store(rejected_steps_ptr + req_idx, accepted_length)
if USE_BLOCK_VERIFICATION and not is_greedy and accepted_length < num_draft_tokens:
# Compute the target and draft log exponential sums for the
# rejected token.
rejected_idx = start_idx + accepted_length
target_lse = _compute_global_logsumexp(
target_local_max_ptr,
target_local_max_stride,
target_local_sumexp_ptr,
target_local_sumexp_stride,
rejected_idx,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
)
if HAS_DRAFT_LOGITS:
draft_lse = _compute_global_logsumexp(
draft_local_max_ptr,
draft_local_max_stride,
draft_local_sumexp_ptr,
draft_local_sumexp_stride,
rejected_idx,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS,
)
tl.store(target_rejected_logsumexp_ptr + req_idx, target_lse)
tl.store(draft_rejected_logsumexp_ptr + req_idx, draft_lse)
@@ -342,10 +686,13 @@ def _resample_kernel(
seed_ptr,
# [num_logits]
pos_ptr,
# [num_logits]
cumulative_log_p_ptr,
vocab_size,
BLOCK_SIZE: tl.constexpr,
HAS_DRAFT_LOGITS: tl.constexpr,
USE_FP64: tl.constexpr,
USE_BLOCK_VERIFICATION: tl.constexpr,
):
req_idx = tl.program_id(0)
resample_idx = tl.load(rejected_step_ptr + req_idx)
@@ -386,6 +733,17 @@ def _resample_kernel(
target_lse = tl.load(target_rejected_logsumexp_ptr + req_idx)
draft_lse = tl.load(draft_rejected_logsumexp_ptr + req_idx)
target_log_probs = target_logits - target_lse
if USE_BLOCK_VERIFICATION:
# Block residual is:
# max(p_tau * M_b(x) - M_s(x), 0) / Z.
# Scale the target logprobs by log(p_tau). p_0 = 1, so skip
# shifting when nothing was accepted (tau == 0).
log_p_tau = 0.0
if resample_idx > 0:
log_p_tau = tl.load(cumulative_log_p_ptr + resample_token_idx - 1).to(
tl.float32
)
target_log_probs += log_p_tau
draft_log_probs = draft_logits - draft_lse
# Compute the residual:
# r(x) = max(p(x) - q(x), 0)
@@ -402,6 +760,11 @@ def _resample_kernel(
else:
# One-hot draft. The residual is just the target distribution with
# the rejected draft token probability zeroed out.
# NOTE: During block verification, the residual becomes:
# 0 if x == rejected_draft_token
# p_tau * M_b(x) / Z otherwise
# Therefore p_tau is a constant that cancels under normalization,
# and does not need to be applied.
rejected_draft_token = tl.load(draft_sampled_ptr + resample_token_idx + 1)
residual_logits = tl.where(
block != rejected_draft_token,
@@ -523,18 +886,20 @@ def rejection_sample(
# [num_speculative_steps]
synthetic_conditional_rates: torch.Tensor | None = None,
use_fp64: bool = False,
use_block_verification: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
num_reqs = cu_num_logits.shape[0] - 1
num_logits, vocab_size = target_logits.shape
has_draft_logits = draft_logits is not None
draft_logits_stride_0 = 0
draft_logits_stride_1 = 0
if has_draft_logits := draft_logits is not None:
draft_logits_stride_0 = draft_logits.stride(0)
draft_logits_stride_1 = draft_logits.stride(1)
# In some cases (e.g. MiMo v2.5 Pro + DFlash) the target model's
# vocab size is larger than the draft's due to padding.
vocab_size = min(vocab_size, draft_logits.size(-1))
if draft_logits is None:
# When draft_logits is None, create a dummy tensor so that Triton
# kernel signatures receive valid pointers/strides. The kernels
# will never read from it when HAS_DRAFT_LOGITS=False.
draft_logits = target_logits.new_empty(1, 1, 1)
# Compute the block-level logits stats, such as target argmax
# Compute the per-vocab-block logits stats, such as target argmax
# (for greedy requests), and target max + softmax exponential
# (for non-greedy requests).
VOCAB_BLOCK_SIZE = 8192
@@ -555,7 +920,7 @@ def rejection_sample(
draft_local_sumexp = target_logits.new_empty(
num_logits, vocab_num_blocks, dtype=torch.float32
)
_compute_block_stats_kernel[(num_logits, vocab_num_blocks)](
_compute_local_logits_stats_kernel[(num_logits, vocab_num_blocks)](
target_local_argmax,
target_local_argmax.stride(0),
target_local_max,
@@ -569,8 +934,8 @@ def rejection_sample(
target_logits,
target_logits.stride(0),
draft_logits,
draft_logits.stride(0),
draft_logits.stride(1),
draft_logits_stride_0,
draft_logits_stride_1,
expanded_idx_mapping,
expanded_local_pos,
temperature,
@@ -580,6 +945,82 @@ def rejection_sample(
HAS_DRAFT_LOGITS=has_draft_logits,
)
# Precompute the running joint ratio and residual mass for block
# verification.
if use_block_verification:
assert synthetic_conditional_rates is None, (
"Block verification is incompatible with synthetic acceptance rates."
)
# Compute the log of the running joint ratio, p_i.
# cumulative_log_p[start + i] = log(p_{i+1}), the cumulative ratio after
# the (i+1)-th draft token.
cumulative_log_p = target_logits.new_empty(num_logits, dtype=torch.float32)
_compute_cumulative_log_p_kernel[(num_reqs,)](
cumulative_log_p,
target_logits,
target_logits.stride(0),
target_local_max,
target_local_max.stride(0),
target_local_sumexp,
target_local_sumexp.stride(0),
draft_sampled,
draft_logits,
draft_logits_stride_0,
draft_logits_stride_1,
draft_local_max,
draft_local_max.stride(0),
draft_local_sumexp,
draft_local_sumexp.stride(0),
cu_num_logits,
idx_mapping,
temperature,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS=padded_vocab_num_blocks,
HAS_DRAFT_LOGITS=has_draft_logits,
num_warps=1,
)
# Compute the per-vocab-block partials of the residual mass, later reduced
# to the total by _compute_global_residual_mass. Only launched for full
# draft logits distributions. One-hot drafts used a closed-form residual
# mass instead.
if has_draft_logits:
local_residual_mass = target_logits.new_empty(
num_logits, vocab_num_blocks, dtype=torch.float32
)
_compute_local_residual_mass_kernel[(num_logits, vocab_num_blocks)](
local_residual_mass,
local_residual_mass.stride(0),
cumulative_log_p,
target_logits,
target_logits.stride(0),
target_local_max,
target_local_max.stride(0),
target_local_sumexp,
target_local_sumexp.stride(0),
draft_logits,
draft_logits_stride_0,
draft_logits_stride_1,
draft_local_max,
draft_local_max.stride(0),
draft_local_sumexp,
draft_local_sumexp.stride(0),
expanded_idx_mapping,
expanded_local_pos,
temperature,
vocab_size,
num_speculative_steps,
vocab_num_blocks,
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
PADDED_VOCAB_NUM_BLOCKS=padded_vocab_num_blocks,
)
else:
local_residual_mass = None
else:
cumulative_log_p = None
local_residual_mass = None
# Sample up until the first rejected/bonus token, and store
# the step.
sampled = draft_sampled.new_empty(
@@ -604,8 +1045,8 @@ def rejection_sample(
target_local_sumexp.stride(0),
draft_sampled,
draft_logits,
draft_logits.stride(0),
draft_logits.stride(1),
draft_logits_stride_0,
draft_logits_stride_1,
draft_local_max,
draft_local_max.stride(0),
draft_local_sumexp,
@@ -616,10 +1057,14 @@ def rejection_sample(
seed,
pos,
synthetic_conditional_rates,
cumulative_log_p,
local_residual_mass,
local_residual_mass.stride(0) if local_residual_mass is not None else 0,
vocab_num_blocks,
PADDED_VOCAB_NUM_BLOCKS=padded_vocab_num_blocks,
HAS_DRAFT_LOGITS=has_draft_logits,
SYNTHETIC_MODE=synthetic_conditional_rates is not None,
USE_BLOCK_VERIFICATION=use_block_verification,
num_warps=1,
)
@@ -644,8 +1089,8 @@ def rejection_sample(
target_logits.stride(0),
target_rejected_logsumexp,
draft_logits,
draft_logits.stride(0),
draft_logits.stride(1),
draft_logits_stride_0,
draft_logits_stride_1,
draft_rejected_logsumexp,
num_sampled,
cu_num_logits,
@@ -654,10 +1099,12 @@ def rejection_sample(
temperature,
seed,
pos,
cumulative_log_p,
vocab_size,
BLOCK_SIZE=RESAMPLE_BLOCK_SIZE,
HAS_DRAFT_LOGITS=has_draft_logits,
USE_FP64=use_fp64,
USE_BLOCK_VERIFICATION=use_block_verification,
)
# Insert the resampled tokens into the output sampled.