From db808b39614384a0349378268a46a1a0feabcec3 Mon Sep 17 00:00:00 2001 From: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Date: Tue, 30 Jun 2026 10:07:24 -0500 Subject: [PATCH] [Model Runner V2][Spec Decode] Implement block verification for rejection sampling (#46781) Signed-off-by: Giancarlo Delfin --- .../test_rejection_sampler_utils.py | 103 +++ vllm/config/speculative.py | 6 +- .../gpu/spec_decode/rejection_sampler.py | 8 +- .../spec_decode/rejection_sampler_utils.py | 609 +++++++++++++++--- 4 files changed, 641 insertions(+), 85 deletions(-) diff --git a/tests/v1/spec_decode/test_rejection_sampler_utils.py b/tests/v1/spec_decode/test_rejection_sampler_utils.py index 613bd846e6..bf9bea80bf 100644 --- a/tests/v1/spec_decode/test_rejection_sampler_utils.py +++ b/tests/v1/spec_decode/test_rejection_sampler_utils.py @@ -309,3 +309,106 @@ def test_placeholder_draft_token_rejected(): assert torch.equal(num_sampled, torch.ones_like(num_sampled)) recovered = sampled[:, 0] assert (recovered >= 0).all() and (recovered < VOCAB_SIZE).all() + + +@pytest.mark.parametrize( + "num_speculative_steps,temperature", + [ + (1, 0.6), + (3, 0.6), + (1, 1.0), + (3, 1.0), + (5, 1.0), + ], +) +@pytest.mark.parametrize("has_draft_logits", [True, False]) +def test_block_verification_rejection_sample( + num_speculative_steps: int, temperature: float, has_draft_logits: bool +): + """ + Verify that block verification (Sun et al.) preserves the target + distribution at every accepted position, for both the full draft-logits + case and the one-hot (no draft logits) case. Block verification changes + *which* prefix is accepted, but the marginal of every output position must + still match the target distribution p(x). + """ + + torch.manual_seed(42) + device = "cuda" + num_trials = 10 * VOCAB_SIZE + + target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32) + draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32) + + if temperature > 0: + target_logits_1d /= temperature + draft_logits_1d /= temperature + + inputs = _build_rejection_sample_inputs( + target_logits_1d, + draft_logits_1d, + num_speculative_steps, + temperature=temperature, + num_trials=num_trials, + ) + if not has_draft_logits: + inputs["draft_logits"] = None + + sampled, num_sampled = rejection_sample( + **inputs, + num_speculative_steps=num_speculative_steps, + use_block_verification=True, + ) + + target_probs = torch.softmax(target_logits_1d, dim=0) + for pos in range(num_speculative_steps + 1): + accepted_mask = num_sampled >= pos + 1 + _assert_distribution_match( + sampled[accepted_mask, pos], target_probs, device, label=f"position {pos}" + ) + + +@pytest.mark.parametrize("num_speculative_steps", [3, 5]) +def test_block_verification_accepts_at_least_as_many(num_speculative_steps: int): + """ + Block verification is designed to accept at least as long a prefix as + token verification in expectation. Verify the mean accepted length is no + worse than the standard method on the same inputs. + """ + + torch.manual_seed(0) + device = "cuda" + num_trials = 20 * VOCAB_SIZE + temperature = 1.0 + + target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32) + # A draft close to the target gives block verification room to recover + # prefixes that token verification would have truncated. + draft_logits_1d = target_logits_1d + 0.5 * torch.randn( + VOCAB_SIZE, device=device, dtype=torch.float32 + ) + + inputs = _build_rejection_sample_inputs( + target_logits_1d, + draft_logits_1d, + num_speculative_steps, + temperature=temperature, + num_trials=num_trials, + ) + + _, num_sampled_standard = rejection_sample( + **inputs, num_speculative_steps=num_speculative_steps + ) + _, num_sampled_block = rejection_sample( + **inputs, + num_speculative_steps=num_speculative_steps, + use_block_verification=True, + ) + + mean_standard = (num_sampled_standard - 1).float().mean().item() + mean_block = (num_sampled_block - 1).float().mean().item() + # Allow a small slack for sampling noise. + assert mean_block >= mean_standard - 1e-2, ( + f"Block verification mean accepted length {mean_block:.4f} is worse " + f"than standard {mean_standard:.4f}." + ) diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index de505e122c..52464753ef 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -67,7 +67,7 @@ SpeculativeMethod = Literal[ EagleModelTypes, NgramGPUTypes, ] -RejectionSampleMethod = Literal["standard", "synthetic"] +RejectionSampleMethod = Literal["standard", "synthetic", "block"] DraftSampleMethod = Literal["greedy", "probabilistic"] @@ -201,7 +201,9 @@ class SpeculativeConfig: """The rejection sampling method to use. 'standard' uses probabilistic rejection sampling (with or without cached draft logits, controlled by draft_sample_method). 'synthetic' accepts draft tokens with a decaying - probability calibrated to synthetic_acceptance_rate.""" + probability calibrated to synthetic_acceptance_rate. 'block' uses block + verification (Sun et al.), which jointly verifies the draft tokens as a + block instead of one at a time.""" synthetic_acceptance_rates: list[float] | None = None """Per-position *unconditional* acceptance rates for synthetic rejection diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 3868604d3a..c56252d55d 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -49,9 +49,10 @@ class RejectionSampler: ): self.sampler = sampler self.num_speculative_steps = spec_config.num_speculative_tokens - self.rejection_sample_method = spec_config.rejection_sample_method + rejection_sample_method = spec_config.rejection_sample_method + self.use_block_verification: bool = False self.synthetic_conditional_rates: torch.Tensor | None = None - if self.rejection_sample_method == "synthetic": + if rejection_sample_method == "synthetic": assert spec_config.synthetic_acceptance_rates is not None self.synthetic_conditional_rates = torch.tensor( unconditional_to_conditional_rates( @@ -60,6 +61,8 @@ class RejectionSampler: dtype=torch.float32, device=device, ) + elif rejection_sample_method == "block": + self.use_block_verification = True def _get_logprobs_tensors( self, @@ -129,6 +132,7 @@ class RejectionSampler: self.num_speculative_steps, self.synthetic_conditional_rates, use_fp64=self.sampler.use_fp64_gumbel, + use_block_verification=self.use_block_verification, ) logprobs_tensors = self._get_logprobs_tensors( input_batch, diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py index 070270c832..eb23717e43 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py @@ -7,18 +7,18 @@ from vllm.v1.worker.gpu.sample.gumbel import gumbel_block_argmax, tl_rand32 @triton.jit -def _compute_block_max_and_sumexp(logits): - block_max = tl.max(logits, axis=0) - block_sumexp = tl.where( - block_max > float("-inf"), - tl.sum(tl.exp(logits - block_max)), +def _compute_max_and_sumexp(logits): + max = tl.max(logits, axis=0) + sumexp = tl.where( + max > float("-inf"), + tl.sum(tl.exp(logits - max)), 0.0, ) - return block_max, block_sumexp + return max, sumexp @triton.jit -def _compute_global_lse( +def _compute_global_logsumexp( local_max_ptr, local_max_stride, local_sumexp_ptr, @@ -45,7 +45,146 @@ def _compute_global_lse( @triton.jit -def _compute_block_stats_kernel( +def _compute_global_residual_mass( + local_residual_mass_ptr, + local_residual_mass_stride, + prefix_joint_ratio, + target_logits_ptr, + target_logits_stride, + target_local_max_ptr, + target_local_max_stride, + target_local_sumexp_ptr, + target_local_sumexp_stride, + draft_sampled_ptr, + logit_idx, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS: tl.constexpr, + HAS_DRAFT_LOGITS: tl.constexpr, +): + if HAS_DRAFT_LOGITS: + blocks = tl.arange(0, PADDED_VOCAB_NUM_BLOCKS) + mask = blocks < vocab_num_blocks + partials = tl.load( + local_residual_mass_ptr + logit_idx * local_residual_mass_stride + blocks, + mask=mask, + other=0.0, + ) + return tl.sum(partials, axis=0) + else: + # One-hot draft. M_s is a point mass at this draft token + # so the residual mass reduces to the closed form: + # p * (1 - M_b(draft_token)). + draft_token = tl.load(draft_sampled_ptr + logit_idx + 1).to(tl.int64) + target_lse = _compute_global_logsumexp( + target_local_max_ptr, + target_local_max_stride, + target_local_sumexp_ptr, + target_local_sumexp_stride, + logit_idx, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS, + ) + target_logit = tl.load( + target_logits_ptr + logit_idx * target_logits_stride + draft_token, + ).to(tl.float32) + m_b = tl.exp(target_logit - target_lse) + return prefix_joint_ratio * (1.0 - m_b) + + +@triton.jit +def _compute_global_target_argmax( + target_local_max_ptr, + target_local_max_stride, + target_local_argmax_ptr, + target_local_argmax_stride, + logit_idx, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS: tl.constexpr, +): + blocks = tl.arange(0, PADDED_VOCAB_NUM_BLOCKS) + blocks_mask = blocks < vocab_num_blocks + local_max = tl.load( + target_local_max_ptr + logit_idx * target_local_max_stride + blocks, + mask=blocks_mask, + other=float("-inf"), + ) + max_block_idx = tl.argmax(local_max, axis=0) + return tl.load( + target_local_argmax_ptr + logit_idx * target_local_argmax_stride + max_block_idx + ).to(tl.int64) + + +@triton.jit +def _compute_global_logprobs_and_logsumexp( + token, + mask, + logit_idx, + req_state_idx, + draft_step, + # [num_logits, V] + target_logits_ptr, + target_logits_stride, + # [num_logits, num_blocks] + target_local_max_ptr, + target_local_max_stride, + target_local_sumexp_ptr, + target_local_sumexp_stride, + # [max_num_reqs, num_speculative_steps, V] + draft_logits_ptr, + draft_logits_stride_0, + draft_logits_stride_1, + # [num_logits, num_blocks] + draft_local_max_ptr, + draft_local_max_stride, + draft_local_sumexp_ptr, + draft_local_sumexp_stride, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS: tl.constexpr, + HAS_DRAFT_LOGITS: tl.constexpr, +): + target_logit = tl.load( + target_logits_ptr + logit_idx * target_logits_stride + token, + mask=mask, + other=float("-inf"), + ).to(tl.float32) + target_lse = _compute_global_logsumexp( + target_local_max_ptr, + target_local_max_stride, + target_local_sumexp_ptr, + target_local_sumexp_stride, + logit_idx, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS, + ) + target_log_prob = target_logit - target_lse + if HAS_DRAFT_LOGITS: + draft_logit = tl.load( + draft_logits_ptr + + req_state_idx * draft_logits_stride_0 + + draft_step * draft_logits_stride_1 + + token, + mask=mask, + other=float("-inf"), + ).to(tl.float32) + draft_lse = _compute_global_logsumexp( + draft_local_max_ptr, + draft_local_max_stride, + draft_local_sumexp_ptr, + draft_local_sumexp_stride, + logit_idx, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS, + ) + draft_log_prob = draft_logit - draft_lse + else: + # One-hot draft: q(token) = 1, log_q = 0. + draft_log_prob = 0.0 + draft_lse = 0.0 + return target_log_prob, draft_log_prob, target_lse, draft_lse + + +@triton.jit +def _compute_local_logits_stats_kernel( # [num_logits, num_blocks] target_local_argmax_ptr, target_local_argmax_stride, @@ -119,7 +258,7 @@ def _compute_block_stats_kernel( mask=mask, other=float("-inf"), ).to(tl.float32) - target_max, target_sumexp = _compute_block_max_and_sumexp(target_logits) + target_max, target_sumexp = _compute_max_and_sumexp(target_logits) tl.store( target_local_max_ptr + logit_idx * target_local_max_stride + block_idx, target_max, @@ -140,7 +279,7 @@ def _compute_block_stats_kernel( mask=mask, other=float("-inf"), ).to(tl.float32) - draft_max, draft_sumexp = _compute_block_max_and_sumexp(draft_logits) + draft_max, draft_sumexp = _compute_max_and_sumexp(draft_logits) tl.store( draft_local_max_ptr + logit_idx * draft_local_max_stride + block_idx, draft_max, @@ -153,6 +292,170 @@ def _compute_block_stats_kernel( ) +@triton.jit +def _compute_cumulative_log_p_kernel( + # [num_logits] + cumulative_log_p_ptr, + # [num_logits, V] + target_logits_ptr, + target_logits_stride, + # [num_logits, num_blocks] + target_local_max_ptr, + target_local_max_stride, + # [num_logits, num_blocks] + target_local_sumexp_ptr, + target_local_sumexp_stride, + # [num_logits] + draft_sampled_ptr, + # [max_num_reqs, num_speculative_steps, V] + draft_logits_ptr, + draft_logits_stride_0, + draft_logits_stride_1, + # [num_logits, num_blocks] + draft_local_max_ptr, + draft_local_max_stride, + # [num_logits, num_blocks] + draft_local_sumexp_ptr, + draft_local_sumexp_stride, + # [num_reqs + 1] + cu_num_logits_ptr, + # [num_reqs] + idx_mapping_ptr, + # [max_num_reqs] + temp_ptr, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS: tl.constexpr, + HAS_DRAFT_LOGITS: tl.constexpr, +): + req_idx = tl.program_id(0) + req_state_idx = tl.load(idx_mapping_ptr + req_idx) + start_idx = tl.load(cu_num_logits_ptr + req_idx) + end_idx = tl.load(cu_num_logits_ptr + req_idx + 1) + num_draft_tokens = end_idx - start_idx - 1 + temp = tl.load(temp_ptr + req_state_idx).to(tl.float32) + if temp == 0.0: + return + + log_p = 0.0 + for step in range(num_draft_tokens): + logit_idx = start_idx + step + draft_token = tl.load(draft_sampled_ptr + logit_idx + 1).to(tl.int64) + target_logprob, draft_logprob, _, _ = _compute_global_logprobs_and_logsumexp( + draft_token, + True, # mask + logit_idx, + req_state_idx, + step, + target_logits_ptr, + target_logits_stride, + target_local_max_ptr, + target_local_max_stride, + target_local_sumexp_ptr, + target_local_sumexp_stride, + draft_logits_ptr, + draft_logits_stride_0, + draft_logits_stride_1, + draft_local_max_ptr, + draft_local_max_stride, + draft_local_sumexp_ptr, + draft_local_sumexp_stride, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS, + HAS_DRAFT_LOGITS, + ) + log_p = tl.minimum(log_p + (target_logprob - draft_logprob), 0.0) + tl.store(cumulative_log_p_ptr + logit_idx, log_p) + + +@triton.jit +def _compute_local_residual_mass_kernel( + # [num_logits, num_blocks] + local_residual_mass_ptr, + local_residual_mass_stride, + # [num_logits] + cumulative_log_p_ptr, + # [num_logits, V] + target_logits_ptr, + target_logits_stride, + # [num_logits, num_blocks] + target_local_max_ptr, + target_local_max_stride, + # [num_logits, num_blocks] + target_local_sumexp_ptr, + target_local_sumexp_stride, + # [max_num_reqs, num_speculative_steps, V] + draft_logits_ptr, + draft_logits_stride_0, + draft_logits_stride_1, + # [num_logits, num_blocks] + draft_local_max_ptr, + draft_local_max_stride, + # [num_logits, num_blocks] + draft_local_sumexp_ptr, + draft_local_sumexp_stride, + # [num_logits] + expanded_idx_mapping_ptr, + # [num_logits] + expanded_local_pos_ptr, + # [max_num_reqs] + temp_ptr, + vocab_size, + num_speculative_steps, + vocab_num_blocks, + BLOCK_SIZE: tl.constexpr, + PADDED_VOCAB_NUM_BLOCKS: tl.constexpr, +): + logit_idx = tl.program_id(0) + draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx) + if draft_step_idx == 0 or draft_step_idx >= num_speculative_steps: + # The acceptance threshold, h, looks one position ahead and sums + # over: max(p_i * M_b(x|x_{ 0.0, residual_mass / denom, 1.0) + else: + h = prefix_joint_ratio + accepted_length = tl.where(u <= h, i + 1, accepted_length) + tl.store(sampled_ptr + req_idx * sampled_stride + i, draft_sampled) + elif accepted: + if is_greedy: # Greedy sampling. Accept IFF draft matches target argmax. # NOTE: Target argmax is stored directly so that resampling # can be skipped upon rejection. - target_blocks = tl.arange(0, PADDED_VOCAB_NUM_BLOCKS) - target_blocks_mask = target_blocks < vocab_num_blocks - target_local_max = tl.load( - target_local_max_ptr - + logit_idx * target_local_max_stride - + target_blocks, - mask=target_blocks_mask, - other=float("-inf"), + target_argmax = _compute_global_target_argmax( + target_local_max_ptr, + target_local_max_stride, + target_local_argmax_ptr, + target_local_argmax_stride, + logit_idx, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS, ) - max_target_block_idx = tl.argmax(target_local_max, axis=0) - target_argmax = tl.load( - target_local_argmax_ptr - + logit_idx * target_local_argmax_stride - + max_target_block_idx - ).to(tl.int64) - if SYNTHETIC_MODE: - pos = tl.load(pos_ptr + logit_idx) - u = tl_rand32(seed, pos, includes_zero=False) rate = tl.load(synthetic_conditional_rates_ptr + i) # -1 is used for padded draft token ids that should be rejected. accepted &= (u < rate) & (draft_sampled >= 0) @@ -254,57 +585,70 @@ def _rejection_kernel( draft_sampled if accepted else target_argmax, ) else: + # Speculative decoding (Leviathan et al., 2023): https://arxiv.org/abs/2211.17192 # -1 is used for padded draft token ids that should be rejected. is_valid_draft = draft_sampled >= 0 # Avoid possible OOB ptr access. draft_sampled = tl.maximum(0, draft_sampled) - target_logit = tl.load( - target_logits_ptr + logit_idx * target_logits_stride + draft_sampled - ).to(tl.float32) - target_lse = _compute_global_lse( - target_local_max_ptr, - target_local_max_stride, - target_local_sumexp_ptr, - target_local_sumexp_stride, - logit_idx, - vocab_num_blocks, - PADDED_VOCAB_NUM_BLOCKS, - ) - target_log_prob = target_logit - target_lse - pos = tl.load(pos_ptr + logit_idx) - u = tl_rand32(seed, pos, includes_zero=False) - if HAS_DRAFT_LOGITS: - draft_logit = tl.load( - draft_logits_ptr - + req_state_idx * draft_logits_stride_0 - + i * draft_logits_stride_1 - + draft_sampled - ).to(tl.float32) - draft_lse = _compute_global_lse( + target_logprob, draft_logprob, target_lse, draft_lse = ( + _compute_global_logprobs_and_logsumexp( + draft_sampled, + True, # mask + logit_idx, + req_state_idx, + i, + target_logits_ptr, + target_logits_stride, + target_local_max_ptr, + target_local_max_stride, + target_local_sumexp_ptr, + target_local_sumexp_stride, + draft_logits_ptr, + draft_logits_stride_0, + draft_logits_stride_1, draft_local_max_ptr, draft_local_max_stride, draft_local_sumexp_ptr, draft_local_sumexp_stride, - logit_idx, vocab_num_blocks, PADDED_VOCAB_NUM_BLOCKS, + HAS_DRAFT_LOGITS, ) - draft_log_prob = draft_logit - draft_lse - else: - # One-hot draft: q(draft_token) = 1, log_q = 0. - draft_log_prob = 0 - + ) if SYNTHETIC_MODE: rate = tl.load(synthetic_conditional_rates_ptr + i) accepted &= u < rate else: # Probability ratio test: p(x) > u * q(x) # Equivalent log form: log_p(x) > log(u) + log_q(x) - accepted &= target_log_prob > tl.log(u) + draft_log_prob + accepted &= target_logprob > tl.log(u) + draft_logprob accepted &= is_valid_draft tl.store(sampled_ptr + req_idx * sampled_stride + i, draft_sampled) - rejected_step += accepted - tl.store(rejected_steps_ptr + req_idx, rejected_step) + accepted_length += accepted + tl.store(rejected_steps_ptr + req_idx, accepted_length) + if USE_BLOCK_VERIFICATION and not is_greedy and accepted_length < num_draft_tokens: + # Compute the target and draft log exponential sums for the + # rejected token. + rejected_idx = start_idx + accepted_length + target_lse = _compute_global_logsumexp( + target_local_max_ptr, + target_local_max_stride, + target_local_sumexp_ptr, + target_local_sumexp_stride, + rejected_idx, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS, + ) + if HAS_DRAFT_LOGITS: + draft_lse = _compute_global_logsumexp( + draft_local_max_ptr, + draft_local_max_stride, + draft_local_sumexp_ptr, + draft_local_sumexp_stride, + rejected_idx, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS, + ) tl.store(target_rejected_logsumexp_ptr + req_idx, target_lse) tl.store(draft_rejected_logsumexp_ptr + req_idx, draft_lse) @@ -342,10 +686,13 @@ def _resample_kernel( seed_ptr, # [num_logits] pos_ptr, + # [num_logits] + cumulative_log_p_ptr, vocab_size, BLOCK_SIZE: tl.constexpr, HAS_DRAFT_LOGITS: tl.constexpr, USE_FP64: tl.constexpr, + USE_BLOCK_VERIFICATION: tl.constexpr, ): req_idx = tl.program_id(0) resample_idx = tl.load(rejected_step_ptr + req_idx) @@ -386,6 +733,17 @@ def _resample_kernel( target_lse = tl.load(target_rejected_logsumexp_ptr + req_idx) draft_lse = tl.load(draft_rejected_logsumexp_ptr + req_idx) target_log_probs = target_logits - target_lse + if USE_BLOCK_VERIFICATION: + # Block residual is: + # max(p_tau * M_b(x) - M_s(x), 0) / Z. + # Scale the target logprobs by log(p_tau). p_0 = 1, so skip + # shifting when nothing was accepted (tau == 0). + log_p_tau = 0.0 + if resample_idx > 0: + log_p_tau = tl.load(cumulative_log_p_ptr + resample_token_idx - 1).to( + tl.float32 + ) + target_log_probs += log_p_tau draft_log_probs = draft_logits - draft_lse # Compute the residual: # r(x) = max(p(x) - q(x), 0) @@ -402,6 +760,11 @@ def _resample_kernel( else: # One-hot draft. The residual is just the target distribution with # the rejected draft token probability zeroed out. + # NOTE: During block verification, the residual becomes: + # 0 if x == rejected_draft_token + # p_tau * M_b(x) / Z otherwise + # Therefore p_tau is a constant that cancels under normalization, + # and does not need to be applied. rejected_draft_token = tl.load(draft_sampled_ptr + resample_token_idx + 1) residual_logits = tl.where( block != rejected_draft_token, @@ -523,18 +886,20 @@ def rejection_sample( # [num_speculative_steps] synthetic_conditional_rates: torch.Tensor | None = None, use_fp64: bool = False, + use_block_verification: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: num_reqs = cu_num_logits.shape[0] - 1 num_logits, vocab_size = target_logits.shape - has_draft_logits = draft_logits is not None + draft_logits_stride_0 = 0 + draft_logits_stride_1 = 0 + if has_draft_logits := draft_logits is not None: + draft_logits_stride_0 = draft_logits.stride(0) + draft_logits_stride_1 = draft_logits.stride(1) + # In some cases (e.g. MiMo v2.5 Pro + DFlash) the target model's + # vocab size is larger than the draft's due to padding. + vocab_size = min(vocab_size, draft_logits.size(-1)) - if draft_logits is None: - # When draft_logits is None, create a dummy tensor so that Triton - # kernel signatures receive valid pointers/strides. The kernels - # will never read from it when HAS_DRAFT_LOGITS=False. - draft_logits = target_logits.new_empty(1, 1, 1) - - # Compute the block-level logits stats, such as target argmax + # Compute the per-vocab-block logits stats, such as target argmax # (for greedy requests), and target max + softmax exponential # (for non-greedy requests). VOCAB_BLOCK_SIZE = 8192 @@ -555,7 +920,7 @@ def rejection_sample( draft_local_sumexp = target_logits.new_empty( num_logits, vocab_num_blocks, dtype=torch.float32 ) - _compute_block_stats_kernel[(num_logits, vocab_num_blocks)]( + _compute_local_logits_stats_kernel[(num_logits, vocab_num_blocks)]( target_local_argmax, target_local_argmax.stride(0), target_local_max, @@ -569,8 +934,8 @@ def rejection_sample( target_logits, target_logits.stride(0), draft_logits, - draft_logits.stride(0), - draft_logits.stride(1), + draft_logits_stride_0, + draft_logits_stride_1, expanded_idx_mapping, expanded_local_pos, temperature, @@ -580,6 +945,82 @@ def rejection_sample( HAS_DRAFT_LOGITS=has_draft_logits, ) + # Precompute the running joint ratio and residual mass for block + # verification. + if use_block_verification: + assert synthetic_conditional_rates is None, ( + "Block verification is incompatible with synthetic acceptance rates." + ) + + # Compute the log of the running joint ratio, p_i. + # cumulative_log_p[start + i] = log(p_{i+1}), the cumulative ratio after + # the (i+1)-th draft token. + cumulative_log_p = target_logits.new_empty(num_logits, dtype=torch.float32) + _compute_cumulative_log_p_kernel[(num_reqs,)]( + cumulative_log_p, + target_logits, + target_logits.stride(0), + target_local_max, + target_local_max.stride(0), + target_local_sumexp, + target_local_sumexp.stride(0), + draft_sampled, + draft_logits, + draft_logits_stride_0, + draft_logits_stride_1, + draft_local_max, + draft_local_max.stride(0), + draft_local_sumexp, + draft_local_sumexp.stride(0), + cu_num_logits, + idx_mapping, + temperature, + vocab_num_blocks, + PADDED_VOCAB_NUM_BLOCKS=padded_vocab_num_blocks, + HAS_DRAFT_LOGITS=has_draft_logits, + num_warps=1, + ) + + # Compute the per-vocab-block partials of the residual mass, later reduced + # to the total by _compute_global_residual_mass. Only launched for full + # draft logits distributions. One-hot drafts used a closed-form residual + # mass instead. + if has_draft_logits: + local_residual_mass = target_logits.new_empty( + num_logits, vocab_num_blocks, dtype=torch.float32 + ) + _compute_local_residual_mass_kernel[(num_logits, vocab_num_blocks)]( + local_residual_mass, + local_residual_mass.stride(0), + cumulative_log_p, + target_logits, + target_logits.stride(0), + target_local_max, + target_local_max.stride(0), + target_local_sumexp, + target_local_sumexp.stride(0), + draft_logits, + draft_logits_stride_0, + draft_logits_stride_1, + draft_local_max, + draft_local_max.stride(0), + draft_local_sumexp, + draft_local_sumexp.stride(0), + expanded_idx_mapping, + expanded_local_pos, + temperature, + vocab_size, + num_speculative_steps, + vocab_num_blocks, + BLOCK_SIZE=VOCAB_BLOCK_SIZE, + PADDED_VOCAB_NUM_BLOCKS=padded_vocab_num_blocks, + ) + else: + local_residual_mass = None + else: + cumulative_log_p = None + local_residual_mass = None + # Sample up until the first rejected/bonus token, and store # the step. sampled = draft_sampled.new_empty( @@ -604,8 +1045,8 @@ def rejection_sample( target_local_sumexp.stride(0), draft_sampled, draft_logits, - draft_logits.stride(0), - draft_logits.stride(1), + draft_logits_stride_0, + draft_logits_stride_1, draft_local_max, draft_local_max.stride(0), draft_local_sumexp, @@ -616,10 +1057,14 @@ def rejection_sample( seed, pos, synthetic_conditional_rates, + cumulative_log_p, + local_residual_mass, + local_residual_mass.stride(0) if local_residual_mass is not None else 0, vocab_num_blocks, PADDED_VOCAB_NUM_BLOCKS=padded_vocab_num_blocks, HAS_DRAFT_LOGITS=has_draft_logits, SYNTHETIC_MODE=synthetic_conditional_rates is not None, + USE_BLOCK_VERIFICATION=use_block_verification, num_warps=1, ) @@ -644,8 +1089,8 @@ def rejection_sample( target_logits.stride(0), target_rejected_logsumexp, draft_logits, - draft_logits.stride(0), - draft_logits.stride(1), + draft_logits_stride_0, + draft_logits_stride_1, draft_rejected_logsumexp, num_sampled, cu_num_logits, @@ -654,10 +1099,12 @@ def rejection_sample( temperature, seed, pos, + cumulative_log_p, vocab_size, BLOCK_SIZE=RESAMPLE_BLOCK_SIZE, HAS_DRAFT_LOGITS=has_draft_logits, USE_FP64=use_fp64, + USE_BLOCK_VERIFICATION=use_block_verification, ) # Insert the resampled tokens into the output sampled.