[Model Runner V2][Spec Decode] Implement block verification for rejection sampling (#46781)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-06-30 08:07:24 -07:00
committed by GitHub
parent 00ebf19cca
commit db808b3961
4 changed files with 641 additions and 85 deletions
@@ -309,3 +309,106 @@ def test_placeholder_draft_token_rejected():
assert torch.equal(num_sampled, torch.ones_like(num_sampled))
recovered = sampled[:, 0]
assert (recovered >= 0).all() and (recovered < VOCAB_SIZE).all()
@pytest.mark.parametrize(
"num_speculative_steps,temperature",
[
(1, 0.6),
(3, 0.6),
(1, 1.0),
(3, 1.0),
(5, 1.0),
],
)
@pytest.mark.parametrize("has_draft_logits", [True, False])
def test_block_verification_rejection_sample(
num_speculative_steps: int, temperature: float, has_draft_logits: bool
):
"""
Verify that block verification (Sun et al.) preserves the target
distribution at every accepted position, for both the full draft-logits
case and the one-hot (no draft logits) case. Block verification changes
*which* prefix is accepted, but the marginal of every output position must
still match the target distribution p(x).
"""
torch.manual_seed(42)
device = "cuda"
num_trials = 10 * VOCAB_SIZE
target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
if temperature > 0:
target_logits_1d /= temperature
draft_logits_1d /= temperature
inputs = _build_rejection_sample_inputs(
target_logits_1d,
draft_logits_1d,
num_speculative_steps,
temperature=temperature,
num_trials=num_trials,
)
if not has_draft_logits:
inputs["draft_logits"] = None
sampled, num_sampled = rejection_sample(
**inputs,
num_speculative_steps=num_speculative_steps,
use_block_verification=True,
)
target_probs = torch.softmax(target_logits_1d, dim=0)
for pos in range(num_speculative_steps + 1):
accepted_mask = num_sampled >= pos + 1
_assert_distribution_match(
sampled[accepted_mask, pos], target_probs, device, label=f"position {pos}"
)
@pytest.mark.parametrize("num_speculative_steps", [3, 5])
def test_block_verification_accepts_at_least_as_many(num_speculative_steps: int):
"""
Block verification is designed to accept at least as long a prefix as
token verification in expectation. Verify the mean accepted length is no
worse than the standard method on the same inputs.
"""
torch.manual_seed(0)
device = "cuda"
num_trials = 20 * VOCAB_SIZE
temperature = 1.0
target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
# A draft close to the target gives block verification room to recover
# prefixes that token verification would have truncated.
draft_logits_1d = target_logits_1d + 0.5 * torch.randn(
VOCAB_SIZE, device=device, dtype=torch.float32
)
inputs = _build_rejection_sample_inputs(
target_logits_1d,
draft_logits_1d,
num_speculative_steps,
temperature=temperature,
num_trials=num_trials,
)
_, num_sampled_standard = rejection_sample(
**inputs, num_speculative_steps=num_speculative_steps
)
_, num_sampled_block = rejection_sample(
**inputs,
num_speculative_steps=num_speculative_steps,
use_block_verification=True,
)
mean_standard = (num_sampled_standard - 1).float().mean().item()
mean_block = (num_sampled_block - 1).float().mean().item()
# Allow a small slack for sampling noise.
assert mean_block >= mean_standard - 1e-2, (
f"Block verification mean accepted length {mean_block:.4f} is worse "
f"than standard {mean_standard:.4f}."
)