[Bugfix][Spec Decode] Avoid reusing random draws across block verification rounds (#61161)

Signed-off-by: DimensionSTP <65501090+DimensionSTP@users.noreply.github.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
DimensionSTP
2026-10-15 14:46:38 -07:00
committed by GitHub
co-authored by Kevin H. Luu
parent 0c25849292
commit 9edcc7fa7e
8 changed files with 192 additions and 5 deletions
@@ -557,6 +557,79 @@ def test_gumbel_drafted_rejection_sample_is_unbiased(num_speculative_steps: int)
)
@pytest.mark.parametrize("use_block_verification", [False, True])
@pytest.mark.parametrize("probabilistic_draft", [False, True])
def test_multiround_rejection_sample_is_unbiased(
use_block_verification: bool, probabilistic_draft: bool
):
"""Uncommitted positions must receive fresh noise on later verification rounds."""
torch.manual_seed(42)
num_trials, num_tokens, k = 32768, 6, 2
device = "cuda"
target_probs = torch.tensor(
[0.3, 0.7] if probabilistic_draft else [0.5, 0.5], device=device
)
draft_logits = torch.tensor(
[0.7, 0.3] if probabilistic_draft else [1.0, 0.0], device=device
).log()
inputs = _build_rejection_sample_inputs(
target_probs.log(), draft_logits, k, 1.0, num_trials
)
seeds = inputs["seed"]
output = torch.full((num_trials, num_tokens + k), -1, device=device)
output[:, 0] = torch.multinomial(target_probs, num_trials, replacement=True)
lengths = torch.ones(num_trials, dtype=torch.int64, device=device)
accepted = 0
while (lengths < num_tokens).any():
active = (lengths < num_tokens).nonzero().flatten()
n = active.numel()
starts = lengths[active]
local_positions = torch.arange(k + 1, device=device).repeat(n)
positions = (starts[:, None] + torch.arange(k + 1, device=device)).flatten()
draft_sampled = torch.zeros((n, k + 1), dtype=torch.int64, device=device)
draft_sampled[:, 0] = output[active, starts - 1]
logits_cache = inputs["draft_logits"] if probabilistic_draft else None
if probabilistic_draft:
for step in range(k):
draft_sampled[:, step + 1] = gumbel_sample(
draft_logits.expand(n, -1),
active,
inputs["temperature"],
seeds,
starts + step,
apply_temperature=True,
is_drafting=True,
logits_cache=logits_cache,
logits_cache_col=torch.tensor(step, device=device),
)
sampled, counts = rejection_sample(
target_logits=target_probs.log().expand(n * (k + 1), -1).contiguous(),
draft_logits=logits_cache,
draft_sampled=draft_sampled.flatten(),
cu_num_logits=torch.arange(n + 1, dtype=torch.int32, device=device)
* (k + 1),
pos=positions,
idx_mapping=active,
expanded_idx_mapping=active.repeat_interleave(k + 1),
expanded_local_pos=local_positions,
temperature=inputs["temperature"],
seed=seeds,
num_speculative_steps=k,
use_block_verification=use_block_verification,
)
for column in range(k + 1):
valid = counts > column
output[active[valid], starts[valid] + column] = sampled[valid, column]
lengths[active] += counts
accepted += (counts - 1).sum().item()
assert accepted > num_trials, "The test must exercise accepted draft tokens."
for position in range(num_tokens):
_assert_distribution_match(
output[:, position], target_probs, device, label=f"position {position}"
)
@pytest.mark.parametrize("num_speculative_steps", [1, 3])
def test_greedy_rejection_sample(num_speculative_steps: int):
"""Verify that greedy (temperature=0) always outputs the target argmax