[Bugfix][Spec Decode] Avoid reusing random draws across block verification rounds (#61161)
Signed-off-by: DimensionSTP <65501090+DimensionSTP@users.noreply.github.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Kevin H. Luu
parent
0c25849292
commit
9edcc7fa7e
@@ -557,6 +557,79 @@ def test_gumbel_drafted_rejection_sample_is_unbiased(num_speculative_steps: int)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_block_verification", [False, True])
|
||||
@pytest.mark.parametrize("probabilistic_draft", [False, True])
|
||||
def test_multiround_rejection_sample_is_unbiased(
|
||||
use_block_verification: bool, probabilistic_draft: bool
|
||||
):
|
||||
"""Uncommitted positions must receive fresh noise on later verification rounds."""
|
||||
torch.manual_seed(42)
|
||||
num_trials, num_tokens, k = 32768, 6, 2
|
||||
device = "cuda"
|
||||
target_probs = torch.tensor(
|
||||
[0.3, 0.7] if probabilistic_draft else [0.5, 0.5], device=device
|
||||
)
|
||||
draft_logits = torch.tensor(
|
||||
[0.7, 0.3] if probabilistic_draft else [1.0, 0.0], device=device
|
||||
).log()
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_probs.log(), draft_logits, k, 1.0, num_trials
|
||||
)
|
||||
seeds = inputs["seed"]
|
||||
output = torch.full((num_trials, num_tokens + k), -1, device=device)
|
||||
output[:, 0] = torch.multinomial(target_probs, num_trials, replacement=True)
|
||||
lengths = torch.ones(num_trials, dtype=torch.int64, device=device)
|
||||
accepted = 0
|
||||
while (lengths < num_tokens).any():
|
||||
active = (lengths < num_tokens).nonzero().flatten()
|
||||
n = active.numel()
|
||||
starts = lengths[active]
|
||||
local_positions = torch.arange(k + 1, device=device).repeat(n)
|
||||
positions = (starts[:, None] + torch.arange(k + 1, device=device)).flatten()
|
||||
draft_sampled = torch.zeros((n, k + 1), dtype=torch.int64, device=device)
|
||||
draft_sampled[:, 0] = output[active, starts - 1]
|
||||
logits_cache = inputs["draft_logits"] if probabilistic_draft else None
|
||||
if probabilistic_draft:
|
||||
for step in range(k):
|
||||
draft_sampled[:, step + 1] = gumbel_sample(
|
||||
draft_logits.expand(n, -1),
|
||||
active,
|
||||
inputs["temperature"],
|
||||
seeds,
|
||||
starts + step,
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache=logits_cache,
|
||||
logits_cache_col=torch.tensor(step, device=device),
|
||||
)
|
||||
sampled, counts = rejection_sample(
|
||||
target_logits=target_probs.log().expand(n * (k + 1), -1).contiguous(),
|
||||
draft_logits=logits_cache,
|
||||
draft_sampled=draft_sampled.flatten(),
|
||||
cu_num_logits=torch.arange(n + 1, dtype=torch.int32, device=device)
|
||||
* (k + 1),
|
||||
pos=positions,
|
||||
idx_mapping=active,
|
||||
expanded_idx_mapping=active.repeat_interleave(k + 1),
|
||||
expanded_local_pos=local_positions,
|
||||
temperature=inputs["temperature"],
|
||||
seed=seeds,
|
||||
num_speculative_steps=k,
|
||||
use_block_verification=use_block_verification,
|
||||
)
|
||||
for column in range(k + 1):
|
||||
valid = counts > column
|
||||
output[active[valid], starts[valid] + column] = sampled[valid, column]
|
||||
lengths[active] += counts
|
||||
accepted += (counts - 1).sum().item()
|
||||
|
||||
assert accepted > num_trials, "The test must exercise accepted draft tokens."
|
||||
for position in range(num_tokens):
|
||||
_assert_distribution_match(
|
||||
output[:, position], target_probs, device, label=f"position {position}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_steps", [1, 3])
|
||||
def test_greedy_rejection_sample(num_speculative_steps: int):
|
||||
"""Verify that greedy (temperature=0) always outputs the target argmax
|
||||
|
||||
Reference in New Issue
Block a user