[Model Runner V2] Fix -1 placeholder draft token ids in rejection sam… (#50939)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Giancarlo Delfin
2026-08-06 16:32:00 -07:00
committed by GitHub
co-authored by Nick Hill
parent 9bca7d840d
commit 27930df9c2
2 changed files with 229 additions and 74 deletions
@@ -367,6 +367,128 @@ def test_placeholder_draft_token_rejected():
assert (recovered >= 0).all() and (recovered < VOCAB_SIZE).all()
@pytest.mark.parametrize("has_draft_logits", [True, False])
@pytest.mark.parametrize("num_placeholders", [1, 2])
def test_block_verification_placeholder_truncates_block(
has_draft_logits: bool, num_placeholders: int
):
"""Trailing placeholder (-1) draft ids end the block early. The last real
draft token must then be verified against the closed-form threshold rather
than a residual mass derived from the placeholder position, whose logits
describe a token that was never drafted.
"""
torch.manual_seed(42)
device = "cuda"
num_trials = 20 * VOCAB_SIZE
K = 3
temperature = 1.0
target_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device)
inputs = _build_rejection_sample_inputs(
target_logits_1d,
draft_logits_1d,
K,
temperature=temperature,
num_trials=num_trials,
)
if not has_draft_logits:
inputs["draft_logits"] = None
inputs["draft_sampled"].view(num_trials, K + 1)[:, K + 1 - num_placeholders :] = -1
sampled, num_sampled = rejection_sample(
**inputs, num_speculative_steps=K, use_block_verification=True
)
num_real = K - num_placeholders
assert (num_sampled <= num_real + 1).all(), (
"Accepted a draft token at or after a placeholder."
)
target_probs = torch.softmax(target_logits_1d, dim=0)
for pos in range(num_real + 1):
accepted_mask = num_sampled >= pos + 1
_assert_distribution_match(
sampled[accepted_mask, pos], target_probs, device, label=f"position {pos}"
)
def test_greedy_placeholder_emits_target_argmax():
"""Greedy sampling skips resampling and relies on the rejection kernel
storing the target argmax at the rejected position. A placeholder must not
bypass that store, or the output slot is returned uninitialized.
"""
torch.manual_seed(0)
device = "cuda"
num_trials = 512
K = 3
target_logits_1d = torch.randn(VOCAB_SIZE, device=device)
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device)
inputs = _build_rejection_sample_inputs(
target_logits_1d,
draft_logits_1d,
K,
temperature=0.0,
num_trials=num_trials,
)
target_argmax = int(target_logits_1d.argmax())
draft_sampled = inputs["draft_sampled"].view(num_trials, K + 1)
# Accept the first draft so that the rejection lands on the placeholder.
draft_sampled[:, 1] = target_argmax
draft_sampled[:, 2:] = -1
sampled, num_sampled = rejection_sample(**inputs, num_speculative_steps=K)
assert torch.equal(num_sampled, torch.full_like(num_sampled, 2))
steps = torch.arange(K + 1, device=device).unsqueeze(0)
emitted = sampled[steps < num_sampled.unsqueeze(1)]
assert (emitted == target_argmax).all(), (
"Greedy sampling emitted a token that is not the target argmax"
)
@pytest.mark.parametrize("use_block_verification", [False, True])
def test_placeholder_blocks_later_draft_tokens(use_block_verification: bool):
"""A placeholder is not necessarily the final draft. Nothing at or after
one may be accepted, even when a valid draft follows it.
"""
torch.manual_seed(0)
device = "cuda"
num_trials = 4 * VOCAB_SIZE
K = 3
temperature = 1.0
# An identical draft is accepted with probability ~1, so any token after
# the placeholder would be accepted unless it is explicitly blocked. The
# draft logits are stored pre-temperature, so pass the unscaled base.
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device)
target_logits_1d = draft_logits_1d / temperature
inputs = _build_rejection_sample_inputs(
target_logits_1d,
draft_logits_1d,
K,
temperature=temperature,
num_trials=num_trials,
)
inputs["draft_sampled"].view(num_trials, K + 1)[:, 2] = -1
_, num_sampled = rejection_sample(
**inputs,
num_speculative_steps=K,
use_block_verification=use_block_verification,
)
# Only the first draft is verifiable, plus one resampled token.
assert (num_sampled <= 2).all(), "Accepted a draft token past a placeholder."
assert (num_sampled == 2).float().mean().item() > 0.9, (
"The first draft was rarely accepted; the test is not exercising "
"acceptance past the placeholder."
)
@pytest.mark.parametrize(
"num_speculative_steps,temperature",
[