[Model Runner V2] Fix -1 placeholder draft token ids in rejection sam… (#50939)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
9bca7d840d
commit
27930df9c2
@@ -367,6 +367,128 @@ def test_placeholder_draft_token_rejected():
|
||||
assert (recovered >= 0).all() and (recovered < VOCAB_SIZE).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_draft_logits", [True, False])
|
||||
@pytest.mark.parametrize("num_placeholders", [1, 2])
|
||||
def test_block_verification_placeholder_truncates_block(
|
||||
has_draft_logits: bool, num_placeholders: int
|
||||
):
|
||||
"""Trailing placeholder (-1) draft ids end the block early. The last real
|
||||
draft token must then be verified against the closed-form threshold rather
|
||||
than a residual mass derived from the placeholder position, whose logits
|
||||
describe a token that was never drafted.
|
||||
"""
|
||||
torch.manual_seed(42)
|
||||
device = "cuda"
|
||||
num_trials = 20 * VOCAB_SIZE
|
||||
K = 3
|
||||
temperature = 1.0
|
||||
|
||||
target_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device)
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
draft_logits_1d,
|
||||
K,
|
||||
temperature=temperature,
|
||||
num_trials=num_trials,
|
||||
)
|
||||
if not has_draft_logits:
|
||||
inputs["draft_logits"] = None
|
||||
inputs["draft_sampled"].view(num_trials, K + 1)[:, K + 1 - num_placeholders :] = -1
|
||||
|
||||
sampled, num_sampled = rejection_sample(
|
||||
**inputs, num_speculative_steps=K, use_block_verification=True
|
||||
)
|
||||
|
||||
num_real = K - num_placeholders
|
||||
assert (num_sampled <= num_real + 1).all(), (
|
||||
"Accepted a draft token at or after a placeholder."
|
||||
)
|
||||
target_probs = torch.softmax(target_logits_1d, dim=0)
|
||||
for pos in range(num_real + 1):
|
||||
accepted_mask = num_sampled >= pos + 1
|
||||
_assert_distribution_match(
|
||||
sampled[accepted_mask, pos], target_probs, device, label=f"position {pos}"
|
||||
)
|
||||
|
||||
|
||||
def test_greedy_placeholder_emits_target_argmax():
|
||||
"""Greedy sampling skips resampling and relies on the rejection kernel
|
||||
storing the target argmax at the rejected position. A placeholder must not
|
||||
bypass that store, or the output slot is returned uninitialized.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
num_trials = 512
|
||||
K = 3
|
||||
|
||||
target_logits_1d = torch.randn(VOCAB_SIZE, device=device)
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device)
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
draft_logits_1d,
|
||||
K,
|
||||
temperature=0.0,
|
||||
num_trials=num_trials,
|
||||
)
|
||||
target_argmax = int(target_logits_1d.argmax())
|
||||
draft_sampled = inputs["draft_sampled"].view(num_trials, K + 1)
|
||||
# Accept the first draft so that the rejection lands on the placeholder.
|
||||
draft_sampled[:, 1] = target_argmax
|
||||
draft_sampled[:, 2:] = -1
|
||||
|
||||
sampled, num_sampled = rejection_sample(**inputs, num_speculative_steps=K)
|
||||
|
||||
assert torch.equal(num_sampled, torch.full_like(num_sampled, 2))
|
||||
steps = torch.arange(K + 1, device=device).unsqueeze(0)
|
||||
emitted = sampled[steps < num_sampled.unsqueeze(1)]
|
||||
assert (emitted == target_argmax).all(), (
|
||||
"Greedy sampling emitted a token that is not the target argmax"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_block_verification", [False, True])
|
||||
def test_placeholder_blocks_later_draft_tokens(use_block_verification: bool):
|
||||
"""A placeholder is not necessarily the final draft. Nothing at or after
|
||||
one may be accepted, even when a valid draft follows it.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
num_trials = 4 * VOCAB_SIZE
|
||||
K = 3
|
||||
temperature = 1.0
|
||||
|
||||
# An identical draft is accepted with probability ~1, so any token after
|
||||
# the placeholder would be accepted unless it is explicitly blocked. The
|
||||
# draft logits are stored pre-temperature, so pass the unscaled base.
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device)
|
||||
target_logits_1d = draft_logits_1d / temperature
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
draft_logits_1d,
|
||||
K,
|
||||
temperature=temperature,
|
||||
num_trials=num_trials,
|
||||
)
|
||||
inputs["draft_sampled"].view(num_trials, K + 1)[:, 2] = -1
|
||||
|
||||
_, num_sampled = rejection_sample(
|
||||
**inputs,
|
||||
num_speculative_steps=K,
|
||||
use_block_verification=use_block_verification,
|
||||
)
|
||||
|
||||
# Only the first draft is verifiable, plus one resampled token.
|
||||
assert (num_sampled <= 2).all(), "Accepted a draft token past a placeholder."
|
||||
assert (num_sampled == 2).float().mean().item() > 0.9, (
|
||||
"The first draft was rarely accepted; the test is not exercising "
|
||||
"acceptance past the placeholder."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_speculative_steps,temperature",
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user