[Spec Decode] Reject placeholder (-1) draft tokens in rejection sampler (#46533)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -748,7 +748,8 @@ def rejection_greedy_sample_kernel(
|
||||
if SYNTHETIC_MODE:
|
||||
uniform_prob = tl.load(uniform_probs_ptr + start_idx + pos)
|
||||
rate = tl.load(synthetic_conditional_rates_ptr + pos)
|
||||
accepted = uniform_prob < rate
|
||||
# -1 is used for padded draft token ids that should be rejected.
|
||||
accepted = (uniform_prob < rate) and draft_token_id >= 0
|
||||
token_id = draft_token_id if accepted else target_argmax_id
|
||||
rejected = not accepted
|
||||
else:
|
||||
@@ -805,7 +806,10 @@ def rejection_random_sample_kernel(
|
||||
if not rejected:
|
||||
draft_token_id = tl.load(draft_token_ids_ptr + start_idx + pos)
|
||||
uniform_prob = tl.load(uniform_probs_ptr + start_idx + pos)
|
||||
if SYNTHETIC_MODE:
|
||||
if draft_token_id < 0:
|
||||
# -1 is used for padded draft token ids that should be rejected.
|
||||
accepted = False
|
||||
elif SYNTHETIC_MODE:
|
||||
rate = tl.load(synthetic_conditional_rates_ptr + pos)
|
||||
accepted = uniform_prob < rate
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user