[Spec Decode] Reject placeholder (-1) draft tokens in rejection sampler (#46533)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -748,7 +748,8 @@ def rejection_greedy_sample_kernel(
|
||||
if SYNTHETIC_MODE:
|
||||
uniform_prob = tl.load(uniform_probs_ptr + start_idx + pos)
|
||||
rate = tl.load(synthetic_conditional_rates_ptr + pos)
|
||||
accepted = uniform_prob < rate
|
||||
# -1 is used for padded draft token ids that should be rejected.
|
||||
accepted = (uniform_prob < rate) and draft_token_id >= 0
|
||||
token_id = draft_token_id if accepted else target_argmax_id
|
||||
rejected = not accepted
|
||||
else:
|
||||
@@ -805,7 +806,10 @@ def rejection_random_sample_kernel(
|
||||
if not rejected:
|
||||
draft_token_id = tl.load(draft_token_ids_ptr + start_idx + pos)
|
||||
uniform_prob = tl.load(uniform_probs_ptr + start_idx + pos)
|
||||
if SYNTHETIC_MODE:
|
||||
if draft_token_id < 0:
|
||||
# -1 is used for padded draft token ids that should be rejected.
|
||||
accepted = False
|
||||
elif SYNTHETIC_MODE:
|
||||
rate = tl.load(synthetic_conditional_rates_ptr + pos)
|
||||
accepted = uniform_prob < rate
|
||||
else:
|
||||
|
||||
@@ -245,7 +245,8 @@ def _rejection_kernel(
|
||||
pos = tl.load(pos_ptr + logit_idx)
|
||||
u = tl_rand64(seed, pos, includes_zero=False)
|
||||
rate = tl.load(synthetic_conditional_rates_ptr + i)
|
||||
accepted &= u < rate
|
||||
# -1 is used for padded draft token ids that should be rejected.
|
||||
accepted &= (u < rate) & (draft_sampled >= 0)
|
||||
else:
|
||||
accepted &= target_argmax == draft_sampled
|
||||
tl.store(
|
||||
@@ -253,6 +254,10 @@ def _rejection_kernel(
|
||||
draft_sampled if accepted else target_argmax,
|
||||
)
|
||||
else:
|
||||
# -1 is used for padded draft token ids that should be rejected.
|
||||
is_valid_draft = draft_sampled >= 0
|
||||
# Avoid possible OOB ptr access.
|
||||
draft_sampled = tl.maximum(0, draft_sampled)
|
||||
target_logit = tl.load(
|
||||
target_logits_ptr + logit_idx * target_logits_stride + draft_sampled
|
||||
).to(tl.float32)
|
||||
@@ -296,6 +301,7 @@ def _rejection_kernel(
|
||||
# Probability ratio test: p(x) > u * q(x)
|
||||
# Equivalent log form: log_p(x) > log(u) + log_q(x)
|
||||
accepted &= target_log_prob > tl.log(u) + draft_log_prob
|
||||
accepted &= is_valid_draft
|
||||
tl.store(sampled_ptr + req_idx * sampled_stride + i, draft_sampled)
|
||||
rejected_step += accepted
|
||||
tl.store(rejected_steps_ptr + req_idx, rejected_step)
|
||||
|
||||
Reference in New Issue
Block a user