[Spec Decode] Reject placeholder (-1) draft tokens in rejection sampler (#46533)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -245,7 +245,8 @@ def _rejection_kernel(
|
||||
pos = tl.load(pos_ptr + logit_idx)
|
||||
u = tl_rand64(seed, pos, includes_zero=False)
|
||||
rate = tl.load(synthetic_conditional_rates_ptr + i)
|
||||
accepted &= u < rate
|
||||
# -1 is used for padded draft token ids that should be rejected.
|
||||
accepted &= (u < rate) & (draft_sampled >= 0)
|
||||
else:
|
||||
accepted &= target_argmax == draft_sampled
|
||||
tl.store(
|
||||
@@ -253,6 +254,10 @@ def _rejection_kernel(
|
||||
draft_sampled if accepted else target_argmax,
|
||||
)
|
||||
else:
|
||||
# -1 is used for padded draft token ids that should be rejected.
|
||||
is_valid_draft = draft_sampled >= 0
|
||||
# Avoid possible OOB ptr access.
|
||||
draft_sampled = tl.maximum(0, draft_sampled)
|
||||
target_logit = tl.load(
|
||||
target_logits_ptr + logit_idx * target_logits_stride + draft_sampled
|
||||
).to(tl.float32)
|
||||
@@ -296,6 +301,7 @@ def _rejection_kernel(
|
||||
# Probability ratio test: p(x) > u * q(x)
|
||||
# Equivalent log form: log_p(x) > log(u) + log_q(x)
|
||||
accepted &= target_log_prob > tl.log(u) + draft_log_prob
|
||||
accepted &= is_valid_draft
|
||||
tl.store(sampled_ptr + req_idx * sampled_stride + i, draft_sampled)
|
||||
rejected_step += accepted
|
||||
tl.store(rejected_steps_ptr + req_idx, rejected_step)
|
||||
|
||||
Reference in New Issue
Block a user