[Perf][Spec Decode] Skip bonus-logit reduction for short drafts in rejection sampling (#61451)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
@@ -194,6 +194,16 @@ def _compute_global_logprobs_and_logsumexp(
|
||||
return target_log_prob, draft_log_prob, target_lse, draft_lse
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _is_bonus_logit(expanded_local_pos_ptr, logit_idx, num_logits):
|
||||
next_local_pos = tl.load(
|
||||
expanded_local_pos_ptr + logit_idx + 1,
|
||||
mask=logit_idx + 1 < num_logits,
|
||||
other=0,
|
||||
)
|
||||
return next_local_pos == 0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compute_local_logits_stats_kernel(
|
||||
# [num_logits, num_blocks]
|
||||
@@ -225,17 +235,17 @@ def _compute_local_logits_stats_kernel(
|
||||
# [max_num_reqs]
|
||||
temp_ptr,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_DRAFT_LOGITS: tl.constexpr,
|
||||
):
|
||||
logit_idx = tl.program_id(0).to(tl.int64)
|
||||
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
|
||||
|
||||
if draft_step_idx >= num_speculative_steps:
|
||||
# Bonus token. Max/argmax and summed exponentials are not needed.
|
||||
if _is_bonus_logit(expanded_local_pos_ptr, logit_idx, num_logits):
|
||||
# Max/argmax and summed exponentials are not needed.
|
||||
return
|
||||
|
||||
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
|
||||
req_state_idx = tl.load(expanded_idx_mapping_ptr + logit_idx).to(tl.int64)
|
||||
temp = tl.load(temp_ptr + req_state_idx).to(tl.float32)
|
||||
|
||||
@@ -423,17 +433,17 @@ def _compute_local_residual_mass_kernel(
|
||||
# [max_num_reqs]
|
||||
temp_ptr,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
vocab_num_blocks,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
PADDED_VOCAB_NUM_BLOCKS: tl.constexpr,
|
||||
):
|
||||
logit_idx = tl.program_id(0).to(tl.int64)
|
||||
# Adaptive verification can end before num_speculative_steps.
|
||||
if logit_idx + 1 >= tl.num_programs(0):
|
||||
return
|
||||
draft_step_idx = tl.load(expanded_local_pos_ptr + logit_idx)
|
||||
if draft_step_idx == 0 or draft_step_idx >= num_speculative_steps:
|
||||
if draft_step_idx == 0 or _is_bonus_logit(
|
||||
expanded_local_pos_ptr, logit_idx, num_logits
|
||||
):
|
||||
# The acceptance threshold, h, looks one position ahead and sums
|
||||
# over: max(p_i * M_b(x|x_{<i}) - M_s(x|x_{<i}), 0). Tokens at the
|
||||
# first and last (bonus) positions aren't needed for this computation.
|
||||
@@ -1138,7 +1148,7 @@ def rejection_sample(
|
||||
expanded_local_pos,
|
||||
temperature,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
|
||||
HAS_DRAFT_LOGITS=has_draft_logits,
|
||||
)
|
||||
@@ -1209,7 +1219,7 @@ def rejection_sample(
|
||||
expanded_local_pos,
|
||||
temperature,
|
||||
vocab_size,
|
||||
num_speculative_steps,
|
||||
num_logits,
|
||||
vocab_num_blocks,
|
||||
BLOCK_SIZE=VOCAB_BLOCK_SIZE,
|
||||
PADDED_VOCAB_NUM_BLOCKS=padded_vocab_num_blocks,
|
||||
|
||||
Reference in New Issue
Block a user