[Bugfix][MRV2][Spec Decode] Reject draft slots that were never proposed (#58784)

Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
qizixi
2026-09-28 13:07:15 -07:00
committed by GitHub
co-authored by Claude Nick Hill
parent 42dbe4578c
commit fedbc3b564
2 changed files with 158 additions and 2 deletions
@@ -74,6 +74,76 @@ def _flatten_sampled_kernel(
tl.store(flat_sampled_ptr + start_idx + i, token_id)
@triton.jit
def _gather_draft_sampled_kernel(
# [num_logits]
draft_sampled_ptr,
# [num_logits]
pos_ptr,
# [num_tokens]
input_ids_ptr,
# [num_tokens]
positions_ptr,
# [num_logits]
logits_indices_ptr,
# [num_logits]
expanded_idx_mapping_ptr,
# [num_logits]
expanded_local_pos_ptr,
# [max_num_reqs]
prefill_len_ptr,
num_logits,
BLOCK_SIZE: tl.constexpr,
):
block = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = block < num_logits
token_idx = tl.load(logits_indices_ptr + block, mask=mask)
token_id = tl.load(input_ids_ptr + token_idx, mask=mask)
pos = tl.load(positions_ptr + token_idx, mask=mask)
req_state_idx = tl.load(expanded_idx_mapping_ptr + block, mask=mask)
local_pos = tl.load(expanded_local_pos_ptr + block, mask=mask)
prefill_len = tl.load(prefill_len_ptr + req_state_idx, mask=mask)
# Drafts are only proposed after sampling past the prefill, so any draft
# slots in a step starting within the prefill are placeholders (e.g. the
# padded first step after a P/D remote KV load).
is_placeholder = (local_pos > 0) & (pos - local_pos < prefill_len)
token_id = tl.where(is_placeholder, -1, token_id)
tl.store(draft_sampled_ptr + block, token_id, mask=mask)
tl.store(pos_ptr + block, pos, mask=mask)
def gather_draft_sampled(
input_ids: torch.Tensor,
positions: torch.Tensor,
logits_indices: torch.Tensor,
expanded_idx_mapping: torch.Tensor,
expanded_local_pos: torch.Tensor,
prefill_len: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Gather the input token and position of each logits row.
Draft rows of requests that have not yet sampled past their prefill are
set to -1 so that the rejection kernels reject them.
"""
num_logits = logits_indices.shape[0]
draft_sampled = input_ids.new_empty(num_logits)
pos = positions.new_empty(num_logits)
BLOCK_SIZE = 1024
_gather_draft_sampled_kernel[(triton.cdiv(num_logits, BLOCK_SIZE),)](
draft_sampled,
pos,
input_ids,
positions,
logits_indices,
expanded_idx_mapping,
expanded_local_pos,
prefill_len,
num_logits,
BLOCK_SIZE=BLOCK_SIZE,
)
return draft_sampled, pos
class RejectionSampler:
def __init__(
self,
@@ -296,8 +366,14 @@ class RejectionSampler:
# that num_nans is computed before applying penalties and temperature.
num_nans = get_num_nans(logits) if self.sampler.compute_nans else None
draft_sampled = input_batch.input_ids[input_batch.logits_indices]
pos = input_batch.positions[input_batch.logits_indices]
draft_sampled, pos = gather_draft_sampled(
input_batch.input_ids,
input_batch.positions,
input_batch.logits_indices,
input_batch.expanded_idx_mapping,
input_batch.expanded_local_pos,
self.sampler.req_states.prefill_len.gpu,
)
max_num_logprobs = self.sampler.sampling_states.max_num_logprobs(
input_batch.idx_mapping_np