[Reasoning] Optimize TPOT for thinking budget when used with speculative decoding (#46662)

Signed-off-by: rishitdholakia13 <rishit+github@cohere.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
rishitdholakia13
2026-07-14 18:55:40 +00:00
committed by GitHub
co-authored by Cursor
parent 7ffb98e248
commit 32e632dfeb
3 changed files with 7 additions and 20 deletions
+3 -7
View File
@@ -292,11 +292,8 @@ class RejectionSampler(nn.Module):
any_penalties_or_bad_words = (
sampling_metadata.bad_words_token_ids or has_penalties
)
holder = sampling_metadata.thinking_budget_state_holder
needs_thinking = holder is not None and holder.has_tracked_requests()
output_token_ids = sampling_metadata.output_token_ids
if any_penalties_or_bad_words or needs_thinking:
if any_penalties_or_bad_words:
output_token_ids = self._combine_outputs_with_spec_tokens(
output_token_ids,
sampling_metadata.spec_token_ids,
@@ -305,9 +302,7 @@ class RejectionSampler(nn.Module):
# Calculate indices of target logits.
repeat_indices: torch.Tensor | None = None
need_repeat_indices = (
sampling_metadata.allowed_token_ids_mask is not None
or has_penalties
or needs_thinking
sampling_metadata.allowed_token_ids_mask is not None or has_penalties
)
if need_repeat_indices:
num_requests = len(metadata.num_draft_tokens)
@@ -337,6 +332,7 @@ class RejectionSampler(nn.Module):
logits = processor.apply_with_spec_decode(
logits, metadata.num_draft_tokens
)
holder = sampling_metadata.thinking_budget_state_holder
if holder is not None and holder.has_tracked_requests():
logits = holder.apply_to_logits(
logits,