[Reasoning] Optimize TPOT for thinking budget when used with speculative decoding (#46662)

Signed-off-by: rishitdholakia13 <rishit+github@cohere.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
rishitdholakia13
2026-07-14 18:55:40 +00:00
committed by GitHub
co-authored by Cursor
parent 7ffb98e248
commit 32e632dfeb
3 changed files with 7 additions and 20 deletions
+4 -7
View File
@@ -378,13 +378,8 @@ class Sampler(nn.Module):
any_penalties_or_bad_words = (
bool(bad_words_token_ids) or not sampling_metadata.no_penalties
)
holder = sampling_metadata.thinking_budget_state_holder
needs_thinking_combine = holder is not None and holder.has_tracked_requests()
output_token_ids = sampling_metadata.output_token_ids
if predict_bonus_token and (
any_penalties_or_bad_words or needs_thinking_combine
):
if predict_bonus_token and any_penalties_or_bad_words:
# Combine base outputs with spec tokens when speculative decoding
# is enabled.
output_token_ids = self._combine_outputs_with_spec_tokens(
@@ -406,9 +401,11 @@ class Sampler(nn.Module):
# Apply penalties (e.g., freq_penalties).
logits = self.apply_penalties(logits, sampling_metadata, output_token_ids)
holder = sampling_metadata.thinking_budget_state_holder
if holder is not None and holder.has_tracked_requests():
# Committed outputs only; spec drafts live in ``spec_token_ids``.
holder.update_state(
output_token_ids,
sampling_metadata.output_token_ids,
sampling_metadata.spec_token_ids,
repeat_indices=None,
)