[Reasoning] Support for speculative decoding with thinking budget (#34668)

Signed-off-by: rishitdholakia13 <rishit+github@cohere.com>
Signed-off-by: rishitdholakia13 <123388671+rishitdholakia13@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
rishitdholakia13
2026-04-29 06:14:52 +00:00
committed by GitHub
co-authored by Nick Hill Cyrus Leung
parent 8a8c9b564e
commit 68dd7db810
12 changed files with 1143 additions and 365 deletions
+23 -2
View File
@@ -198,22 +198,43 @@ class LogitsprocsTestFakes(NamedTuple):
def fake_update_logitsprocs_state(
test_fakes: LogitsprocsTestFakes,
batch_update: BatchUpdate,
batch_update: BatchUpdate | None,
) -> None:
"""Imitate logits processors persistent batch state update
in engine core"""
for logitproc in test_fakes.get_logitsprocs():
logitproc.update_state(batch_update)
holder = test_fakes.sampling_metadata.thinking_budget_state_holder
if holder is not None:
holder.sync_batch(batch_update)
def fake_apply_logitsprocs(
test_fakes: LogitsprocsTestFakes,
slice_indices: list[int],
slot_output_token_ids: list[list[int]] | None = None,
) -> torch.Tensor:
"""Imitate application of logits processors in engine core"""
"""Imitate application of logits processors in engine core.
When ``thinking_budget_state_holder`` has tracked requests, this mirrors
:meth:`Sampler.apply_logits_processors` by refreshing per-slot
``output_token_ids`` (if ``slot_output_token_ids`` is provided), then
``update_state`` + ``apply_to_logits`` on the holder after built-in logits
processors.
"""
logits = test_fakes.logits[torch.tensor(slice_indices, dtype=torch.long)].clone()
for processor in test_fakes.get_logitsprocs():
logits = processor.apply(logits)
md = test_fakes.sampling_metadata
holder = md.thinking_budget_state_holder
if holder is not None and holder.has_tracked_requests():
if slot_output_token_ids is not None:
for i, toks in enumerate(slot_output_token_ids):
if i < len(md.output_token_ids):
md.output_token_ids[i] = list(toks)
holder.update_state(md.output_token_ids, md.spec_token_ids, None)
logits = holder.apply_to_logits(logits, False, md.spec_token_ids)
return logits