[Reasoning] Support for speculative decoding with thinking budget (#34668)

Signed-off-by: rishitdholakia13 <rishit+github@cohere.com>
Signed-off-by: rishitdholakia13 <123388671+rishitdholakia13@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
rishitdholakia13
2026-04-29 06:14:52 +00:00
committed by GitHub
co-authored by Nick Hill Cyrus Leung
parent 8a8c9b564e
commit 68dd7db810
12 changed files with 1143 additions and 365 deletions
+30 -2
View File
@@ -8,6 +8,7 @@ from typing import cast
import numpy as np
import torch
from vllm.config.reasoning import ReasoningConfig
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.pooling_params import PoolingParams
@@ -22,6 +23,9 @@ from vllm.v1.sample.logits_processor import (
MoveDirectionality,
)
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.thinking_budget_state import (
maybe_create_thinking_budget_state_holder,
)
from vllm.v1.utils import copy_slice
from vllm.v1.worker.block_table import MultiGroupBlockTable
@@ -92,12 +96,20 @@ class InputBatch:
max_num_blocks_per_req: list[int] | None = None,
logitsprocs: LogitsProcessors | None = None,
logitsprocs_need_output_token_ids: bool = False,
is_spec_decode: bool = False,
num_spec_tokens: int = 0,
is_pooling_model: bool = False,
cp_kv_cache_interleave_size: int = 1,
reasoning_config: ReasoningConfig | None = None,
):
self.thinking_budget_state_holder = maybe_create_thinking_budget_state_holder(
reasoning_config,
max_num_reqs,
num_spec_tokens,
device,
pin_memory,
)
self.thinking_token_budget_reqs: set[str] = set()
self.is_pooling_model = is_pooling_model
self.is_spec_decode = is_spec_decode
self.max_num_reqs = max_num_reqs
self.max_model_len = max_model_len
self.max_num_batched_tokens = max_num_batched_tokens
@@ -540,6 +552,7 @@ class InputBatch:
# False means we don't fill with -inf.
self.allowed_token_ids_mask_cpu_tensor[req_index].fill_(False)
self.bad_words_token_ids.pop(req_index, None)
self.thinking_token_budget_reqs.discard(req_id)
return req_index
def swap_states(self, i1: int, i2: int) -> None:
@@ -800,6 +813,8 @@ class InputBatch:
# reset batch update tracking.
# Update sampling metadata if batch state is changed.
batch_update = self.batch_update_builder.get_and_reset(self.num_reqs)
if self.thinking_budget_state_holder is not None and batch_update:
self.thinking_budget_state_holder.sync_batch(batch_update)
for logit_proc in self.logitsprocs.all:
logit_proc.update_state(batch_update)
if batch_update:
@@ -853,10 +868,15 @@ class InputBatch:
# Only set output_token_ids if required by the current requests'
# sampling parameters.
holder = self.thinking_budget_state_holder
thinking_budget_tracks_reqs = (
holder is not None and holder.has_tracked_requests()
)
needs_output_token_ids = (
not self.no_penalties
or bool(self.bad_words_token_ids)
or self.logitsprocs_need_output_token_ids
or not thinking_budget_tracks_reqs
)
output_token_ids = (
cast(list[list[int]], self.req_output_token_ids)
@@ -902,6 +922,7 @@ class InputBatch:
allowed_token_ids_mask=allowed_token_ids_mask,
bad_words_token_ids=self.bad_words_token_ids,
logitsprocs=self.logitsprocs,
thinking_budget_state_holder=self.thinking_budget_state_holder,
)
def get_pooling_params(self) -> list[PoolingParams]:
@@ -1076,6 +1097,13 @@ class InputBatch:
and len(self.repetition_penalties_reqs) == 0
)
@property
def no_thinking_budget(self) -> bool:
return (
self.thinking_budget_state_holder is None
or len(self.thinking_token_budget_reqs) == 0
)
@property
def max_num_logprobs(self) -> int | None:
return max(self.num_logprobs.values()) if self.num_logprobs else None