[Reasoning] Support for speculative decoding with thinking budget (#34668)
Signed-off-by: rishitdholakia13 <rishit+github@cohere.com> Signed-off-by: rishitdholakia13 <123388671+rishitdholakia13@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Nick Hill
Cyrus Leung
parent
8a8c9b564e
commit
68dd7db810
@@ -8,6 +8,7 @@ from typing import cast
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config.reasoning import ReasoningConfig
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.pooling_params import PoolingParams
|
||||
@@ -22,6 +23,9 @@ from vllm.v1.sample.logits_processor import (
|
||||
MoveDirectionality,
|
||||
)
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.thinking_budget_state import (
|
||||
maybe_create_thinking_budget_state_holder,
|
||||
)
|
||||
from vllm.v1.utils import copy_slice
|
||||
from vllm.v1.worker.block_table import MultiGroupBlockTable
|
||||
|
||||
@@ -92,12 +96,20 @@ class InputBatch:
|
||||
max_num_blocks_per_req: list[int] | None = None,
|
||||
logitsprocs: LogitsProcessors | None = None,
|
||||
logitsprocs_need_output_token_ids: bool = False,
|
||||
is_spec_decode: bool = False,
|
||||
num_spec_tokens: int = 0,
|
||||
is_pooling_model: bool = False,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
reasoning_config: ReasoningConfig | None = None,
|
||||
):
|
||||
self.thinking_budget_state_holder = maybe_create_thinking_budget_state_holder(
|
||||
reasoning_config,
|
||||
max_num_reqs,
|
||||
num_spec_tokens,
|
||||
device,
|
||||
pin_memory,
|
||||
)
|
||||
self.thinking_token_budget_reqs: set[str] = set()
|
||||
self.is_pooling_model = is_pooling_model
|
||||
self.is_spec_decode = is_spec_decode
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.max_model_len = max_model_len
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
@@ -540,6 +552,7 @@ class InputBatch:
|
||||
# False means we don't fill with -inf.
|
||||
self.allowed_token_ids_mask_cpu_tensor[req_index].fill_(False)
|
||||
self.bad_words_token_ids.pop(req_index, None)
|
||||
self.thinking_token_budget_reqs.discard(req_id)
|
||||
return req_index
|
||||
|
||||
def swap_states(self, i1: int, i2: int) -> None:
|
||||
@@ -800,6 +813,8 @@ class InputBatch:
|
||||
# reset batch update tracking.
|
||||
# Update sampling metadata if batch state is changed.
|
||||
batch_update = self.batch_update_builder.get_and_reset(self.num_reqs)
|
||||
if self.thinking_budget_state_holder is not None and batch_update:
|
||||
self.thinking_budget_state_holder.sync_batch(batch_update)
|
||||
for logit_proc in self.logitsprocs.all:
|
||||
logit_proc.update_state(batch_update)
|
||||
if batch_update:
|
||||
@@ -853,10 +868,15 @@ class InputBatch:
|
||||
|
||||
# Only set output_token_ids if required by the current requests'
|
||||
# sampling parameters.
|
||||
holder = self.thinking_budget_state_holder
|
||||
thinking_budget_tracks_reqs = (
|
||||
holder is not None and holder.has_tracked_requests()
|
||||
)
|
||||
needs_output_token_ids = (
|
||||
not self.no_penalties
|
||||
or bool(self.bad_words_token_ids)
|
||||
or self.logitsprocs_need_output_token_ids
|
||||
or not thinking_budget_tracks_reqs
|
||||
)
|
||||
output_token_ids = (
|
||||
cast(list[list[int]], self.req_output_token_ids)
|
||||
@@ -902,6 +922,7 @@ class InputBatch:
|
||||
allowed_token_ids_mask=allowed_token_ids_mask,
|
||||
bad_words_token_ids=self.bad_words_token_ids,
|
||||
logitsprocs=self.logitsprocs,
|
||||
thinking_budget_state_holder=self.thinking_budget_state_holder,
|
||||
)
|
||||
|
||||
def get_pooling_params(self) -> list[PoolingParams]:
|
||||
@@ -1076,6 +1097,13 @@ class InputBatch:
|
||||
and len(self.repetition_penalties_reqs) == 0
|
||||
)
|
||||
|
||||
@property
|
||||
def no_thinking_budget(self) -> bool:
|
||||
return (
|
||||
self.thinking_budget_state_holder is None
|
||||
or len(self.thinking_token_budget_reqs) == 0
|
||||
)
|
||||
|
||||
@property
|
||||
def max_num_logprobs(self) -> int | None:
|
||||
return max(self.num_logprobs.values()) if self.num_logprobs else None
|
||||
|
||||
Reference in New Issue
Block a user