[Reasoning] Support for speculative decoding with thinking budget (#34668)

Signed-off-by: rishitdholakia13 <rishit+github@cohere.com>
Signed-off-by: rishitdholakia13 <123388671+rishitdholakia13@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
rishitdholakia13
2026-04-29 06:14:52 +00:00
committed by GitHub
co-authored by Nick Hill Cyrus Leung
parent 8a8c9b564e
commit 68dd7db810
12 changed files with 1143 additions and 365 deletions
@@ -1,18 +1,62 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for thinking_token_budget with reasoning models."""
"""E2E tests for ``thinking_token_budget`` with reasoning models.
Covers Qwen3-0.6B and Qwen3.5 FP8 + MTP.
"""
import asyncio
import json
from typing import Literal
import openai
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
from tests.utils import RemoteOpenAIServer, multi_gpu_only, requires_fp8
from vllm.platforms import current_platform
from vllm.tokenizers import get_tokenizer
MODEL_NAME = "Qwen/Qwen3-0.6B"
QWEN35_FP8_MTP_MODEL = "Qwen/Qwen3.5-35B-A3B-FP8"
MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
THINK_BUDGET = 5
REASONING_START_STR = "<think>"
REASONING_END_STR = "</think>"
def _count_reasoning_decode_token_ids_between_markers(
full_token_ids: list[int],
reasoning_start_ids: list[int],
reasoning_end_ids: list[int],
) -> int | None:
"""Count decode tokens in the thinking span (after last start, before first end)."""
if not reasoning_start_ids or not reasoning_end_ids:
raise ValueError("reasoning marker token id lists must be non-empty")
def _last_subseq_index(haystack: list[int], needle: list[int]) -> int:
n = len(needle)
if n > len(haystack):
return -1
for i in range(len(haystack) - n, -1, -1):
if haystack[i : i + n] == needle:
return i
return -1
last_start = _last_subseq_index(full_token_ids, reasoning_start_ids)
if last_start < 0:
return None
pos_after_start = last_start + len(reasoning_start_ids)
end_n = len(reasoning_end_ids)
for j in range(pos_after_start, len(full_token_ids) - end_n + 1):
if full_token_ids[j : j + end_n] == reasoning_end_ids:
return j - pos_after_start
return len(full_token_ids) - pos_after_start
@pytest.fixture(scope="module")
def server():
@@ -48,6 +92,51 @@ def server_with_auto_reasoning_config():
yield remote_server
@pytest.fixture(scope="module")
def server_qwen35_fp8_mtp_tp2():
"""Qwen3.5-35B FP8 with MTP speculative decoding and tensor parallel size 2."""
if current_platform.device_count() < 2:
pytest.skip("Need at least 2 GPUs for --tensor-parallel-size 2")
if not current_platform.supports_fp8():
pytest.skip("FP8 is not supported on this platform")
spec_cfg = {
"method": "mtp",
"num_speculative_tokens": 2,
"max_model_len": 32768,
}
args = [
"--tensor-parallel-size",
"2",
"--max-model-len",
"32768",
"--speculative-config",
json.dumps(spec_cfg),
"--reasoning-parser",
"qwen3",
"--reasoning-config",
json.dumps(
{
"reasoning_start_str": REASONING_START_STR,
"reasoning_end_str": REASONING_END_STR,
}
),
]
# With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers
# on 0,1 do not exhaust memory on the same devices as this worker.
env_dict = None
if current_platform.device_count() >= 4:
env_dict = {"CUDA_VISIBLE_DEVICES": "2,3"}
with RemoteOpenAIServer(
QWEN35_FP8_MTP_MODEL,
args,
max_wait_seconds=3000,
env_dict=env_dict,
) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(request, server, server_with_auto_reasoning_config):
server_map = {
@@ -89,8 +178,10 @@ async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
"""Test that thinking_token_budget limits the number of reasoning tokens.
In streaming mode each reasoning delta corresponds to one token, so
counting non-empty reasoning_content chunks gives the exact token count.
Counts non-empty streaming ``delta.reasoning`` chunks (coarse proxy; each
chunk may represent multiple decode tokens — see
``_count_reasoning_decode_token_ids_between_markers`` and the Qwen3.5 MTP
test for id-based checks).
"""
reasoning_token_count = 0
@@ -110,3 +201,89 @@ async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI
f"reasoning tokens ({reasoning_token_count}) exceeded "
f"thinking_token_budget ({THINK_BUDGET})"
)
@pytest.mark.asyncio
@multi_gpu_only(num_gpus=2)
@requires_fp8
async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_plain(
server_qwen35_fp8_mtp_tp2,
):
"""Concurrent chat requests: some with ``thinking_token_budget``, some without.
Exercises the scheduler / input processor under a mixed batch on the same
Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with
``_count_reasoning_decode_token_ids_between_markers`` on full token ids.
"""
_batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [
("budget", 1),
("budget", 12),
("plain", None),
("budget", 20),
("budget", 14),
("plain", None),
("plain", None),
("budget", 12),
("plain", None),
]
tokenizer = get_tokenizer(tokenizer_name=QWEN35_FP8_MTP_MODEL)
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
async with server_qwen35_fp8_mtp_tp2.get_async_client() as client:
async def budgeted_call(expected_budget: int):
return await client.chat.completions.create(
model=QWEN35_FP8_MTP_MODEL,
messages=MESSAGES,
max_tokens=256,
stream=False,
extra_body={
"thinking_token_budget": expected_budget,
"return_token_ids": True,
},
)
async def plain_call():
return await client.chat.completions.create(
model=QWEN35_FP8_MTP_MODEL,
messages=MESSAGES,
max_tokens=256,
stream=False,
)
coros = []
for row in _batch_spec:
if row[0] == "budget":
b = row[1]
assert isinstance(b, int)
coros.append(budgeted_call(b))
else:
coros.append(plain_call())
results = await asyncio.gather(*coros)
for i, (response, (kind, expected_budget)) in enumerate(
zip(results, _batch_spec, strict=True)
):
msg = response.choices[0].message
assert msg.content or getattr(msg, "reasoning", None), (
f"index {i} ({kind}): empty message"
)
if kind == "budget":
assert expected_budget is not None
assert response.prompt_token_ids is not None
assert response.choices[0].token_ids is not None
full_ids = list(response.prompt_token_ids) + list(
response.choices[0].token_ids
)
n_reason = _count_reasoning_decode_token_ids_between_markers(
full_ids, start_ids, end_ids
)
assert n_reason is not None, f"index {i}: missing reasoning start in ids"
assert n_reason == expected_budget, (
f"index {i}: reasoning decode token ids ({n_reason}) != "
f"thinking_token_budget ({expected_budget})"
)
+338 -85
View File
@@ -30,10 +30,13 @@ from vllm.v1.sample.logits_processor import (
MinPLogitsProcessor,
MinTokensLogitsProcessor,
MoveDirectionality,
ThinkingTokenBudgetLogitsProcessor,
build_logitsprocs,
)
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.thinking_budget_state import (
ThinkingBudgetStateHolder,
maybe_create_thinking_budget_state_holder,
)
PIN_MEMORY_AVAILABLE = is_pin_memory_available()
MAX_NUM_REQS = 256
@@ -48,8 +51,10 @@ MAX_NUM_PROMPT_TOKENS = 64
MIN_TOKENS_LEN_THRESHOLD = 5
REQS_PER_LOGITPROC = 50
STR_NO_LOGITPROC = "none"
# Thinking budget uses ``ThinkingBudgetStateHolder`` (not a logits processor).
STR_THINKING_BUDGET = "thinking_budget"
# ThinkingTokenBudgetLogitsProcessor testing constants
# Thinking token budget testing constants
THINKING_TOKEN_BUDGET = 5
THINK_START_TOKEN_ID = 999
THINK_END_TOKEN_ID = 998
@@ -80,15 +85,8 @@ class LogitsProcsRequestParams:
if num_tokens > 0:
# Use diverse random tokens
self.out_tokens = [random.randint(1, 950) for _ in range(num_tokens)]
# Set first token for ThinkingTokenBudget testing
is_thinking_processor = (
logitproc_type is ThinkingTokenBudgetLogitsProcessor
or (
hasattr(logitproc_type, "__name__")
and logitproc_type.__name__ == "ThinkingTokenBudgetLogitsProcessor"
)
)
if is_thinking_processor:
# Think-start seed for ``STR_THINKING_BUDGET`` rows.
if logitproc_type == STR_THINKING_BUDGET:
self.out_tokens[0] = THINK_START_TOKEN_ID
else:
self.out_tokens = []
@@ -102,7 +100,7 @@ class LogitsProcsRequestParams:
class MockReasoningConfig:
"""Mock reasoning config for testing ThinkingTokenBudgetLogitsProcessor."""
"""Minimal reasoning config for ``ThinkingBudgetStateHolder`` tests."""
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
reasoning_end_token_ids = [THINK_END_TOKEN_ID]
@@ -137,6 +135,18 @@ def _generate_fake_sampling_metadata(
is_pin_memory=PIN_MEMORY_AVAILABLE,
is_pooling_model=False,
)
num_spec = (
vllm_config.speculative_config.num_speculative_tokens
if vllm_config.speculative_config
else 0
)
thinking_holder = maybe_create_thinking_budget_state_holder(
vllm_config.reasoning_config,
vllm_config.scheduler_config.max_num_seqs,
num_spec,
device,
PIN_MEMORY_AVAILABLE,
)
fake_sampling_metadata = SamplingMetadata(
temperature=torch.full((batch_size,), 0.0),
all_greedy=True,
@@ -156,6 +166,7 @@ def _generate_fake_sampling_metadata(
allowed_token_ids_mask=None,
bad_words_token_ids={},
logitsprocs=logitsprocs,
thinking_budget_state_holder=thinking_holder,
)
return fake_sampling_metadata
@@ -187,7 +198,7 @@ def _sampling_params_from_logitproc(logitproc_type: LogitprocType) -> SamplingPa
def _generate_mixed_logitsprocs_batch_params(
reqs_per_logitproc: int,
logitsprocs_types: list[str],
logitsprocs_types: list[LogitprocType],
) -> list[LogitsProcsRequestParams]:
"""Define key params for a batch of requests with a different
logitproc enabled per request.
@@ -450,23 +461,21 @@ def _thinking_budget_validate(
request_params: LogitsProcsRequestParams,
step_idx: int,
) -> None:
"""Validate thinking token budget processor behavior"""
# Get the ThinkingTokenBudgetLogitsProcessor instance
tb_processor: ThinkingTokenBudgetLogitsProcessor = next(
test_fakes.get_logitsprocs_by_cls(ThinkingTokenBudgetLogitsProcessor)
)
"""Validate ``ThinkingBudgetStateHolder`` thinking-budget behavior.
# Get current request state
state = tb_processor._state.get(batch_index)
State is keyed by **batch slot** (same index space as logits rows), matching
``sync_batch`` / sampler integration (see PR #34668 discussion).
"""
holder = test_fakes.sampling_metadata.thinking_budget_state_holder
assert holder is not None
state = holder._state.get(batch_index)
params = request_params.params
# Validate thinking token budget configuration
if hasattr(params, "thinking_token_budget") and params.thinking_token_budget:
# State should exist for requests with thinking_token_budget
if state is None:
_raise_error_invalid(
msg_suffix=(
f"Expected state for batch {batch_index} "
f"Expected holder state for batch slot {batch_index} "
f"with thinking_token_budget={params.thinking_token_budget}"
),
batch_index=batch_index,
@@ -474,10 +483,8 @@ def _thinking_budget_validate(
step_idx=step_idx,
)
# Validate budget matches what was set
expected_budget = params.thinking_token_budget
actual_budget = state["thinking_token_budget"]
if actual_budget != expected_budget:
_raise_error_invalid(
msg_suffix=(
@@ -488,13 +495,9 @@ def _thinking_budget_validate(
step_idx=step_idx,
)
# Check if we're in thinking mode and validate token counting
output_tokens = request_params.out_tokens
# Find if thinking has started in output tokens
start_tokens = holder.think_start_token_ids
thinking_started = False
start_tokens = tb_processor.reasoning_start_token_ids
if len(start_tokens) > 0:
for i in range(len(output_tokens) - len(start_tokens) + 1):
if output_tokens[i : i + len(start_tokens)] == start_tokens:
@@ -502,61 +505,42 @@ def _thinking_budget_validate(
break
if thinking_started:
# If budget is exceeded, validate end token forcing
think_count = state["think_count"]
budget = state["thinking_token_budget"]
if think_count >= budget and not state["in_end"]:
_raise_error_invalid(
msg_suffix=(
f"Budget exceeded ({think_count} >= {budget}) but "
"in_end is false"
),
batch_index=batch_index,
request_params=request_params,
step_idx=step_idx,
)
if think_count >= budget:
if not state["in_end"]:
end_tokens = holder.think_end_token_ids
if (
think_count >= budget
and state["in_end"]
and len(end_tokens) > 0
and holder.has_tracked_requests()
):
expected_end_token_id = end_tokens[
min(state["end_count"], len(end_tokens) - 1)
]
# Holder bumps forced vocab positions to 1e9 (does not -inf others).
forced_logit = float(logits_new[batch_index, expected_end_token_id])
if forced_logit < 1.0e8:
_raise_error_invalid(
msg_suffix=(
f"Budget exceeded ({think_count} >= "
f"{budget}) but not "
"forcing end tokens"
f"Expected forced end token {expected_end_token_id} "
f"with large logit, got {forced_logit}"
),
batch_index=batch_index,
request_params=request_params,
step_idx=step_idx,
)
# Validate that only end tokens are allowed
end_tokens = tb_processor.reasoning_end_token_ids
if len(end_tokens) > 0:
expected_end_token_id = end_tokens[
min(state["end_count"], len(end_tokens) - 1)
]
# Check logits masking
batch_logits = logits_new[batch_index]
for token_id in range(len(batch_logits)):
logit_value = batch_logits[token_id]
if token_id == expected_end_token_id:
# End token should not be masked
if logit_value == -float("inf"):
_raise_error_invalid(
msg_suffix=(
f"End token {token_id} should not be "
"masked but is"
),
batch_index=batch_index,
request_params=request_params,
step_idx=step_idx,
)
else:
# All other tokens should be masked when forcing end
if logit_value != -float("inf"):
_raise_error_invalid(
msg_suffix=(
f"Token {token_id} should be masked "
f"when forcing end tokens, but "
f"logit={logit_value}"
),
batch_index=batch_index,
request_params=request_params,
step_idx=step_idx,
)
def _none_validate(
test_fakes: LogitsprocsTestFakes,
@@ -604,7 +588,7 @@ logitsprocs_test_mapping = {
MinTokensLogitsProcessor: LogitsprocTestHelpers(
gen_request_fxn=_min_tokens_params, eval_fxn=_min_tokens_validate
),
ThinkingTokenBudgetLogitsProcessor: LogitsprocTestHelpers(
STR_THINKING_BUDGET: LogitsprocTestHelpers(
gen_request_fxn=_thinking_budget_params, eval_fxn=_thinking_budget_validate
),
}
@@ -614,20 +598,17 @@ def _get_test_cases() -> list[list[str]]:
"""Each test case is a set of logitsprocs"""
logitsprocs_types = list(logitsprocs_test_mapping.keys())
# Isolate ThinkingTokenBudgetLogitsProcessor from all other processors
# to avoid unexpected modification of logits interference
thinking_processor = ThinkingTokenBudgetLogitsProcessor
# Isolate thinking-budget handling from other processors to avoid cross-talk.
thinking_id: LogitprocType = STR_THINKING_BUDGET
other_processors = [
p
for p in logitsprocs_types
if p != STR_NO_LOGITPROC and p != thinking_processor
p for p in logitsprocs_types if p != STR_NO_LOGITPROC and p != thinking_id
]
return (
[[STR_NO_LOGITPROC]]
+ [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors]
+ [other_processors]
+ [[thinking_processor]]
+ [[thinking_id]]
)
@@ -802,12 +783,23 @@ def _assert_valid(
)
def _slot_outputs_for_metadata(
persistent_batch: list[LogitsProcsRequestParams], pad_len: int
) -> list[list[int]]:
"""Per-batch-slot output token ids aligned with ``SamplingMetadata`` rows."""
rows: list[list[int]] = [[] for _ in range(pad_len)]
for i, req in enumerate(persistent_batch):
if i < pad_len:
rows[i] = list(req.out_tokens)
return rows
@create_new_process_for_each_test()
@pytest.mark.parametrize("device", DEVICES)
@pytest.mark.parametrize("reqs_per_logitproc", [REQS_PER_LOGITPROC])
@pytest.mark.parametrize("logitsprocs_under_test", _get_test_cases())
def test_logitsprocs(
device: str, reqs_per_logitproc: int, logitsprocs_under_test: list[str]
device: str, reqs_per_logitproc: int, logitsprocs_under_test: list[LogitprocType]
):
random.seed(40)
torch.set_default_device(device)
@@ -855,9 +847,10 @@ def test_logitsprocs(
# Apply fake batch update to logitsprocs
fake_update_logitsprocs_state(test_fakes, batch_update)
# Emulate application of logits processors in engine
# Emulate application of logits processors + thinking holder (sampler order).
slice_idxs = [req.workload_index for req in persistent_batch]
logits_w_lp = fake_apply_logitsprocs(test_fakes, slice_idxs).cpu()
slot_rows = _slot_outputs_for_metadata(persistent_batch, workload_size)
logits_w_lp = fake_apply_logitsprocs(test_fakes, slice_idxs, slot_rows).cpu()
_assert_valid(
batch_size=batch_size,
@@ -869,3 +862,263 @@ def test_logitsprocs(
)
step_idx += 1
class MockReasoningNoEndTokens:
"""Reasoning config with no end token ids (disables enforcement in holder)."""
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
reasoning_end_token_ids: list[int] = []
def test_maybe_create_thinking_budget_holder_without_reasoning():
cfg = VllmConfig()
assert cfg.reasoning_config is None
assert (
maybe_create_thinking_budget_state_holder(
None,
cfg.scheduler_config.max_num_seqs,
0,
torch.device("cpu"),
False,
)
is None
)
def test_thinking_budget_holder_has_tracked_after_sync_add():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
0,
torch.device("cpu"),
False,
)
assert not h.has_tracked_requests()
h.sync_batch(
BatchUpdate(
batch_size=1,
removed=(),
added=[
(
0,
SamplingParams(thinking_token_budget=3),
None,
[THINK_START_TOKEN_ID],
)
],
moved=(),
)
)
assert h.has_tracked_requests()
assert h._state[0]["thinking_token_budget"] == 3
def test_thinking_budget_holder_sync_remove_clears_state():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
0,
torch.device("cpu"),
False,
)
h.sync_batch(
BatchUpdate(
batch_size=1,
removed=(),
added=[
(
0,
SamplingParams(thinking_token_budget=3),
None,
[],
)
],
moved=(),
)
)
assert h.has_tracked_requests()
h.sync_batch(BatchUpdate(batch_size=0, removed=(0,), added=(), moved=()))
assert not h.has_tracked_requests()
def test_thinking_budget_holder_sync_add_without_budget_drops_row():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
0,
torch.device("cpu"),
False,
)
h.sync_batch(
BatchUpdate(
batch_size=1,
removed=(),
added=[(0, SamplingParams(), None, [])],
moved=(),
)
)
assert not h.has_tracked_requests()
def test_thinking_budget_holder_swap_exchanges_state():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
0,
torch.device("cpu"),
False,
)
h.sync_batch(
BatchUpdate(
batch_size=2,
removed=(),
added=[
(
0,
SamplingParams(thinking_token_budget=3),
None,
[],
),
(
1,
SamplingParams(thinking_token_budget=7),
None,
[],
),
],
moved=(),
)
)
b0, b1 = h._state[0]["thinking_token_budget"], h._state[1]["thinking_token_budget"]
h.sync_batch(
BatchUpdate(
batch_size=2,
removed=(),
added=(),
moved=[(0, 1, MoveDirectionality.SWAP)],
)
)
assert h._state[0]["thinking_token_budget"] == b1
assert h._state[1]["thinking_token_budget"] == b0
def test_thinking_budget_holder_unidirectional_move():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
0,
torch.device("cpu"),
False,
)
h.sync_batch(
BatchUpdate(
batch_size=2,
removed=(),
added=[
(
1,
SamplingParams(thinking_token_budget=4),
None,
[],
),
],
moved=(),
)
)
assert 1 in h._state and 0 not in h._state
h.sync_batch(
BatchUpdate(
batch_size=2,
removed=(),
added=(),
moved=[(1, 0, MoveDirectionality.UNIDIRECTIONAL)],
)
)
assert 0 in h._state and 1 not in h._state
assert h._state[0]["thinking_token_budget"] == 4
def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
vc = VllmConfig()
vc.reasoning_config = MockReasoningConfig()
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
0,
torch.device("cpu"),
False,
)
h.sync_batch(
BatchUpdate(
batch_size=1,
removed=(),
added=[
(
0,
SamplingParams(thinking_token_budget=5),
None,
[THINK_START_TOKEN_ID],
)
],
moved=(),
)
)
out_lists = [[THINK_START_TOKEN_ID], [THINK_START_TOKEN_ID, 10, 11, 12, 13, 14]]
h.update_state(
out_lists,
None,
torch.tensor([0, 0], dtype=torch.long),
)
assert h._state[0]["output_tok_ids"] == out_lists[1]
def test_thinking_budget_holder_spec_mode_tensor_layout():
h = ThinkingBudgetStateHolder(
MockReasoningConfig(),
8,
2,
torch.device("cpu"),
False,
)
assert h.in_spec_mode
assert h.mask.shape[0] == 8 * (2 + 1)
def test_thinking_budget_holder_empty_end_tokens_disables_row():
vc = VllmConfig()
vc.reasoning_config = MockReasoningNoEndTokens()
h = ThinkingBudgetStateHolder(
vc.reasoning_config,
vc.scheduler_config.max_num_seqs,
0,
torch.device("cpu"),
False,
)
h.sync_batch(
BatchUpdate(
batch_size=1,
removed=(),
added=[
(
0,
SamplingParams(thinking_token_budget=5),
None,
[THINK_START_TOKEN_ID],
)
],
moved=(),
)
)
h.update_state([[THINK_START_TOKEN_ID, 1]], None, None)
assert h._state[0]["thinking_token_budget"] == -1
+23 -2
View File
@@ -198,22 +198,43 @@ class LogitsprocsTestFakes(NamedTuple):
def fake_update_logitsprocs_state(
test_fakes: LogitsprocsTestFakes,
batch_update: BatchUpdate,
batch_update: BatchUpdate | None,
) -> None:
"""Imitate logits processors persistent batch state update
in engine core"""
for logitproc in test_fakes.get_logitsprocs():
logitproc.update_state(batch_update)
holder = test_fakes.sampling_metadata.thinking_budget_state_holder
if holder is not None:
holder.sync_batch(batch_update)
def fake_apply_logitsprocs(
test_fakes: LogitsprocsTestFakes,
slice_indices: list[int],
slot_output_token_ids: list[list[int]] | None = None,
) -> torch.Tensor:
"""Imitate application of logits processors in engine core"""
"""Imitate application of logits processors in engine core.
When ``thinking_budget_state_holder`` has tracked requests, this mirrors
:meth:`Sampler.apply_logits_processors` by refreshing per-slot
``output_token_ids`` (if ``slot_output_token_ids`` is provided), then
``update_state`` + ``apply_to_logits`` on the holder after built-in logits
processors.
"""
logits = test_fakes.logits[torch.tensor(slice_indices, dtype=torch.long)].clone()
for processor in test_fakes.get_logitsprocs():
logits = processor.apply(logits)
md = test_fakes.sampling_metadata
holder = md.thinking_budget_state_holder
if holder is not None and holder.has_tracked_requests():
if slot_output_token_ids is not None:
for i, toks in enumerate(slot_output_token_ids):
if i < len(md.output_token_ids):
md.output_token_ids[i] = list(toks)
holder.update_state(md.output_token_ids, md.spec_token_ids, None)
logits = holder.apply_to_logits(logits, False, md.spec_token_ids)
return logits
@@ -39,7 +39,6 @@ def mock_model_runner_with_input_batch():
vocab_size=32000,
block_sizes=[16],
kernel_block_sizes=[16],
is_spec_decode=False,
logitsprocs=None,
is_pooling_model=False,
)
@@ -18,7 +18,6 @@ from vllm.v1.sample.logits_processor.builtin import (
LogitBiasLogitsProcessor,
MinPLogitsProcessor,
MinTokensLogitsProcessor,
ThinkingTokenBudgetLogitsProcessor,
process_dict_updates,
)
from vllm.v1.sample.logits_processor.interface import (
@@ -51,7 +50,6 @@ BUILTIN_LOGITS_PROCESSORS: list[type[LogitsProcessor]] = [
MinTokensLogitsProcessor,
LogitBiasLogitsProcessor,
MinPLogitsProcessor,
ThinkingTokenBudgetLogitsProcessor,
]
@@ -356,5 +354,4 @@ __all__ = [
"STR_POOLING_REJECTS_LOGITSPROCS",
"LOGITSPROCS_GROUP",
"AdapterLogitsProcessor",
"ThinkingTokenBudgetLogitsProcessor",
]
+1 -262
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Sequence
from typing import TYPE_CHECKING, Any, TypeVar
from typing import TYPE_CHECKING, TypeVar
import numpy as np
import torch
@@ -291,267 +291,6 @@ class MinTokensLogitsProcessor(LogitsProcessor):
return logits
class ThinkingTokenBudgetLogitsProcessor(LogitsProcessor):
"""Limits the number of tokens allowed inside a 'thinking' section."""
def __init__(
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
):
reasoning_config = vllm_config.reasoning_config
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
# Check if thinking is enabled
self.is_enabled = reasoning_config is not None and reasoning_config.enabled
self.reasoning_start_token_ids = getattr(
reasoning_config, "reasoning_start_token_ids", []
)
self.reasoning_end_token_ids = getattr(
reasoning_config, "reasoning_end_token_ids", []
)
self.pin_memory = is_pin_memory
self.device = device
# Per-request state tracking for thinking token management
# Key: request_index, Value: state dict containing:
# "in_think": bool - currently in thinking mode
# "in_end": bool - currently forcing end tokens output
# "check_count_down": int - steps remaining until next think
# start/end token parsing
# "think_count": int - number of thinking tokens generated
# "end_count": int - number of end tokens forced so far
# "thinking_token_budget": int - max allowed thinking tokens
# "output_tok_ids": list[int] - generated output tokens
# "prev_output_length": int - previous output length for
# incremental processing
self._state: dict[int, dict[str, Any]] = {}
# Preallocate reusable tensors
self.mask = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
self.force_token_ids = torch.full(
(max_num_reqs,), -1, dtype=torch.long, device=device
)
@staticmethod
def _find_last_sequence_index(target_list: list[int], token_ids: list[int]) -> int:
"""
Returns the index of the last occurrence of token_ids in target_list.
Args:
target_list (list[int]): The list of token IDs.
token_ids (list[int]): The sequence of token IDs to find.
"""
if not token_ids:
return -1
for i in range(len(target_list) - len(token_ids), -1, -1):
if target_list[i : i + len(token_ids)] == token_ids:
return i
return -1
def _init_state_entry(
self, prompt_tok_ids: list[int] | None, thinking_token_budget: int
) -> dict[str, Any]:
"""Initializes the tracking state for a given sequence index."""
if prompt_tok_ids is None:
last_start = -1
last_end = -1
in_think = False
think_count = 0
else:
last_start = self._find_last_sequence_index(
prompt_tok_ids, self.reasoning_start_token_ids
)
last_end = self._find_last_sequence_index(
prompt_tok_ids, self.reasoning_end_token_ids
)
in_think = last_start > last_end
if in_think:
think_count = len(prompt_tok_ids) - (
last_start + len(self.reasoning_start_token_ids)
)
else:
think_count = 0
return {
"in_think": in_think, # Currently in thinking mode
"in_end": in_think and thinking_token_budget == 0,
"check_count_down": thinking_token_budget,
"think_count": think_count, # Number of tokens in thinking section
"end_count": 0, # Number of end tokens forced so far
"prompt_tok_ids": prompt_tok_ids,
"output_tok_ids": [],
"thinking_token_budget": thinking_token_budget,
"prev_output_length": 0,
# Track previous output length for incremental updates
}
def _update_think_state(self, state: dict[str, Any]):
"""Updates the state based on newly generated output tokens."""
if not state.get("in_end", False) and state.get("check_count_down", 0) > 0:
state["check_count_down"] -= 1
return
output = state.get("output_tok_ids", [])
if not output:
return
# Track previous output length for incremental processing
prev_length = state.get("prev_output_length", 0)
current_length = len(output)
if current_length <= prev_length:
return
# Process only newly added tokens
new_tokens = output[prev_length:]
state["prev_output_length"] = current_length
# Check if new tokens contain think start or end sequences
start_len = len(self.reasoning_start_token_ids)
end_len = len(self.reasoning_end_token_ids)
# Look for think sequences in recent tokens (including boundary)
# Check overlapping regions where sequences might span boundaries
check_start_idx = max(0, prev_length - max(start_len, end_len) + 1)
recent_tokens = output[check_start_idx:]
# Find any think start/end sequences in recent tokens
recent_start_pos = self._find_last_sequence_index(
recent_tokens, self.reasoning_start_token_ids
)
recent_end_pos = self._find_last_sequence_index(
recent_tokens, self.reasoning_end_token_ids
)
# Update state based on recent sequences
if not state["in_end"]:
if recent_start_pos >= 0 and recent_end_pos >= 0:
if recent_start_pos > recent_end_pos:
# Case: ...<end>...<start>... - entering think mode
absolute_start_pos = check_start_idx + recent_start_pos
new_think_count = current_length - (absolute_start_pos + start_len)
state["in_think"] = True
state["think_count"] = new_think_count
else:
# Case: ...<start>...<end>... - exiting think mode
state["in_think"] = False
state["think_count"] = 0
elif recent_start_pos >= 0:
# Found think start - entering think mode
absolute_start_pos = check_start_idx + recent_start_pos
new_think_count = current_length - (absolute_start_pos + start_len)
state["in_think"] = True
state["think_count"] = new_think_count
elif recent_end_pos >= 0:
# Found think end - exiting think mode
state["in_think"] = False
state["think_count"] = 0
elif state["in_think"]:
# Continue thinking mode, increment count by new tokens
state["think_count"] += len(new_tokens)
# Set countdown based on current state
if state["in_think"]:
remaining_budget = max(
0, state["thinking_token_budget"] - state["think_count"]
)
state["check_count_down"] = max(0, remaining_budget - 1)
else:
state["check_count_down"] = state["thinking_token_budget"]
# Check if need to transition to end mode
if (
state["in_think"]
and state["think_count"] >= state["thinking_token_budget"]
):
state["in_think"] = False
state["in_end"] = True
state["end_count"] = 0
state["check_count_down"] = state["thinking_token_budget"]
else:
# In end mode
state["end_count"] += 1
if state["end_count"] >= len(self.reasoning_end_token_ids):
state.update(
{
"in_end": False,
"end_count": 0,
"check_count_down": state["thinking_token_budget"],
}
)
def is_argmax_invariant(self) -> bool:
"""This logits processor can change the outcome of
greedy sampling by forcing that the thinking section
ends after a certain number of tokens."""
return False
def update_state(self, batch_update: BatchUpdate | None):
if not self.is_enabled:
return
if batch_update:
for index, params, prompt_tok_ids, output_tok_ids in batch_update.added:
thinking_token_budget = params.thinking_token_budget
if thinking_token_budget is not None:
self._state[index] = self._init_state_entry(
prompt_tok_ids, thinking_token_budget
)
self._state[index]["output_tok_ids"] = output_tok_ids
else:
# Remove state if no thinking budget
self._state.pop(index, None)
for index in batch_update.removed:
self._state.pop(index, {})
for i1, i2, direction in batch_update.moved:
if direction == MoveDirectionality.SWAP:
state1 = self._state.pop(i1, None)
state2 = self._state.pop(i2, None)
if state1 is not None:
self._state[i2] = state1
if state2 is not None:
self._state[i1] = state2
else:
state = self._state.pop(i1, None)
if state is not None:
self._state[i2] = state
for state in self._state.values():
self._update_think_state(state)
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if not self.is_enabled or not self._state:
return logits
batch_size = logits.size(0)
self.mask[:batch_size] = False
for i in range(batch_size):
state = self._state.get(i)
if state and state["in_end"]:
self.mask[i] = True
self.force_token_ids[i] = self.reasoning_end_token_ids[
state["end_count"]
]
# Check in CPU first not to sync with GPU
has_active_thinking = any(
state.get("in_end", False) for state in self._state.values()
)
if has_active_thinking:
current_mask = self.mask[:batch_size]
active_indices = current_mask.nonzero(as_tuple=False).view(-1)
if len(active_indices) > 0:
force_tokens = self.force_token_ids[active_indices]
# Apply a large value for the end thinking token id index
logits[active_indices, force_tokens] = 1e9
return logits
def process_dict_updates(
req_entries: dict[int, T],
batch_update: BatchUpdate | None,
+6
View File
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from dataclasses import dataclass
import torch
from vllm.v1.sample.logits_processor import LogitsProcessors
from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder
@dataclass
@@ -47,3 +50,6 @@ class SamplingMetadata:
# Speculative token ids
spec_token_ids: list[list[int]] | None = None
# When non-None, use ``holder.has_tracked_requests()`` to see if this batch applies
# thinking-token-budget logits (holder may exist with an empty tracking set).
thinking_budget_state_holder: ThinkingBudgetStateHolder | None = None
+16 -3
View File
@@ -290,16 +290,24 @@ class RejectionSampler(nn.Module):
any_penalties_or_bad_words = (
sampling_metadata.bad_words_token_ids or has_penalties
)
holder = sampling_metadata.thinking_budget_state_holder
needs_thinking = holder is not None and holder.has_tracked_requests()
output_token_ids = sampling_metadata.output_token_ids
if any_penalties_or_bad_words:
if any_penalties_or_bad_words or needs_thinking:
output_token_ids = self._combine_outputs_with_spec_tokens(
output_token_ids,
sampling_metadata.spec_token_ids,
)
# Calculate indices of target logits.
if sampling_metadata.allowed_token_ids_mask is not None or has_penalties:
repeat_indices: torch.Tensor | None = None
need_repeat_indices = (
sampling_metadata.allowed_token_ids_mask is not None
or has_penalties
or needs_thinking
)
if need_repeat_indices:
num_requests = len(metadata.num_draft_tokens)
num_draft_tokens = torch.tensor(metadata.num_draft_tokens, device="cpu")
original_indices = torch.arange(num_requests, device="cpu")
@@ -327,7 +335,12 @@ class RejectionSampler(nn.Module):
logits = processor.apply_with_spec_decode(
logits, metadata.num_draft_tokens
)
if holder is not None and holder.has_tracked_requests():
logits = holder.apply_to_logits(
logits,
predict_bonus_token=False,
spec_token_ids=sampling_metadata.spec_token_ids,
)
return logits
@staticmethod
+16 -1
View File
@@ -364,9 +364,13 @@ class Sampler(nn.Module):
any_penalties_or_bad_words = (
bool(bad_words_token_ids) or not sampling_metadata.no_penalties
)
holder = sampling_metadata.thinking_budget_state_holder
needs_thinking_combine = holder is not None and holder.has_tracked_requests()
output_token_ids = sampling_metadata.output_token_ids
if predict_bonus_token and any_penalties_or_bad_words:
if predict_bonus_token and (
any_penalties_or_bad_words or needs_thinking_combine
):
# Combine base outputs with spec tokens when speculative decoding
# is enabled.
output_token_ids = self._combine_outputs_with_spec_tokens(
@@ -388,6 +392,17 @@ class Sampler(nn.Module):
# Apply penalties (e.g., freq_penalties).
logits = self.apply_penalties(logits, sampling_metadata, output_token_ids)
if holder is not None and holder.has_tracked_requests():
holder.update_state(
output_token_ids,
sampling_metadata.spec_token_ids,
repeat_indices=None,
)
logits = holder.apply_to_logits(
logits,
predict_bonus_token,
sampling_metadata.spec_token_ids,
)
return logits
@staticmethod
+528
View File
@@ -0,0 +1,528 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-batch thinking token budget state; applied after penalties at sample time."""
from typing import TYPE_CHECKING, Any
import torch
from vllm.v1.sample.logits_processor.interface import (
BatchUpdate,
MoveDirectionality,
)
if TYPE_CHECKING:
from vllm.config.reasoning import ReasoningConfig
def maybe_create_thinking_budget_state_holder(
reasoning_config: "ReasoningConfig | None",
max_num_seqs: int,
num_spec_tokens: int,
device: torch.device,
is_pin_memory: bool,
) -> "ThinkingBudgetStateHolder | None":
if reasoning_config is None:
return None
return ThinkingBudgetStateHolder(
reasoning_config, max_num_seqs, num_spec_tokens, device, is_pin_memory
)
class ThinkingBudgetStateHolder:
"""Tracks thinking sections and forces end tokens when budget is exceeded."""
think_start_token_ids: list[int]
think_end_token_ids: list[int]
def __init__(
self,
reasoning_config: "ReasoningConfig | None",
max_num_seqs: int,
num_spec_tokens: int,
device: torch.device,
is_pin_memory: bool,
):
_ = is_pin_memory # API parity with logits processors
max_num_reqs = max_num_seqs
self.in_spec_mode = num_spec_tokens > 0
self.num_spec_tokens = num_spec_tokens
# No separate enable flag: a non-``None`` ``reasoning_config`` is the switch.
self.is_enabled = reasoning_config is not None
if reasoning_config is None:
self.think_start_token_ids = []
self.think_end_token_ids = []
else:
rs = reasoning_config.reasoning_start_token_ids
re = reasoning_config.reasoning_end_token_ids
self.think_start_token_ids = rs if rs else []
self.think_end_token_ids = re if re else []
self.device = device
self._state: dict[int, dict[str, Any]] = {}
self.cu_num_tokens: dict[int, int] = {}
if self.num_spec_tokens > 0:
self.mask = torch.zeros(
max_num_reqs * (self.num_spec_tokens + 1),
dtype=torch.bool,
device=device,
)
self.force_token_ids = torch.full(
(max_num_reqs * (self.num_spec_tokens + 1),),
-1,
dtype=torch.long,
device=device,
)
else:
self.mask = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
self.force_token_ids = torch.full(
(max_num_reqs,), -1, dtype=torch.long, device=device
)
def has_tracked_requests(self) -> bool:
"""True when ``sync_batch`` has state for a ``thinking_token_budget`` row.
Used to decide whether sampling needs output-token rows and spec combining;
distinct from merely having a holder instance (reasoning may be on with no
budgeted requests in this batch).
"""
return bool(self._state)
def sync_batch(self, batch_update: BatchUpdate | None) -> None:
"""Add/remove/move per-request state only (no _update_think_state)."""
if not self.is_enabled or not batch_update:
return
for index in batch_update.removed:
self._state.pop(index, None)
for index, params, prompt_tok_ids, output_tok_ids in batch_update.added:
thinking_token_budget = params.thinking_token_budget
if thinking_token_budget is not None:
self._state[index] = self._init_state_entry(
prompt_tok_ids, thinking_token_budget
)
self._state[index]["output_tok_ids"] = output_tok_ids
self._state[index]["spec_token_ids"] = []
else:
self._state.pop(index, None)
for i1, i2, direction in batch_update.moved:
if direction == MoveDirectionality.SWAP:
state1 = self._state.get(i1)
state2 = self._state.get(i2)
if state1 is not None:
self._state[i2] = state1
if state2 is not None:
self._state[i1] = state2
else:
state = self._state.pop(i1, None)
if state is not None:
self._state[i2] = state
def update_state(
self,
output_token_ids: list[list[int]],
spec_token_ids: list[list[int]] | None,
repeat_indices: torch.Tensor | None = None,
) -> None:
"""Refresh output/spec from sampling rows and recompute think state."""
if not self.is_enabled or not self._state:
return
spec_lists = spec_token_ids or []
last_row_for_req: dict[int, int] | None = None
if repeat_indices is not None:
last_row_for_req = {}
rpt = repeat_indices.cpu().tolist()
for batch_row, req_i in enumerate(rpt):
last_row_for_req[req_i] = batch_row
for seq_idx, state in list(self._state.items()):
if last_row_for_req is not None:
output_row: int | None = last_row_for_req.get(seq_idx)
if output_row is None or output_row >= len(output_token_ids):
continue
state["output_tok_ids"] = output_token_ids[output_row]
elif seq_idx >= len(output_token_ids):
continue
else:
state["output_tok_ids"] = output_token_ids[seq_idx]
if seq_idx < len(spec_lists):
state["spec_token_ids"] = list(spec_lists[seq_idx])
else:
state["spec_token_ids"] = []
state["in_spec_mode"] = self.in_spec_mode
state["force_index"] = []
if len(state["output_tok_ids"]) > 0:
spec_len = len(state["spec_token_ids"])
# Only strip draft suffix when there are spec tokens; ``[:-0]`` would
# clear the whole list (Python treats stop index 0 as "up to empty").
if spec_len > 0 and len(state["output_tok_ids"]) >= spec_len:
state["output_tok_ids"] = state["output_tok_ids"][:-spec_len]
self._update_think_state(state)
def apply_to_logits(
self,
logits: torch.Tensor,
predict_bonus_token: bool,
spec_token_ids: list[list[int]] | None,
) -> torch.Tensor:
"""Mask and bump logits for forced end-of-thinking tokens."""
if not self.is_enabled or not self._state:
return logits
spec_lists = spec_token_ids or []
return self._apply_forcing_to_logits(logits, predict_bonus_token, spec_lists)
@staticmethod
def _find_last_sequence_index(target_list: list[int], token_ids: list[int]) -> int:
if not token_ids:
return -1
for i in range(len(target_list) - len(token_ids), -1, -1):
if target_list[i : i + len(token_ids)] == token_ids:
return i
return -1
def _init_state_entry(
self, prompt_tok_ids: list[int] | None, thinking_token_budget: int
) -> dict[str, Any]:
if prompt_tok_ids is None:
last_start = -1
last_end = -1
in_think = False
think_count = 0
start_thinking = -1
countdown = thinking_token_budget
continue_thinking = False
in_end = False
else:
start_thinking = -1
countdown = thinking_token_budget
continue_thinking = False
in_end = False
last_start = self._find_last_sequence_index(
prompt_tok_ids, self.think_start_token_ids
)
last_end = self._find_last_sequence_index(
prompt_tok_ids, self.think_end_token_ids
)
in_think = last_start > last_end
# load metrics such as think count, start thinking
# if request is in thinking mode, already
if in_think:
think_count = len(prompt_tok_ids) - (
last_start + len(self.think_start_token_ids)
)
start_thinking = len(prompt_tok_ids) - think_count - 1
countdown -= think_count
continue_thinking = True
# check if the token is exhausted within prompt
token_exhausted = thinking_token_budget - think_count
in_end = token_exhausted <= 0
else:
think_count = 0
return {
"in_think": in_think,
"in_end": in_end,
"check_count_down": countdown,
"think_count": think_count,
"end_count": 0,
"prompt_tok_ids": prompt_tok_ids,
"output_tok_ids": [],
"thinking_token_budget": thinking_token_budget,
"prev_output_length": 0,
"spec_token_ids": [],
"force_index": [],
"start_thinking": start_thinking,
"end_thinking": -1,
"in_spec_mode": False,
"bonus_token_forced": False,
"continue_thinking": continue_thinking,
}
def _update_think_state(self, state: dict[str, Any]) -> None:
if state.get("thinking_token_budget", -1) == -1:
return
if len(self.think_end_token_ids) == 0:
state["thinking_token_budget"] = -1
state["in_end"] = False
state["force_index"] = []
return
if state["start_thinking"] == -1:
start_thinking = self._find_last_sequence_index(
state.get("output_tok_ids", []), self.think_start_token_ids
)
state["start_thinking"] = start_thinking
if state["end_thinking"] == -1:
end_thinking = self._find_last_sequence_index(
state.get("output_tok_ids", []), self.think_end_token_ids
)
state["end_thinking"] = end_thinking
if state["start_thinking"] == -1:
return
if state["continue_thinking"]:
sampled_tokens_from_previous_step = len(
state.get("output_tok_ids", [])
) - state.get("prev_output_length", 0)
else:
if state["prev_output_length"] == 0:
sampled_tokens_from_previous_step = len(
state.get("output_tok_ids", [])
) - len(self.think_start_token_ids)
else:
sampled_tokens_from_previous_step = (
len(state.get("output_tok_ids", [])) - state["prev_output_length"]
)
current_step_countdown = (
state["check_count_down"] - sampled_tokens_from_previous_step
)
predicted_countdown = current_step_countdown - len(state["spec_token_ids"]) - 1
# We only proceed further if we have counted down the thinking budget
# to 0 or less and when we are in the "in think" mode.
if (
not state.get("in_end", False)
and predicted_countdown >= 0
and state["start_thinking"] > -1
):
state["check_count_down"] = current_step_countdown
state["prev_output_length"] = len(state.get("output_tok_ids", []))
return
output = state.get("output_tok_ids", [])
if not output:
# When in_end was set at init (budget=0, prompt already in think),
# we must force the first generated token to be the end token;
# otherwise apply() sees in_end=True but force_index=[] and
# allows an extra thinking token.
if state.get("in_end", False):
state["force_index"] = [0]
return
# Track previous output length for incremental processing
prev_length = state.get("prev_output_length", 0)
current_length = len(output)
if current_length <= prev_length:
if state.get("in_end", False):
remaining_budget = state["thinking_token_budget"] - state["think_count"]
spec_len = len(state["spec_token_ids"])
if spec_len > 0:
if 0 < remaining_budget < spec_len:
state["force_index"] = [remaining_budget]
elif remaining_budget <= 0:
state["force_index"] = [0]
else:
state["force_index"] = [spec_len]
else:
state["force_index"] = [0]
return
state["prev_output_length"] = current_length
start_len = len(self.think_start_token_ids)
absolute_start_pos = state["start_thinking"]
if state["continue_thinking"] and state["end_thinking"] > -1:
absolute_end_pos = state["end_thinking"] + len(
state.get("prompt_tok_ids") or []
)
else:
absolute_end_pos = state["end_thinking"]
# Update state based on recent sequences
# This is the case where we are in end mode, but the rejection sampler
# rejected a token before the end token,
# so we need to go back to think mode and wait for the next end token
# eg with 999: [2,4,5,999] -> [3,-1,-1,-1]
if state["in_end"] and state["end_count"] == 0:
new_tokens = output[prev_length:]
stopping_thinking = (
self.think_end_token_ids[state["end_count"]] in new_tokens
)
if not stopping_thinking:
state["in_think"] = True
state["in_end"] = False
state["end_count"] = 0
state["bonus_token_forced"] = False
if not state["in_end"]:
if absolute_start_pos >= 0 and absolute_end_pos >= 0:
# Case: ...<end>...<start>... - entering think mode
if absolute_start_pos > absolute_end_pos:
new_think_count = current_length - (absolute_start_pos + start_len)
state["in_think"] = True
state["think_count"] = new_think_count
else:
# Case: ...<start>...<end>... - exiting think mode
state["in_think"] = False
state["think_count"] = 0
elif absolute_start_pos >= 0 and not state["continue_thinking"]:
# Found think start - entering think mode
new_think_count = current_length - (absolute_start_pos + start_len)
state["in_think"] = True
state["think_count"] = new_think_count
elif absolute_end_pos >= 0:
# Found think end - exiting think mode
state["in_think"] = False
state["think_count"] = 0
elif state["in_think"]:
# Continue thinking mode, increment count by new tokens
prompt_tok_ids = state.get("prompt_tok_ids") or []
think_tokens_in_prompt = len(prompt_tok_ids) - (
absolute_start_pos + start_len
)
state["think_count"] = (
len(state["output_tok_ids"]) + think_tokens_in_prompt
)
if state["in_think"]:
remaining_budget = max(
0, state["thinking_token_budget"] - state["think_count"]
)
state["check_count_down"] = remaining_budget
else:
state["check_count_down"] = state["thinking_token_budget"]
total_thinking_tokens = (
state["think_count"] + len(state["spec_token_ids"]) + 1
)
# Check if need to transition to end mode
# If we have more thinking tokens than the budget,
# we need to transition to end mode
if (
state["in_think"]
and total_thinking_tokens > state["thinking_token_budget"]
):
# Calculate force_index: position within spec_token_ids where
# forcing starts. If we're already over budget without spec
# tokens, force from position 0. Force from the position
# where budget is exceeded.
state["in_think"] = False
state["in_end"] = True
state["end_count"] = 0
state["check_count_down"] = state["thinking_token_budget"]
remaining_budget = state["thinking_token_budget"] - state["think_count"]
spec_len = len(state["spec_token_ids"])
if 0 < remaining_budget < spec_len:
state["force_index"] = [remaining_budget]
elif remaining_budget <= 0:
state["force_index"] = [0]
else:
# remaining_budget >= spec_len: all spec tokens are within
# budget; force the bonus token position
state["force_index"] = [len(state["spec_token_ids"])]
else:
state["force_index"] = []
if len(state["spec_token_ids"]) > 0:
for i, token_id in enumerate(state["spec_token_ids"]):
if state["end_count"] + 1 < len(self.think_end_token_ids):
if token_id == self.think_end_token_ids[state["end_count"] + 1]:
state["end_count"] += 1
else:
state["end_count"] += 1
state["force_index"] = [i]
break
else:
state["end_count"] += 1
if len(state["force_index"]) == 0:
state["end_count"] += 1
state["force_index"] = [len(state["spec_token_ids"])]
else:
state["end_count"] += 1
state["force_index"] = [0]
if state["end_count"] >= len(self.think_end_token_ids):
state.update(
{
"in_end": False,
"end_count": 0,
"check_count_down": state["thinking_token_budget"],
}
)
def _apply_forcing_to_logits(
self,
logits: torch.Tensor,
predict_bonus_token: bool,
spec_token_ids_for_layout: list[list[int]],
) -> torch.Tensor:
self.mask[:] = False
cumulative_total = 0
self.cu_num_tokens.clear()
n_layout = len(spec_token_ids_for_layout)
if self._state:
n_layout = max(n_layout, max(self._state.keys()) + 1)
for index in range(n_layout):
self.cu_num_tokens[index] = cumulative_total
spec_tokens = (
spec_token_ids_for_layout[index]
if index < len(spec_token_ids_for_layout)
else []
)
if self.in_spec_mode:
cumulative_total += len(spec_tokens) if not predict_bonus_token else 1
else:
cumulative_total += 1
for seq_idx in sorted(self._state.keys()):
if seq_idx not in self.cu_num_tokens:
continue
state = self._state[seq_idx]
if state.get("in_end", False):
# logits processor in spec mode are called twice
# once for bonus token logits and
# second time for the target logits
# in case the force index is bonus token index
# we change the force index to 0
if predict_bonus_token:
if state.get("force_index") and state["force_index"][0] < len(
state["spec_token_ids"]
):
continue
else:
state["force_index"] = [0]
# continue enforcing the end thinking tokens
if state["end_count"] > 0:
state["bonus_token_forced"] = False
if state and not state["bonus_token_forced"]:
force_index = state.get("force_index", [])
if len(force_index) == 0:
continue
end_count = state.get("end_count", 0)
for force_idx in force_index:
if end_count < len(self.think_end_token_ids):
mask_idx = self.cu_num_tokens[seq_idx] + force_idx
if mask_idx < len(self.mask) and mask_idx < logits.shape[0]:
self.mask[mask_idx] = True
self.force_token_ids[mask_idx] = (
self.think_end_token_ids[end_count]
)
if predict_bonus_token:
if state["end_count"] > 0:
state["bonus_token_forced"] = False
state["force_index"] = []
else:
state["bonus_token_forced"] = True
has_active_thinking = any(
state.get("in_end", False) for state in self._state.values()
)
if has_active_thinking:
active_indices = self.mask.nonzero(as_tuple=False).view(-1)
if len(active_indices) > 0:
force_tokens = self.force_token_ids[active_indices]
logits[active_indices, force_tokens] = 1e9
return logits
+30 -2
View File
@@ -8,6 +8,7 @@ from typing import cast
import numpy as np
import torch
from vllm.config.reasoning import ReasoningConfig
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.pooling_params import PoolingParams
@@ -22,6 +23,9 @@ from vllm.v1.sample.logits_processor import (
MoveDirectionality,
)
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.thinking_budget_state import (
maybe_create_thinking_budget_state_holder,
)
from vllm.v1.utils import copy_slice
from vllm.v1.worker.block_table import MultiGroupBlockTable
@@ -92,12 +96,20 @@ class InputBatch:
max_num_blocks_per_req: list[int] | None = None,
logitsprocs: LogitsProcessors | None = None,
logitsprocs_need_output_token_ids: bool = False,
is_spec_decode: bool = False,
num_spec_tokens: int = 0,
is_pooling_model: bool = False,
cp_kv_cache_interleave_size: int = 1,
reasoning_config: ReasoningConfig | None = None,
):
self.thinking_budget_state_holder = maybe_create_thinking_budget_state_holder(
reasoning_config,
max_num_reqs,
num_spec_tokens,
device,
pin_memory,
)
self.thinking_token_budget_reqs: set[str] = set()
self.is_pooling_model = is_pooling_model
self.is_spec_decode = is_spec_decode
self.max_num_reqs = max_num_reqs
self.max_model_len = max_model_len
self.max_num_batched_tokens = max_num_batched_tokens
@@ -540,6 +552,7 @@ class InputBatch:
# False means we don't fill with -inf.
self.allowed_token_ids_mask_cpu_tensor[req_index].fill_(False)
self.bad_words_token_ids.pop(req_index, None)
self.thinking_token_budget_reqs.discard(req_id)
return req_index
def swap_states(self, i1: int, i2: int) -> None:
@@ -800,6 +813,8 @@ class InputBatch:
# reset batch update tracking.
# Update sampling metadata if batch state is changed.
batch_update = self.batch_update_builder.get_and_reset(self.num_reqs)
if self.thinking_budget_state_holder is not None and batch_update:
self.thinking_budget_state_holder.sync_batch(batch_update)
for logit_proc in self.logitsprocs.all:
logit_proc.update_state(batch_update)
if batch_update:
@@ -853,10 +868,15 @@ class InputBatch:
# Only set output_token_ids if required by the current requests'
# sampling parameters.
holder = self.thinking_budget_state_holder
thinking_budget_tracks_reqs = (
holder is not None and holder.has_tracked_requests()
)
needs_output_token_ids = (
not self.no_penalties
or bool(self.bad_words_token_ids)
or self.logitsprocs_need_output_token_ids
or not thinking_budget_tracks_reqs
)
output_token_ids = (
cast(list[list[int]], self.req_output_token_ids)
@@ -902,6 +922,7 @@ class InputBatch:
allowed_token_ids_mask=allowed_token_ids_mask,
bad_words_token_ids=self.bad_words_token_ids,
logitsprocs=self.logitsprocs,
thinking_budget_state_holder=self.thinking_budget_state_holder,
)
def get_pooling_params(self) -> list[PoolingParams]:
@@ -1076,6 +1097,13 @@ class InputBatch:
and len(self.repetition_penalties_reqs) == 0
)
@property
def no_thinking_budget(self) -> bool:
return (
self.thinking_budget_state_holder is None
or len(self.thinking_token_budget_reqs) == 0
)
@property
def max_num_logprobs(self) -> int | None:
return max(self.num_logprobs.values()) if self.num_logprobs else None
+4 -2
View File
@@ -629,7 +629,7 @@ class GPUModelRunner(
vocab_size=self.model_config.get_vocab_size(),
block_sizes=[placeholder_block_size],
kernel_block_sizes=[placeholder_block_size],
is_spec_decode=bool(self.vllm_config.speculative_config),
num_spec_tokens=self.num_spec_tokens,
logitsprocs=build_logitsprocs(
self.vllm_config,
self.device,
@@ -645,6 +645,7 @@ class GPUModelRunner(
or self.vllm_config.reasoning_config is not None,
is_pooling_model=self.is_pooling_model,
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
reasoning_config=self.vllm_config.reasoning_config,
)
# Separate cuda stream for overlapping transfer of sampled token ids from
@@ -6504,10 +6505,11 @@ class GPUModelRunner(
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
max_num_blocks_per_req=max_num_blocks,
is_spec_decode=bool(self.vllm_config.speculative_config),
num_spec_tokens=self.num_spec_tokens,
logitsprocs=self.input_batch.logitsprocs,
logitsprocs_need_output_token_ids=self.input_batch.logitsprocs_need_output_token_ids,
is_pooling_model=self.is_pooling_model,
reasoning_config=self.vllm_config.reasoning_config,
)
assert self._init_block_sizes == block_sizes, (