diff --git a/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py b/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py
index d2db50082a..d7a601114b 100644
--- a/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py
+++ b/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py
@@ -1,18 +1,62 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""E2E tests for thinking_token_budget with reasoning models."""
+"""E2E tests for ``thinking_token_budget`` with reasoning models.
+
+Covers Qwen3-0.6B and Qwen3.5 FP8 + MTP.
+"""
+
+import asyncio
+import json
+from typing import Literal
import openai
import pytest
import pytest_asyncio
-from tests.utils import RemoteOpenAIServer
+from tests.utils import RemoteOpenAIServer, multi_gpu_only, requires_fp8
+from vllm.platforms import current_platform
+from vllm.tokenizers import get_tokenizer
MODEL_NAME = "Qwen/Qwen3-0.6B"
+QWEN35_FP8_MTP_MODEL = "Qwen/Qwen3.5-35B-A3B-FP8"
MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
THINK_BUDGET = 5
+REASONING_START_STR = ""
+REASONING_END_STR = ""
+
+
+def _count_reasoning_decode_token_ids_between_markers(
+ full_token_ids: list[int],
+ reasoning_start_ids: list[int],
+ reasoning_end_ids: list[int],
+) -> int | None:
+ """Count decode tokens in the thinking span (after last start, before first end)."""
+
+ if not reasoning_start_ids or not reasoning_end_ids:
+ raise ValueError("reasoning marker token id lists must be non-empty")
+
+ def _last_subseq_index(haystack: list[int], needle: list[int]) -> int:
+ n = len(needle)
+ if n > len(haystack):
+ return -1
+ for i in range(len(haystack) - n, -1, -1):
+ if haystack[i : i + n] == needle:
+ return i
+ return -1
+
+ last_start = _last_subseq_index(full_token_ids, reasoning_start_ids)
+ if last_start < 0:
+ return None
+
+ pos_after_start = last_start + len(reasoning_start_ids)
+ end_n = len(reasoning_end_ids)
+ for j in range(pos_after_start, len(full_token_ids) - end_n + 1):
+ if full_token_ids[j : j + end_n] == reasoning_end_ids:
+ return j - pos_after_start
+ return len(full_token_ids) - pos_after_start
+
@pytest.fixture(scope="module")
def server():
@@ -48,6 +92,51 @@ def server_with_auto_reasoning_config():
yield remote_server
+@pytest.fixture(scope="module")
+def server_qwen35_fp8_mtp_tp2():
+ """Qwen3.5-35B FP8 with MTP speculative decoding and tensor parallel size 2."""
+ if current_platform.device_count() < 2:
+ pytest.skip("Need at least 2 GPUs for --tensor-parallel-size 2")
+ if not current_platform.supports_fp8():
+ pytest.skip("FP8 is not supported on this platform")
+
+ spec_cfg = {
+ "method": "mtp",
+ "num_speculative_tokens": 2,
+ "max_model_len": 32768,
+ }
+ args = [
+ "--tensor-parallel-size",
+ "2",
+ "--max-model-len",
+ "32768",
+ "--speculative-config",
+ json.dumps(spec_cfg),
+ "--reasoning-parser",
+ "qwen3",
+ "--reasoning-config",
+ json.dumps(
+ {
+ "reasoning_start_str": REASONING_START_STR,
+ "reasoning_end_str": REASONING_END_STR,
+ }
+ ),
+ ]
+ # With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers
+ # on 0,1 do not exhaust memory on the same devices as this worker.
+ env_dict = None
+ if current_platform.device_count() >= 4:
+ env_dict = {"CUDA_VISIBLE_DEVICES": "2,3"}
+
+ with RemoteOpenAIServer(
+ QWEN35_FP8_MTP_MODEL,
+ args,
+ max_wait_seconds=3000,
+ env_dict=env_dict,
+ ) as remote_server:
+ yield remote_server
+
+
@pytest_asyncio.fixture
async def client(request, server, server_with_auto_reasoning_config):
server_map = {
@@ -89,8 +178,10 @@ async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
"""Test that thinking_token_budget limits the number of reasoning tokens.
- In streaming mode each reasoning delta corresponds to one token, so
- counting non-empty reasoning_content chunks gives the exact token count.
+ Counts non-empty streaming ``delta.reasoning`` chunks (coarse proxy; each
+ chunk may represent multiple decode tokens — see
+ ``_count_reasoning_decode_token_ids_between_markers`` and the Qwen3.5 MTP
+ test for id-based checks).
"""
reasoning_token_count = 0
@@ -110,3 +201,89 @@ async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI
f"reasoning tokens ({reasoning_token_count}) exceeded "
f"thinking_token_budget ({THINK_BUDGET})"
)
+
+
+@pytest.mark.asyncio
+@multi_gpu_only(num_gpus=2)
+@requires_fp8
+async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_plain(
+ server_qwen35_fp8_mtp_tp2,
+):
+ """Concurrent chat requests: some with ``thinking_token_budget``, some without.
+
+ Exercises the scheduler / input processor under a mixed batch on the same
+ Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with
+ ``_count_reasoning_decode_token_ids_between_markers`` on full token ids.
+ """
+
+ _batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [
+ ("budget", 1),
+ ("budget", 12),
+ ("plain", None),
+ ("budget", 20),
+ ("budget", 14),
+ ("plain", None),
+ ("plain", None),
+ ("budget", 12),
+ ("plain", None),
+ ]
+
+ tokenizer = get_tokenizer(tokenizer_name=QWEN35_FP8_MTP_MODEL)
+ start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
+ end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
+
+ async with server_qwen35_fp8_mtp_tp2.get_async_client() as client:
+
+ async def budgeted_call(expected_budget: int):
+ return await client.chat.completions.create(
+ model=QWEN35_FP8_MTP_MODEL,
+ messages=MESSAGES,
+ max_tokens=256,
+ stream=False,
+ extra_body={
+ "thinking_token_budget": expected_budget,
+ "return_token_ids": True,
+ },
+ )
+
+ async def plain_call():
+ return await client.chat.completions.create(
+ model=QWEN35_FP8_MTP_MODEL,
+ messages=MESSAGES,
+ max_tokens=256,
+ stream=False,
+ )
+
+ coros = []
+ for row in _batch_spec:
+ if row[0] == "budget":
+ b = row[1]
+ assert isinstance(b, int)
+ coros.append(budgeted_call(b))
+ else:
+ coros.append(plain_call())
+ results = await asyncio.gather(*coros)
+
+ for i, (response, (kind, expected_budget)) in enumerate(
+ zip(results, _batch_spec, strict=True)
+ ):
+ msg = response.choices[0].message
+ assert msg.content or getattr(msg, "reasoning", None), (
+ f"index {i} ({kind}): empty message"
+ )
+
+ if kind == "budget":
+ assert expected_budget is not None
+ assert response.prompt_token_ids is not None
+ assert response.choices[0].token_ids is not None
+ full_ids = list(response.prompt_token_ids) + list(
+ response.choices[0].token_ids
+ )
+ n_reason = _count_reasoning_decode_token_ids_between_markers(
+ full_ids, start_ids, end_ids
+ )
+ assert n_reason is not None, f"index {i}: missing reasoning start in ids"
+ assert n_reason == expected_budget, (
+ f"index {i}: reasoning decode token ids ({n_reason}) != "
+ f"thinking_token_budget ({expected_budget})"
+ )
diff --git a/tests/v1/logits_processors/test_correctness.py b/tests/v1/logits_processors/test_correctness.py
index 9ee6a70abe..1326b96346 100644
--- a/tests/v1/logits_processors/test_correctness.py
+++ b/tests/v1/logits_processors/test_correctness.py
@@ -30,10 +30,13 @@ from vllm.v1.sample.logits_processor import (
MinPLogitsProcessor,
MinTokensLogitsProcessor,
MoveDirectionality,
- ThinkingTokenBudgetLogitsProcessor,
build_logitsprocs,
)
from vllm.v1.sample.metadata import SamplingMetadata
+from vllm.v1.sample.thinking_budget_state import (
+ ThinkingBudgetStateHolder,
+ maybe_create_thinking_budget_state_holder,
+)
PIN_MEMORY_AVAILABLE = is_pin_memory_available()
MAX_NUM_REQS = 256
@@ -48,8 +51,10 @@ MAX_NUM_PROMPT_TOKENS = 64
MIN_TOKENS_LEN_THRESHOLD = 5
REQS_PER_LOGITPROC = 50
STR_NO_LOGITPROC = "none"
+# Thinking budget uses ``ThinkingBudgetStateHolder`` (not a logits processor).
+STR_THINKING_BUDGET = "thinking_budget"
-# ThinkingTokenBudgetLogitsProcessor testing constants
+# Thinking token budget testing constants
THINKING_TOKEN_BUDGET = 5
THINK_START_TOKEN_ID = 999
THINK_END_TOKEN_ID = 998
@@ -80,15 +85,8 @@ class LogitsProcsRequestParams:
if num_tokens > 0:
# Use diverse random tokens
self.out_tokens = [random.randint(1, 950) for _ in range(num_tokens)]
- # Set first token for ThinkingTokenBudget testing
- is_thinking_processor = (
- logitproc_type is ThinkingTokenBudgetLogitsProcessor
- or (
- hasattr(logitproc_type, "__name__")
- and logitproc_type.__name__ == "ThinkingTokenBudgetLogitsProcessor"
- )
- )
- if is_thinking_processor:
+ # Think-start seed for ``STR_THINKING_BUDGET`` rows.
+ if logitproc_type == STR_THINKING_BUDGET:
self.out_tokens[0] = THINK_START_TOKEN_ID
else:
self.out_tokens = []
@@ -102,7 +100,7 @@ class LogitsProcsRequestParams:
class MockReasoningConfig:
- """Mock reasoning config for testing ThinkingTokenBudgetLogitsProcessor."""
+ """Minimal reasoning config for ``ThinkingBudgetStateHolder`` tests."""
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
reasoning_end_token_ids = [THINK_END_TOKEN_ID]
@@ -137,6 +135,18 @@ def _generate_fake_sampling_metadata(
is_pin_memory=PIN_MEMORY_AVAILABLE,
is_pooling_model=False,
)
+ num_spec = (
+ vllm_config.speculative_config.num_speculative_tokens
+ if vllm_config.speculative_config
+ else 0
+ )
+ thinking_holder = maybe_create_thinking_budget_state_holder(
+ vllm_config.reasoning_config,
+ vllm_config.scheduler_config.max_num_seqs,
+ num_spec,
+ device,
+ PIN_MEMORY_AVAILABLE,
+ )
fake_sampling_metadata = SamplingMetadata(
temperature=torch.full((batch_size,), 0.0),
all_greedy=True,
@@ -156,6 +166,7 @@ def _generate_fake_sampling_metadata(
allowed_token_ids_mask=None,
bad_words_token_ids={},
logitsprocs=logitsprocs,
+ thinking_budget_state_holder=thinking_holder,
)
return fake_sampling_metadata
@@ -187,7 +198,7 @@ def _sampling_params_from_logitproc(logitproc_type: LogitprocType) -> SamplingPa
def _generate_mixed_logitsprocs_batch_params(
reqs_per_logitproc: int,
- logitsprocs_types: list[str],
+ logitsprocs_types: list[LogitprocType],
) -> list[LogitsProcsRequestParams]:
"""Define key params for a batch of requests with a different
logitproc enabled per request.
@@ -450,23 +461,21 @@ def _thinking_budget_validate(
request_params: LogitsProcsRequestParams,
step_idx: int,
) -> None:
- """Validate thinking token budget processor behavior"""
- # Get the ThinkingTokenBudgetLogitsProcessor instance
- tb_processor: ThinkingTokenBudgetLogitsProcessor = next(
- test_fakes.get_logitsprocs_by_cls(ThinkingTokenBudgetLogitsProcessor)
- )
+ """Validate ``ThinkingBudgetStateHolder`` thinking-budget behavior.
- # Get current request state
- state = tb_processor._state.get(batch_index)
+ State is keyed by **batch slot** (same index space as logits rows), matching
+ ``sync_batch`` / sampler integration (see PR #34668 discussion).
+ """
+ holder = test_fakes.sampling_metadata.thinking_budget_state_holder
+ assert holder is not None
+ state = holder._state.get(batch_index)
params = request_params.params
- # Validate thinking token budget configuration
if hasattr(params, "thinking_token_budget") and params.thinking_token_budget:
- # State should exist for requests with thinking_token_budget
if state is None:
_raise_error_invalid(
msg_suffix=(
- f"Expected state for batch {batch_index} "
+ f"Expected holder state for batch slot {batch_index} "
f"with thinking_token_budget={params.thinking_token_budget}"
),
batch_index=batch_index,
@@ -474,10 +483,8 @@ def _thinking_budget_validate(
step_idx=step_idx,
)
- # Validate budget matches what was set
expected_budget = params.thinking_token_budget
actual_budget = state["thinking_token_budget"]
-
if actual_budget != expected_budget:
_raise_error_invalid(
msg_suffix=(
@@ -488,13 +495,9 @@ def _thinking_budget_validate(
step_idx=step_idx,
)
- # Check if we're in thinking mode and validate token counting
output_tokens = request_params.out_tokens
-
- # Find if thinking has started in output tokens
+ start_tokens = holder.think_start_token_ids
thinking_started = False
- start_tokens = tb_processor.reasoning_start_token_ids
-
if len(start_tokens) > 0:
for i in range(len(output_tokens) - len(start_tokens) + 1):
if output_tokens[i : i + len(start_tokens)] == start_tokens:
@@ -502,61 +505,42 @@ def _thinking_budget_validate(
break
if thinking_started:
- # If budget is exceeded, validate end token forcing
think_count = state["think_count"]
budget = state["thinking_token_budget"]
+ if think_count >= budget and not state["in_end"]:
+ _raise_error_invalid(
+ msg_suffix=(
+ f"Budget exceeded ({think_count} >= {budget}) but "
+ "in_end is false"
+ ),
+ batch_index=batch_index,
+ request_params=request_params,
+ step_idx=step_idx,
+ )
- if think_count >= budget:
- if not state["in_end"]:
+ end_tokens = holder.think_end_token_ids
+ if (
+ think_count >= budget
+ and state["in_end"]
+ and len(end_tokens) > 0
+ and holder.has_tracked_requests()
+ ):
+ expected_end_token_id = end_tokens[
+ min(state["end_count"], len(end_tokens) - 1)
+ ]
+ # Holder bumps forced vocab positions to 1e9 (does not -inf others).
+ forced_logit = float(logits_new[batch_index, expected_end_token_id])
+ if forced_logit < 1.0e8:
_raise_error_invalid(
msg_suffix=(
- f"Budget exceeded ({think_count} >= "
- f"{budget}) but not "
- "forcing end tokens"
+ f"Expected forced end token {expected_end_token_id} "
+ f"with large logit, got {forced_logit}"
),
batch_index=batch_index,
request_params=request_params,
step_idx=step_idx,
)
- # Validate that only end tokens are allowed
- end_tokens = tb_processor.reasoning_end_token_ids
- if len(end_tokens) > 0:
- expected_end_token_id = end_tokens[
- min(state["end_count"], len(end_tokens) - 1)
- ]
-
- # Check logits masking
- batch_logits = logits_new[batch_index]
- for token_id in range(len(batch_logits)):
- logit_value = batch_logits[token_id]
-
- if token_id == expected_end_token_id:
- # End token should not be masked
- if logit_value == -float("inf"):
- _raise_error_invalid(
- msg_suffix=(
- f"End token {token_id} should not be "
- "masked but is"
- ),
- batch_index=batch_index,
- request_params=request_params,
- step_idx=step_idx,
- )
- else:
- # All other tokens should be masked when forcing end
- if logit_value != -float("inf"):
- _raise_error_invalid(
- msg_suffix=(
- f"Token {token_id} should be masked "
- f"when forcing end tokens, but "
- f"logit={logit_value}"
- ),
- batch_index=batch_index,
- request_params=request_params,
- step_idx=step_idx,
- )
-
def _none_validate(
test_fakes: LogitsprocsTestFakes,
@@ -604,7 +588,7 @@ logitsprocs_test_mapping = {
MinTokensLogitsProcessor: LogitsprocTestHelpers(
gen_request_fxn=_min_tokens_params, eval_fxn=_min_tokens_validate
),
- ThinkingTokenBudgetLogitsProcessor: LogitsprocTestHelpers(
+ STR_THINKING_BUDGET: LogitsprocTestHelpers(
gen_request_fxn=_thinking_budget_params, eval_fxn=_thinking_budget_validate
),
}
@@ -614,20 +598,17 @@ def _get_test_cases() -> list[list[str]]:
"""Each test case is a set of logitsprocs"""
logitsprocs_types = list(logitsprocs_test_mapping.keys())
- # Isolate ThinkingTokenBudgetLogitsProcessor from all other processors
- # to avoid unexpected modification of logits interference
- thinking_processor = ThinkingTokenBudgetLogitsProcessor
+ # Isolate thinking-budget handling from other processors to avoid cross-talk.
+ thinking_id: LogitprocType = STR_THINKING_BUDGET
other_processors = [
- p
- for p in logitsprocs_types
- if p != STR_NO_LOGITPROC and p != thinking_processor
+ p for p in logitsprocs_types if p != STR_NO_LOGITPROC and p != thinking_id
]
return (
[[STR_NO_LOGITPROC]]
+ [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors]
+ [other_processors]
- + [[thinking_processor]]
+ + [[thinking_id]]
)
@@ -802,12 +783,23 @@ def _assert_valid(
)
+def _slot_outputs_for_metadata(
+ persistent_batch: list[LogitsProcsRequestParams], pad_len: int
+) -> list[list[int]]:
+ """Per-batch-slot output token ids aligned with ``SamplingMetadata`` rows."""
+ rows: list[list[int]] = [[] for _ in range(pad_len)]
+ for i, req in enumerate(persistent_batch):
+ if i < pad_len:
+ rows[i] = list(req.out_tokens)
+ return rows
+
+
@create_new_process_for_each_test()
@pytest.mark.parametrize("device", DEVICES)
@pytest.mark.parametrize("reqs_per_logitproc", [REQS_PER_LOGITPROC])
@pytest.mark.parametrize("logitsprocs_under_test", _get_test_cases())
def test_logitsprocs(
- device: str, reqs_per_logitproc: int, logitsprocs_under_test: list[str]
+ device: str, reqs_per_logitproc: int, logitsprocs_under_test: list[LogitprocType]
):
random.seed(40)
torch.set_default_device(device)
@@ -855,9 +847,10 @@ def test_logitsprocs(
# Apply fake batch update to logitsprocs
fake_update_logitsprocs_state(test_fakes, batch_update)
- # Emulate application of logits processors in engine
+ # Emulate application of logits processors + thinking holder (sampler order).
slice_idxs = [req.workload_index for req in persistent_batch]
- logits_w_lp = fake_apply_logitsprocs(test_fakes, slice_idxs).cpu()
+ slot_rows = _slot_outputs_for_metadata(persistent_batch, workload_size)
+ logits_w_lp = fake_apply_logitsprocs(test_fakes, slice_idxs, slot_rows).cpu()
_assert_valid(
batch_size=batch_size,
@@ -869,3 +862,263 @@ def test_logitsprocs(
)
step_idx += 1
+
+
+class MockReasoningNoEndTokens:
+ """Reasoning config with no end token ids (disables enforcement in holder)."""
+
+ reasoning_start_token_ids = [THINK_START_TOKEN_ID]
+ reasoning_end_token_ids: list[int] = []
+
+
+def test_maybe_create_thinking_budget_holder_without_reasoning():
+ cfg = VllmConfig()
+ assert cfg.reasoning_config is None
+ assert (
+ maybe_create_thinking_budget_state_holder(
+ None,
+ cfg.scheduler_config.max_num_seqs,
+ 0,
+ torch.device("cpu"),
+ False,
+ )
+ is None
+ )
+
+
+def test_thinking_budget_holder_has_tracked_after_sync_add():
+ vc = VllmConfig()
+ vc.reasoning_config = MockReasoningConfig()
+ h = ThinkingBudgetStateHolder(
+ vc.reasoning_config,
+ vc.scheduler_config.max_num_seqs,
+ 0,
+ torch.device("cpu"),
+ False,
+ )
+ assert not h.has_tracked_requests()
+ h.sync_batch(
+ BatchUpdate(
+ batch_size=1,
+ removed=(),
+ added=[
+ (
+ 0,
+ SamplingParams(thinking_token_budget=3),
+ None,
+ [THINK_START_TOKEN_ID],
+ )
+ ],
+ moved=(),
+ )
+ )
+ assert h.has_tracked_requests()
+ assert h._state[0]["thinking_token_budget"] == 3
+
+
+def test_thinking_budget_holder_sync_remove_clears_state():
+ vc = VllmConfig()
+ vc.reasoning_config = MockReasoningConfig()
+ h = ThinkingBudgetStateHolder(
+ vc.reasoning_config,
+ vc.scheduler_config.max_num_seqs,
+ 0,
+ torch.device("cpu"),
+ False,
+ )
+ h.sync_batch(
+ BatchUpdate(
+ batch_size=1,
+ removed=(),
+ added=[
+ (
+ 0,
+ SamplingParams(thinking_token_budget=3),
+ None,
+ [],
+ )
+ ],
+ moved=(),
+ )
+ )
+ assert h.has_tracked_requests()
+ h.sync_batch(BatchUpdate(batch_size=0, removed=(0,), added=(), moved=()))
+ assert not h.has_tracked_requests()
+
+
+def test_thinking_budget_holder_sync_add_without_budget_drops_row():
+ vc = VllmConfig()
+ vc.reasoning_config = MockReasoningConfig()
+ h = ThinkingBudgetStateHolder(
+ vc.reasoning_config,
+ vc.scheduler_config.max_num_seqs,
+ 0,
+ torch.device("cpu"),
+ False,
+ )
+ h.sync_batch(
+ BatchUpdate(
+ batch_size=1,
+ removed=(),
+ added=[(0, SamplingParams(), None, [])],
+ moved=(),
+ )
+ )
+ assert not h.has_tracked_requests()
+
+
+def test_thinking_budget_holder_swap_exchanges_state():
+ vc = VllmConfig()
+ vc.reasoning_config = MockReasoningConfig()
+ h = ThinkingBudgetStateHolder(
+ vc.reasoning_config,
+ vc.scheduler_config.max_num_seqs,
+ 0,
+ torch.device("cpu"),
+ False,
+ )
+ h.sync_batch(
+ BatchUpdate(
+ batch_size=2,
+ removed=(),
+ added=[
+ (
+ 0,
+ SamplingParams(thinking_token_budget=3),
+ None,
+ [],
+ ),
+ (
+ 1,
+ SamplingParams(thinking_token_budget=7),
+ None,
+ [],
+ ),
+ ],
+ moved=(),
+ )
+ )
+ b0, b1 = h._state[0]["thinking_token_budget"], h._state[1]["thinking_token_budget"]
+ h.sync_batch(
+ BatchUpdate(
+ batch_size=2,
+ removed=(),
+ added=(),
+ moved=[(0, 1, MoveDirectionality.SWAP)],
+ )
+ )
+ assert h._state[0]["thinking_token_budget"] == b1
+ assert h._state[1]["thinking_token_budget"] == b0
+
+
+def test_thinking_budget_holder_unidirectional_move():
+ vc = VllmConfig()
+ vc.reasoning_config = MockReasoningConfig()
+ h = ThinkingBudgetStateHolder(
+ vc.reasoning_config,
+ vc.scheduler_config.max_num_seqs,
+ 0,
+ torch.device("cpu"),
+ False,
+ )
+ h.sync_batch(
+ BatchUpdate(
+ batch_size=2,
+ removed=(),
+ added=[
+ (
+ 1,
+ SamplingParams(thinking_token_budget=4),
+ None,
+ [],
+ ),
+ ],
+ moved=(),
+ )
+ )
+ assert 1 in h._state and 0 not in h._state
+ h.sync_batch(
+ BatchUpdate(
+ batch_size=2,
+ removed=(),
+ added=(),
+ moved=[(1, 0, MoveDirectionality.UNIDIRECTIONAL)],
+ )
+ )
+ assert 0 in h._state and 1 not in h._state
+ assert h._state[0]["thinking_token_budget"] == 4
+
+
+def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
+ vc = VllmConfig()
+ vc.reasoning_config = MockReasoningConfig()
+ h = ThinkingBudgetStateHolder(
+ vc.reasoning_config,
+ vc.scheduler_config.max_num_seqs,
+ 0,
+ torch.device("cpu"),
+ False,
+ )
+ h.sync_batch(
+ BatchUpdate(
+ batch_size=1,
+ removed=(),
+ added=[
+ (
+ 0,
+ SamplingParams(thinking_token_budget=5),
+ None,
+ [THINK_START_TOKEN_ID],
+ )
+ ],
+ moved=(),
+ )
+ )
+ out_lists = [[THINK_START_TOKEN_ID], [THINK_START_TOKEN_ID, 10, 11, 12, 13, 14]]
+ h.update_state(
+ out_lists,
+ None,
+ torch.tensor([0, 0], dtype=torch.long),
+ )
+ assert h._state[0]["output_tok_ids"] == out_lists[1]
+
+
+def test_thinking_budget_holder_spec_mode_tensor_layout():
+ h = ThinkingBudgetStateHolder(
+ MockReasoningConfig(),
+ 8,
+ 2,
+ torch.device("cpu"),
+ False,
+ )
+ assert h.in_spec_mode
+ assert h.mask.shape[0] == 8 * (2 + 1)
+
+
+def test_thinking_budget_holder_empty_end_tokens_disables_row():
+ vc = VllmConfig()
+ vc.reasoning_config = MockReasoningNoEndTokens()
+ h = ThinkingBudgetStateHolder(
+ vc.reasoning_config,
+ vc.scheduler_config.max_num_seqs,
+ 0,
+ torch.device("cpu"),
+ False,
+ )
+ h.sync_batch(
+ BatchUpdate(
+ batch_size=1,
+ removed=(),
+ added=[
+ (
+ 0,
+ SamplingParams(thinking_token_budget=5),
+ None,
+ [THINK_START_TOKEN_ID],
+ )
+ ],
+ moved=(),
+ )
+ )
+ h.update_state([[THINK_START_TOKEN_ID, 1]], None, None)
+ assert h._state[0]["thinking_token_budget"] == -1
diff --git a/tests/v1/sample/utils.py b/tests/v1/sample/utils.py
index a0abb3b4c6..907be3614b 100644
--- a/tests/v1/sample/utils.py
+++ b/tests/v1/sample/utils.py
@@ -198,22 +198,43 @@ class LogitsprocsTestFakes(NamedTuple):
def fake_update_logitsprocs_state(
test_fakes: LogitsprocsTestFakes,
- batch_update: BatchUpdate,
+ batch_update: BatchUpdate | None,
) -> None:
"""Imitate logits processors persistent batch state update
in engine core"""
for logitproc in test_fakes.get_logitsprocs():
logitproc.update_state(batch_update)
+ holder = test_fakes.sampling_metadata.thinking_budget_state_holder
+ if holder is not None:
+ holder.sync_batch(batch_update)
def fake_apply_logitsprocs(
test_fakes: LogitsprocsTestFakes,
slice_indices: list[int],
+ slot_output_token_ids: list[list[int]] | None = None,
) -> torch.Tensor:
- """Imitate application of logits processors in engine core"""
+ """Imitate application of logits processors in engine core.
+
+ When ``thinking_budget_state_holder`` has tracked requests, this mirrors
+ :meth:`Sampler.apply_logits_processors` by refreshing per-slot
+ ``output_token_ids`` (if ``slot_output_token_ids`` is provided), then
+ ``update_state`` + ``apply_to_logits`` on the holder after built-in logits
+ processors.
+ """
logits = test_fakes.logits[torch.tensor(slice_indices, dtype=torch.long)].clone()
for processor in test_fakes.get_logitsprocs():
logits = processor.apply(logits)
+
+ md = test_fakes.sampling_metadata
+ holder = md.thinking_budget_state_holder
+ if holder is not None and holder.has_tracked_requests():
+ if slot_output_token_ids is not None:
+ for i, toks in enumerate(slot_output_token_ids):
+ if i < len(md.output_token_ids):
+ md.output_token_ids[i] = list(toks)
+ holder.update_state(md.output_token_ids, md.spec_token_ids, None)
+ logits = holder.apply_to_logits(logits, False, md.spec_token_ids)
return logits
diff --git a/tests/v1/streaming_input/test_gpu_model_runner_streaming.py b/tests/v1/streaming_input/test_gpu_model_runner_streaming.py
index 0ed7b6cb3e..946ca99507 100644
--- a/tests/v1/streaming_input/test_gpu_model_runner_streaming.py
+++ b/tests/v1/streaming_input/test_gpu_model_runner_streaming.py
@@ -39,7 +39,6 @@ def mock_model_runner_with_input_batch():
vocab_size=32000,
block_sizes=[16],
kernel_block_sizes=[16],
- is_spec_decode=False,
logitsprocs=None,
is_pooling_model=False,
)
diff --git a/vllm/v1/sample/logits_processor/__init__.py b/vllm/v1/sample/logits_processor/__init__.py
index fb4a046fc0..2cb89e1ea9 100644
--- a/vllm/v1/sample/logits_processor/__init__.py
+++ b/vllm/v1/sample/logits_processor/__init__.py
@@ -18,7 +18,6 @@ from vllm.v1.sample.logits_processor.builtin import (
LogitBiasLogitsProcessor,
MinPLogitsProcessor,
MinTokensLogitsProcessor,
- ThinkingTokenBudgetLogitsProcessor,
process_dict_updates,
)
from vllm.v1.sample.logits_processor.interface import (
@@ -51,7 +50,6 @@ BUILTIN_LOGITS_PROCESSORS: list[type[LogitsProcessor]] = [
MinTokensLogitsProcessor,
LogitBiasLogitsProcessor,
MinPLogitsProcessor,
- ThinkingTokenBudgetLogitsProcessor,
]
@@ -356,5 +354,4 @@ __all__ = [
"STR_POOLING_REJECTS_LOGITSPROCS",
"LOGITSPROCS_GROUP",
"AdapterLogitsProcessor",
- "ThinkingTokenBudgetLogitsProcessor",
]
diff --git a/vllm/v1/sample/logits_processor/builtin.py b/vllm/v1/sample/logits_processor/builtin.py
index 1739452b44..11a52711d6 100644
--- a/vllm/v1/sample/logits_processor/builtin.py
+++ b/vllm/v1/sample/logits_processor/builtin.py
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Sequence
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, TypeVar
import numpy as np
import torch
@@ -291,267 +291,6 @@ class MinTokensLogitsProcessor(LogitsProcessor):
return logits
-class ThinkingTokenBudgetLogitsProcessor(LogitsProcessor):
- """Limits the number of tokens allowed inside a 'thinking' section."""
-
- def __init__(
- self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
- ):
- reasoning_config = vllm_config.reasoning_config
- max_num_reqs = vllm_config.scheduler_config.max_num_seqs
-
- # Check if thinking is enabled
- self.is_enabled = reasoning_config is not None and reasoning_config.enabled
-
- self.reasoning_start_token_ids = getattr(
- reasoning_config, "reasoning_start_token_ids", []
- )
- self.reasoning_end_token_ids = getattr(
- reasoning_config, "reasoning_end_token_ids", []
- )
-
- self.pin_memory = is_pin_memory
- self.device = device
- # Per-request state tracking for thinking token management
- # Key: request_index, Value: state dict containing:
- # "in_think": bool - currently in thinking mode
- # "in_end": bool - currently forcing end tokens output
- # "check_count_down": int - steps remaining until next think
- # start/end token parsing
- # "think_count": int - number of thinking tokens generated
- # "end_count": int - number of end tokens forced so far
- # "thinking_token_budget": int - max allowed thinking tokens
- # "output_tok_ids": list[int] - generated output tokens
- # "prev_output_length": int - previous output length for
- # incremental processing
- self._state: dict[int, dict[str, Any]] = {}
-
- # Preallocate reusable tensors
- self.mask = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
- self.force_token_ids = torch.full(
- (max_num_reqs,), -1, dtype=torch.long, device=device
- )
-
- @staticmethod
- def _find_last_sequence_index(target_list: list[int], token_ids: list[int]) -> int:
- """
- Returns the index of the last occurrence of token_ids in target_list.
-
- Args:
- target_list (list[int]): The list of token IDs.
- token_ids (list[int]): The sequence of token IDs to find.
- """
- if not token_ids:
- return -1
- for i in range(len(target_list) - len(token_ids), -1, -1):
- if target_list[i : i + len(token_ids)] == token_ids:
- return i
- return -1
-
- def _init_state_entry(
- self, prompt_tok_ids: list[int] | None, thinking_token_budget: int
- ) -> dict[str, Any]:
- """Initializes the tracking state for a given sequence index."""
- if prompt_tok_ids is None:
- last_start = -1
- last_end = -1
- in_think = False
- think_count = 0
- else:
- last_start = self._find_last_sequence_index(
- prompt_tok_ids, self.reasoning_start_token_ids
- )
- last_end = self._find_last_sequence_index(
- prompt_tok_ids, self.reasoning_end_token_ids
- )
- in_think = last_start > last_end
- if in_think:
- think_count = len(prompt_tok_ids) - (
- last_start + len(self.reasoning_start_token_ids)
- )
- else:
- think_count = 0
-
- return {
- "in_think": in_think, # Currently in thinking mode
- "in_end": in_think and thinking_token_budget == 0,
- "check_count_down": thinking_token_budget,
- "think_count": think_count, # Number of tokens in thinking section
- "end_count": 0, # Number of end tokens forced so far
- "prompt_tok_ids": prompt_tok_ids,
- "output_tok_ids": [],
- "thinking_token_budget": thinking_token_budget,
- "prev_output_length": 0,
- # Track previous output length for incremental updates
- }
-
- def _update_think_state(self, state: dict[str, Any]):
- """Updates the state based on newly generated output tokens."""
- if not state.get("in_end", False) and state.get("check_count_down", 0) > 0:
- state["check_count_down"] -= 1
- return
-
- output = state.get("output_tok_ids", [])
- if not output:
- return
-
- # Track previous output length for incremental processing
- prev_length = state.get("prev_output_length", 0)
- current_length = len(output)
-
- if current_length <= prev_length:
- return
-
- # Process only newly added tokens
- new_tokens = output[prev_length:]
- state["prev_output_length"] = current_length
-
- # Check if new tokens contain think start or end sequences
- start_len = len(self.reasoning_start_token_ids)
- end_len = len(self.reasoning_end_token_ids)
-
- # Look for think sequences in recent tokens (including boundary)
- # Check overlapping regions where sequences might span boundaries
- check_start_idx = max(0, prev_length - max(start_len, end_len) + 1)
- recent_tokens = output[check_start_idx:]
-
- # Find any think start/end sequences in recent tokens
- recent_start_pos = self._find_last_sequence_index(
- recent_tokens, self.reasoning_start_token_ids
- )
- recent_end_pos = self._find_last_sequence_index(
- recent_tokens, self.reasoning_end_token_ids
- )
-
- # Update state based on recent sequences
- if not state["in_end"]:
- if recent_start_pos >= 0 and recent_end_pos >= 0:
- if recent_start_pos > recent_end_pos:
- # Case: ......... - entering think mode
- absolute_start_pos = check_start_idx + recent_start_pos
- new_think_count = current_length - (absolute_start_pos + start_len)
- state["in_think"] = True
- state["think_count"] = new_think_count
- else:
- # Case: ......... - exiting think mode
- state["in_think"] = False
- state["think_count"] = 0
- elif recent_start_pos >= 0:
- # Found think start - entering think mode
- absolute_start_pos = check_start_idx + recent_start_pos
- new_think_count = current_length - (absolute_start_pos + start_len)
- state["in_think"] = True
- state["think_count"] = new_think_count
- elif recent_end_pos >= 0:
- # Found think end - exiting think mode
- state["in_think"] = False
- state["think_count"] = 0
- elif state["in_think"]:
- # Continue thinking mode, increment count by new tokens
- state["think_count"] += len(new_tokens)
-
- # Set countdown based on current state
- if state["in_think"]:
- remaining_budget = max(
- 0, state["thinking_token_budget"] - state["think_count"]
- )
- state["check_count_down"] = max(0, remaining_budget - 1)
- else:
- state["check_count_down"] = state["thinking_token_budget"]
-
- # Check if need to transition to end mode
- if (
- state["in_think"]
- and state["think_count"] >= state["thinking_token_budget"]
- ):
- state["in_think"] = False
- state["in_end"] = True
- state["end_count"] = 0
- state["check_count_down"] = state["thinking_token_budget"]
- else:
- # In end mode
- state["end_count"] += 1
- if state["end_count"] >= len(self.reasoning_end_token_ids):
- state.update(
- {
- "in_end": False,
- "end_count": 0,
- "check_count_down": state["thinking_token_budget"],
- }
- )
-
- def is_argmax_invariant(self) -> bool:
- """This logits processor can change the outcome of
- greedy sampling by forcing that the thinking section
- ends after a certain number of tokens."""
- return False
-
- def update_state(self, batch_update: BatchUpdate | None):
- if not self.is_enabled:
- return
- if batch_update:
- for index, params, prompt_tok_ids, output_tok_ids in batch_update.added:
- thinking_token_budget = params.thinking_token_budget
-
- if thinking_token_budget is not None:
- self._state[index] = self._init_state_entry(
- prompt_tok_ids, thinking_token_budget
- )
- self._state[index]["output_tok_ids"] = output_tok_ids
- else:
- # Remove state if no thinking budget
- self._state.pop(index, None)
-
- for index in batch_update.removed:
- self._state.pop(index, {})
-
- for i1, i2, direction in batch_update.moved:
- if direction == MoveDirectionality.SWAP:
- state1 = self._state.pop(i1, None)
- state2 = self._state.pop(i2, None)
- if state1 is not None:
- self._state[i2] = state1
- if state2 is not None:
- self._state[i1] = state2
- else:
- state = self._state.pop(i1, None)
- if state is not None:
- self._state[i2] = state
-
- for state in self._state.values():
- self._update_think_state(state)
-
- def apply(self, logits: torch.Tensor) -> torch.Tensor:
- if not self.is_enabled or not self._state:
- return logits
-
- batch_size = logits.size(0)
- self.mask[:batch_size] = False
-
- for i in range(batch_size):
- state = self._state.get(i)
- if state and state["in_end"]:
- self.mask[i] = True
- self.force_token_ids[i] = self.reasoning_end_token_ids[
- state["end_count"]
- ]
-
- # Check in CPU first not to sync with GPU
- has_active_thinking = any(
- state.get("in_end", False) for state in self._state.values()
- )
-
- if has_active_thinking:
- current_mask = self.mask[:batch_size]
- active_indices = current_mask.nonzero(as_tuple=False).view(-1)
- if len(active_indices) > 0:
- force_tokens = self.force_token_ids[active_indices]
- # Apply a large value for the end thinking token id index
- logits[active_indices, force_tokens] = 1e9
-
- return logits
-
-
def process_dict_updates(
req_entries: dict[int, T],
batch_update: BatchUpdate | None,
diff --git a/vllm/v1/sample/metadata.py b/vllm/v1/sample/metadata.py
index 4682cde109..fa4ceac8e7 100644
--- a/vllm/v1/sample/metadata.py
+++ b/vllm/v1/sample/metadata.py
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from __future__ import annotations
+
from dataclasses import dataclass
import torch
from vllm.v1.sample.logits_processor import LogitsProcessors
+from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder
@dataclass
@@ -47,3 +50,6 @@ class SamplingMetadata:
# Speculative token ids
spec_token_ids: list[list[int]] | None = None
+ # When non-None, use ``holder.has_tracked_requests()`` to see if this batch applies
+ # thinking-token-budget logits (holder may exist with an empty tracking set).
+ thinking_budget_state_holder: ThinkingBudgetStateHolder | None = None
diff --git a/vllm/v1/sample/rejection_sampler.py b/vllm/v1/sample/rejection_sampler.py
index 2b63893c04..678654cb78 100644
--- a/vllm/v1/sample/rejection_sampler.py
+++ b/vllm/v1/sample/rejection_sampler.py
@@ -290,16 +290,24 @@ class RejectionSampler(nn.Module):
any_penalties_or_bad_words = (
sampling_metadata.bad_words_token_ids or has_penalties
)
+ holder = sampling_metadata.thinking_budget_state_holder
+ needs_thinking = holder is not None and holder.has_tracked_requests()
output_token_ids = sampling_metadata.output_token_ids
- if any_penalties_or_bad_words:
+ if any_penalties_or_bad_words or needs_thinking:
output_token_ids = self._combine_outputs_with_spec_tokens(
output_token_ids,
sampling_metadata.spec_token_ids,
)
# Calculate indices of target logits.
- if sampling_metadata.allowed_token_ids_mask is not None or has_penalties:
+ repeat_indices: torch.Tensor | None = None
+ need_repeat_indices = (
+ sampling_metadata.allowed_token_ids_mask is not None
+ or has_penalties
+ or needs_thinking
+ )
+ if need_repeat_indices:
num_requests = len(metadata.num_draft_tokens)
num_draft_tokens = torch.tensor(metadata.num_draft_tokens, device="cpu")
original_indices = torch.arange(num_requests, device="cpu")
@@ -327,7 +335,12 @@ class RejectionSampler(nn.Module):
logits = processor.apply_with_spec_decode(
logits, metadata.num_draft_tokens
)
-
+ if holder is not None and holder.has_tracked_requests():
+ logits = holder.apply_to_logits(
+ logits,
+ predict_bonus_token=False,
+ spec_token_ids=sampling_metadata.spec_token_ids,
+ )
return logits
@staticmethod
diff --git a/vllm/v1/sample/sampler.py b/vllm/v1/sample/sampler.py
index 5341351352..a77eafba25 100644
--- a/vllm/v1/sample/sampler.py
+++ b/vllm/v1/sample/sampler.py
@@ -364,9 +364,13 @@ class Sampler(nn.Module):
any_penalties_or_bad_words = (
bool(bad_words_token_ids) or not sampling_metadata.no_penalties
)
+ holder = sampling_metadata.thinking_budget_state_holder
+ needs_thinking_combine = holder is not None and holder.has_tracked_requests()
output_token_ids = sampling_metadata.output_token_ids
- if predict_bonus_token and any_penalties_or_bad_words:
+ if predict_bonus_token and (
+ any_penalties_or_bad_words or needs_thinking_combine
+ ):
# Combine base outputs with spec tokens when speculative decoding
# is enabled.
output_token_ids = self._combine_outputs_with_spec_tokens(
@@ -388,6 +392,17 @@ class Sampler(nn.Module):
# Apply penalties (e.g., freq_penalties).
logits = self.apply_penalties(logits, sampling_metadata, output_token_ids)
+ if holder is not None and holder.has_tracked_requests():
+ holder.update_state(
+ output_token_ids,
+ sampling_metadata.spec_token_ids,
+ repeat_indices=None,
+ )
+ logits = holder.apply_to_logits(
+ logits,
+ predict_bonus_token,
+ sampling_metadata.spec_token_ids,
+ )
return logits
@staticmethod
diff --git a/vllm/v1/sample/thinking_budget_state.py b/vllm/v1/sample/thinking_budget_state.py
new file mode 100644
index 0000000000..74599a1e8c
--- /dev/null
+++ b/vllm/v1/sample/thinking_budget_state.py
@@ -0,0 +1,528 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Per-batch thinking token budget state; applied after penalties at sample time."""
+
+from typing import TYPE_CHECKING, Any
+
+import torch
+
+from vllm.v1.sample.logits_processor.interface import (
+ BatchUpdate,
+ MoveDirectionality,
+)
+
+if TYPE_CHECKING:
+ from vllm.config.reasoning import ReasoningConfig
+
+
+def maybe_create_thinking_budget_state_holder(
+ reasoning_config: "ReasoningConfig | None",
+ max_num_seqs: int,
+ num_spec_tokens: int,
+ device: torch.device,
+ is_pin_memory: bool,
+) -> "ThinkingBudgetStateHolder | None":
+ if reasoning_config is None:
+ return None
+ return ThinkingBudgetStateHolder(
+ reasoning_config, max_num_seqs, num_spec_tokens, device, is_pin_memory
+ )
+
+
+class ThinkingBudgetStateHolder:
+ """Tracks thinking sections and forces end tokens when budget is exceeded."""
+
+ think_start_token_ids: list[int]
+ think_end_token_ids: list[int]
+
+ def __init__(
+ self,
+ reasoning_config: "ReasoningConfig | None",
+ max_num_seqs: int,
+ num_spec_tokens: int,
+ device: torch.device,
+ is_pin_memory: bool,
+ ):
+ _ = is_pin_memory # API parity with logits processors
+ max_num_reqs = max_num_seqs
+ self.in_spec_mode = num_spec_tokens > 0
+ self.num_spec_tokens = num_spec_tokens
+
+ # No separate enable flag: a non-``None`` ``reasoning_config`` is the switch.
+ self.is_enabled = reasoning_config is not None
+
+ if reasoning_config is None:
+ self.think_start_token_ids = []
+ self.think_end_token_ids = []
+ else:
+ rs = reasoning_config.reasoning_start_token_ids
+ re = reasoning_config.reasoning_end_token_ids
+ self.think_start_token_ids = rs if rs else []
+ self.think_end_token_ids = re if re else []
+
+ self.device = device
+ self._state: dict[int, dict[str, Any]] = {}
+ self.cu_num_tokens: dict[int, int] = {}
+
+ if self.num_spec_tokens > 0:
+ self.mask = torch.zeros(
+ max_num_reqs * (self.num_spec_tokens + 1),
+ dtype=torch.bool,
+ device=device,
+ )
+ self.force_token_ids = torch.full(
+ (max_num_reqs * (self.num_spec_tokens + 1),),
+ -1,
+ dtype=torch.long,
+ device=device,
+ )
+ else:
+ self.mask = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
+ self.force_token_ids = torch.full(
+ (max_num_reqs,), -1, dtype=torch.long, device=device
+ )
+
+ def has_tracked_requests(self) -> bool:
+ """True when ``sync_batch`` has state for a ``thinking_token_budget`` row.
+
+ Used to decide whether sampling needs output-token rows and spec combining;
+ distinct from merely having a holder instance (reasoning may be on with no
+ budgeted requests in this batch).
+ """
+ return bool(self._state)
+
+ def sync_batch(self, batch_update: BatchUpdate | None) -> None:
+ """Add/remove/move per-request state only (no _update_think_state)."""
+ if not self.is_enabled or not batch_update:
+ return
+ for index in batch_update.removed:
+ self._state.pop(index, None)
+
+ for index, params, prompt_tok_ids, output_tok_ids in batch_update.added:
+ thinking_token_budget = params.thinking_token_budget
+ if thinking_token_budget is not None:
+ self._state[index] = self._init_state_entry(
+ prompt_tok_ids, thinking_token_budget
+ )
+ self._state[index]["output_tok_ids"] = output_tok_ids
+ self._state[index]["spec_token_ids"] = []
+ else:
+ self._state.pop(index, None)
+
+ for i1, i2, direction in batch_update.moved:
+ if direction == MoveDirectionality.SWAP:
+ state1 = self._state.get(i1)
+ state2 = self._state.get(i2)
+ if state1 is not None:
+ self._state[i2] = state1
+ if state2 is not None:
+ self._state[i1] = state2
+ else:
+ state = self._state.pop(i1, None)
+ if state is not None:
+ self._state[i2] = state
+
+ def update_state(
+ self,
+ output_token_ids: list[list[int]],
+ spec_token_ids: list[list[int]] | None,
+ repeat_indices: torch.Tensor | None = None,
+ ) -> None:
+ """Refresh output/spec from sampling rows and recompute think state."""
+ if not self.is_enabled or not self._state:
+ return
+
+ spec_lists = spec_token_ids or []
+ last_row_for_req: dict[int, int] | None = None
+ if repeat_indices is not None:
+ last_row_for_req = {}
+ rpt = repeat_indices.cpu().tolist()
+ for batch_row, req_i in enumerate(rpt):
+ last_row_for_req[req_i] = batch_row
+
+ for seq_idx, state in list(self._state.items()):
+ if last_row_for_req is not None:
+ output_row: int | None = last_row_for_req.get(seq_idx)
+ if output_row is None or output_row >= len(output_token_ids):
+ continue
+ state["output_tok_ids"] = output_token_ids[output_row]
+ elif seq_idx >= len(output_token_ids):
+ continue
+ else:
+ state["output_tok_ids"] = output_token_ids[seq_idx]
+ if seq_idx < len(spec_lists):
+ state["spec_token_ids"] = list(spec_lists[seq_idx])
+ else:
+ state["spec_token_ids"] = []
+ state["in_spec_mode"] = self.in_spec_mode
+ state["force_index"] = []
+ if len(state["output_tok_ids"]) > 0:
+ spec_len = len(state["spec_token_ids"])
+ # Only strip draft suffix when there are spec tokens; ``[:-0]`` would
+ # clear the whole list (Python treats stop index 0 as "up to empty").
+ if spec_len > 0 and len(state["output_tok_ids"]) >= spec_len:
+ state["output_tok_ids"] = state["output_tok_ids"][:-spec_len]
+ self._update_think_state(state)
+
+ def apply_to_logits(
+ self,
+ logits: torch.Tensor,
+ predict_bonus_token: bool,
+ spec_token_ids: list[list[int]] | None,
+ ) -> torch.Tensor:
+ """Mask and bump logits for forced end-of-thinking tokens."""
+ if not self.is_enabled or not self._state:
+ return logits
+ spec_lists = spec_token_ids or []
+ return self._apply_forcing_to_logits(logits, predict_bonus_token, spec_lists)
+
+ @staticmethod
+ def _find_last_sequence_index(target_list: list[int], token_ids: list[int]) -> int:
+ if not token_ids:
+ return -1
+ for i in range(len(target_list) - len(token_ids), -1, -1):
+ if target_list[i : i + len(token_ids)] == token_ids:
+ return i
+ return -1
+
+ def _init_state_entry(
+ self, prompt_tok_ids: list[int] | None, thinking_token_budget: int
+ ) -> dict[str, Any]:
+ if prompt_tok_ids is None:
+ last_start = -1
+ last_end = -1
+ in_think = False
+ think_count = 0
+ start_thinking = -1
+ countdown = thinking_token_budget
+ continue_thinking = False
+ in_end = False
+ else:
+ start_thinking = -1
+ countdown = thinking_token_budget
+ continue_thinking = False
+ in_end = False
+ last_start = self._find_last_sequence_index(
+ prompt_tok_ids, self.think_start_token_ids
+ )
+ last_end = self._find_last_sequence_index(
+ prompt_tok_ids, self.think_end_token_ids
+ )
+ in_think = last_start > last_end
+ # load metrics such as think count, start thinking
+ # if request is in thinking mode, already
+ if in_think:
+ think_count = len(prompt_tok_ids) - (
+ last_start + len(self.think_start_token_ids)
+ )
+ start_thinking = len(prompt_tok_ids) - think_count - 1
+ countdown -= think_count
+ continue_thinking = True
+ # check if the token is exhausted within prompt
+ token_exhausted = thinking_token_budget - think_count
+ in_end = token_exhausted <= 0
+ else:
+ think_count = 0
+
+ return {
+ "in_think": in_think,
+ "in_end": in_end,
+ "check_count_down": countdown,
+ "think_count": think_count,
+ "end_count": 0,
+ "prompt_tok_ids": prompt_tok_ids,
+ "output_tok_ids": [],
+ "thinking_token_budget": thinking_token_budget,
+ "prev_output_length": 0,
+ "spec_token_ids": [],
+ "force_index": [],
+ "start_thinking": start_thinking,
+ "end_thinking": -1,
+ "in_spec_mode": False,
+ "bonus_token_forced": False,
+ "continue_thinking": continue_thinking,
+ }
+
+ def _update_think_state(self, state: dict[str, Any]) -> None:
+ if state.get("thinking_token_budget", -1) == -1:
+ return
+ if len(self.think_end_token_ids) == 0:
+ state["thinking_token_budget"] = -1
+ state["in_end"] = False
+ state["force_index"] = []
+ return
+
+ if state["start_thinking"] == -1:
+ start_thinking = self._find_last_sequence_index(
+ state.get("output_tok_ids", []), self.think_start_token_ids
+ )
+ state["start_thinking"] = start_thinking
+ if state["end_thinking"] == -1:
+ end_thinking = self._find_last_sequence_index(
+ state.get("output_tok_ids", []), self.think_end_token_ids
+ )
+ state["end_thinking"] = end_thinking
+
+ if state["start_thinking"] == -1:
+ return
+
+ if state["continue_thinking"]:
+ sampled_tokens_from_previous_step = len(
+ state.get("output_tok_ids", [])
+ ) - state.get("prev_output_length", 0)
+ else:
+ if state["prev_output_length"] == 0:
+ sampled_tokens_from_previous_step = len(
+ state.get("output_tok_ids", [])
+ ) - len(self.think_start_token_ids)
+ else:
+ sampled_tokens_from_previous_step = (
+ len(state.get("output_tok_ids", [])) - state["prev_output_length"]
+ )
+ current_step_countdown = (
+ state["check_count_down"] - sampled_tokens_from_previous_step
+ )
+ predicted_countdown = current_step_countdown - len(state["spec_token_ids"]) - 1
+ # We only proceed further if we have counted down the thinking budget
+ # to 0 or less and when we are in the "in think" mode.
+ if (
+ not state.get("in_end", False)
+ and predicted_countdown >= 0
+ and state["start_thinking"] > -1
+ ):
+ state["check_count_down"] = current_step_countdown
+ state["prev_output_length"] = len(state.get("output_tok_ids", []))
+ return
+ output = state.get("output_tok_ids", [])
+ if not output:
+ # When in_end was set at init (budget=0, prompt already in think),
+ # we must force the first generated token to be the end token;
+ # otherwise apply() sees in_end=True but force_index=[] and
+ # allows an extra thinking token.
+ if state.get("in_end", False):
+ state["force_index"] = [0]
+ return
+
+ # Track previous output length for incremental processing
+ prev_length = state.get("prev_output_length", 0)
+ current_length = len(output)
+
+ if current_length <= prev_length:
+ if state.get("in_end", False):
+ remaining_budget = state["thinking_token_budget"] - state["think_count"]
+ spec_len = len(state["spec_token_ids"])
+ if spec_len > 0:
+ if 0 < remaining_budget < spec_len:
+ state["force_index"] = [remaining_budget]
+ elif remaining_budget <= 0:
+ state["force_index"] = [0]
+ else:
+ state["force_index"] = [spec_len]
+ else:
+ state["force_index"] = [0]
+ return
+
+ state["prev_output_length"] = current_length
+
+ start_len = len(self.think_start_token_ids)
+ absolute_start_pos = state["start_thinking"]
+
+ if state["continue_thinking"] and state["end_thinking"] > -1:
+ absolute_end_pos = state["end_thinking"] + len(
+ state.get("prompt_tok_ids") or []
+ )
+ else:
+ absolute_end_pos = state["end_thinking"]
+ # Update state based on recent sequences
+ # This is the case where we are in end mode, but the rejection sampler
+ # rejected a token before the end token,
+ # so we need to go back to think mode and wait for the next end token
+ # eg with 999: [2,4,5,999] -> [3,-1,-1,-1]
+ if state["in_end"] and state["end_count"] == 0:
+ new_tokens = output[prev_length:]
+ stopping_thinking = (
+ self.think_end_token_ids[state["end_count"]] in new_tokens
+ )
+ if not stopping_thinking:
+ state["in_think"] = True
+ state["in_end"] = False
+ state["end_count"] = 0
+ state["bonus_token_forced"] = False
+
+ if not state["in_end"]:
+ if absolute_start_pos >= 0 and absolute_end_pos >= 0:
+ # Case: ......... - entering think mode
+ if absolute_start_pos > absolute_end_pos:
+ new_think_count = current_length - (absolute_start_pos + start_len)
+ state["in_think"] = True
+ state["think_count"] = new_think_count
+ else:
+ # Case: ......... - exiting think mode
+ state["in_think"] = False
+ state["think_count"] = 0
+
+ elif absolute_start_pos >= 0 and not state["continue_thinking"]:
+ # Found think start - entering think mode
+ new_think_count = current_length - (absolute_start_pos + start_len)
+ state["in_think"] = True
+ state["think_count"] = new_think_count
+
+ elif absolute_end_pos >= 0:
+ # Found think end - exiting think mode
+ state["in_think"] = False
+ state["think_count"] = 0
+
+ elif state["in_think"]:
+ # Continue thinking mode, increment count by new tokens
+ prompt_tok_ids = state.get("prompt_tok_ids") or []
+ think_tokens_in_prompt = len(prompt_tok_ids) - (
+ absolute_start_pos + start_len
+ )
+ state["think_count"] = (
+ len(state["output_tok_ids"]) + think_tokens_in_prompt
+ )
+ if state["in_think"]:
+ remaining_budget = max(
+ 0, state["thinking_token_budget"] - state["think_count"]
+ )
+ state["check_count_down"] = remaining_budget
+ else:
+ state["check_count_down"] = state["thinking_token_budget"]
+
+ total_thinking_tokens = (
+ state["think_count"] + len(state["spec_token_ids"]) + 1
+ )
+ # Check if need to transition to end mode
+ # If we have more thinking tokens than the budget,
+ # we need to transition to end mode
+ if (
+ state["in_think"]
+ and total_thinking_tokens > state["thinking_token_budget"]
+ ):
+ # Calculate force_index: position within spec_token_ids where
+ # forcing starts. If we're already over budget without spec
+ # tokens, force from position 0. Force from the position
+ # where budget is exceeded.
+ state["in_think"] = False
+ state["in_end"] = True
+ state["end_count"] = 0
+ state["check_count_down"] = state["thinking_token_budget"]
+ remaining_budget = state["thinking_token_budget"] - state["think_count"]
+ spec_len = len(state["spec_token_ids"])
+ if 0 < remaining_budget < spec_len:
+ state["force_index"] = [remaining_budget]
+
+ elif remaining_budget <= 0:
+ state["force_index"] = [0]
+
+ else:
+ # remaining_budget >= spec_len: all spec tokens are within
+ # budget; force the bonus token position
+ state["force_index"] = [len(state["spec_token_ids"])]
+
+ else:
+ state["force_index"] = []
+ if len(state["spec_token_ids"]) > 0:
+ for i, token_id in enumerate(state["spec_token_ids"]):
+ if state["end_count"] + 1 < len(self.think_end_token_ids):
+ if token_id == self.think_end_token_ids[state["end_count"] + 1]:
+ state["end_count"] += 1
+ else:
+ state["end_count"] += 1
+ state["force_index"] = [i]
+ break
+ else:
+ state["end_count"] += 1
+ if len(state["force_index"]) == 0:
+ state["end_count"] += 1
+ state["force_index"] = [len(state["spec_token_ids"])]
+ else:
+ state["end_count"] += 1
+ state["force_index"] = [0]
+ if state["end_count"] >= len(self.think_end_token_ids):
+ state.update(
+ {
+ "in_end": False,
+ "end_count": 0,
+ "check_count_down": state["thinking_token_budget"],
+ }
+ )
+
+ def _apply_forcing_to_logits(
+ self,
+ logits: torch.Tensor,
+ predict_bonus_token: bool,
+ spec_token_ids_for_layout: list[list[int]],
+ ) -> torch.Tensor:
+ self.mask[:] = False
+ cumulative_total = 0
+ self.cu_num_tokens.clear()
+
+ n_layout = len(spec_token_ids_for_layout)
+ if self._state:
+ n_layout = max(n_layout, max(self._state.keys()) + 1)
+
+ for index in range(n_layout):
+ self.cu_num_tokens[index] = cumulative_total
+ spec_tokens = (
+ spec_token_ids_for_layout[index]
+ if index < len(spec_token_ids_for_layout)
+ else []
+ )
+ if self.in_spec_mode:
+ cumulative_total += len(spec_tokens) if not predict_bonus_token else 1
+ else:
+ cumulative_total += 1
+
+ for seq_idx in sorted(self._state.keys()):
+ if seq_idx not in self.cu_num_tokens:
+ continue
+ state = self._state[seq_idx]
+ if state.get("in_end", False):
+ # logits processor in spec mode are called twice
+ # once for bonus token logits and
+ # second time for the target logits
+ # in case the force index is bonus token index
+ # we change the force index to 0
+ if predict_bonus_token:
+ if state.get("force_index") and state["force_index"][0] < len(
+ state["spec_token_ids"]
+ ):
+ continue
+ else:
+ state["force_index"] = [0]
+ # continue enforcing the end thinking tokens
+ if state["end_count"] > 0:
+ state["bonus_token_forced"] = False
+ if state and not state["bonus_token_forced"]:
+ force_index = state.get("force_index", [])
+ if len(force_index) == 0:
+ continue
+ end_count = state.get("end_count", 0)
+ for force_idx in force_index:
+ if end_count < len(self.think_end_token_ids):
+ mask_idx = self.cu_num_tokens[seq_idx] + force_idx
+ if mask_idx < len(self.mask) and mask_idx < logits.shape[0]:
+ self.mask[mask_idx] = True
+ self.force_token_ids[mask_idx] = (
+ self.think_end_token_ids[end_count]
+ )
+ if predict_bonus_token:
+ if state["end_count"] > 0:
+ state["bonus_token_forced"] = False
+ state["force_index"] = []
+ else:
+ state["bonus_token_forced"] = True
+
+ has_active_thinking = any(
+ state.get("in_end", False) for state in self._state.values()
+ )
+
+ if has_active_thinking:
+ active_indices = self.mask.nonzero(as_tuple=False).view(-1)
+
+ if len(active_indices) > 0:
+ force_tokens = self.force_token_ids[active_indices]
+ logits[active_indices, force_tokens] = 1e9
+
+ return logits
diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py
index 89e63f3def..5930a1aaba 100644
--- a/vllm/v1/worker/gpu_input_batch.py
+++ b/vllm/v1/worker/gpu_input_batch.py
@@ -8,6 +8,7 @@ from typing import cast
import numpy as np
import torch
+from vllm.config.reasoning import ReasoningConfig
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.pooling_params import PoolingParams
@@ -22,6 +23,9 @@ from vllm.v1.sample.logits_processor import (
MoveDirectionality,
)
from vllm.v1.sample.metadata import SamplingMetadata
+from vllm.v1.sample.thinking_budget_state import (
+ maybe_create_thinking_budget_state_holder,
+)
from vllm.v1.utils import copy_slice
from vllm.v1.worker.block_table import MultiGroupBlockTable
@@ -92,12 +96,20 @@ class InputBatch:
max_num_blocks_per_req: list[int] | None = None,
logitsprocs: LogitsProcessors | None = None,
logitsprocs_need_output_token_ids: bool = False,
- is_spec_decode: bool = False,
+ num_spec_tokens: int = 0,
is_pooling_model: bool = False,
cp_kv_cache_interleave_size: int = 1,
+ reasoning_config: ReasoningConfig | None = None,
):
+ self.thinking_budget_state_holder = maybe_create_thinking_budget_state_holder(
+ reasoning_config,
+ max_num_reqs,
+ num_spec_tokens,
+ device,
+ pin_memory,
+ )
+ self.thinking_token_budget_reqs: set[str] = set()
self.is_pooling_model = is_pooling_model
- self.is_spec_decode = is_spec_decode
self.max_num_reqs = max_num_reqs
self.max_model_len = max_model_len
self.max_num_batched_tokens = max_num_batched_tokens
@@ -540,6 +552,7 @@ class InputBatch:
# False means we don't fill with -inf.
self.allowed_token_ids_mask_cpu_tensor[req_index].fill_(False)
self.bad_words_token_ids.pop(req_index, None)
+ self.thinking_token_budget_reqs.discard(req_id)
return req_index
def swap_states(self, i1: int, i2: int) -> None:
@@ -800,6 +813,8 @@ class InputBatch:
# reset batch update tracking.
# Update sampling metadata if batch state is changed.
batch_update = self.batch_update_builder.get_and_reset(self.num_reqs)
+ if self.thinking_budget_state_holder is not None and batch_update:
+ self.thinking_budget_state_holder.sync_batch(batch_update)
for logit_proc in self.logitsprocs.all:
logit_proc.update_state(batch_update)
if batch_update:
@@ -853,10 +868,15 @@ class InputBatch:
# Only set output_token_ids if required by the current requests'
# sampling parameters.
+ holder = self.thinking_budget_state_holder
+ thinking_budget_tracks_reqs = (
+ holder is not None and holder.has_tracked_requests()
+ )
needs_output_token_ids = (
not self.no_penalties
or bool(self.bad_words_token_ids)
or self.logitsprocs_need_output_token_ids
+ or not thinking_budget_tracks_reqs
)
output_token_ids = (
cast(list[list[int]], self.req_output_token_ids)
@@ -902,6 +922,7 @@ class InputBatch:
allowed_token_ids_mask=allowed_token_ids_mask,
bad_words_token_ids=self.bad_words_token_ids,
logitsprocs=self.logitsprocs,
+ thinking_budget_state_holder=self.thinking_budget_state_holder,
)
def get_pooling_params(self) -> list[PoolingParams]:
@@ -1076,6 +1097,13 @@ class InputBatch:
and len(self.repetition_penalties_reqs) == 0
)
+ @property
+ def no_thinking_budget(self) -> bool:
+ return (
+ self.thinking_budget_state_holder is None
+ or len(self.thinking_token_budget_reqs) == 0
+ )
+
@property
def max_num_logprobs(self) -> int | None:
return max(self.num_logprobs.values()) if self.num_logprobs else None
diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py
index caf3bfdfc3..e2af34eecb 100644
--- a/vllm/v1/worker/gpu_model_runner.py
+++ b/vllm/v1/worker/gpu_model_runner.py
@@ -629,7 +629,7 @@ class GPUModelRunner(
vocab_size=self.model_config.get_vocab_size(),
block_sizes=[placeholder_block_size],
kernel_block_sizes=[placeholder_block_size],
- is_spec_decode=bool(self.vllm_config.speculative_config),
+ num_spec_tokens=self.num_spec_tokens,
logitsprocs=build_logitsprocs(
self.vllm_config,
self.device,
@@ -645,6 +645,7 @@ class GPUModelRunner(
or self.vllm_config.reasoning_config is not None,
is_pooling_model=self.is_pooling_model,
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
+ reasoning_config=self.vllm_config.reasoning_config,
)
# Separate cuda stream for overlapping transfer of sampled token ids from
@@ -6504,10 +6505,11 @@ class GPUModelRunner(
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
max_num_blocks_per_req=max_num_blocks,
- is_spec_decode=bool(self.vllm_config.speculative_config),
+ num_spec_tokens=self.num_spec_tokens,
logitsprocs=self.input_batch.logitsprocs,
logitsprocs_need_output_token_ids=self.input_batch.logitsprocs_need_output_token_ids,
is_pooling_model=self.is_pooling_model,
+ reasoning_config=self.vllm_config.reasoning_config,
)
assert self._init_block_sizes == block_sizes, (