[Bugfix] Fix thinking_token_budget not enforced after natural </think> re-entry (#45984)

Signed-off-by: Ashwin Giridharan <girida@amazon.com>
This commit is contained in:
Ashwin Giridharan
2026-07-10 22:47:51 +00:00
committed by GitHub
parent 26ff616bbf
commit ed908cf0a0
3 changed files with 453 additions and 47 deletions
+407 -3
View File
@@ -145,6 +145,7 @@ def _generate_fake_sampling_metadata(
vllm_config.scheduler_config.max_num_seqs,
num_spec,
device,
is_pin_memory=False,
)
fake_sampling_metadata = SamplingMetadata(
temperature=torch.full((batch_size,), 0.0),
@@ -879,6 +880,7 @@ def test_maybe_create_thinking_budget_holder_without_reasoning():
cfg.scheduler_config.max_num_seqs,
0,
torch.device("cpu"),
is_pin_memory=False,
)
is None
)
@@ -1251,12 +1253,20 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index
out.append(tok)
h.update_state([out], None, None)
assert h._state[0]["end_thinking"] == -1 # not present yet
expected_end_idx = len(out) # marker appended next
out.extend(end)
h.update_state([out], None, None)
assert h._state[0]["start_thinking"] == 0
assert h._state[0]["end_thinking"] == expected_end_idx
# After a natural exit, start_thinking and end_thinking are reset to -1
# (state machine prepared for next block). Verify the exit happened at the
# correct position by checking scan_offset (set to len(output) after exit).
assert not h._state[0]["in_think"], "Should have exited think mode after </think>"
assert h._state[0]["start_thinking"] == -1, (
"start_thinking should be reset after natural exit"
)
assert h._state[0]["scan_offset"] == len(out), (
f"scan_offset should point past the end marker; "
f"expected {len(out)}, got {h._state[0]['scan_offset']}"
)
# --- Thinking budget re-entry tests (issue #43708) ---
@@ -1381,3 +1391,397 @@ class TestThinkingBudgetReentry:
assert not holder._state[0]["in_end"]
assert not holder._state[0]["in_think"]
# --- Thinking budget natural-end re-entry tests (issue #45974) ---
# After a model naturally emits </think> before exhausting its budget,
# subsequent <think> blocks must still be tracked and budget-enforced.
class TestThinkingBudgetNaturalEndReentry:
"""Tests for thinking budget enforcement across multiple think blocks.
Covers both natural-end re-entry (issue #45974) and forced-end re-entry
(issue #43708).
"""
THINK_START = 100
THINK_END_SINGLE = [200]
THINK_END_MULTI = [200, 201, 202]
BUDGET = 10
CONTENT_TOKEN = 50
THINK_TOKEN = 60
@staticmethod
def _make_holder(end_token_ids):
from dataclasses import dataclass
from vllm.v1.sample.thinking_budget_state import (
ThinkingBudgetStateHolder,
)
@dataclass
class FakeReasoningConfig:
reasoning_start_token_ids: list[int]
reasoning_end_token_ids: list[int]
enabled: bool = True
cfg = FakeReasoningConfig(
reasoning_start_token_ids=[TestThinkingBudgetNaturalEndReentry.THINK_START],
reasoning_end_token_ids=end_token_ids,
)
return ThinkingBudgetStateHolder(
reasoning_config=cfg,
max_num_seqs=8,
num_spec_tokens=0,
device=torch.device("cpu"),
is_pin_memory=False,
)
@staticmethod
def _sync_batch(holder, budget):
from unittest.mock import MagicMock
params = MagicMock()
params.thinking_token_budget = budget
batch_update = MagicMock(
removed=[],
added=[(0, params, None, [])],
moved=[],
)
holder.sync_batch(batch_update)
@staticmethod
def _step(holder, output_tok_ids):
holder.update_state(
output_token_ids=[output_tok_ids],
spec_token_ids=None,
repeat_indices=None,
)
# --- Natural-end re-entry tests ---
def test_natural_end_reentry_single_token(self):
"""After natural </think>, a new <think> block must be enforced."""
holder = self._make_holder(self.THINK_END_SINGLE)
self._sync_batch(holder, self.BUDGET)
output = []
# Block 1: 6 tokens (under budget) + natural </think>
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(6):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
output.append(self.THINK_END_SINGLE[0])
self._step(holder, list(output))
assert not holder._state[0]["in_end"]
assert not holder._state[0]["in_think"]
# Some content
for _ in range(3):
output.append(self.CONTENT_TOKEN)
self._step(holder, list(output))
# Block 2: re-entry — should be budget-tracked
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(self.BUDGET + 1):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
assert holder._state[0]["in_end"], (
"Second thinking block after natural end must be budget-enforced"
)
def test_natural_end_reentry_multi_token(self):
"""Multi-token </think> natural end still allows Block 2 enforcement."""
holder = self._make_holder(self.THINK_END_MULTI)
self._sync_batch(holder, self.BUDGET)
output = []
# Block 1: 4 tokens + natural multi-token </think>
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(4):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
for tok in self.THINK_END_MULTI:
output.append(tok)
self._step(holder, list(output))
assert not holder._state[0]["in_think"]
# Content
for _ in range(5):
output.append(self.CONTENT_TOKEN)
self._step(holder, list(output))
# Block 2: re-entry
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(self.BUDGET + 1):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
assert holder._state[0]["in_end"], (
"Second block after multi-token natural end must be enforced"
)
def test_natural_end_immediate_reentry(self):
"""Natural </think> followed immediately by <think> with no content."""
holder = self._make_holder(self.THINK_END_SINGLE)
self._sync_batch(holder, self.BUDGET)
output = []
# Block 1: 5 tokens + natural end
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(5):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
output.append(self.THINK_END_SINGLE[0])
self._step(holder, list(output))
# Immediate re-entry — NO content between blocks
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(self.BUDGET + 1):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
assert holder._state[0]["in_end"], (
"Immediate re-entry after natural end must be budget-enforced"
)
def test_multiple_natural_reentries(self):
"""Block 1 natural -> Block 2 natural -> Block 3 must be enforced."""
holder = self._make_holder(self.THINK_END_SINGLE)
self._sync_batch(holder, self.BUDGET)
output = []
# Block 1: 4 tokens + natural end
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(4):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
output.append(self.THINK_END_SINGLE[0])
self._step(holder, list(output))
# Content
for _ in range(2):
output.append(self.CONTENT_TOKEN)
self._step(holder, list(output))
# Block 2: 3 tokens + natural end
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(3):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
output.append(self.THINK_END_SINGLE[0])
self._step(holder, list(output))
# Content
for _ in range(2):
output.append(self.CONTENT_TOKEN)
self._step(holder, list(output))
# Block 3: exceed budget
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(self.BUDGET + 1):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
assert holder._state[0]["in_end"], (
"Third block after two natural ends must be budget-enforced"
)
def test_natural_then_forced_in_block_2(self):
"""Block 1 ends naturally, Block 2 exceeds budget -> forced end."""
budget = 5
holder = self._make_holder(self.THINK_END_SINGLE)
self._sync_batch(holder, budget)
output = []
# Block 1: 3 tokens + natural end (under budget)
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(3):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
output.append(self.THINK_END_SINGLE[0])
self._step(holder, list(output))
assert not holder._state[0]["in_end"]
# Content
for _ in range(3):
output.append(self.CONTENT_TOKEN)
self._step(holder, list(output))
# Block 2: exceed budget
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(budget + 1):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
assert holder._state[0]["in_end"], (
"Block 2 must trigger forced end after natural end in Block 1"
)
def test_budget_one_natural_end_reentry(self):
"""Budget = 1: natural end with 0 think tokens, re-entry enforced."""
budget = 1
holder = self._make_holder(self.THINK_END_SINGLE)
self._sync_batch(holder, budget)
output = []
# Block 1: immediate natural end (0 think tokens)
output.append(self.THINK_START)
self._step(holder, list(output))
output.append(self.THINK_END_SINGLE[0])
self._step(holder, list(output))
assert not holder._state[0]["in_end"]
# Content
for _ in range(3):
output.append(self.CONTENT_TOKEN)
self._step(holder, list(output))
# Block 2: 2 think tokens (should be enforced at budget=1)
output.append(self.THINK_START)
self._step(holder, list(output))
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
assert holder._state[0]["in_end"], (
"Budget=1 must enforce after 1 token in re-entry block"
)
def test_partial_end_sequence_in_content(self):
"""Partial end sequence in content shouldn't trigger false exit."""
holder = self._make_holder(self.THINK_END_MULTI)
self._sync_batch(holder, self.BUDGET)
output = []
# Block 1: natural end with full sequence [200, 201, 202]
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(3):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
for tok in self.THINK_END_MULTI:
output.append(tok)
self._step(holder, list(output))
# Content with partial end sequence [200, 201] but not [200, 201, 202]
output.append(self.CONTENT_TOKEN)
self._step(holder, list(output))
output.append(200)
self._step(holder, list(output))
output.append(201)
self._step(holder, list(output))
output.append(self.CONTENT_TOKEN)
self._step(holder, list(output))
# Block 2: re-entry must still be enforced
output.append(self.THINK_START)
self._step(holder, list(output))
for _ in range(self.BUDGET + 1):
output.append(self.THINK_TOKEN)
self._step(holder, list(output))
assert holder._state[0]["in_end"], (
"Partial end tokens in content must not prevent Block 2 enforcement"
)
def test_continue_thinking_natural_end_reentry(self):
"""Prompt-prefill inside <think>: after natural end, Block 2 must be enforced.
When a request arrives with the prompt already inside a <think> block,
continue_thinking is set to True. This test verifies that after a
natural </think> exit, continue_thinking is cleared and Block 2
enforcement works correctly.
"""
from dataclasses import dataclass
from unittest.mock import MagicMock
from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder
@dataclass
class FakeReasoningConfig:
reasoning_start_token_ids: list[int]
reasoning_end_token_ids: list[int]
enabled: bool = True
cfg = FakeReasoningConfig(
reasoning_start_token_ids=[self.THINK_START],
reasoning_end_token_ids=self.THINK_END_SINGLE,
)
holder = ThinkingBudgetStateHolder(
reasoning_config=cfg,
max_num_seqs=8,
num_spec_tokens=0,
device=torch.device("cpu"),
is_pin_memory=False,
)
# Simulate: prompt already inside <think> (sets continue_thinking=True)
prompt_tok_ids = [self.THINK_START]
params = MagicMock()
params.thinking_token_budget = self.BUDGET
batch_update = MagicMock(
removed=[],
added=[(0, params, prompt_tok_ids, [])],
moved=[],
)
holder.sync_batch(batch_update)
assert holder._state[0]["continue_thinking"], (
"Prompt inside <think> must set continue_thinking=True"
)
# Block 1: emit some think tokens then natural </think>
output = list(prompt_tok_ids)
for _ in range(4):
output.append(self.THINK_TOKEN)
holder.update_state([output], None, None)
output.append(self.THINK_END_SINGLE[0])
holder.update_state([output], None, None)
assert not holder._state[0]["in_think"], "Should have exited think mode"
assert not holder._state[0]["continue_thinking"], (
"continue_thinking must be cleared after natural end"
)
# Content tokens
for _ in range(3):
output.append(self.CONTENT_TOKEN)
holder.update_state([output], None, None)
# Block 2: re-entry must be detected and budget enforced
output.append(self.THINK_START)
holder.update_state([output], None, None)
for _ in range(self.BUDGET + 1):
output.append(self.THINK_TOKEN)
holder.update_state([output], None, None)
assert holder._state[0]["in_end"], (
"Block 2 after natural end of prompt-prefill session must be enforced"
)
# --- Forced-end re-entry tests ---
+45 -44
View File
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.sample.logits_processor.interface import (
BatchUpdate,
MoveDirectionality,
@@ -22,11 +22,12 @@ def maybe_create_thinking_budget_state_holder(
max_num_seqs: int,
num_spec_tokens: int,
device: torch.device,
is_pin_memory: bool,
) -> "ThinkingBudgetStateHolder | None":
if reasoning_config is None:
return None
return ThinkingBudgetStateHolder(
reasoning_config, max_num_seqs, num_spec_tokens, device, PIN_MEMORY
reasoning_config, max_num_seqs, num_spec_tokens, device, is_pin_memory
)
@@ -173,19 +174,6 @@ class ThinkingBudgetStateHolder:
return i
return -1
@staticmethod
def _find_last_sequence_index_from(
target_list: list[int], token_ids: list[int], search_start: int
) -> int:
"""Last occurrence of ``token_ids`` at or after ``search_start``."""
if not token_ids:
return -1
lo = max(0, search_start)
for i in range(len(target_list) - len(token_ids), lo - 1, -1):
if target_list[i : i + len(token_ids)] == token_ids:
return i
return -1
def _init_state_entry(
self, prompt_tok_ids: list[int] | None, thinking_token_budget: int
) -> dict[str, Any]:
@@ -239,8 +227,6 @@ class ThinkingBudgetStateHolder:
"force_index": [],
"start_thinking": start_thinking,
"end_thinking": -1,
"start_search_pos": 0,
"end_search_pos": 0,
"in_spec_mode": False,
"bonus_token_forced": False,
"continue_thinking": continue_thinking,
@@ -256,41 +242,40 @@ class ThinkingBudgetStateHolder:
state["force_index"] = []
return
output_tok_ids = state.get("output_tok_ids", [])
if state["start_thinking"] == -1:
seq_len = len(self.think_start_token_ids)
scan_offset = state.get("scan_offset", 0)
start_thinking = self._find_last_sequence_index_from(
output_tok_ids,
self.think_start_token_ids,
max(scan_offset, state["start_search_pos"] - (seq_len - 1)),
output_slice = state.get("output_tok_ids", [])[scan_offset:]
start_thinking = self._find_last_sequence_index(
output_slice, self.think_start_token_ids
)
if start_thinking >= 0 and scan_offset > 0:
# Re-entry after a forced end: budget was already exhausted
# in a prior block, so immediately force-close this one.
# scan_offset > 0 is only set after forced-end completion
# (never after natural end), so this won't block legitimate
# re-entries where budget remains.
state["start_thinking"] = start_thinking
state["in_think"] = False
state["in_end"] = True
state["end_count"] = 0
state["force_index"] = [0]
return
if start_thinking >= 0:
start_thinking += scan_offset
state["start_thinking"] = start_thinking
if start_thinking == -1:
state["start_search_pos"] = len(output_tok_ids)
if state["end_thinking"] == -1:
seq_len = len(self.think_end_token_ids)
scan_offset = state.get("scan_offset", 0)
end_thinking = self._find_last_sequence_index_from(
output_tok_ids,
self.think_end_token_ids,
max(scan_offset, state["end_search_pos"] - (seq_len - 1)),
output_slice = state.get("output_tok_ids", [])[scan_offset:]
end_thinking = self._find_last_sequence_index(
output_slice, self.think_end_token_ids
)
if end_thinking >= 0:
end_thinking += scan_offset
state["end_thinking"] = end_thinking
if end_thinking == -1:
state["end_search_pos"] = len(output_tok_ids)
if (
not state.get("in_end", False)
and state["start_thinking"] >= 0
and state["end_thinking"] >= 0
and state["end_thinking"] > state["start_thinking"]
and not state.get("continue_thinking", False)
):
state["in_think"] = False
state["think_count"] = 0
state["continue_thinking"] = False
state["start_thinking"] = -1
state["end_thinking"] = -1
state["scan_offset"] = len(state.get("output_tok_ids", []))
state["check_count_down"] = state["thinking_token_budget"]
return
if state["start_thinking"] == -1:
return
@@ -314,10 +299,18 @@ class ThinkingBudgetStateHolder:
predicted_countdown = current_step_countdown - len(state["spec_token_ids"]) - 1
# We only proceed further if we have counted down the thinking budget
# to 0 or less and when we are in the "in think" mode.
# Exception: when continue_thinking=True and a natural </think> is
# detected (end_thinking != -1), fall through to handle the exit —
# even if the budget hasn't expired yet. For continue_thinking=False,
# the early natural-end detection block above already handles it.
natural_end_with_continue = (
state.get("continue_thinking", False) and state["end_thinking"] != -1
)
if (
not state.get("in_end", False)
and predicted_countdown >= 0
and state["start_thinking"] > -1
and not natural_end_with_continue
):
state["check_count_down"] = current_step_countdown
state["prev_output_length"] = len(state.get("output_tok_ids", []))
@@ -389,6 +382,10 @@ class ThinkingBudgetStateHolder:
# Case: ...<start>...<end>... - exiting think mode
state["in_think"] = False
state["think_count"] = 0
state["continue_thinking"] = False
state["start_thinking"] = -1
state["end_thinking"] = -1
state["scan_offset"] = len(state.get("output_tok_ids", []))
elif absolute_start_pos >= 0 and not state["continue_thinking"]:
# Found think start - entering think mode
@@ -400,6 +397,10 @@ class ThinkingBudgetStateHolder:
# Found think end - exiting think mode
state["in_think"] = False
state["think_count"] = 0
state["continue_thinking"] = False
state["start_thinking"] = -1
state["end_thinking"] = -1
state["scan_offset"] = len(state.get("output_tok_ids", []))
elif state["in_think"]:
# Continue thinking mode, increment count by new tokens
+1
View File
@@ -113,6 +113,7 @@ class InputBatch:
max_num_reqs,
num_spec_tokens,
device,
PIN_MEMORY,
)
self.thinking_token_budget_reqs: set[str] = set()
self.is_pooling_model = is_pooling_model