From ed908cf0a08af20839b82dc0638dae91b1ad630b Mon Sep 17 00:00:00 2001 From: Ashwin Giridharan Date: Fri, 10 Jul 2026 15:47:51 -0700 Subject: [PATCH] [Bugfix] Fix thinking_token_budget not enforced after natural re-entry (#45984) Signed-off-by: Ashwin Giridharan --- .../v1/logits_processors/test_correctness.py | 410 +++++++++++++++++- vllm/v1/sample/thinking_budget_state.py | 89 ++-- vllm/v1/worker/gpu_input_batch.py | 1 + 3 files changed, 453 insertions(+), 47 deletions(-) diff --git a/tests/v1/logits_processors/test_correctness.py b/tests/v1/logits_processors/test_correctness.py index 17dc624fe4..a51157557e 100644 --- a/tests/v1/logits_processors/test_correctness.py +++ b/tests/v1/logits_processors/test_correctness.py @@ -145,6 +145,7 @@ def _generate_fake_sampling_metadata( vllm_config.scheduler_config.max_num_seqs, num_spec, device, + is_pin_memory=False, ) fake_sampling_metadata = SamplingMetadata( temperature=torch.full((batch_size,), 0.0), @@ -879,6 +880,7 @@ def test_maybe_create_thinking_budget_holder_without_reasoning(): cfg.scheduler_config.max_num_seqs, 0, torch.device("cpu"), + is_pin_memory=False, ) is None ) @@ -1251,12 +1253,20 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index out.append(tok) h.update_state([out], None, None) assert h._state[0]["end_thinking"] == -1 # not present yet - expected_end_idx = len(out) # marker appended next out.extend(end) h.update_state([out], None, None) - assert h._state[0]["start_thinking"] == 0 - assert h._state[0]["end_thinking"] == expected_end_idx + # After a natural exit, start_thinking and end_thinking are reset to -1 + # (state machine prepared for next block). Verify the exit happened at the + # correct position by checking scan_offset (set to len(output) after exit). + assert not h._state[0]["in_think"], "Should have exited think mode after " + assert h._state[0]["start_thinking"] == -1, ( + "start_thinking should be reset after natural exit" + ) + assert h._state[0]["scan_offset"] == len(out), ( + f"scan_offset should point past the end marker; " + f"expected {len(out)}, got {h._state[0]['scan_offset']}" + ) # --- Thinking budget re-entry tests (issue #43708) --- @@ -1381,3 +1391,397 @@ class TestThinkingBudgetReentry: assert not holder._state[0]["in_end"] assert not holder._state[0]["in_think"] + + +# --- Thinking budget natural-end re-entry tests (issue #45974) --- +# After a model naturally emits before exhausting its budget, +# subsequent blocks must still be tracked and budget-enforced. + + +class TestThinkingBudgetNaturalEndReentry: + """Tests for thinking budget enforcement across multiple think blocks. + + Covers both natural-end re-entry (issue #45974) and forced-end re-entry + (issue #43708). + """ + + THINK_START = 100 + THINK_END_SINGLE = [200] + THINK_END_MULTI = [200, 201, 202] + BUDGET = 10 + CONTENT_TOKEN = 50 + THINK_TOKEN = 60 + + @staticmethod + def _make_holder(end_token_ids): + from dataclasses import dataclass + + from vllm.v1.sample.thinking_budget_state import ( + ThinkingBudgetStateHolder, + ) + + @dataclass + class FakeReasoningConfig: + reasoning_start_token_ids: list[int] + reasoning_end_token_ids: list[int] + enabled: bool = True + + cfg = FakeReasoningConfig( + reasoning_start_token_ids=[TestThinkingBudgetNaturalEndReentry.THINK_START], + reasoning_end_token_ids=end_token_ids, + ) + return ThinkingBudgetStateHolder( + reasoning_config=cfg, + max_num_seqs=8, + num_spec_tokens=0, + device=torch.device("cpu"), + is_pin_memory=False, + ) + + @staticmethod + def _sync_batch(holder, budget): + from unittest.mock import MagicMock + + params = MagicMock() + params.thinking_token_budget = budget + batch_update = MagicMock( + removed=[], + added=[(0, params, None, [])], + moved=[], + ) + holder.sync_batch(batch_update) + + @staticmethod + def _step(holder, output_tok_ids): + holder.update_state( + output_token_ids=[output_tok_ids], + spec_token_ids=None, + repeat_indices=None, + ) + + # --- Natural-end re-entry tests --- + + def test_natural_end_reentry_single_token(self): + """After natural , a new block must be enforced.""" + holder = self._make_holder(self.THINK_END_SINGLE) + self._sync_batch(holder, self.BUDGET) + + output = [] + + # Block 1: 6 tokens (under budget) + natural + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(6): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + output.append(self.THINK_END_SINGLE[0]) + self._step(holder, list(output)) + + assert not holder._state[0]["in_end"] + assert not holder._state[0]["in_think"] + + # Some content + for _ in range(3): + output.append(self.CONTENT_TOKEN) + self._step(holder, list(output)) + + # Block 2: re-entry — should be budget-tracked + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(self.BUDGET + 1): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + + assert holder._state[0]["in_end"], ( + "Second thinking block after natural end must be budget-enforced" + ) + + def test_natural_end_reentry_multi_token(self): + """Multi-token natural end still allows Block 2 enforcement.""" + holder = self._make_holder(self.THINK_END_MULTI) + self._sync_batch(holder, self.BUDGET) + + output = [] + + # Block 1: 4 tokens + natural multi-token + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(4): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + for tok in self.THINK_END_MULTI: + output.append(tok) + self._step(holder, list(output)) + + assert not holder._state[0]["in_think"] + + # Content + for _ in range(5): + output.append(self.CONTENT_TOKEN) + self._step(holder, list(output)) + + # Block 2: re-entry + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(self.BUDGET + 1): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + + assert holder._state[0]["in_end"], ( + "Second block after multi-token natural end must be enforced" + ) + + def test_natural_end_immediate_reentry(self): + """Natural followed immediately by with no content.""" + holder = self._make_holder(self.THINK_END_SINGLE) + self._sync_batch(holder, self.BUDGET) + + output = [] + + # Block 1: 5 tokens + natural end + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(5): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + output.append(self.THINK_END_SINGLE[0]) + self._step(holder, list(output)) + + # Immediate re-entry — NO content between blocks + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(self.BUDGET + 1): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + + assert holder._state[0]["in_end"], ( + "Immediate re-entry after natural end must be budget-enforced" + ) + + def test_multiple_natural_reentries(self): + """Block 1 natural -> Block 2 natural -> Block 3 must be enforced.""" + holder = self._make_holder(self.THINK_END_SINGLE) + self._sync_batch(holder, self.BUDGET) + + output = [] + + # Block 1: 4 tokens + natural end + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(4): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + output.append(self.THINK_END_SINGLE[0]) + self._step(holder, list(output)) + + # Content + for _ in range(2): + output.append(self.CONTENT_TOKEN) + self._step(holder, list(output)) + + # Block 2: 3 tokens + natural end + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(3): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + output.append(self.THINK_END_SINGLE[0]) + self._step(holder, list(output)) + + # Content + for _ in range(2): + output.append(self.CONTENT_TOKEN) + self._step(holder, list(output)) + + # Block 3: exceed budget + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(self.BUDGET + 1): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + + assert holder._state[0]["in_end"], ( + "Third block after two natural ends must be budget-enforced" + ) + + def test_natural_then_forced_in_block_2(self): + """Block 1 ends naturally, Block 2 exceeds budget -> forced end.""" + budget = 5 + holder = self._make_holder(self.THINK_END_SINGLE) + self._sync_batch(holder, budget) + + output = [] + + # Block 1: 3 tokens + natural end (under budget) + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(3): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + output.append(self.THINK_END_SINGLE[0]) + self._step(holder, list(output)) + + assert not holder._state[0]["in_end"] + + # Content + for _ in range(3): + output.append(self.CONTENT_TOKEN) + self._step(holder, list(output)) + + # Block 2: exceed budget + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(budget + 1): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + + assert holder._state[0]["in_end"], ( + "Block 2 must trigger forced end after natural end in Block 1" + ) + + def test_budget_one_natural_end_reentry(self): + """Budget = 1: natural end with 0 think tokens, re-entry enforced.""" + budget = 1 + holder = self._make_holder(self.THINK_END_SINGLE) + self._sync_batch(holder, budget) + + output = [] + + # Block 1: immediate natural end (0 think tokens) + output.append(self.THINK_START) + self._step(holder, list(output)) + output.append(self.THINK_END_SINGLE[0]) + self._step(holder, list(output)) + + assert not holder._state[0]["in_end"] + + # Content + for _ in range(3): + output.append(self.CONTENT_TOKEN) + self._step(holder, list(output)) + + # Block 2: 2 think tokens (should be enforced at budget=1) + output.append(self.THINK_START) + self._step(holder, list(output)) + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + + assert holder._state[0]["in_end"], ( + "Budget=1 must enforce after 1 token in re-entry block" + ) + + def test_partial_end_sequence_in_content(self): + """Partial end sequence in content shouldn't trigger false exit.""" + holder = self._make_holder(self.THINK_END_MULTI) + self._sync_batch(holder, self.BUDGET) + + output = [] + + # Block 1: natural end with full sequence [200, 201, 202] + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(3): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + for tok in self.THINK_END_MULTI: + output.append(tok) + self._step(holder, list(output)) + + # Content with partial end sequence [200, 201] but not [200, 201, 202] + output.append(self.CONTENT_TOKEN) + self._step(holder, list(output)) + output.append(200) + self._step(holder, list(output)) + output.append(201) + self._step(holder, list(output)) + output.append(self.CONTENT_TOKEN) + self._step(holder, list(output)) + + # Block 2: re-entry must still be enforced + output.append(self.THINK_START) + self._step(holder, list(output)) + for _ in range(self.BUDGET + 1): + output.append(self.THINK_TOKEN) + self._step(holder, list(output)) + + assert holder._state[0]["in_end"], ( + "Partial end tokens in content must not prevent Block 2 enforcement" + ) + + def test_continue_thinking_natural_end_reentry(self): + """Prompt-prefill inside : after natural end, Block 2 must be enforced. + + When a request arrives with the prompt already inside a block, + continue_thinking is set to True. This test verifies that after a + natural exit, continue_thinking is cleared and Block 2 + enforcement works correctly. + """ + from dataclasses import dataclass + from unittest.mock import MagicMock + + from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder + + @dataclass + class FakeReasoningConfig: + reasoning_start_token_ids: list[int] + reasoning_end_token_ids: list[int] + enabled: bool = True + + cfg = FakeReasoningConfig( + reasoning_start_token_ids=[self.THINK_START], + reasoning_end_token_ids=self.THINK_END_SINGLE, + ) + holder = ThinkingBudgetStateHolder( + reasoning_config=cfg, + max_num_seqs=8, + num_spec_tokens=0, + device=torch.device("cpu"), + is_pin_memory=False, + ) + + # Simulate: prompt already inside (sets continue_thinking=True) + prompt_tok_ids = [self.THINK_START] + params = MagicMock() + params.thinking_token_budget = self.BUDGET + batch_update = MagicMock( + removed=[], + added=[(0, params, prompt_tok_ids, [])], + moved=[], + ) + holder.sync_batch(batch_update) + assert holder._state[0]["continue_thinking"], ( + "Prompt inside must set continue_thinking=True" + ) + + # Block 1: emit some think tokens then natural + output = list(prompt_tok_ids) + for _ in range(4): + output.append(self.THINK_TOKEN) + holder.update_state([output], None, None) + output.append(self.THINK_END_SINGLE[0]) + holder.update_state([output], None, None) + + assert not holder._state[0]["in_think"], "Should have exited think mode" + assert not holder._state[0]["continue_thinking"], ( + "continue_thinking must be cleared after natural end" + ) + + # Content tokens + for _ in range(3): + output.append(self.CONTENT_TOKEN) + holder.update_state([output], None, None) + + # Block 2: re-entry must be detected and budget enforced + output.append(self.THINK_START) + holder.update_state([output], None, None) + for _ in range(self.BUDGET + 1): + output.append(self.THINK_TOKEN) + holder.update_state([output], None, None) + + assert holder._state[0]["in_end"], ( + "Block 2 after natural end of prompt-prefill session must be enforced" + ) + + # --- Forced-end re-entry tests --- diff --git a/vllm/v1/sample/thinking_budget_state.py b/vllm/v1/sample/thinking_budget_state.py index 95c3406b02..f19f612f48 100644 --- a/vllm/v1/sample/thinking_budget_state.py +++ b/vllm/v1/sample/thinking_budget_state.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any import torch from vllm.platforms import current_platform -from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.sample.logits_processor.interface import ( BatchUpdate, MoveDirectionality, @@ -22,11 +22,12 @@ def maybe_create_thinking_budget_state_holder( max_num_seqs: int, num_spec_tokens: int, device: torch.device, + is_pin_memory: bool, ) -> "ThinkingBudgetStateHolder | None": if reasoning_config is None: return None return ThinkingBudgetStateHolder( - reasoning_config, max_num_seqs, num_spec_tokens, device, PIN_MEMORY + reasoning_config, max_num_seqs, num_spec_tokens, device, is_pin_memory ) @@ -173,19 +174,6 @@ class ThinkingBudgetStateHolder: return i return -1 - @staticmethod - def _find_last_sequence_index_from( - target_list: list[int], token_ids: list[int], search_start: int - ) -> int: - """Last occurrence of ``token_ids`` at or after ``search_start``.""" - if not token_ids: - return -1 - lo = max(0, search_start) - for i in range(len(target_list) - len(token_ids), lo - 1, -1): - if target_list[i : i + len(token_ids)] == token_ids: - return i - return -1 - def _init_state_entry( self, prompt_tok_ids: list[int] | None, thinking_token_budget: int ) -> dict[str, Any]: @@ -239,8 +227,6 @@ class ThinkingBudgetStateHolder: "force_index": [], "start_thinking": start_thinking, "end_thinking": -1, - "start_search_pos": 0, - "end_search_pos": 0, "in_spec_mode": False, "bonus_token_forced": False, "continue_thinking": continue_thinking, @@ -256,41 +242,40 @@ class ThinkingBudgetStateHolder: state["force_index"] = [] return - output_tok_ids = state.get("output_tok_ids", []) if state["start_thinking"] == -1: - seq_len = len(self.think_start_token_ids) scan_offset = state.get("scan_offset", 0) - start_thinking = self._find_last_sequence_index_from( - output_tok_ids, - self.think_start_token_ids, - max(scan_offset, state["start_search_pos"] - (seq_len - 1)), + output_slice = state.get("output_tok_ids", [])[scan_offset:] + start_thinking = self._find_last_sequence_index( + output_slice, self.think_start_token_ids ) - if start_thinking >= 0 and scan_offset > 0: - # Re-entry after a forced end: budget was already exhausted - # in a prior block, so immediately force-close this one. - # scan_offset > 0 is only set after forced-end completion - # (never after natural end), so this won't block legitimate - # re-entries where budget remains. - state["start_thinking"] = start_thinking - state["in_think"] = False - state["in_end"] = True - state["end_count"] = 0 - state["force_index"] = [0] - return + if start_thinking >= 0: + start_thinking += scan_offset state["start_thinking"] = start_thinking - if start_thinking == -1: - state["start_search_pos"] = len(output_tok_ids) if state["end_thinking"] == -1: - seq_len = len(self.think_end_token_ids) scan_offset = state.get("scan_offset", 0) - end_thinking = self._find_last_sequence_index_from( - output_tok_ids, - self.think_end_token_ids, - max(scan_offset, state["end_search_pos"] - (seq_len - 1)), + output_slice = state.get("output_tok_ids", [])[scan_offset:] + end_thinking = self._find_last_sequence_index( + output_slice, self.think_end_token_ids ) + if end_thinking >= 0: + end_thinking += scan_offset state["end_thinking"] = end_thinking - if end_thinking == -1: - state["end_search_pos"] = len(output_tok_ids) + + if ( + not state.get("in_end", False) + and state["start_thinking"] >= 0 + and state["end_thinking"] >= 0 + and state["end_thinking"] > state["start_thinking"] + and not state.get("continue_thinking", False) + ): + state["in_think"] = False + state["think_count"] = 0 + state["continue_thinking"] = False + state["start_thinking"] = -1 + state["end_thinking"] = -1 + state["scan_offset"] = len(state.get("output_tok_ids", [])) + state["check_count_down"] = state["thinking_token_budget"] + return if state["start_thinking"] == -1: return @@ -314,10 +299,18 @@ class ThinkingBudgetStateHolder: predicted_countdown = current_step_countdown - len(state["spec_token_ids"]) - 1 # We only proceed further if we have counted down the thinking budget # to 0 or less and when we are in the "in think" mode. + # Exception: when continue_thinking=True and a natural is + # detected (end_thinking != -1), fall through to handle the exit — + # even if the budget hasn't expired yet. For continue_thinking=False, + # the early natural-end detection block above already handles it. + natural_end_with_continue = ( + state.get("continue_thinking", False) and state["end_thinking"] != -1 + ) if ( not state.get("in_end", False) and predicted_countdown >= 0 and state["start_thinking"] > -1 + and not natural_end_with_continue ): state["check_count_down"] = current_step_countdown state["prev_output_length"] = len(state.get("output_tok_ids", [])) @@ -389,6 +382,10 @@ class ThinkingBudgetStateHolder: # Case: ......... - exiting think mode state["in_think"] = False state["think_count"] = 0 + state["continue_thinking"] = False + state["start_thinking"] = -1 + state["end_thinking"] = -1 + state["scan_offset"] = len(state.get("output_tok_ids", [])) elif absolute_start_pos >= 0 and not state["continue_thinking"]: # Found think start - entering think mode @@ -400,6 +397,10 @@ class ThinkingBudgetStateHolder: # Found think end - exiting think mode state["in_think"] = False state["think_count"] = 0 + state["continue_thinking"] = False + state["start_thinking"] = -1 + state["end_thinking"] = -1 + state["scan_offset"] = len(state.get("output_tok_ids", [])) elif state["in_think"]: # Continue thinking mode, increment count by new tokens diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index bb4fb3fe45..baa502ee81 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -113,6 +113,7 @@ class InputBatch: max_num_reqs, num_spec_tokens, device, + PIN_MEMORY, ) self.thinking_token_budget_reqs: set[str] = set() self.is_pooling_model = is_pooling_model