[Perf][ThinkingBudget] reduce search space for thinking tokens (#46425)

Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Walter Beller-Morales
2026-06-24 23:02:25 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 2801b11156
commit 7f99e80c3b
2 changed files with 61 additions and 4 deletions
@@ -1226,3 +1226,34 @@ def test_thinking_budget_invalid_budget_rejected(invalid_budget):
with pytest.raises(VLLMValidationError, match="thinking_token_budget"):
SamplingParams(thinking_token_budget=invalid_budget)
def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index():
"""Test thinking budget enforced for a long thinking run,
then a natural end marker."""
h = ThinkingBudgetStateHolder(
MockReasoningConfig(), 8, 0, torch.device("cpu"), False
)
h.sync_batch(
BatchUpdate(
batch_size=1,
removed=(),
added=[(0, SamplingParams(thinking_token_budget=10_000), None, [])],
moved=(),
)
)
start = MockReasoningConfig.reasoning_start_token_ids
end = MockReasoningConfig.reasoning_end_token_ids
out: list[int] = list(start)
h.update_state([out], None, None)
for tok in range(500): # 500 filler thinking tokens, one decode step each
out.append(tok)
h.update_state([out], None, None)
assert h._state[0]["end_thinking"] == -1 # not present yet
expected_end_idx = len(out) # marker appended next
out.extend(end)
h.update_state([out], None, None)
assert h._state[0]["start_thinking"] == 0
assert h._state[0]["end_thinking"] == expected_end_idx
+30 -4
View File
@@ -173,6 +173,19 @@ class ThinkingBudgetStateHolder:
return i
return -1
@staticmethod
def _find_last_sequence_index_from(
target_list: list[int], token_ids: list[int], search_start: int
) -> int:
"""Last occurrence of ``token_ids`` at or after ``search_start``."""
if not token_ids:
return -1
lo = max(0, search_start)
for i in range(len(target_list) - len(token_ids), lo - 1, -1):
if target_list[i : i + len(token_ids)] == token_ids:
return i
return -1
def _init_state_entry(
self, prompt_tok_ids: list[int] | None, thinking_token_budget: int
) -> dict[str, Any]:
@@ -226,6 +239,8 @@ class ThinkingBudgetStateHolder:
"force_index": [],
"start_thinking": start_thinking,
"end_thinking": -1,
"start_search_pos": 0,
"end_search_pos": 0,
"in_spec_mode": False,
"bonus_token_forced": False,
"continue_thinking": continue_thinking,
@@ -240,16 +255,27 @@ class ThinkingBudgetStateHolder:
state["force_index"] = []
return
output_tok_ids = state.get("output_tok_ids", [])
if state["start_thinking"] == -1:
start_thinking = self._find_last_sequence_index(
state.get("output_tok_ids", []), self.think_start_token_ids
seq_len = len(self.think_start_token_ids)
start_thinking = self._find_last_sequence_index_from(
output_tok_ids,
self.think_start_token_ids,
state["start_search_pos"] - (seq_len - 1),
)
state["start_thinking"] = start_thinking
if start_thinking == -1:
state["start_search_pos"] = len(output_tok_ids)
if state["end_thinking"] == -1:
end_thinking = self._find_last_sequence_index(
state.get("output_tok_ids", []), self.think_end_token_ids
seq_len = len(self.think_end_token_ids)
end_thinking = self._find_last_sequence_index_from(
output_tok_ids,
self.think_end_token_ids,
state["end_search_pos"] - (seq_len - 1),
)
state["end_thinking"] = end_thinking
if end_thinking == -1:
state["end_search_pos"] = len(output_tok_ids)
if state["start_thinking"] == -1:
return