[Perf][ThinkingBudget] reduce search space for thinking tokens (#46425)
Signed-off-by: walterbm <walter.beller.morales@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
2801b11156
commit
7f99e80c3b
@@ -1226,3 +1226,34 @@ def test_thinking_budget_invalid_budget_rejected(invalid_budget):
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="thinking_token_budget"):
|
||||
SamplingParams(thinking_token_budget=invalid_budget)
|
||||
|
||||
|
||||
def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index():
|
||||
"""Test thinking budget enforced for a long thinking run,
|
||||
then a natural end marker."""
|
||||
h = ThinkingBudgetStateHolder(
|
||||
MockReasoningConfig(), 8, 0, torch.device("cpu"), False
|
||||
)
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[(0, SamplingParams(thinking_token_budget=10_000), None, [])],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
start = MockReasoningConfig.reasoning_start_token_ids
|
||||
end = MockReasoningConfig.reasoning_end_token_ids
|
||||
|
||||
out: list[int] = list(start)
|
||||
h.update_state([out], None, None)
|
||||
for tok in range(500): # 500 filler thinking tokens, one decode step each
|
||||
out.append(tok)
|
||||
h.update_state([out], None, None)
|
||||
assert h._state[0]["end_thinking"] == -1 # not present yet
|
||||
expected_end_idx = len(out) # marker appended next
|
||||
out.extend(end)
|
||||
h.update_state([out], None, None)
|
||||
|
||||
assert h._state[0]["start_thinking"] == 0
|
||||
assert h._state[0]["end_thinking"] == expected_end_idx
|
||||
|
||||
@@ -173,6 +173,19 @@ class ThinkingBudgetStateHolder:
|
||||
return i
|
||||
return -1
|
||||
|
||||
@staticmethod
|
||||
def _find_last_sequence_index_from(
|
||||
target_list: list[int], token_ids: list[int], search_start: int
|
||||
) -> int:
|
||||
"""Last occurrence of ``token_ids`` at or after ``search_start``."""
|
||||
if not token_ids:
|
||||
return -1
|
||||
lo = max(0, search_start)
|
||||
for i in range(len(target_list) - len(token_ids), lo - 1, -1):
|
||||
if target_list[i : i + len(token_ids)] == token_ids:
|
||||
return i
|
||||
return -1
|
||||
|
||||
def _init_state_entry(
|
||||
self, prompt_tok_ids: list[int] | None, thinking_token_budget: int
|
||||
) -> dict[str, Any]:
|
||||
@@ -226,6 +239,8 @@ class ThinkingBudgetStateHolder:
|
||||
"force_index": [],
|
||||
"start_thinking": start_thinking,
|
||||
"end_thinking": -1,
|
||||
"start_search_pos": 0,
|
||||
"end_search_pos": 0,
|
||||
"in_spec_mode": False,
|
||||
"bonus_token_forced": False,
|
||||
"continue_thinking": continue_thinking,
|
||||
@@ -240,16 +255,27 @@ class ThinkingBudgetStateHolder:
|
||||
state["force_index"] = []
|
||||
return
|
||||
|
||||
output_tok_ids = state.get("output_tok_ids", [])
|
||||
if state["start_thinking"] == -1:
|
||||
start_thinking = self._find_last_sequence_index(
|
||||
state.get("output_tok_ids", []), self.think_start_token_ids
|
||||
seq_len = len(self.think_start_token_ids)
|
||||
start_thinking = self._find_last_sequence_index_from(
|
||||
output_tok_ids,
|
||||
self.think_start_token_ids,
|
||||
state["start_search_pos"] - (seq_len - 1),
|
||||
)
|
||||
state["start_thinking"] = start_thinking
|
||||
if start_thinking == -1:
|
||||
state["start_search_pos"] = len(output_tok_ids)
|
||||
if state["end_thinking"] == -1:
|
||||
end_thinking = self._find_last_sequence_index(
|
||||
state.get("output_tok_ids", []), self.think_end_token_ids
|
||||
seq_len = len(self.think_end_token_ids)
|
||||
end_thinking = self._find_last_sequence_index_from(
|
||||
output_tok_ids,
|
||||
self.think_end_token_ids,
|
||||
state["end_search_pos"] - (seq_len - 1),
|
||||
)
|
||||
state["end_thinking"] = end_thinking
|
||||
if end_thinking == -1:
|
||||
state["end_search_pos"] = len(output_tok_ids)
|
||||
|
||||
if state["start_thinking"] == -1:
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user