[Bugfix][Reasoning] Fix thinking_token_budget not enforced on re-entry after forced end (#43757)
Signed-off-by: Ashwin Giridharan <girida@amazon.com> Signed-off-by: Cursor Agent <cursor-agent@cursor.com> Co-authored-by: Cursor Agent <cursor-agent@cursor.com> Co-authored-by: Simon Mo <simon.mo@hey.com>
This commit is contained in:
co-authored by
Cursor Agent
Simon Mo
parent
77654d080c
commit
cda05ee8c4
@@ -1257,3 +1257,127 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index
|
||||
|
||||
assert h._state[0]["start_thinking"] == 0
|
||||
assert h._state[0]["end_thinking"] == expected_end_idx
|
||||
|
||||
|
||||
# --- Thinking budget re-entry tests (issue #43708) ---
|
||||
# Regression tests: after budget forces end-of-thinking token sequence,
|
||||
# the state machine must detect and enforce budget on subsequent blocks.
|
||||
|
||||
|
||||
class TestThinkingBudgetReentry:
|
||||
THINK_START = 100
|
||||
THINK_END_SINGLE = [200]
|
||||
THINK_END_MULTI = [200, 201, 202]
|
||||
BUDGET = 5
|
||||
CONTENT_TOKEN = 50
|
||||
THINK_TOKEN = 60
|
||||
|
||||
@staticmethod
|
||||
def _make_holder(end_token_ids: list[int]) -> ThinkingBudgetStateHolder:
|
||||
class FakeReasoningConfig:
|
||||
reasoning_start_token_ids = [TestThinkingBudgetReentry.THINK_START]
|
||||
reasoning_end_token_ids: list[int] = []
|
||||
enabled = True
|
||||
|
||||
cfg = FakeReasoningConfig()
|
||||
cfg.reasoning_end_token_ids = end_token_ids
|
||||
return ThinkingBudgetStateHolder(
|
||||
reasoning_config=cfg,
|
||||
max_num_seqs=8,
|
||||
num_spec_tokens=0,
|
||||
device=torch.device("cpu"),
|
||||
is_pin_memory=False,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _sync_batch(holder: ThinkingBudgetStateHolder, budget: int) -> None:
|
||||
holder.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[(0, SamplingParams(thinking_token_budget=budget), None, [])],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _step(holder: ThinkingBudgetStateHolder, output_tok_ids: list[int]) -> None:
|
||||
holder.update_state(
|
||||
output_token_ids=[output_tok_ids],
|
||||
spec_token_ids=None,
|
||||
repeat_indices=None,
|
||||
)
|
||||
|
||||
def _exhaust_budget(self, holder: ThinkingBudgetStateHolder) -> list[int]:
|
||||
output = [self.THINK_START]
|
||||
self._step(holder, list(output))
|
||||
for _ in range(self.BUDGET):
|
||||
output.append(self.THINK_TOKEN)
|
||||
self._step(holder, list(output))
|
||||
|
||||
assert holder._state[0]["in_end"]
|
||||
return output
|
||||
|
||||
def _accept_end_tokens(
|
||||
self,
|
||||
holder: ThinkingBudgetStateHolder,
|
||||
output: list[int],
|
||||
end_token_ids: list[int],
|
||||
) -> None:
|
||||
for tok in end_token_ids:
|
||||
output.append(tok)
|
||||
self._step(holder, list(output))
|
||||
|
||||
def test_single_token_end_reentry(self):
|
||||
holder = self._make_holder(self.THINK_END_SINGLE)
|
||||
self._sync_batch(holder, self.BUDGET)
|
||||
|
||||
output = self._exhaust_budget(holder)
|
||||
self._accept_end_tokens(holder, output, self.THINK_END_SINGLE)
|
||||
|
||||
for _ in range(3):
|
||||
output.append(self.CONTENT_TOKEN)
|
||||
self._step(holder, list(output))
|
||||
|
||||
output.append(self.THINK_START)
|
||||
self._step(holder, list(output))
|
||||
for _ in range(self.BUDGET):
|
||||
output.append(self.THINK_TOKEN)
|
||||
self._step(holder, list(output))
|
||||
|
||||
assert holder._state[0]["in_end"], (
|
||||
"Second thinking block must also be budget-enforced"
|
||||
)
|
||||
|
||||
def test_multi_token_end_reentry(self):
|
||||
holder = self._make_holder(self.THINK_END_MULTI)
|
||||
self._sync_batch(holder, self.BUDGET)
|
||||
|
||||
output = self._exhaust_budget(holder)
|
||||
self._accept_end_tokens(holder, output, self.THINK_END_MULTI)
|
||||
|
||||
assert not holder._state[0]["in_end"]
|
||||
|
||||
output.append(self.THINK_START)
|
||||
self._step(holder, list(output))
|
||||
for _ in range(self.BUDGET):
|
||||
output.append(self.THINK_TOKEN)
|
||||
self._step(holder, list(output))
|
||||
|
||||
assert holder._state[0]["in_end"], (
|
||||
"Immediate re-entry after multi-token end must be enforced"
|
||||
)
|
||||
|
||||
def test_single_block_not_broken(self):
|
||||
holder = self._make_holder(self.THINK_END_SINGLE)
|
||||
self._sync_batch(holder, self.BUDGET)
|
||||
|
||||
output = self._exhaust_budget(holder)
|
||||
self._accept_end_tokens(holder, output, self.THINK_END_SINGLE)
|
||||
|
||||
for _ in range(20):
|
||||
output.append(self.CONTENT_TOKEN)
|
||||
self._step(holder, list(output))
|
||||
|
||||
assert not holder._state[0]["in_end"]
|
||||
assert not holder._state[0]["in_think"]
|
||||
|
||||
@@ -244,6 +244,7 @@ class ThinkingBudgetStateHolder:
|
||||
"in_spec_mode": False,
|
||||
"bonus_token_forced": False,
|
||||
"continue_thinking": continue_thinking,
|
||||
"scan_offset": 0,
|
||||
}
|
||||
|
||||
def _update_think_state(self, state: dict[str, Any]) -> None:
|
||||
@@ -258,20 +259,34 @@ class ThinkingBudgetStateHolder:
|
||||
output_tok_ids = state.get("output_tok_ids", [])
|
||||
if state["start_thinking"] == -1:
|
||||
seq_len = len(self.think_start_token_ids)
|
||||
scan_offset = state.get("scan_offset", 0)
|
||||
start_thinking = self._find_last_sequence_index_from(
|
||||
output_tok_ids,
|
||||
self.think_start_token_ids,
|
||||
state["start_search_pos"] - (seq_len - 1),
|
||||
max(scan_offset, state["start_search_pos"] - (seq_len - 1)),
|
||||
)
|
||||
if start_thinking >= 0 and scan_offset > 0:
|
||||
# Re-entry after a forced end: budget was already exhausted
|
||||
# in a prior block, so immediately force-close this one.
|
||||
# scan_offset > 0 is only set after forced-end completion
|
||||
# (never after natural end), so this won't block legitimate
|
||||
# re-entries where budget remains.
|
||||
state["start_thinking"] = start_thinking
|
||||
state["in_think"] = False
|
||||
state["in_end"] = True
|
||||
state["end_count"] = 0
|
||||
state["force_index"] = [0]
|
||||
return
|
||||
state["start_thinking"] = start_thinking
|
||||
if start_thinking == -1:
|
||||
state["start_search_pos"] = len(output_tok_ids)
|
||||
if state["end_thinking"] == -1:
|
||||
seq_len = len(self.think_end_token_ids)
|
||||
scan_offset = state.get("scan_offset", 0)
|
||||
end_thinking = self._find_last_sequence_index_from(
|
||||
output_tok_ids,
|
||||
self.think_end_token_ids,
|
||||
state["end_search_pos"] - (seq_len - 1),
|
||||
max(scan_offset, state["end_search_pos"] - (seq_len - 1)),
|
||||
)
|
||||
state["end_thinking"] = end_thinking
|
||||
if end_thinking == -1:
|
||||
@@ -459,6 +474,11 @@ class ThinkingBudgetStateHolder:
|
||||
"in_end": False,
|
||||
"end_count": 0,
|
||||
"check_count_down": state["thinking_token_budget"],
|
||||
"start_thinking": -1,
|
||||
"end_thinking": -1,
|
||||
"think_count": 0,
|
||||
"continue_thinking": False,
|
||||
"scan_offset": len(state.get("output_tok_ids", [])),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user