[Bugfix] Fix inverted condition causing thinking_token_budget to be silently ignored (#41674)
Signed-off-by: Keyi Li <likey6688@gmail.com> Co-authored-by: Keyi Li <likey6688@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Keyi Li
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
faa4b76afa
commit
bf610c2f56
@@ -1122,3 +1122,75 @@ def test_thinking_budget_holder_empty_end_tokens_disables_row():
|
||||
)
|
||||
h.update_state([[THINK_START_TOKEN_ID, 1]], None, None)
|
||||
assert h._state[0]["thinking_token_budget"] == -1
|
||||
|
||||
|
||||
def test_thinking_budget_enforced_without_penalties():
|
||||
"""Regression test for gpu_input_batch.py bug.
|
||||
|
||||
When thinking_budget_tracks_reqs=True and no penalties/bad_words are set,
|
||||
the old code computed needs_output_token_ids=False (inverted condition:
|
||||
``or not thinking_budget_tracks_reqs``), causing update_state to receive
|
||||
an empty list and skip _update_think_state for every request.
|
||||
|
||||
Fix: changed ``or not thinking_budget_tracks_reqs`` to
|
||||
``or thinking_budget_tracks_reqs`` so that output_token_ids is populated
|
||||
whenever the thinking budget state holder has tracked requests.
|
||||
|
||||
This test verifies that update_state correctly calls _update_think_state
|
||||
(setting in_end=True) when given the real output_token_ids, and that
|
||||
passing an empty list (the pre-fix behavior) prevents budget enforcement.
|
||||
"""
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
budget = 3 # allow 3 thinking tokens
|
||||
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
output_token_ids: list[int] = []
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(thinking_token_budget=budget),
|
||||
None,
|
||||
output_token_ids,
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
assert h.has_tracked_requests()
|
||||
|
||||
# Simulate the buggy behavior: update_state receives empty list.
|
||||
# _update_think_state is skipped → in_end stays False → no budget enforcement.
|
||||
h.update_state([], None, None)
|
||||
assert not h._state[0].get("in_end", False), (
|
||||
"With empty output_token_ids, in_end should stay False (budget not yet tracked)"
|
||||
)
|
||||
|
||||
# Simulate the correct behavior: output_token_ids is the live list.
|
||||
# Step 1: think-start token appears.
|
||||
output_token_ids.append(THINK_START_TOKEN_ID)
|
||||
h.update_state([output_token_ids], None, None)
|
||||
assert not h._state[0].get("in_end", False), (
|
||||
"Still within budget after 0 think tokens"
|
||||
)
|
||||
|
||||
# Steps 2–4: 3 thinking tokens (hits the budget exactly).
|
||||
for tok in [1, 2, 3]:
|
||||
output_token_ids.append(tok)
|
||||
h.update_state([output_token_ids], None, None)
|
||||
|
||||
# After exactly `budget` thinking tokens the holder should force end token.
|
||||
assert h._state[0].get("in_end", False), (
|
||||
"Budget exceeded: in_end should be True so that apply_to_logits "
|
||||
"forces the end token"
|
||||
)
|
||||
|
||||
@@ -884,7 +884,7 @@ class InputBatch:
|
||||
not self.no_penalties
|
||||
or bool(self.bad_words_token_ids)
|
||||
or self.logitsprocs_need_output_token_ids
|
||||
or not thinking_budget_tracks_reqs
|
||||
or thinking_budget_tracks_reqs
|
||||
)
|
||||
output_token_ids = (
|
||||
cast(list[list[int]], self.req_output_token_ids)
|
||||
|
||||
Reference in New Issue
Block a user