[Feature] limit thinking tokens (hard limit) (#20859)

Signed-off-by: Sungjae Lee <33976427+llsj14@users.noreply.github.com>
Signed-off-by: Sungjae Lee <sung-jae.lee@navercorp.com>
Signed-off-by: Chauncey <chaunceyjiang@gmail.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Sungjae Lee
2026-03-24 09:53:07 -07:00
committed by GitHub
co-authored by Chauncey Harry Mellor
parent 8de5261e69
commit 4731884796
13 changed files with 702 additions and 12 deletions
@@ -0,0 +1,87 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for thinking_token_budget with reasoning models."""
import openai
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen3-0.6B"
MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
THINK_BUDGET = 5
@pytest.fixture(scope="module")
def server():
args = [
"--reasoning-parser",
"qwen3",
"--reasoning-config",
'{"think_start_str": "<think>", "think_end_str": "</think>"}',
"--max-model-len",
"2048",
"--enforce-eager",
"--no-async-scheduling",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server):
async with server.get_async_client() as async_client:
yield async_client
@pytest.mark.asyncio
async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
"""Test that mixed requests (some with thinking_token_budget, some without)
complete successfully without errors."""
response_with_budget = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=100,
extra_body={"thinking_token_budget": THINK_BUDGET},
)
response_without_budget = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=100,
)
msg_with = response_with_budget.choices[0].message
msg_without = response_without_budget.choices[0].message
assert msg_with.content or getattr(msg_with, "reasoning", None)
assert msg_without.content or getattr(msg_without, "reasoning", None)
@pytest.mark.asyncio
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
"""Test that thinking_token_budget limits the number of reasoning tokens.
In streaming mode each reasoning delta corresponds to one token, so
counting non-empty reasoning_content chunks gives the exact token count.
"""
reasoning_token_count = 0
stream = await client.chat.completions.create(
model=MODEL_NAME,
messages=MESSAGES,
max_tokens=100,
stream=True,
extra_body={"thinking_token_budget": THINK_BUDGET},
)
async for chunk in stream:
delta = chunk.choices[0].delta
if getattr(delta, "reasoning", None):
reasoning_token_count += 1
assert reasoning_token_count == THINK_BUDGET, (
f"reasoning tokens ({reasoning_token_count}) != "
f"thinking_token_budget ({THINK_BUDGET})"
)
+173 -10
View File
@@ -30,6 +30,7 @@ from vllm.v1.sample.logits_processor import (
MinPLogitsProcessor,
MinTokensLogitsProcessor,
MoveDirectionality,
ThinkingTokenBudgetLogitsProcessor,
build_logitsprocs,
)
from vllm.v1.sample.metadata import SamplingMetadata
@@ -47,6 +48,11 @@ MIN_TOKENS_LEN_THRESHOLD = 5
REQS_PER_LOGITPROC = 50
STR_NO_LOGITPROC = "none"
# ThinkingTokenBudgetLogitsProcessor testing constants
THINKING_TOKEN_BUDGET = 5
THINK_START_TOKEN_ID = 999
THINK_END_TOKEN_ID = 998
# LogitsProcessor subclass or "none"
LogitprocType: TypeAlias = type[LogitsProcessor] | str
@@ -67,9 +73,24 @@ class LogitsProcsRequestParams:
self.workload_index = workload_index
self.logitproc_type = logitproc_type
# Number of output tokens is randomly 0 or twice the min-tokens
# threshold which will be used in testing. Output token values
# don't matter *for these tests* so use 0 as a dummy value
self.out_tokens = [0] * (MIN_TOKENS_LEN_THRESHOLD * random.randint(0, 2))
# threshold which will be used in testing.
# Generate diverse random tokens for all processors (more realistic)
num_tokens = MIN_TOKENS_LEN_THRESHOLD * random.randint(0, 2)
if num_tokens > 0:
# Use diverse random tokens
self.out_tokens = [random.randint(1, 950) for _ in range(num_tokens)]
# Set first token for ThinkingTokenBudget testing
is_thinking_processor = (
logitproc_type is ThinkingTokenBudgetLogitsProcessor
or (
hasattr(logitproc_type, "__name__")
and logitproc_type.__name__ == "ThinkingTokenBudgetLogitsProcessor"
)
)
if is_thinking_processor:
self.out_tokens[0] = THINK_START_TOKEN_ID
else:
self.out_tokens = []
self.prompt_tokens = []
self.params = _sampling_params_from_logitproc(logitproc_type)
@@ -79,6 +100,13 @@ class LogitsProcsRequestParams:
return f"MyClass({summ})"
class MockReasoningConfig:
"""Mock reasoning config for testing ThinkingTokenBudgetLogitsProcessor."""
think_start_token_ids = [THINK_START_TOKEN_ID]
think_end_token_ids = [THINK_END_TOKEN_ID]
def _generate_fake_sampling_metadata(
num_output_tokens: int,
batch_size: int,
@@ -97,8 +125,12 @@ def _generate_fake_sampling_metadata(
0, vocab_size, size=np.random.randint(1, MAX_NUM_PROMPT_TOKENS)
).tolist()
)
vllm_config = VllmConfig()
vllm_config.reasoning_config = MockReasoningConfig()
logitsprocs = build_logitsprocs(
vllm_config=VllmConfig(),
vllm_config=vllm_config,
device=device,
is_pin_memory=PIN_MEMORY_AVAILABLE,
is_pooling_model=False,
@@ -403,6 +435,127 @@ def _min_tokens_validate(
)
def _thinking_budget_params(kwargs: dict) -> None:
"""Set SamplingParams kwargs for thinking token budget tests"""
kwargs["thinking_token_budget"] = THINKING_TOKEN_BUDGET
def _thinking_budget_validate(
test_fakes: LogitsprocsTestFakes,
persistent_batch: list[LogitsProcsRequestParams],
logits_new: torch.Tensor,
batch_index: int,
request_params: LogitsProcsRequestParams,
step_idx: int,
) -> None:
"""Validate thinking token budget processor behavior"""
# Get the ThinkingTokenBudgetLogitsProcessor instance
tb_processor: ThinkingTokenBudgetLogitsProcessor = next(
test_fakes.get_logitsprocs_by_cls(ThinkingTokenBudgetLogitsProcessor)
)
# Get current request state
state = tb_processor._state.get(batch_index)
params = request_params.params
# Validate thinking token budget configuration
if hasattr(params, "thinking_token_budget") and params.thinking_token_budget:
# State should exist for requests with thinking_token_budget
if state is None:
_raise_error_invalid(
msg_suffix=(
f"Expected state for batch {batch_index} "
f"with thinking_token_budget={params.thinking_token_budget}"
),
batch_index=batch_index,
request_params=request_params,
step_idx=step_idx,
)
# Validate budget matches what was set
expected_budget = params.thinking_token_budget
actual_budget = state["thinking_token_budget"]
if actual_budget != expected_budget:
_raise_error_invalid(
msg_suffix=(
f"Budget mismatch: expected {expected_budget}, got {actual_budget}"
),
batch_index=batch_index,
request_params=request_params,
step_idx=step_idx,
)
# Check if we're in thinking mode and validate token counting
output_tokens = request_params.out_tokens
# Find if thinking has started in output tokens
thinking_started = False
start_tokens = tb_processor.think_start_token_ids
if len(start_tokens) > 0:
for i in range(len(output_tokens) - len(start_tokens) + 1):
if output_tokens[i : i + len(start_tokens)] == start_tokens:
thinking_started = True
break
if thinking_started:
# If budget is exceeded, validate end token forcing
think_count = state["think_count"]
budget = state["thinking_token_budget"]
if think_count >= budget:
if not state["in_end"]:
_raise_error_invalid(
msg_suffix=(
f"Budget exceeded ({think_count} >= "
f"{budget}) but not "
"forcing end tokens"
),
batch_index=batch_index,
request_params=request_params,
step_idx=step_idx,
)
# Validate that only end tokens are allowed
end_tokens = tb_processor.think_end_token_ids
if len(end_tokens) > 0:
expected_end_token_id = end_tokens[
min(state["end_count"], len(end_tokens) - 1)
]
# Check logits masking
batch_logits = logits_new[batch_index]
for token_id in range(len(batch_logits)):
logit_value = batch_logits[token_id]
if token_id == expected_end_token_id:
# End token should not be masked
if logit_value == -float("inf"):
_raise_error_invalid(
msg_suffix=(
f"End token {token_id} should not be "
"masked but is"
),
batch_index=batch_index,
request_params=request_params,
step_idx=step_idx,
)
else:
# All other tokens should be masked when forcing end
if logit_value != -float("inf"):
_raise_error_invalid(
msg_suffix=(
f"Token {token_id} should be masked "
f"when forcing end tokens, but "
f"logit={logit_value}"
),
batch_index=batch_index,
request_params=request_params,
step_idx=step_idx,
)
def _none_validate(
test_fakes: LogitsprocsTestFakes,
persistent_batch: list[LogitsProcsRequestParams],
@@ -449,20 +602,30 @@ logitsprocs_test_mapping = {
MinTokensLogitsProcessor: LogitsprocTestHelpers(
gen_request_fxn=_min_tokens_params, eval_fxn=_min_tokens_validate
),
ThinkingTokenBudgetLogitsProcessor: LogitsprocTestHelpers(
gen_request_fxn=_thinking_budget_params, eval_fxn=_thinking_budget_validate
),
}
def _get_test_cases() -> list[list[str]]:
"""Each test case is a set of logitsprocs"""
logitsprocs_types = list(logitsprocs_test_mapping.keys())
# Isolate ThinkingTokenBudgetLogitsProcessor from all other processors
# to avoid unexpected modification of logits interference
thinking_processor = ThinkingTokenBudgetLogitsProcessor
other_processors = [
p
for p in logitsprocs_types
if p != STR_NO_LOGITPROC and p != thinking_processor
]
return (
[[STR_NO_LOGITPROC]]
+ [
[logitproc_type, STR_NO_LOGITPROC]
for logitproc_type in logitsprocs_types
if logitproc_type != STR_NO_LOGITPROC
]
+ [logitsprocs_types]
+ [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors]
+ [other_processors]
+ [[thinking_processor]]
)