[Feature] limit thinking tokens (hard limit) (#20859)
Signed-off-by: Sungjae Lee <33976427+llsj14@users.noreply.github.com> Signed-off-by: Sungjae Lee <sung-jae.lee@navercorp.com> Signed-off-by: Chauncey <chaunceyjiang@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Chauncey
Harry Mellor
parent
8de5261e69
commit
4731884796
@@ -0,0 +1,87 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for thinking_token_budget with reasoning models."""
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
|
||||
THINK_BUDGET = 5
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--reasoning-config",
|
||||
'{"think_start_str": "<think>", "think_end_str": "</think>"}',
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--no-async-scheduling",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
|
||||
"""Test that mixed requests (some with thinking_token_budget, some without)
|
||||
complete successfully without errors."""
|
||||
|
||||
response_with_budget = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=100,
|
||||
extra_body={"thinking_token_budget": THINK_BUDGET},
|
||||
)
|
||||
response_without_budget = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=100,
|
||||
)
|
||||
|
||||
msg_with = response_with_budget.choices[0].message
|
||||
msg_without = response_without_budget.choices[0].message
|
||||
|
||||
assert msg_with.content or getattr(msg_with, "reasoning", None)
|
||||
assert msg_without.content or getattr(msg_without, "reasoning", None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
|
||||
"""Test that thinking_token_budget limits the number of reasoning tokens.
|
||||
|
||||
In streaming mode each reasoning delta corresponds to one token, so
|
||||
counting non-empty reasoning_content chunks gives the exact token count.
|
||||
"""
|
||||
|
||||
reasoning_token_count = 0
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=100,
|
||||
stream=True,
|
||||
extra_body={"thinking_token_budget": THINK_BUDGET},
|
||||
)
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta
|
||||
if getattr(delta, "reasoning", None):
|
||||
reasoning_token_count += 1
|
||||
|
||||
assert reasoning_token_count == THINK_BUDGET, (
|
||||
f"reasoning tokens ({reasoning_token_count}) != "
|
||||
f"thinking_token_budget ({THINK_BUDGET})"
|
||||
)
|
||||
@@ -30,6 +30,7 @@ from vllm.v1.sample.logits_processor import (
|
||||
MinPLogitsProcessor,
|
||||
MinTokensLogitsProcessor,
|
||||
MoveDirectionality,
|
||||
ThinkingTokenBudgetLogitsProcessor,
|
||||
build_logitsprocs,
|
||||
)
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
@@ -47,6 +48,11 @@ MIN_TOKENS_LEN_THRESHOLD = 5
|
||||
REQS_PER_LOGITPROC = 50
|
||||
STR_NO_LOGITPROC = "none"
|
||||
|
||||
# ThinkingTokenBudgetLogitsProcessor testing constants
|
||||
THINKING_TOKEN_BUDGET = 5
|
||||
THINK_START_TOKEN_ID = 999
|
||||
THINK_END_TOKEN_ID = 998
|
||||
|
||||
# LogitsProcessor subclass or "none"
|
||||
LogitprocType: TypeAlias = type[LogitsProcessor] | str
|
||||
|
||||
@@ -67,9 +73,24 @@ class LogitsProcsRequestParams:
|
||||
self.workload_index = workload_index
|
||||
self.logitproc_type = logitproc_type
|
||||
# Number of output tokens is randomly 0 or twice the min-tokens
|
||||
# threshold which will be used in testing. Output token values
|
||||
# don't matter *for these tests* so use 0 as a dummy value
|
||||
self.out_tokens = [0] * (MIN_TOKENS_LEN_THRESHOLD * random.randint(0, 2))
|
||||
# threshold which will be used in testing.
|
||||
# Generate diverse random tokens for all processors (more realistic)
|
||||
num_tokens = MIN_TOKENS_LEN_THRESHOLD * random.randint(0, 2)
|
||||
if num_tokens > 0:
|
||||
# Use diverse random tokens
|
||||
self.out_tokens = [random.randint(1, 950) for _ in range(num_tokens)]
|
||||
# Set first token for ThinkingTokenBudget testing
|
||||
is_thinking_processor = (
|
||||
logitproc_type is ThinkingTokenBudgetLogitsProcessor
|
||||
or (
|
||||
hasattr(logitproc_type, "__name__")
|
||||
and logitproc_type.__name__ == "ThinkingTokenBudgetLogitsProcessor"
|
||||
)
|
||||
)
|
||||
if is_thinking_processor:
|
||||
self.out_tokens[0] = THINK_START_TOKEN_ID
|
||||
else:
|
||||
self.out_tokens = []
|
||||
self.prompt_tokens = []
|
||||
self.params = _sampling_params_from_logitproc(logitproc_type)
|
||||
|
||||
@@ -79,6 +100,13 @@ class LogitsProcsRequestParams:
|
||||
return f"MyClass({summ})"
|
||||
|
||||
|
||||
class MockReasoningConfig:
|
||||
"""Mock reasoning config for testing ThinkingTokenBudgetLogitsProcessor."""
|
||||
|
||||
think_start_token_ids = [THINK_START_TOKEN_ID]
|
||||
think_end_token_ids = [THINK_END_TOKEN_ID]
|
||||
|
||||
|
||||
def _generate_fake_sampling_metadata(
|
||||
num_output_tokens: int,
|
||||
batch_size: int,
|
||||
@@ -97,8 +125,12 @@ def _generate_fake_sampling_metadata(
|
||||
0, vocab_size, size=np.random.randint(1, MAX_NUM_PROMPT_TOKENS)
|
||||
).tolist()
|
||||
)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.reasoning_config = MockReasoningConfig()
|
||||
|
||||
logitsprocs = build_logitsprocs(
|
||||
vllm_config=VllmConfig(),
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
is_pin_memory=PIN_MEMORY_AVAILABLE,
|
||||
is_pooling_model=False,
|
||||
@@ -403,6 +435,127 @@ def _min_tokens_validate(
|
||||
)
|
||||
|
||||
|
||||
def _thinking_budget_params(kwargs: dict) -> None:
|
||||
"""Set SamplingParams kwargs for thinking token budget tests"""
|
||||
kwargs["thinking_token_budget"] = THINKING_TOKEN_BUDGET
|
||||
|
||||
|
||||
def _thinking_budget_validate(
|
||||
test_fakes: LogitsprocsTestFakes,
|
||||
persistent_batch: list[LogitsProcsRequestParams],
|
||||
logits_new: torch.Tensor,
|
||||
batch_index: int,
|
||||
request_params: LogitsProcsRequestParams,
|
||||
step_idx: int,
|
||||
) -> None:
|
||||
"""Validate thinking token budget processor behavior"""
|
||||
# Get the ThinkingTokenBudgetLogitsProcessor instance
|
||||
tb_processor: ThinkingTokenBudgetLogitsProcessor = next(
|
||||
test_fakes.get_logitsprocs_by_cls(ThinkingTokenBudgetLogitsProcessor)
|
||||
)
|
||||
|
||||
# Get current request state
|
||||
state = tb_processor._state.get(batch_index)
|
||||
params = request_params.params
|
||||
|
||||
# Validate thinking token budget configuration
|
||||
if hasattr(params, "thinking_token_budget") and params.thinking_token_budget:
|
||||
# State should exist for requests with thinking_token_budget
|
||||
if state is None:
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
f"Expected state for batch {batch_index} "
|
||||
f"with thinking_token_budget={params.thinking_token_budget}"
|
||||
),
|
||||
batch_index=batch_index,
|
||||
request_params=request_params,
|
||||
step_idx=step_idx,
|
||||
)
|
||||
|
||||
# Validate budget matches what was set
|
||||
expected_budget = params.thinking_token_budget
|
||||
actual_budget = state["thinking_token_budget"]
|
||||
|
||||
if actual_budget != expected_budget:
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
f"Budget mismatch: expected {expected_budget}, got {actual_budget}"
|
||||
),
|
||||
batch_index=batch_index,
|
||||
request_params=request_params,
|
||||
step_idx=step_idx,
|
||||
)
|
||||
|
||||
# Check if we're in thinking mode and validate token counting
|
||||
output_tokens = request_params.out_tokens
|
||||
|
||||
# Find if thinking has started in output tokens
|
||||
thinking_started = False
|
||||
start_tokens = tb_processor.think_start_token_ids
|
||||
|
||||
if len(start_tokens) > 0:
|
||||
for i in range(len(output_tokens) - len(start_tokens) + 1):
|
||||
if output_tokens[i : i + len(start_tokens)] == start_tokens:
|
||||
thinking_started = True
|
||||
break
|
||||
|
||||
if thinking_started:
|
||||
# If budget is exceeded, validate end token forcing
|
||||
think_count = state["think_count"]
|
||||
budget = state["thinking_token_budget"]
|
||||
|
||||
if think_count >= budget:
|
||||
if not state["in_end"]:
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
f"Budget exceeded ({think_count} >= "
|
||||
f"{budget}) but not "
|
||||
"forcing end tokens"
|
||||
),
|
||||
batch_index=batch_index,
|
||||
request_params=request_params,
|
||||
step_idx=step_idx,
|
||||
)
|
||||
|
||||
# Validate that only end tokens are allowed
|
||||
end_tokens = tb_processor.think_end_token_ids
|
||||
if len(end_tokens) > 0:
|
||||
expected_end_token_id = end_tokens[
|
||||
min(state["end_count"], len(end_tokens) - 1)
|
||||
]
|
||||
|
||||
# Check logits masking
|
||||
batch_logits = logits_new[batch_index]
|
||||
for token_id in range(len(batch_logits)):
|
||||
logit_value = batch_logits[token_id]
|
||||
|
||||
if token_id == expected_end_token_id:
|
||||
# End token should not be masked
|
||||
if logit_value == -float("inf"):
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
f"End token {token_id} should not be "
|
||||
"masked but is"
|
||||
),
|
||||
batch_index=batch_index,
|
||||
request_params=request_params,
|
||||
step_idx=step_idx,
|
||||
)
|
||||
else:
|
||||
# All other tokens should be masked when forcing end
|
||||
if logit_value != -float("inf"):
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
f"Token {token_id} should be masked "
|
||||
f"when forcing end tokens, but "
|
||||
f"logit={logit_value}"
|
||||
),
|
||||
batch_index=batch_index,
|
||||
request_params=request_params,
|
||||
step_idx=step_idx,
|
||||
)
|
||||
|
||||
|
||||
def _none_validate(
|
||||
test_fakes: LogitsprocsTestFakes,
|
||||
persistent_batch: list[LogitsProcsRequestParams],
|
||||
@@ -449,20 +602,30 @@ logitsprocs_test_mapping = {
|
||||
MinTokensLogitsProcessor: LogitsprocTestHelpers(
|
||||
gen_request_fxn=_min_tokens_params, eval_fxn=_min_tokens_validate
|
||||
),
|
||||
ThinkingTokenBudgetLogitsProcessor: LogitsprocTestHelpers(
|
||||
gen_request_fxn=_thinking_budget_params, eval_fxn=_thinking_budget_validate
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _get_test_cases() -> list[list[str]]:
|
||||
"""Each test case is a set of logitsprocs"""
|
||||
logitsprocs_types = list(logitsprocs_test_mapping.keys())
|
||||
|
||||
# Isolate ThinkingTokenBudgetLogitsProcessor from all other processors
|
||||
# to avoid unexpected modification of logits interference
|
||||
thinking_processor = ThinkingTokenBudgetLogitsProcessor
|
||||
other_processors = [
|
||||
p
|
||||
for p in logitsprocs_types
|
||||
if p != STR_NO_LOGITPROC and p != thinking_processor
|
||||
]
|
||||
|
||||
return (
|
||||
[[STR_NO_LOGITPROC]]
|
||||
+ [
|
||||
[logitproc_type, STR_NO_LOGITPROC]
|
||||
for logitproc_type in logitsprocs_types
|
||||
if logitproc_type != STR_NO_LOGITPROC
|
||||
]
|
||||
+ [logitsprocs_types]
|
||||
+ [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors]
|
||||
+ [other_processors]
|
||||
+ [[thinking_processor]]
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user