[Reasoning] Support for speculative decoding with thinking budget (#34668)
Signed-off-by: rishitdholakia13 <rishit+github@cohere.com> Signed-off-by: rishitdholakia13 <123388671+rishitdholakia13@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Nick Hill
Cyrus Leung
parent
8a8c9b564e
commit
68dd7db810
@@ -1,18 +1,62 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for thinking_token_budget with reasoning models."""
|
||||
"""E2E tests for ``thinking_token_budget`` with reasoning models.
|
||||
|
||||
Covers Qwen3-0.6B and Qwen3.5 FP8 + MTP.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Literal
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from tests.utils import RemoteOpenAIServer, multi_gpu_only, requires_fp8
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
QWEN35_FP8_MTP_MODEL = "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
|
||||
THINK_BUDGET = 5
|
||||
|
||||
REASONING_START_STR = "<think>"
|
||||
REASONING_END_STR = "</think>"
|
||||
|
||||
|
||||
def _count_reasoning_decode_token_ids_between_markers(
|
||||
full_token_ids: list[int],
|
||||
reasoning_start_ids: list[int],
|
||||
reasoning_end_ids: list[int],
|
||||
) -> int | None:
|
||||
"""Count decode tokens in the thinking span (after last start, before first end)."""
|
||||
|
||||
if not reasoning_start_ids or not reasoning_end_ids:
|
||||
raise ValueError("reasoning marker token id lists must be non-empty")
|
||||
|
||||
def _last_subseq_index(haystack: list[int], needle: list[int]) -> int:
|
||||
n = len(needle)
|
||||
if n > len(haystack):
|
||||
return -1
|
||||
for i in range(len(haystack) - n, -1, -1):
|
||||
if haystack[i : i + n] == needle:
|
||||
return i
|
||||
return -1
|
||||
|
||||
last_start = _last_subseq_index(full_token_ids, reasoning_start_ids)
|
||||
if last_start < 0:
|
||||
return None
|
||||
|
||||
pos_after_start = last_start + len(reasoning_start_ids)
|
||||
end_n = len(reasoning_end_ids)
|
||||
for j in range(pos_after_start, len(full_token_ids) - end_n + 1):
|
||||
if full_token_ids[j : j + end_n] == reasoning_end_ids:
|
||||
return j - pos_after_start
|
||||
return len(full_token_ids) - pos_after_start
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
@@ -48,6 +92,51 @@ def server_with_auto_reasoning_config():
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server_qwen35_fp8_mtp_tp2():
|
||||
"""Qwen3.5-35B FP8 with MTP speculative decoding and tensor parallel size 2."""
|
||||
if current_platform.device_count() < 2:
|
||||
pytest.skip("Need at least 2 GPUs for --tensor-parallel-size 2")
|
||||
if not current_platform.supports_fp8():
|
||||
pytest.skip("FP8 is not supported on this platform")
|
||||
|
||||
spec_cfg = {
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 2,
|
||||
"max_model_len": 32768,
|
||||
}
|
||||
args = [
|
||||
"--tensor-parallel-size",
|
||||
"2",
|
||||
"--max-model-len",
|
||||
"32768",
|
||||
"--speculative-config",
|
||||
json.dumps(spec_cfg),
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--reasoning-config",
|
||||
json.dumps(
|
||||
{
|
||||
"reasoning_start_str": REASONING_START_STR,
|
||||
"reasoning_end_str": REASONING_END_STR,
|
||||
}
|
||||
),
|
||||
]
|
||||
# With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers
|
||||
# on 0,1 do not exhaust memory on the same devices as this worker.
|
||||
env_dict = None
|
||||
if current_platform.device_count() >= 4:
|
||||
env_dict = {"CUDA_VISIBLE_DEVICES": "2,3"}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
QWEN35_FP8_MTP_MODEL,
|
||||
args,
|
||||
max_wait_seconds=3000,
|
||||
env_dict=env_dict,
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(request, server, server_with_auto_reasoning_config):
|
||||
server_map = {
|
||||
@@ -89,8 +178,10 @@ async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI):
|
||||
async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI):
|
||||
"""Test that thinking_token_budget limits the number of reasoning tokens.
|
||||
|
||||
In streaming mode each reasoning delta corresponds to one token, so
|
||||
counting non-empty reasoning_content chunks gives the exact token count.
|
||||
Counts non-empty streaming ``delta.reasoning`` chunks (coarse proxy; each
|
||||
chunk may represent multiple decode tokens — see
|
||||
``_count_reasoning_decode_token_ids_between_markers`` and the Qwen3.5 MTP
|
||||
test for id-based checks).
|
||||
"""
|
||||
|
||||
reasoning_token_count = 0
|
||||
@@ -110,3 +201,89 @@ async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI
|
||||
f"reasoning tokens ({reasoning_token_count}) exceeded "
|
||||
f"thinking_token_budget ({THINK_BUDGET})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
@requires_fp8
|
||||
async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_plain(
|
||||
server_qwen35_fp8_mtp_tp2,
|
||||
):
|
||||
"""Concurrent chat requests: some with ``thinking_token_budget``, some without.
|
||||
|
||||
Exercises the scheduler / input processor under a mixed batch on the same
|
||||
Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with
|
||||
``_count_reasoning_decode_token_ids_between_markers`` on full token ids.
|
||||
"""
|
||||
|
||||
_batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [
|
||||
("budget", 1),
|
||||
("budget", 12),
|
||||
("plain", None),
|
||||
("budget", 20),
|
||||
("budget", 14),
|
||||
("plain", None),
|
||||
("plain", None),
|
||||
("budget", 12),
|
||||
("plain", None),
|
||||
]
|
||||
|
||||
tokenizer = get_tokenizer(tokenizer_name=QWEN35_FP8_MTP_MODEL)
|
||||
start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False))
|
||||
end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False))
|
||||
|
||||
async with server_qwen35_fp8_mtp_tp2.get_async_client() as client:
|
||||
|
||||
async def budgeted_call(expected_budget: int):
|
||||
return await client.chat.completions.create(
|
||||
model=QWEN35_FP8_MTP_MODEL,
|
||||
messages=MESSAGES,
|
||||
max_tokens=256,
|
||||
stream=False,
|
||||
extra_body={
|
||||
"thinking_token_budget": expected_budget,
|
||||
"return_token_ids": True,
|
||||
},
|
||||
)
|
||||
|
||||
async def plain_call():
|
||||
return await client.chat.completions.create(
|
||||
model=QWEN35_FP8_MTP_MODEL,
|
||||
messages=MESSAGES,
|
||||
max_tokens=256,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
coros = []
|
||||
for row in _batch_spec:
|
||||
if row[0] == "budget":
|
||||
b = row[1]
|
||||
assert isinstance(b, int)
|
||||
coros.append(budgeted_call(b))
|
||||
else:
|
||||
coros.append(plain_call())
|
||||
results = await asyncio.gather(*coros)
|
||||
|
||||
for i, (response, (kind, expected_budget)) in enumerate(
|
||||
zip(results, _batch_spec, strict=True)
|
||||
):
|
||||
msg = response.choices[0].message
|
||||
assert msg.content or getattr(msg, "reasoning", None), (
|
||||
f"index {i} ({kind}): empty message"
|
||||
)
|
||||
|
||||
if kind == "budget":
|
||||
assert expected_budget is not None
|
||||
assert response.prompt_token_ids is not None
|
||||
assert response.choices[0].token_ids is not None
|
||||
full_ids = list(response.prompt_token_ids) + list(
|
||||
response.choices[0].token_ids
|
||||
)
|
||||
n_reason = _count_reasoning_decode_token_ids_between_markers(
|
||||
full_ids, start_ids, end_ids
|
||||
)
|
||||
assert n_reason is not None, f"index {i}: missing reasoning start in ids"
|
||||
assert n_reason == expected_budget, (
|
||||
f"index {i}: reasoning decode token ids ({n_reason}) != "
|
||||
f"thinking_token_budget ({expected_budget})"
|
||||
)
|
||||
|
||||
@@ -30,10 +30,13 @@ from vllm.v1.sample.logits_processor import (
|
||||
MinPLogitsProcessor,
|
||||
MinTokensLogitsProcessor,
|
||||
MoveDirectionality,
|
||||
ThinkingTokenBudgetLogitsProcessor,
|
||||
build_logitsprocs,
|
||||
)
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.thinking_budget_state import (
|
||||
ThinkingBudgetStateHolder,
|
||||
maybe_create_thinking_budget_state_holder,
|
||||
)
|
||||
|
||||
PIN_MEMORY_AVAILABLE = is_pin_memory_available()
|
||||
MAX_NUM_REQS = 256
|
||||
@@ -48,8 +51,10 @@ MAX_NUM_PROMPT_TOKENS = 64
|
||||
MIN_TOKENS_LEN_THRESHOLD = 5
|
||||
REQS_PER_LOGITPROC = 50
|
||||
STR_NO_LOGITPROC = "none"
|
||||
# Thinking budget uses ``ThinkingBudgetStateHolder`` (not a logits processor).
|
||||
STR_THINKING_BUDGET = "thinking_budget"
|
||||
|
||||
# ThinkingTokenBudgetLogitsProcessor testing constants
|
||||
# Thinking token budget testing constants
|
||||
THINKING_TOKEN_BUDGET = 5
|
||||
THINK_START_TOKEN_ID = 999
|
||||
THINK_END_TOKEN_ID = 998
|
||||
@@ -80,15 +85,8 @@ class LogitsProcsRequestParams:
|
||||
if num_tokens > 0:
|
||||
# Use diverse random tokens
|
||||
self.out_tokens = [random.randint(1, 950) for _ in range(num_tokens)]
|
||||
# Set first token for ThinkingTokenBudget testing
|
||||
is_thinking_processor = (
|
||||
logitproc_type is ThinkingTokenBudgetLogitsProcessor
|
||||
or (
|
||||
hasattr(logitproc_type, "__name__")
|
||||
and logitproc_type.__name__ == "ThinkingTokenBudgetLogitsProcessor"
|
||||
)
|
||||
)
|
||||
if is_thinking_processor:
|
||||
# Think-start seed for ``STR_THINKING_BUDGET`` rows.
|
||||
if logitproc_type == STR_THINKING_BUDGET:
|
||||
self.out_tokens[0] = THINK_START_TOKEN_ID
|
||||
else:
|
||||
self.out_tokens = []
|
||||
@@ -102,7 +100,7 @@ class LogitsProcsRequestParams:
|
||||
|
||||
|
||||
class MockReasoningConfig:
|
||||
"""Mock reasoning config for testing ThinkingTokenBudgetLogitsProcessor."""
|
||||
"""Minimal reasoning config for ``ThinkingBudgetStateHolder`` tests."""
|
||||
|
||||
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
|
||||
reasoning_end_token_ids = [THINK_END_TOKEN_ID]
|
||||
@@ -137,6 +135,18 @@ def _generate_fake_sampling_metadata(
|
||||
is_pin_memory=PIN_MEMORY_AVAILABLE,
|
||||
is_pooling_model=False,
|
||||
)
|
||||
num_spec = (
|
||||
vllm_config.speculative_config.num_speculative_tokens
|
||||
if vllm_config.speculative_config
|
||||
else 0
|
||||
)
|
||||
thinking_holder = maybe_create_thinking_budget_state_holder(
|
||||
vllm_config.reasoning_config,
|
||||
vllm_config.scheduler_config.max_num_seqs,
|
||||
num_spec,
|
||||
device,
|
||||
PIN_MEMORY_AVAILABLE,
|
||||
)
|
||||
fake_sampling_metadata = SamplingMetadata(
|
||||
temperature=torch.full((batch_size,), 0.0),
|
||||
all_greedy=True,
|
||||
@@ -156,6 +166,7 @@ def _generate_fake_sampling_metadata(
|
||||
allowed_token_ids_mask=None,
|
||||
bad_words_token_ids={},
|
||||
logitsprocs=logitsprocs,
|
||||
thinking_budget_state_holder=thinking_holder,
|
||||
)
|
||||
return fake_sampling_metadata
|
||||
|
||||
@@ -187,7 +198,7 @@ def _sampling_params_from_logitproc(logitproc_type: LogitprocType) -> SamplingPa
|
||||
|
||||
def _generate_mixed_logitsprocs_batch_params(
|
||||
reqs_per_logitproc: int,
|
||||
logitsprocs_types: list[str],
|
||||
logitsprocs_types: list[LogitprocType],
|
||||
) -> list[LogitsProcsRequestParams]:
|
||||
"""Define key params for a batch of requests with a different
|
||||
logitproc enabled per request.
|
||||
@@ -450,23 +461,21 @@ def _thinking_budget_validate(
|
||||
request_params: LogitsProcsRequestParams,
|
||||
step_idx: int,
|
||||
) -> None:
|
||||
"""Validate thinking token budget processor behavior"""
|
||||
# Get the ThinkingTokenBudgetLogitsProcessor instance
|
||||
tb_processor: ThinkingTokenBudgetLogitsProcessor = next(
|
||||
test_fakes.get_logitsprocs_by_cls(ThinkingTokenBudgetLogitsProcessor)
|
||||
)
|
||||
"""Validate ``ThinkingBudgetStateHolder`` thinking-budget behavior.
|
||||
|
||||
# Get current request state
|
||||
state = tb_processor._state.get(batch_index)
|
||||
State is keyed by **batch slot** (same index space as logits rows), matching
|
||||
``sync_batch`` / sampler integration (see PR #34668 discussion).
|
||||
"""
|
||||
holder = test_fakes.sampling_metadata.thinking_budget_state_holder
|
||||
assert holder is not None
|
||||
state = holder._state.get(batch_index)
|
||||
params = request_params.params
|
||||
|
||||
# Validate thinking token budget configuration
|
||||
if hasattr(params, "thinking_token_budget") and params.thinking_token_budget:
|
||||
# State should exist for requests with thinking_token_budget
|
||||
if state is None:
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
f"Expected state for batch {batch_index} "
|
||||
f"Expected holder state for batch slot {batch_index} "
|
||||
f"with thinking_token_budget={params.thinking_token_budget}"
|
||||
),
|
||||
batch_index=batch_index,
|
||||
@@ -474,10 +483,8 @@ def _thinking_budget_validate(
|
||||
step_idx=step_idx,
|
||||
)
|
||||
|
||||
# Validate budget matches what was set
|
||||
expected_budget = params.thinking_token_budget
|
||||
actual_budget = state["thinking_token_budget"]
|
||||
|
||||
if actual_budget != expected_budget:
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
@@ -488,13 +495,9 @@ def _thinking_budget_validate(
|
||||
step_idx=step_idx,
|
||||
)
|
||||
|
||||
# Check if we're in thinking mode and validate token counting
|
||||
output_tokens = request_params.out_tokens
|
||||
|
||||
# Find if thinking has started in output tokens
|
||||
start_tokens = holder.think_start_token_ids
|
||||
thinking_started = False
|
||||
start_tokens = tb_processor.reasoning_start_token_ids
|
||||
|
||||
if len(start_tokens) > 0:
|
||||
for i in range(len(output_tokens) - len(start_tokens) + 1):
|
||||
if output_tokens[i : i + len(start_tokens)] == start_tokens:
|
||||
@@ -502,61 +505,42 @@ def _thinking_budget_validate(
|
||||
break
|
||||
|
||||
if thinking_started:
|
||||
# If budget is exceeded, validate end token forcing
|
||||
think_count = state["think_count"]
|
||||
budget = state["thinking_token_budget"]
|
||||
if think_count >= budget and not state["in_end"]:
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
f"Budget exceeded ({think_count} >= {budget}) but "
|
||||
"in_end is false"
|
||||
),
|
||||
batch_index=batch_index,
|
||||
request_params=request_params,
|
||||
step_idx=step_idx,
|
||||
)
|
||||
|
||||
if think_count >= budget:
|
||||
if not state["in_end"]:
|
||||
end_tokens = holder.think_end_token_ids
|
||||
if (
|
||||
think_count >= budget
|
||||
and state["in_end"]
|
||||
and len(end_tokens) > 0
|
||||
and holder.has_tracked_requests()
|
||||
):
|
||||
expected_end_token_id = end_tokens[
|
||||
min(state["end_count"], len(end_tokens) - 1)
|
||||
]
|
||||
# Holder bumps forced vocab positions to 1e9 (does not -inf others).
|
||||
forced_logit = float(logits_new[batch_index, expected_end_token_id])
|
||||
if forced_logit < 1.0e8:
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
f"Budget exceeded ({think_count} >= "
|
||||
f"{budget}) but not "
|
||||
"forcing end tokens"
|
||||
f"Expected forced end token {expected_end_token_id} "
|
||||
f"with large logit, got {forced_logit}"
|
||||
),
|
||||
batch_index=batch_index,
|
||||
request_params=request_params,
|
||||
step_idx=step_idx,
|
||||
)
|
||||
|
||||
# Validate that only end tokens are allowed
|
||||
end_tokens = tb_processor.reasoning_end_token_ids
|
||||
if len(end_tokens) > 0:
|
||||
expected_end_token_id = end_tokens[
|
||||
min(state["end_count"], len(end_tokens) - 1)
|
||||
]
|
||||
|
||||
# Check logits masking
|
||||
batch_logits = logits_new[batch_index]
|
||||
for token_id in range(len(batch_logits)):
|
||||
logit_value = batch_logits[token_id]
|
||||
|
||||
if token_id == expected_end_token_id:
|
||||
# End token should not be masked
|
||||
if logit_value == -float("inf"):
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
f"End token {token_id} should not be "
|
||||
"masked but is"
|
||||
),
|
||||
batch_index=batch_index,
|
||||
request_params=request_params,
|
||||
step_idx=step_idx,
|
||||
)
|
||||
else:
|
||||
# All other tokens should be masked when forcing end
|
||||
if logit_value != -float("inf"):
|
||||
_raise_error_invalid(
|
||||
msg_suffix=(
|
||||
f"Token {token_id} should be masked "
|
||||
f"when forcing end tokens, but "
|
||||
f"logit={logit_value}"
|
||||
),
|
||||
batch_index=batch_index,
|
||||
request_params=request_params,
|
||||
step_idx=step_idx,
|
||||
)
|
||||
|
||||
|
||||
def _none_validate(
|
||||
test_fakes: LogitsprocsTestFakes,
|
||||
@@ -604,7 +588,7 @@ logitsprocs_test_mapping = {
|
||||
MinTokensLogitsProcessor: LogitsprocTestHelpers(
|
||||
gen_request_fxn=_min_tokens_params, eval_fxn=_min_tokens_validate
|
||||
),
|
||||
ThinkingTokenBudgetLogitsProcessor: LogitsprocTestHelpers(
|
||||
STR_THINKING_BUDGET: LogitsprocTestHelpers(
|
||||
gen_request_fxn=_thinking_budget_params, eval_fxn=_thinking_budget_validate
|
||||
),
|
||||
}
|
||||
@@ -614,20 +598,17 @@ def _get_test_cases() -> list[list[str]]:
|
||||
"""Each test case is a set of logitsprocs"""
|
||||
logitsprocs_types = list(logitsprocs_test_mapping.keys())
|
||||
|
||||
# Isolate ThinkingTokenBudgetLogitsProcessor from all other processors
|
||||
# to avoid unexpected modification of logits interference
|
||||
thinking_processor = ThinkingTokenBudgetLogitsProcessor
|
||||
# Isolate thinking-budget handling from other processors to avoid cross-talk.
|
||||
thinking_id: LogitprocType = STR_THINKING_BUDGET
|
||||
other_processors = [
|
||||
p
|
||||
for p in logitsprocs_types
|
||||
if p != STR_NO_LOGITPROC and p != thinking_processor
|
||||
p for p in logitsprocs_types if p != STR_NO_LOGITPROC and p != thinking_id
|
||||
]
|
||||
|
||||
return (
|
||||
[[STR_NO_LOGITPROC]]
|
||||
+ [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors]
|
||||
+ [other_processors]
|
||||
+ [[thinking_processor]]
|
||||
+ [[thinking_id]]
|
||||
)
|
||||
|
||||
|
||||
@@ -802,12 +783,23 @@ def _assert_valid(
|
||||
)
|
||||
|
||||
|
||||
def _slot_outputs_for_metadata(
|
||||
persistent_batch: list[LogitsProcsRequestParams], pad_len: int
|
||||
) -> list[list[int]]:
|
||||
"""Per-batch-slot output token ids aligned with ``SamplingMetadata`` rows."""
|
||||
rows: list[list[int]] = [[] for _ in range(pad_len)]
|
||||
for i, req in enumerate(persistent_batch):
|
||||
if i < pad_len:
|
||||
rows[i] = list(req.out_tokens)
|
||||
return rows
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
@pytest.mark.parametrize("reqs_per_logitproc", [REQS_PER_LOGITPROC])
|
||||
@pytest.mark.parametrize("logitsprocs_under_test", _get_test_cases())
|
||||
def test_logitsprocs(
|
||||
device: str, reqs_per_logitproc: int, logitsprocs_under_test: list[str]
|
||||
device: str, reqs_per_logitproc: int, logitsprocs_under_test: list[LogitprocType]
|
||||
):
|
||||
random.seed(40)
|
||||
torch.set_default_device(device)
|
||||
@@ -855,9 +847,10 @@ def test_logitsprocs(
|
||||
# Apply fake batch update to logitsprocs
|
||||
fake_update_logitsprocs_state(test_fakes, batch_update)
|
||||
|
||||
# Emulate application of logits processors in engine
|
||||
# Emulate application of logits processors + thinking holder (sampler order).
|
||||
slice_idxs = [req.workload_index for req in persistent_batch]
|
||||
logits_w_lp = fake_apply_logitsprocs(test_fakes, slice_idxs).cpu()
|
||||
slot_rows = _slot_outputs_for_metadata(persistent_batch, workload_size)
|
||||
logits_w_lp = fake_apply_logitsprocs(test_fakes, slice_idxs, slot_rows).cpu()
|
||||
|
||||
_assert_valid(
|
||||
batch_size=batch_size,
|
||||
@@ -869,3 +862,263 @@ def test_logitsprocs(
|
||||
)
|
||||
|
||||
step_idx += 1
|
||||
|
||||
|
||||
class MockReasoningNoEndTokens:
|
||||
"""Reasoning config with no end token ids (disables enforcement in holder)."""
|
||||
|
||||
reasoning_start_token_ids = [THINK_START_TOKEN_ID]
|
||||
reasoning_end_token_ids: list[int] = []
|
||||
|
||||
|
||||
def test_maybe_create_thinking_budget_holder_without_reasoning():
|
||||
cfg = VllmConfig()
|
||||
assert cfg.reasoning_config is None
|
||||
assert (
|
||||
maybe_create_thinking_budget_state_holder(
|
||||
None,
|
||||
cfg.scheduler_config.max_num_seqs,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_thinking_budget_holder_has_tracked_after_sync_add():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
assert not h.has_tracked_requests()
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(thinking_token_budget=3),
|
||||
None,
|
||||
[THINK_START_TOKEN_ID],
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
assert h.has_tracked_requests()
|
||||
assert h._state[0]["thinking_token_budget"] == 3
|
||||
|
||||
|
||||
def test_thinking_budget_holder_sync_remove_clears_state():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(thinking_token_budget=3),
|
||||
None,
|
||||
[],
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
assert h.has_tracked_requests()
|
||||
h.sync_batch(BatchUpdate(batch_size=0, removed=(0,), added=(), moved=()))
|
||||
assert not h.has_tracked_requests()
|
||||
|
||||
|
||||
def test_thinking_budget_holder_sync_add_without_budget_drops_row():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[(0, SamplingParams(), None, [])],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
assert not h.has_tracked_requests()
|
||||
|
||||
|
||||
def test_thinking_budget_holder_swap_exchanges_state():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=2,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(thinking_token_budget=3),
|
||||
None,
|
||||
[],
|
||||
),
|
||||
(
|
||||
1,
|
||||
SamplingParams(thinking_token_budget=7),
|
||||
None,
|
||||
[],
|
||||
),
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
b0, b1 = h._state[0]["thinking_token_budget"], h._state[1]["thinking_token_budget"]
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=2,
|
||||
removed=(),
|
||||
added=(),
|
||||
moved=[(0, 1, MoveDirectionality.SWAP)],
|
||||
)
|
||||
)
|
||||
assert h._state[0]["thinking_token_budget"] == b1
|
||||
assert h._state[1]["thinking_token_budget"] == b0
|
||||
|
||||
|
||||
def test_thinking_budget_holder_unidirectional_move():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=2,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
1,
|
||||
SamplingParams(thinking_token_budget=4),
|
||||
None,
|
||||
[],
|
||||
),
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
assert 1 in h._state and 0 not in h._state
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=2,
|
||||
removed=(),
|
||||
added=(),
|
||||
moved=[(1, 0, MoveDirectionality.UNIDIRECTIONAL)],
|
||||
)
|
||||
)
|
||||
assert 0 in h._state and 1 not in h._state
|
||||
assert h._state[0]["thinking_token_budget"] == 4
|
||||
|
||||
|
||||
def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningConfig()
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(thinking_token_budget=5),
|
||||
None,
|
||||
[THINK_START_TOKEN_ID],
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
out_lists = [[THINK_START_TOKEN_ID], [THINK_START_TOKEN_ID, 10, 11, 12, 13, 14]]
|
||||
h.update_state(
|
||||
out_lists,
|
||||
None,
|
||||
torch.tensor([0, 0], dtype=torch.long),
|
||||
)
|
||||
assert h._state[0]["output_tok_ids"] == out_lists[1]
|
||||
|
||||
|
||||
def test_thinking_budget_holder_spec_mode_tensor_layout():
|
||||
h = ThinkingBudgetStateHolder(
|
||||
MockReasoningConfig(),
|
||||
8,
|
||||
2,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
assert h.in_spec_mode
|
||||
assert h.mask.shape[0] == 8 * (2 + 1)
|
||||
|
||||
|
||||
def test_thinking_budget_holder_empty_end_tokens_disables_row():
|
||||
vc = VllmConfig()
|
||||
vc.reasoning_config = MockReasoningNoEndTokens()
|
||||
h = ThinkingBudgetStateHolder(
|
||||
vc.reasoning_config,
|
||||
vc.scheduler_config.max_num_seqs,
|
||||
0,
|
||||
torch.device("cpu"),
|
||||
False,
|
||||
)
|
||||
h.sync_batch(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(thinking_token_budget=5),
|
||||
None,
|
||||
[THINK_START_TOKEN_ID],
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
h.update_state([[THINK_START_TOKEN_ID, 1]], None, None)
|
||||
assert h._state[0]["thinking_token_budget"] == -1
|
||||
|
||||
@@ -198,22 +198,43 @@ class LogitsprocsTestFakes(NamedTuple):
|
||||
|
||||
def fake_update_logitsprocs_state(
|
||||
test_fakes: LogitsprocsTestFakes,
|
||||
batch_update: BatchUpdate,
|
||||
batch_update: BatchUpdate | None,
|
||||
) -> None:
|
||||
"""Imitate logits processors persistent batch state update
|
||||
in engine core"""
|
||||
for logitproc in test_fakes.get_logitsprocs():
|
||||
logitproc.update_state(batch_update)
|
||||
holder = test_fakes.sampling_metadata.thinking_budget_state_holder
|
||||
if holder is not None:
|
||||
holder.sync_batch(batch_update)
|
||||
|
||||
|
||||
def fake_apply_logitsprocs(
|
||||
test_fakes: LogitsprocsTestFakes,
|
||||
slice_indices: list[int],
|
||||
slot_output_token_ids: list[list[int]] | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Imitate application of logits processors in engine core"""
|
||||
"""Imitate application of logits processors in engine core.
|
||||
|
||||
When ``thinking_budget_state_holder`` has tracked requests, this mirrors
|
||||
:meth:`Sampler.apply_logits_processors` by refreshing per-slot
|
||||
``output_token_ids`` (if ``slot_output_token_ids`` is provided), then
|
||||
``update_state`` + ``apply_to_logits`` on the holder after built-in logits
|
||||
processors.
|
||||
"""
|
||||
logits = test_fakes.logits[torch.tensor(slice_indices, dtype=torch.long)].clone()
|
||||
for processor in test_fakes.get_logitsprocs():
|
||||
logits = processor.apply(logits)
|
||||
|
||||
md = test_fakes.sampling_metadata
|
||||
holder = md.thinking_budget_state_holder
|
||||
if holder is not None and holder.has_tracked_requests():
|
||||
if slot_output_token_ids is not None:
|
||||
for i, toks in enumerate(slot_output_token_ids):
|
||||
if i < len(md.output_token_ids):
|
||||
md.output_token_ids[i] = list(toks)
|
||||
holder.update_state(md.output_token_ids, md.spec_token_ids, None)
|
||||
logits = holder.apply_to_logits(logits, False, md.spec_token_ids)
|
||||
return logits
|
||||
|
||||
|
||||
|
||||
@@ -39,7 +39,6 @@ def mock_model_runner_with_input_batch():
|
||||
vocab_size=32000,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
is_spec_decode=False,
|
||||
logitsprocs=None,
|
||||
is_pooling_model=False,
|
||||
)
|
||||
|
||||
@@ -18,7 +18,6 @@ from vllm.v1.sample.logits_processor.builtin import (
|
||||
LogitBiasLogitsProcessor,
|
||||
MinPLogitsProcessor,
|
||||
MinTokensLogitsProcessor,
|
||||
ThinkingTokenBudgetLogitsProcessor,
|
||||
process_dict_updates,
|
||||
)
|
||||
from vllm.v1.sample.logits_processor.interface import (
|
||||
@@ -51,7 +50,6 @@ BUILTIN_LOGITS_PROCESSORS: list[type[LogitsProcessor]] = [
|
||||
MinTokensLogitsProcessor,
|
||||
LogitBiasLogitsProcessor,
|
||||
MinPLogitsProcessor,
|
||||
ThinkingTokenBudgetLogitsProcessor,
|
||||
]
|
||||
|
||||
|
||||
@@ -356,5 +354,4 @@ __all__ = [
|
||||
"STR_POOLING_REJECTS_LOGITSPROCS",
|
||||
"LOGITSPROCS_GROUP",
|
||||
"AdapterLogitsProcessor",
|
||||
"ThinkingTokenBudgetLogitsProcessor",
|
||||
]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import TYPE_CHECKING, Any, TypeVar
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -291,267 +291,6 @@ class MinTokensLogitsProcessor(LogitsProcessor):
|
||||
return logits
|
||||
|
||||
|
||||
class ThinkingTokenBudgetLogitsProcessor(LogitsProcessor):
|
||||
"""Limits the number of tokens allowed inside a 'thinking' section."""
|
||||
|
||||
def __init__(
|
||||
self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool
|
||||
):
|
||||
reasoning_config = vllm_config.reasoning_config
|
||||
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
|
||||
|
||||
# Check if thinking is enabled
|
||||
self.is_enabled = reasoning_config is not None and reasoning_config.enabled
|
||||
|
||||
self.reasoning_start_token_ids = getattr(
|
||||
reasoning_config, "reasoning_start_token_ids", []
|
||||
)
|
||||
self.reasoning_end_token_ids = getattr(
|
||||
reasoning_config, "reasoning_end_token_ids", []
|
||||
)
|
||||
|
||||
self.pin_memory = is_pin_memory
|
||||
self.device = device
|
||||
# Per-request state tracking for thinking token management
|
||||
# Key: request_index, Value: state dict containing:
|
||||
# "in_think": bool - currently in thinking mode
|
||||
# "in_end": bool - currently forcing end tokens output
|
||||
# "check_count_down": int - steps remaining until next think
|
||||
# start/end token parsing
|
||||
# "think_count": int - number of thinking tokens generated
|
||||
# "end_count": int - number of end tokens forced so far
|
||||
# "thinking_token_budget": int - max allowed thinking tokens
|
||||
# "output_tok_ids": list[int] - generated output tokens
|
||||
# "prev_output_length": int - previous output length for
|
||||
# incremental processing
|
||||
self._state: dict[int, dict[str, Any]] = {}
|
||||
|
||||
# Preallocate reusable tensors
|
||||
self.mask = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
|
||||
self.force_token_ids = torch.full(
|
||||
(max_num_reqs,), -1, dtype=torch.long, device=device
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _find_last_sequence_index(target_list: list[int], token_ids: list[int]) -> int:
|
||||
"""
|
||||
Returns the index of the last occurrence of token_ids in target_list.
|
||||
|
||||
Args:
|
||||
target_list (list[int]): The list of token IDs.
|
||||
token_ids (list[int]): The sequence of token IDs to find.
|
||||
"""
|
||||
if not token_ids:
|
||||
return -1
|
||||
for i in range(len(target_list) - len(token_ids), -1, -1):
|
||||
if target_list[i : i + len(token_ids)] == token_ids:
|
||||
return i
|
||||
return -1
|
||||
|
||||
def _init_state_entry(
|
||||
self, prompt_tok_ids: list[int] | None, thinking_token_budget: int
|
||||
) -> dict[str, Any]:
|
||||
"""Initializes the tracking state for a given sequence index."""
|
||||
if prompt_tok_ids is None:
|
||||
last_start = -1
|
||||
last_end = -1
|
||||
in_think = False
|
||||
think_count = 0
|
||||
else:
|
||||
last_start = self._find_last_sequence_index(
|
||||
prompt_tok_ids, self.reasoning_start_token_ids
|
||||
)
|
||||
last_end = self._find_last_sequence_index(
|
||||
prompt_tok_ids, self.reasoning_end_token_ids
|
||||
)
|
||||
in_think = last_start > last_end
|
||||
if in_think:
|
||||
think_count = len(prompt_tok_ids) - (
|
||||
last_start + len(self.reasoning_start_token_ids)
|
||||
)
|
||||
else:
|
||||
think_count = 0
|
||||
|
||||
return {
|
||||
"in_think": in_think, # Currently in thinking mode
|
||||
"in_end": in_think and thinking_token_budget == 0,
|
||||
"check_count_down": thinking_token_budget,
|
||||
"think_count": think_count, # Number of tokens in thinking section
|
||||
"end_count": 0, # Number of end tokens forced so far
|
||||
"prompt_tok_ids": prompt_tok_ids,
|
||||
"output_tok_ids": [],
|
||||
"thinking_token_budget": thinking_token_budget,
|
||||
"prev_output_length": 0,
|
||||
# Track previous output length for incremental updates
|
||||
}
|
||||
|
||||
def _update_think_state(self, state: dict[str, Any]):
|
||||
"""Updates the state based on newly generated output tokens."""
|
||||
if not state.get("in_end", False) and state.get("check_count_down", 0) > 0:
|
||||
state["check_count_down"] -= 1
|
||||
return
|
||||
|
||||
output = state.get("output_tok_ids", [])
|
||||
if not output:
|
||||
return
|
||||
|
||||
# Track previous output length for incremental processing
|
||||
prev_length = state.get("prev_output_length", 0)
|
||||
current_length = len(output)
|
||||
|
||||
if current_length <= prev_length:
|
||||
return
|
||||
|
||||
# Process only newly added tokens
|
||||
new_tokens = output[prev_length:]
|
||||
state["prev_output_length"] = current_length
|
||||
|
||||
# Check if new tokens contain think start or end sequences
|
||||
start_len = len(self.reasoning_start_token_ids)
|
||||
end_len = len(self.reasoning_end_token_ids)
|
||||
|
||||
# Look for think sequences in recent tokens (including boundary)
|
||||
# Check overlapping regions where sequences might span boundaries
|
||||
check_start_idx = max(0, prev_length - max(start_len, end_len) + 1)
|
||||
recent_tokens = output[check_start_idx:]
|
||||
|
||||
# Find any think start/end sequences in recent tokens
|
||||
recent_start_pos = self._find_last_sequence_index(
|
||||
recent_tokens, self.reasoning_start_token_ids
|
||||
)
|
||||
recent_end_pos = self._find_last_sequence_index(
|
||||
recent_tokens, self.reasoning_end_token_ids
|
||||
)
|
||||
|
||||
# Update state based on recent sequences
|
||||
if not state["in_end"]:
|
||||
if recent_start_pos >= 0 and recent_end_pos >= 0:
|
||||
if recent_start_pos > recent_end_pos:
|
||||
# Case: ...<end>...<start>... - entering think mode
|
||||
absolute_start_pos = check_start_idx + recent_start_pos
|
||||
new_think_count = current_length - (absolute_start_pos + start_len)
|
||||
state["in_think"] = True
|
||||
state["think_count"] = new_think_count
|
||||
else:
|
||||
# Case: ...<start>...<end>... - exiting think mode
|
||||
state["in_think"] = False
|
||||
state["think_count"] = 0
|
||||
elif recent_start_pos >= 0:
|
||||
# Found think start - entering think mode
|
||||
absolute_start_pos = check_start_idx + recent_start_pos
|
||||
new_think_count = current_length - (absolute_start_pos + start_len)
|
||||
state["in_think"] = True
|
||||
state["think_count"] = new_think_count
|
||||
elif recent_end_pos >= 0:
|
||||
# Found think end - exiting think mode
|
||||
state["in_think"] = False
|
||||
state["think_count"] = 0
|
||||
elif state["in_think"]:
|
||||
# Continue thinking mode, increment count by new tokens
|
||||
state["think_count"] += len(new_tokens)
|
||||
|
||||
# Set countdown based on current state
|
||||
if state["in_think"]:
|
||||
remaining_budget = max(
|
||||
0, state["thinking_token_budget"] - state["think_count"]
|
||||
)
|
||||
state["check_count_down"] = max(0, remaining_budget - 1)
|
||||
else:
|
||||
state["check_count_down"] = state["thinking_token_budget"]
|
||||
|
||||
# Check if need to transition to end mode
|
||||
if (
|
||||
state["in_think"]
|
||||
and state["think_count"] >= state["thinking_token_budget"]
|
||||
):
|
||||
state["in_think"] = False
|
||||
state["in_end"] = True
|
||||
state["end_count"] = 0
|
||||
state["check_count_down"] = state["thinking_token_budget"]
|
||||
else:
|
||||
# In end mode
|
||||
state["end_count"] += 1
|
||||
if state["end_count"] >= len(self.reasoning_end_token_ids):
|
||||
state.update(
|
||||
{
|
||||
"in_end": False,
|
||||
"end_count": 0,
|
||||
"check_count_down": state["thinking_token_budget"],
|
||||
}
|
||||
)
|
||||
|
||||
def is_argmax_invariant(self) -> bool:
|
||||
"""This logits processor can change the outcome of
|
||||
greedy sampling by forcing that the thinking section
|
||||
ends after a certain number of tokens."""
|
||||
return False
|
||||
|
||||
def update_state(self, batch_update: BatchUpdate | None):
|
||||
if not self.is_enabled:
|
||||
return
|
||||
if batch_update:
|
||||
for index, params, prompt_tok_ids, output_tok_ids in batch_update.added:
|
||||
thinking_token_budget = params.thinking_token_budget
|
||||
|
||||
if thinking_token_budget is not None:
|
||||
self._state[index] = self._init_state_entry(
|
||||
prompt_tok_ids, thinking_token_budget
|
||||
)
|
||||
self._state[index]["output_tok_ids"] = output_tok_ids
|
||||
else:
|
||||
# Remove state if no thinking budget
|
||||
self._state.pop(index, None)
|
||||
|
||||
for index in batch_update.removed:
|
||||
self._state.pop(index, {})
|
||||
|
||||
for i1, i2, direction in batch_update.moved:
|
||||
if direction == MoveDirectionality.SWAP:
|
||||
state1 = self._state.pop(i1, None)
|
||||
state2 = self._state.pop(i2, None)
|
||||
if state1 is not None:
|
||||
self._state[i2] = state1
|
||||
if state2 is not None:
|
||||
self._state[i1] = state2
|
||||
else:
|
||||
state = self._state.pop(i1, None)
|
||||
if state is not None:
|
||||
self._state[i2] = state
|
||||
|
||||
for state in self._state.values():
|
||||
self._update_think_state(state)
|
||||
|
||||
def apply(self, logits: torch.Tensor) -> torch.Tensor:
|
||||
if not self.is_enabled or not self._state:
|
||||
return logits
|
||||
|
||||
batch_size = logits.size(0)
|
||||
self.mask[:batch_size] = False
|
||||
|
||||
for i in range(batch_size):
|
||||
state = self._state.get(i)
|
||||
if state and state["in_end"]:
|
||||
self.mask[i] = True
|
||||
self.force_token_ids[i] = self.reasoning_end_token_ids[
|
||||
state["end_count"]
|
||||
]
|
||||
|
||||
# Check in CPU first not to sync with GPU
|
||||
has_active_thinking = any(
|
||||
state.get("in_end", False) for state in self._state.values()
|
||||
)
|
||||
|
||||
if has_active_thinking:
|
||||
current_mask = self.mask[:batch_size]
|
||||
active_indices = current_mask.nonzero(as_tuple=False).view(-1)
|
||||
if len(active_indices) > 0:
|
||||
force_tokens = self.force_token_ids[active_indices]
|
||||
# Apply a large value for the end thinking token id index
|
||||
logits[active_indices, force_tokens] = 1e9
|
||||
|
||||
return logits
|
||||
|
||||
|
||||
def process_dict_updates(
|
||||
req_entries: dict[int, T],
|
||||
batch_update: BatchUpdate | None,
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.sample.logits_processor import LogitsProcessors
|
||||
from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -47,3 +50,6 @@ class SamplingMetadata:
|
||||
|
||||
# Speculative token ids
|
||||
spec_token_ids: list[list[int]] | None = None
|
||||
# When non-None, use ``holder.has_tracked_requests()`` to see if this batch applies
|
||||
# thinking-token-budget logits (holder may exist with an empty tracking set).
|
||||
thinking_budget_state_holder: ThinkingBudgetStateHolder | None = None
|
||||
|
||||
@@ -290,16 +290,24 @@ class RejectionSampler(nn.Module):
|
||||
any_penalties_or_bad_words = (
|
||||
sampling_metadata.bad_words_token_ids or has_penalties
|
||||
)
|
||||
holder = sampling_metadata.thinking_budget_state_holder
|
||||
needs_thinking = holder is not None and holder.has_tracked_requests()
|
||||
|
||||
output_token_ids = sampling_metadata.output_token_ids
|
||||
if any_penalties_or_bad_words:
|
||||
if any_penalties_or_bad_words or needs_thinking:
|
||||
output_token_ids = self._combine_outputs_with_spec_tokens(
|
||||
output_token_ids,
|
||||
sampling_metadata.spec_token_ids,
|
||||
)
|
||||
|
||||
# Calculate indices of target logits.
|
||||
if sampling_metadata.allowed_token_ids_mask is not None or has_penalties:
|
||||
repeat_indices: torch.Tensor | None = None
|
||||
need_repeat_indices = (
|
||||
sampling_metadata.allowed_token_ids_mask is not None
|
||||
or has_penalties
|
||||
or needs_thinking
|
||||
)
|
||||
if need_repeat_indices:
|
||||
num_requests = len(metadata.num_draft_tokens)
|
||||
num_draft_tokens = torch.tensor(metadata.num_draft_tokens, device="cpu")
|
||||
original_indices = torch.arange(num_requests, device="cpu")
|
||||
@@ -327,7 +335,12 @@ class RejectionSampler(nn.Module):
|
||||
logits = processor.apply_with_spec_decode(
|
||||
logits, metadata.num_draft_tokens
|
||||
)
|
||||
|
||||
if holder is not None and holder.has_tracked_requests():
|
||||
logits = holder.apply_to_logits(
|
||||
logits,
|
||||
predict_bonus_token=False,
|
||||
spec_token_ids=sampling_metadata.spec_token_ids,
|
||||
)
|
||||
return logits
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -364,9 +364,13 @@ class Sampler(nn.Module):
|
||||
any_penalties_or_bad_words = (
|
||||
bool(bad_words_token_ids) or not sampling_metadata.no_penalties
|
||||
)
|
||||
holder = sampling_metadata.thinking_budget_state_holder
|
||||
needs_thinking_combine = holder is not None and holder.has_tracked_requests()
|
||||
|
||||
output_token_ids = sampling_metadata.output_token_ids
|
||||
if predict_bonus_token and any_penalties_or_bad_words:
|
||||
if predict_bonus_token and (
|
||||
any_penalties_or_bad_words or needs_thinking_combine
|
||||
):
|
||||
# Combine base outputs with spec tokens when speculative decoding
|
||||
# is enabled.
|
||||
output_token_ids = self._combine_outputs_with_spec_tokens(
|
||||
@@ -388,6 +392,17 @@ class Sampler(nn.Module):
|
||||
|
||||
# Apply penalties (e.g., freq_penalties).
|
||||
logits = self.apply_penalties(logits, sampling_metadata, output_token_ids)
|
||||
if holder is not None and holder.has_tracked_requests():
|
||||
holder.update_state(
|
||||
output_token_ids,
|
||||
sampling_metadata.spec_token_ids,
|
||||
repeat_indices=None,
|
||||
)
|
||||
logits = holder.apply_to_logits(
|
||||
logits,
|
||||
predict_bonus_token,
|
||||
sampling_metadata.spec_token_ids,
|
||||
)
|
||||
return logits
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -0,0 +1,528 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-batch thinking token budget state; applied after penalties at sample time."""
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.sample.logits_processor.interface import (
|
||||
BatchUpdate,
|
||||
MoveDirectionality,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.reasoning import ReasoningConfig
|
||||
|
||||
|
||||
def maybe_create_thinking_budget_state_holder(
|
||||
reasoning_config: "ReasoningConfig | None",
|
||||
max_num_seqs: int,
|
||||
num_spec_tokens: int,
|
||||
device: torch.device,
|
||||
is_pin_memory: bool,
|
||||
) -> "ThinkingBudgetStateHolder | None":
|
||||
if reasoning_config is None:
|
||||
return None
|
||||
return ThinkingBudgetStateHolder(
|
||||
reasoning_config, max_num_seqs, num_spec_tokens, device, is_pin_memory
|
||||
)
|
||||
|
||||
|
||||
class ThinkingBudgetStateHolder:
|
||||
"""Tracks thinking sections and forces end tokens when budget is exceeded."""
|
||||
|
||||
think_start_token_ids: list[int]
|
||||
think_end_token_ids: list[int]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
reasoning_config: "ReasoningConfig | None",
|
||||
max_num_seqs: int,
|
||||
num_spec_tokens: int,
|
||||
device: torch.device,
|
||||
is_pin_memory: bool,
|
||||
):
|
||||
_ = is_pin_memory # API parity with logits processors
|
||||
max_num_reqs = max_num_seqs
|
||||
self.in_spec_mode = num_spec_tokens > 0
|
||||
self.num_spec_tokens = num_spec_tokens
|
||||
|
||||
# No separate enable flag: a non-``None`` ``reasoning_config`` is the switch.
|
||||
self.is_enabled = reasoning_config is not None
|
||||
|
||||
if reasoning_config is None:
|
||||
self.think_start_token_ids = []
|
||||
self.think_end_token_ids = []
|
||||
else:
|
||||
rs = reasoning_config.reasoning_start_token_ids
|
||||
re = reasoning_config.reasoning_end_token_ids
|
||||
self.think_start_token_ids = rs if rs else []
|
||||
self.think_end_token_ids = re if re else []
|
||||
|
||||
self.device = device
|
||||
self._state: dict[int, dict[str, Any]] = {}
|
||||
self.cu_num_tokens: dict[int, int] = {}
|
||||
|
||||
if self.num_spec_tokens > 0:
|
||||
self.mask = torch.zeros(
|
||||
max_num_reqs * (self.num_spec_tokens + 1),
|
||||
dtype=torch.bool,
|
||||
device=device,
|
||||
)
|
||||
self.force_token_ids = torch.full(
|
||||
(max_num_reqs * (self.num_spec_tokens + 1),),
|
||||
-1,
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
)
|
||||
else:
|
||||
self.mask = torch.zeros(max_num_reqs, dtype=torch.bool, device=device)
|
||||
self.force_token_ids = torch.full(
|
||||
(max_num_reqs,), -1, dtype=torch.long, device=device
|
||||
)
|
||||
|
||||
def has_tracked_requests(self) -> bool:
|
||||
"""True when ``sync_batch`` has state for a ``thinking_token_budget`` row.
|
||||
|
||||
Used to decide whether sampling needs output-token rows and spec combining;
|
||||
distinct from merely having a holder instance (reasoning may be on with no
|
||||
budgeted requests in this batch).
|
||||
"""
|
||||
return bool(self._state)
|
||||
|
||||
def sync_batch(self, batch_update: BatchUpdate | None) -> None:
|
||||
"""Add/remove/move per-request state only (no _update_think_state)."""
|
||||
if not self.is_enabled or not batch_update:
|
||||
return
|
||||
for index in batch_update.removed:
|
||||
self._state.pop(index, None)
|
||||
|
||||
for index, params, prompt_tok_ids, output_tok_ids in batch_update.added:
|
||||
thinking_token_budget = params.thinking_token_budget
|
||||
if thinking_token_budget is not None:
|
||||
self._state[index] = self._init_state_entry(
|
||||
prompt_tok_ids, thinking_token_budget
|
||||
)
|
||||
self._state[index]["output_tok_ids"] = output_tok_ids
|
||||
self._state[index]["spec_token_ids"] = []
|
||||
else:
|
||||
self._state.pop(index, None)
|
||||
|
||||
for i1, i2, direction in batch_update.moved:
|
||||
if direction == MoveDirectionality.SWAP:
|
||||
state1 = self._state.get(i1)
|
||||
state2 = self._state.get(i2)
|
||||
if state1 is not None:
|
||||
self._state[i2] = state1
|
||||
if state2 is not None:
|
||||
self._state[i1] = state2
|
||||
else:
|
||||
state = self._state.pop(i1, None)
|
||||
if state is not None:
|
||||
self._state[i2] = state
|
||||
|
||||
def update_state(
|
||||
self,
|
||||
output_token_ids: list[list[int]],
|
||||
spec_token_ids: list[list[int]] | None,
|
||||
repeat_indices: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Refresh output/spec from sampling rows and recompute think state."""
|
||||
if not self.is_enabled or not self._state:
|
||||
return
|
||||
|
||||
spec_lists = spec_token_ids or []
|
||||
last_row_for_req: dict[int, int] | None = None
|
||||
if repeat_indices is not None:
|
||||
last_row_for_req = {}
|
||||
rpt = repeat_indices.cpu().tolist()
|
||||
for batch_row, req_i in enumerate(rpt):
|
||||
last_row_for_req[req_i] = batch_row
|
||||
|
||||
for seq_idx, state in list(self._state.items()):
|
||||
if last_row_for_req is not None:
|
||||
output_row: int | None = last_row_for_req.get(seq_idx)
|
||||
if output_row is None or output_row >= len(output_token_ids):
|
||||
continue
|
||||
state["output_tok_ids"] = output_token_ids[output_row]
|
||||
elif seq_idx >= len(output_token_ids):
|
||||
continue
|
||||
else:
|
||||
state["output_tok_ids"] = output_token_ids[seq_idx]
|
||||
if seq_idx < len(spec_lists):
|
||||
state["spec_token_ids"] = list(spec_lists[seq_idx])
|
||||
else:
|
||||
state["spec_token_ids"] = []
|
||||
state["in_spec_mode"] = self.in_spec_mode
|
||||
state["force_index"] = []
|
||||
if len(state["output_tok_ids"]) > 0:
|
||||
spec_len = len(state["spec_token_ids"])
|
||||
# Only strip draft suffix when there are spec tokens; ``[:-0]`` would
|
||||
# clear the whole list (Python treats stop index 0 as "up to empty").
|
||||
if spec_len > 0 and len(state["output_tok_ids"]) >= spec_len:
|
||||
state["output_tok_ids"] = state["output_tok_ids"][:-spec_len]
|
||||
self._update_think_state(state)
|
||||
|
||||
def apply_to_logits(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
predict_bonus_token: bool,
|
||||
spec_token_ids: list[list[int]] | None,
|
||||
) -> torch.Tensor:
|
||||
"""Mask and bump logits for forced end-of-thinking tokens."""
|
||||
if not self.is_enabled or not self._state:
|
||||
return logits
|
||||
spec_lists = spec_token_ids or []
|
||||
return self._apply_forcing_to_logits(logits, predict_bonus_token, spec_lists)
|
||||
|
||||
@staticmethod
|
||||
def _find_last_sequence_index(target_list: list[int], token_ids: list[int]) -> int:
|
||||
if not token_ids:
|
||||
return -1
|
||||
for i in range(len(target_list) - len(token_ids), -1, -1):
|
||||
if target_list[i : i + len(token_ids)] == token_ids:
|
||||
return i
|
||||
return -1
|
||||
|
||||
def _init_state_entry(
|
||||
self, prompt_tok_ids: list[int] | None, thinking_token_budget: int
|
||||
) -> dict[str, Any]:
|
||||
if prompt_tok_ids is None:
|
||||
last_start = -1
|
||||
last_end = -1
|
||||
in_think = False
|
||||
think_count = 0
|
||||
start_thinking = -1
|
||||
countdown = thinking_token_budget
|
||||
continue_thinking = False
|
||||
in_end = False
|
||||
else:
|
||||
start_thinking = -1
|
||||
countdown = thinking_token_budget
|
||||
continue_thinking = False
|
||||
in_end = False
|
||||
last_start = self._find_last_sequence_index(
|
||||
prompt_tok_ids, self.think_start_token_ids
|
||||
)
|
||||
last_end = self._find_last_sequence_index(
|
||||
prompt_tok_ids, self.think_end_token_ids
|
||||
)
|
||||
in_think = last_start > last_end
|
||||
# load metrics such as think count, start thinking
|
||||
# if request is in thinking mode, already
|
||||
if in_think:
|
||||
think_count = len(prompt_tok_ids) - (
|
||||
last_start + len(self.think_start_token_ids)
|
||||
)
|
||||
start_thinking = len(prompt_tok_ids) - think_count - 1
|
||||
countdown -= think_count
|
||||
continue_thinking = True
|
||||
# check if the token is exhausted within prompt
|
||||
token_exhausted = thinking_token_budget - think_count
|
||||
in_end = token_exhausted <= 0
|
||||
else:
|
||||
think_count = 0
|
||||
|
||||
return {
|
||||
"in_think": in_think,
|
||||
"in_end": in_end,
|
||||
"check_count_down": countdown,
|
||||
"think_count": think_count,
|
||||
"end_count": 0,
|
||||
"prompt_tok_ids": prompt_tok_ids,
|
||||
"output_tok_ids": [],
|
||||
"thinking_token_budget": thinking_token_budget,
|
||||
"prev_output_length": 0,
|
||||
"spec_token_ids": [],
|
||||
"force_index": [],
|
||||
"start_thinking": start_thinking,
|
||||
"end_thinking": -1,
|
||||
"in_spec_mode": False,
|
||||
"bonus_token_forced": False,
|
||||
"continue_thinking": continue_thinking,
|
||||
}
|
||||
|
||||
def _update_think_state(self, state: dict[str, Any]) -> None:
|
||||
if state.get("thinking_token_budget", -1) == -1:
|
||||
return
|
||||
if len(self.think_end_token_ids) == 0:
|
||||
state["thinking_token_budget"] = -1
|
||||
state["in_end"] = False
|
||||
state["force_index"] = []
|
||||
return
|
||||
|
||||
if state["start_thinking"] == -1:
|
||||
start_thinking = self._find_last_sequence_index(
|
||||
state.get("output_tok_ids", []), self.think_start_token_ids
|
||||
)
|
||||
state["start_thinking"] = start_thinking
|
||||
if state["end_thinking"] == -1:
|
||||
end_thinking = self._find_last_sequence_index(
|
||||
state.get("output_tok_ids", []), self.think_end_token_ids
|
||||
)
|
||||
state["end_thinking"] = end_thinking
|
||||
|
||||
if state["start_thinking"] == -1:
|
||||
return
|
||||
|
||||
if state["continue_thinking"]:
|
||||
sampled_tokens_from_previous_step = len(
|
||||
state.get("output_tok_ids", [])
|
||||
) - state.get("prev_output_length", 0)
|
||||
else:
|
||||
if state["prev_output_length"] == 0:
|
||||
sampled_tokens_from_previous_step = len(
|
||||
state.get("output_tok_ids", [])
|
||||
) - len(self.think_start_token_ids)
|
||||
else:
|
||||
sampled_tokens_from_previous_step = (
|
||||
len(state.get("output_tok_ids", [])) - state["prev_output_length"]
|
||||
)
|
||||
current_step_countdown = (
|
||||
state["check_count_down"] - sampled_tokens_from_previous_step
|
||||
)
|
||||
predicted_countdown = current_step_countdown - len(state["spec_token_ids"]) - 1
|
||||
# We only proceed further if we have counted down the thinking budget
|
||||
# to 0 or less and when we are in the "in think" mode.
|
||||
if (
|
||||
not state.get("in_end", False)
|
||||
and predicted_countdown >= 0
|
||||
and state["start_thinking"] > -1
|
||||
):
|
||||
state["check_count_down"] = current_step_countdown
|
||||
state["prev_output_length"] = len(state.get("output_tok_ids", []))
|
||||
return
|
||||
output = state.get("output_tok_ids", [])
|
||||
if not output:
|
||||
# When in_end was set at init (budget=0, prompt already in think),
|
||||
# we must force the first generated token to be the end token;
|
||||
# otherwise apply() sees in_end=True but force_index=[] and
|
||||
# allows an extra thinking token.
|
||||
if state.get("in_end", False):
|
||||
state["force_index"] = [0]
|
||||
return
|
||||
|
||||
# Track previous output length for incremental processing
|
||||
prev_length = state.get("prev_output_length", 0)
|
||||
current_length = len(output)
|
||||
|
||||
if current_length <= prev_length:
|
||||
if state.get("in_end", False):
|
||||
remaining_budget = state["thinking_token_budget"] - state["think_count"]
|
||||
spec_len = len(state["spec_token_ids"])
|
||||
if spec_len > 0:
|
||||
if 0 < remaining_budget < spec_len:
|
||||
state["force_index"] = [remaining_budget]
|
||||
elif remaining_budget <= 0:
|
||||
state["force_index"] = [0]
|
||||
else:
|
||||
state["force_index"] = [spec_len]
|
||||
else:
|
||||
state["force_index"] = [0]
|
||||
return
|
||||
|
||||
state["prev_output_length"] = current_length
|
||||
|
||||
start_len = len(self.think_start_token_ids)
|
||||
absolute_start_pos = state["start_thinking"]
|
||||
|
||||
if state["continue_thinking"] and state["end_thinking"] > -1:
|
||||
absolute_end_pos = state["end_thinking"] + len(
|
||||
state.get("prompt_tok_ids") or []
|
||||
)
|
||||
else:
|
||||
absolute_end_pos = state["end_thinking"]
|
||||
# Update state based on recent sequences
|
||||
# This is the case where we are in end mode, but the rejection sampler
|
||||
# rejected a token before the end token,
|
||||
# so we need to go back to think mode and wait for the next end token
|
||||
# eg with 999: [2,4,5,999] -> [3,-1,-1,-1]
|
||||
if state["in_end"] and state["end_count"] == 0:
|
||||
new_tokens = output[prev_length:]
|
||||
stopping_thinking = (
|
||||
self.think_end_token_ids[state["end_count"]] in new_tokens
|
||||
)
|
||||
if not stopping_thinking:
|
||||
state["in_think"] = True
|
||||
state["in_end"] = False
|
||||
state["end_count"] = 0
|
||||
state["bonus_token_forced"] = False
|
||||
|
||||
if not state["in_end"]:
|
||||
if absolute_start_pos >= 0 and absolute_end_pos >= 0:
|
||||
# Case: ...<end>...<start>... - entering think mode
|
||||
if absolute_start_pos > absolute_end_pos:
|
||||
new_think_count = current_length - (absolute_start_pos + start_len)
|
||||
state["in_think"] = True
|
||||
state["think_count"] = new_think_count
|
||||
else:
|
||||
# Case: ...<start>...<end>... - exiting think mode
|
||||
state["in_think"] = False
|
||||
state["think_count"] = 0
|
||||
|
||||
elif absolute_start_pos >= 0 and not state["continue_thinking"]:
|
||||
# Found think start - entering think mode
|
||||
new_think_count = current_length - (absolute_start_pos + start_len)
|
||||
state["in_think"] = True
|
||||
state["think_count"] = new_think_count
|
||||
|
||||
elif absolute_end_pos >= 0:
|
||||
# Found think end - exiting think mode
|
||||
state["in_think"] = False
|
||||
state["think_count"] = 0
|
||||
|
||||
elif state["in_think"]:
|
||||
# Continue thinking mode, increment count by new tokens
|
||||
prompt_tok_ids = state.get("prompt_tok_ids") or []
|
||||
think_tokens_in_prompt = len(prompt_tok_ids) - (
|
||||
absolute_start_pos + start_len
|
||||
)
|
||||
state["think_count"] = (
|
||||
len(state["output_tok_ids"]) + think_tokens_in_prompt
|
||||
)
|
||||
if state["in_think"]:
|
||||
remaining_budget = max(
|
||||
0, state["thinking_token_budget"] - state["think_count"]
|
||||
)
|
||||
state["check_count_down"] = remaining_budget
|
||||
else:
|
||||
state["check_count_down"] = state["thinking_token_budget"]
|
||||
|
||||
total_thinking_tokens = (
|
||||
state["think_count"] + len(state["spec_token_ids"]) + 1
|
||||
)
|
||||
# Check if need to transition to end mode
|
||||
# If we have more thinking tokens than the budget,
|
||||
# we need to transition to end mode
|
||||
if (
|
||||
state["in_think"]
|
||||
and total_thinking_tokens > state["thinking_token_budget"]
|
||||
):
|
||||
# Calculate force_index: position within spec_token_ids where
|
||||
# forcing starts. If we're already over budget without spec
|
||||
# tokens, force from position 0. Force from the position
|
||||
# where budget is exceeded.
|
||||
state["in_think"] = False
|
||||
state["in_end"] = True
|
||||
state["end_count"] = 0
|
||||
state["check_count_down"] = state["thinking_token_budget"]
|
||||
remaining_budget = state["thinking_token_budget"] - state["think_count"]
|
||||
spec_len = len(state["spec_token_ids"])
|
||||
if 0 < remaining_budget < spec_len:
|
||||
state["force_index"] = [remaining_budget]
|
||||
|
||||
elif remaining_budget <= 0:
|
||||
state["force_index"] = [0]
|
||||
|
||||
else:
|
||||
# remaining_budget >= spec_len: all spec tokens are within
|
||||
# budget; force the bonus token position
|
||||
state["force_index"] = [len(state["spec_token_ids"])]
|
||||
|
||||
else:
|
||||
state["force_index"] = []
|
||||
if len(state["spec_token_ids"]) > 0:
|
||||
for i, token_id in enumerate(state["spec_token_ids"]):
|
||||
if state["end_count"] + 1 < len(self.think_end_token_ids):
|
||||
if token_id == self.think_end_token_ids[state["end_count"] + 1]:
|
||||
state["end_count"] += 1
|
||||
else:
|
||||
state["end_count"] += 1
|
||||
state["force_index"] = [i]
|
||||
break
|
||||
else:
|
||||
state["end_count"] += 1
|
||||
if len(state["force_index"]) == 0:
|
||||
state["end_count"] += 1
|
||||
state["force_index"] = [len(state["spec_token_ids"])]
|
||||
else:
|
||||
state["end_count"] += 1
|
||||
state["force_index"] = [0]
|
||||
if state["end_count"] >= len(self.think_end_token_ids):
|
||||
state.update(
|
||||
{
|
||||
"in_end": False,
|
||||
"end_count": 0,
|
||||
"check_count_down": state["thinking_token_budget"],
|
||||
}
|
||||
)
|
||||
|
||||
def _apply_forcing_to_logits(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
predict_bonus_token: bool,
|
||||
spec_token_ids_for_layout: list[list[int]],
|
||||
) -> torch.Tensor:
|
||||
self.mask[:] = False
|
||||
cumulative_total = 0
|
||||
self.cu_num_tokens.clear()
|
||||
|
||||
n_layout = len(spec_token_ids_for_layout)
|
||||
if self._state:
|
||||
n_layout = max(n_layout, max(self._state.keys()) + 1)
|
||||
|
||||
for index in range(n_layout):
|
||||
self.cu_num_tokens[index] = cumulative_total
|
||||
spec_tokens = (
|
||||
spec_token_ids_for_layout[index]
|
||||
if index < len(spec_token_ids_for_layout)
|
||||
else []
|
||||
)
|
||||
if self.in_spec_mode:
|
||||
cumulative_total += len(spec_tokens) if not predict_bonus_token else 1
|
||||
else:
|
||||
cumulative_total += 1
|
||||
|
||||
for seq_idx in sorted(self._state.keys()):
|
||||
if seq_idx not in self.cu_num_tokens:
|
||||
continue
|
||||
state = self._state[seq_idx]
|
||||
if state.get("in_end", False):
|
||||
# logits processor in spec mode are called twice
|
||||
# once for bonus token logits and
|
||||
# second time for the target logits
|
||||
# in case the force index is bonus token index
|
||||
# we change the force index to 0
|
||||
if predict_bonus_token:
|
||||
if state.get("force_index") and state["force_index"][0] < len(
|
||||
state["spec_token_ids"]
|
||||
):
|
||||
continue
|
||||
else:
|
||||
state["force_index"] = [0]
|
||||
# continue enforcing the end thinking tokens
|
||||
if state["end_count"] > 0:
|
||||
state["bonus_token_forced"] = False
|
||||
if state and not state["bonus_token_forced"]:
|
||||
force_index = state.get("force_index", [])
|
||||
if len(force_index) == 0:
|
||||
continue
|
||||
end_count = state.get("end_count", 0)
|
||||
for force_idx in force_index:
|
||||
if end_count < len(self.think_end_token_ids):
|
||||
mask_idx = self.cu_num_tokens[seq_idx] + force_idx
|
||||
if mask_idx < len(self.mask) and mask_idx < logits.shape[0]:
|
||||
self.mask[mask_idx] = True
|
||||
self.force_token_ids[mask_idx] = (
|
||||
self.think_end_token_ids[end_count]
|
||||
)
|
||||
if predict_bonus_token:
|
||||
if state["end_count"] > 0:
|
||||
state["bonus_token_forced"] = False
|
||||
state["force_index"] = []
|
||||
else:
|
||||
state["bonus_token_forced"] = True
|
||||
|
||||
has_active_thinking = any(
|
||||
state.get("in_end", False) for state in self._state.values()
|
||||
)
|
||||
|
||||
if has_active_thinking:
|
||||
active_indices = self.mask.nonzero(as_tuple=False).view(-1)
|
||||
|
||||
if len(active_indices) > 0:
|
||||
force_tokens = self.force_token_ids[active_indices]
|
||||
logits[active_indices, force_tokens] = 1e9
|
||||
|
||||
return logits
|
||||
@@ -8,6 +8,7 @@ from typing import cast
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config.reasoning import ReasoningConfig
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.pooling_params import PoolingParams
|
||||
@@ -22,6 +23,9 @@ from vllm.v1.sample.logits_processor import (
|
||||
MoveDirectionality,
|
||||
)
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.thinking_budget_state import (
|
||||
maybe_create_thinking_budget_state_holder,
|
||||
)
|
||||
from vllm.v1.utils import copy_slice
|
||||
from vllm.v1.worker.block_table import MultiGroupBlockTable
|
||||
|
||||
@@ -92,12 +96,20 @@ class InputBatch:
|
||||
max_num_blocks_per_req: list[int] | None = None,
|
||||
logitsprocs: LogitsProcessors | None = None,
|
||||
logitsprocs_need_output_token_ids: bool = False,
|
||||
is_spec_decode: bool = False,
|
||||
num_spec_tokens: int = 0,
|
||||
is_pooling_model: bool = False,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
reasoning_config: ReasoningConfig | None = None,
|
||||
):
|
||||
self.thinking_budget_state_holder = maybe_create_thinking_budget_state_holder(
|
||||
reasoning_config,
|
||||
max_num_reqs,
|
||||
num_spec_tokens,
|
||||
device,
|
||||
pin_memory,
|
||||
)
|
||||
self.thinking_token_budget_reqs: set[str] = set()
|
||||
self.is_pooling_model = is_pooling_model
|
||||
self.is_spec_decode = is_spec_decode
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.max_model_len = max_model_len
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
@@ -540,6 +552,7 @@ class InputBatch:
|
||||
# False means we don't fill with -inf.
|
||||
self.allowed_token_ids_mask_cpu_tensor[req_index].fill_(False)
|
||||
self.bad_words_token_ids.pop(req_index, None)
|
||||
self.thinking_token_budget_reqs.discard(req_id)
|
||||
return req_index
|
||||
|
||||
def swap_states(self, i1: int, i2: int) -> None:
|
||||
@@ -800,6 +813,8 @@ class InputBatch:
|
||||
# reset batch update tracking.
|
||||
# Update sampling metadata if batch state is changed.
|
||||
batch_update = self.batch_update_builder.get_and_reset(self.num_reqs)
|
||||
if self.thinking_budget_state_holder is not None and batch_update:
|
||||
self.thinking_budget_state_holder.sync_batch(batch_update)
|
||||
for logit_proc in self.logitsprocs.all:
|
||||
logit_proc.update_state(batch_update)
|
||||
if batch_update:
|
||||
@@ -853,10 +868,15 @@ class InputBatch:
|
||||
|
||||
# Only set output_token_ids if required by the current requests'
|
||||
# sampling parameters.
|
||||
holder = self.thinking_budget_state_holder
|
||||
thinking_budget_tracks_reqs = (
|
||||
holder is not None and holder.has_tracked_requests()
|
||||
)
|
||||
needs_output_token_ids = (
|
||||
not self.no_penalties
|
||||
or bool(self.bad_words_token_ids)
|
||||
or self.logitsprocs_need_output_token_ids
|
||||
or not thinking_budget_tracks_reqs
|
||||
)
|
||||
output_token_ids = (
|
||||
cast(list[list[int]], self.req_output_token_ids)
|
||||
@@ -902,6 +922,7 @@ class InputBatch:
|
||||
allowed_token_ids_mask=allowed_token_ids_mask,
|
||||
bad_words_token_ids=self.bad_words_token_ids,
|
||||
logitsprocs=self.logitsprocs,
|
||||
thinking_budget_state_holder=self.thinking_budget_state_holder,
|
||||
)
|
||||
|
||||
def get_pooling_params(self) -> list[PoolingParams]:
|
||||
@@ -1076,6 +1097,13 @@ class InputBatch:
|
||||
and len(self.repetition_penalties_reqs) == 0
|
||||
)
|
||||
|
||||
@property
|
||||
def no_thinking_budget(self) -> bool:
|
||||
return (
|
||||
self.thinking_budget_state_holder is None
|
||||
or len(self.thinking_token_budget_reqs) == 0
|
||||
)
|
||||
|
||||
@property
|
||||
def max_num_logprobs(self) -> int | None:
|
||||
return max(self.num_logprobs.values()) if self.num_logprobs else None
|
||||
|
||||
@@ -629,7 +629,7 @@ class GPUModelRunner(
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=[placeholder_block_size],
|
||||
kernel_block_sizes=[placeholder_block_size],
|
||||
is_spec_decode=bool(self.vllm_config.speculative_config),
|
||||
num_spec_tokens=self.num_spec_tokens,
|
||||
logitsprocs=build_logitsprocs(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
@@ -645,6 +645,7 @@ class GPUModelRunner(
|
||||
or self.vllm_config.reasoning_config is not None,
|
||||
is_pooling_model=self.is_pooling_model,
|
||||
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
|
||||
reasoning_config=self.vllm_config.reasoning_config,
|
||||
)
|
||||
|
||||
# Separate cuda stream for overlapping transfer of sampled token ids from
|
||||
@@ -6504,10 +6505,11 @@ class GPUModelRunner(
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks_per_req=max_num_blocks,
|
||||
is_spec_decode=bool(self.vllm_config.speculative_config),
|
||||
num_spec_tokens=self.num_spec_tokens,
|
||||
logitsprocs=self.input_batch.logitsprocs,
|
||||
logitsprocs_need_output_token_ids=self.input_batch.logitsprocs_need_output_token_ids,
|
||||
is_pooling_model=self.is_pooling_model,
|
||||
reasoning_config=self.vllm_config.reasoning_config,
|
||||
)
|
||||
|
||||
assert self._init_block_sizes == block_sizes, (
|
||||
|
||||
Reference in New Issue
Block a user