diff --git a/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py b/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py index d2db50082a..d7a601114b 100644 --- a/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py +++ b/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py @@ -1,18 +1,62 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""E2E tests for thinking_token_budget with reasoning models.""" +"""E2E tests for ``thinking_token_budget`` with reasoning models. + +Covers Qwen3-0.6B and Qwen3.5 FP8 + MTP. +""" + +import asyncio +import json +from typing import Literal import openai import pytest import pytest_asyncio -from tests.utils import RemoteOpenAIServer +from tests.utils import RemoteOpenAIServer, multi_gpu_only, requires_fp8 +from vllm.platforms import current_platform +from vllm.tokenizers import get_tokenizer MODEL_NAME = "Qwen/Qwen3-0.6B" +QWEN35_FP8_MTP_MODEL = "Qwen/Qwen3.5-35B-A3B-FP8" MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}] THINK_BUDGET = 5 +REASONING_START_STR = "" +REASONING_END_STR = "" + + +def _count_reasoning_decode_token_ids_between_markers( + full_token_ids: list[int], + reasoning_start_ids: list[int], + reasoning_end_ids: list[int], +) -> int | None: + """Count decode tokens in the thinking span (after last start, before first end).""" + + if not reasoning_start_ids or not reasoning_end_ids: + raise ValueError("reasoning marker token id lists must be non-empty") + + def _last_subseq_index(haystack: list[int], needle: list[int]) -> int: + n = len(needle) + if n > len(haystack): + return -1 + for i in range(len(haystack) - n, -1, -1): + if haystack[i : i + n] == needle: + return i + return -1 + + last_start = _last_subseq_index(full_token_ids, reasoning_start_ids) + if last_start < 0: + return None + + pos_after_start = last_start + len(reasoning_start_ids) + end_n = len(reasoning_end_ids) + for j in range(pos_after_start, len(full_token_ids) - end_n + 1): + if full_token_ids[j : j + end_n] == reasoning_end_ids: + return j - pos_after_start + return len(full_token_ids) - pos_after_start + @pytest.fixture(scope="module") def server(): @@ -48,6 +92,51 @@ def server_with_auto_reasoning_config(): yield remote_server +@pytest.fixture(scope="module") +def server_qwen35_fp8_mtp_tp2(): + """Qwen3.5-35B FP8 with MTP speculative decoding and tensor parallel size 2.""" + if current_platform.device_count() < 2: + pytest.skip("Need at least 2 GPUs for --tensor-parallel-size 2") + if not current_platform.supports_fp8(): + pytest.skip("FP8 is not supported on this platform") + + spec_cfg = { + "method": "mtp", + "num_speculative_tokens": 2, + "max_model_len": 32768, + } + args = [ + "--tensor-parallel-size", + "2", + "--max-model-len", + "32768", + "--speculative-config", + json.dumps(spec_cfg), + "--reasoning-parser", + "qwen3", + "--reasoning-config", + json.dumps( + { + "reasoning_start_str": REASONING_START_STR, + "reasoning_end_str": REASONING_END_STR, + } + ), + ] + # With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers + # on 0,1 do not exhaust memory on the same devices as this worker. + env_dict = None + if current_platform.device_count() >= 4: + env_dict = {"CUDA_VISIBLE_DEVICES": "2,3"} + + with RemoteOpenAIServer( + QWEN35_FP8_MTP_MODEL, + args, + max_wait_seconds=3000, + env_dict=env_dict, + ) as remote_server: + yield remote_server + + @pytest_asyncio.fixture async def client(request, server, server_with_auto_reasoning_config): server_map = { @@ -89,8 +178,10 @@ async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI): async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI): """Test that thinking_token_budget limits the number of reasoning tokens. - In streaming mode each reasoning delta corresponds to one token, so - counting non-empty reasoning_content chunks gives the exact token count. + Counts non-empty streaming ``delta.reasoning`` chunks (coarse proxy; each + chunk may represent multiple decode tokens — see + ``_count_reasoning_decode_token_ids_between_markers`` and the Qwen3.5 MTP + test for id-based checks). """ reasoning_token_count = 0 @@ -110,3 +201,89 @@ async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI f"reasoning tokens ({reasoning_token_count}) exceeded " f"thinking_token_budget ({THINK_BUDGET})" ) + + +@pytest.mark.asyncio +@multi_gpu_only(num_gpus=2) +@requires_fp8 +async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_plain( + server_qwen35_fp8_mtp_tp2, +): + """Concurrent chat requests: some with ``thinking_token_budget``, some without. + + Exercises the scheduler / input processor under a mixed batch on the same + Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with + ``_count_reasoning_decode_token_ids_between_markers`` on full token ids. + """ + + _batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [ + ("budget", 1), + ("budget", 12), + ("plain", None), + ("budget", 20), + ("budget", 14), + ("plain", None), + ("plain", None), + ("budget", 12), + ("plain", None), + ] + + tokenizer = get_tokenizer(tokenizer_name=QWEN35_FP8_MTP_MODEL) + start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False)) + end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False)) + + async with server_qwen35_fp8_mtp_tp2.get_async_client() as client: + + async def budgeted_call(expected_budget: int): + return await client.chat.completions.create( + model=QWEN35_FP8_MTP_MODEL, + messages=MESSAGES, + max_tokens=256, + stream=False, + extra_body={ + "thinking_token_budget": expected_budget, + "return_token_ids": True, + }, + ) + + async def plain_call(): + return await client.chat.completions.create( + model=QWEN35_FP8_MTP_MODEL, + messages=MESSAGES, + max_tokens=256, + stream=False, + ) + + coros = [] + for row in _batch_spec: + if row[0] == "budget": + b = row[1] + assert isinstance(b, int) + coros.append(budgeted_call(b)) + else: + coros.append(plain_call()) + results = await asyncio.gather(*coros) + + for i, (response, (kind, expected_budget)) in enumerate( + zip(results, _batch_spec, strict=True) + ): + msg = response.choices[0].message + assert msg.content or getattr(msg, "reasoning", None), ( + f"index {i} ({kind}): empty message" + ) + + if kind == "budget": + assert expected_budget is not None + assert response.prompt_token_ids is not None + assert response.choices[0].token_ids is not None + full_ids = list(response.prompt_token_ids) + list( + response.choices[0].token_ids + ) + n_reason = _count_reasoning_decode_token_ids_between_markers( + full_ids, start_ids, end_ids + ) + assert n_reason is not None, f"index {i}: missing reasoning start in ids" + assert n_reason == expected_budget, ( + f"index {i}: reasoning decode token ids ({n_reason}) != " + f"thinking_token_budget ({expected_budget})" + ) diff --git a/tests/v1/logits_processors/test_correctness.py b/tests/v1/logits_processors/test_correctness.py index 9ee6a70abe..1326b96346 100644 --- a/tests/v1/logits_processors/test_correctness.py +++ b/tests/v1/logits_processors/test_correctness.py @@ -30,10 +30,13 @@ from vllm.v1.sample.logits_processor import ( MinPLogitsProcessor, MinTokensLogitsProcessor, MoveDirectionality, - ThinkingTokenBudgetLogitsProcessor, build_logitsprocs, ) from vllm.v1.sample.metadata import SamplingMetadata +from vllm.v1.sample.thinking_budget_state import ( + ThinkingBudgetStateHolder, + maybe_create_thinking_budget_state_holder, +) PIN_MEMORY_AVAILABLE = is_pin_memory_available() MAX_NUM_REQS = 256 @@ -48,8 +51,10 @@ MAX_NUM_PROMPT_TOKENS = 64 MIN_TOKENS_LEN_THRESHOLD = 5 REQS_PER_LOGITPROC = 50 STR_NO_LOGITPROC = "none" +# Thinking budget uses ``ThinkingBudgetStateHolder`` (not a logits processor). +STR_THINKING_BUDGET = "thinking_budget" -# ThinkingTokenBudgetLogitsProcessor testing constants +# Thinking token budget testing constants THINKING_TOKEN_BUDGET = 5 THINK_START_TOKEN_ID = 999 THINK_END_TOKEN_ID = 998 @@ -80,15 +85,8 @@ class LogitsProcsRequestParams: if num_tokens > 0: # Use diverse random tokens self.out_tokens = [random.randint(1, 950) for _ in range(num_tokens)] - # Set first token for ThinkingTokenBudget testing - is_thinking_processor = ( - logitproc_type is ThinkingTokenBudgetLogitsProcessor - or ( - hasattr(logitproc_type, "__name__") - and logitproc_type.__name__ == "ThinkingTokenBudgetLogitsProcessor" - ) - ) - if is_thinking_processor: + # Think-start seed for ``STR_THINKING_BUDGET`` rows. + if logitproc_type == STR_THINKING_BUDGET: self.out_tokens[0] = THINK_START_TOKEN_ID else: self.out_tokens = [] @@ -102,7 +100,7 @@ class LogitsProcsRequestParams: class MockReasoningConfig: - """Mock reasoning config for testing ThinkingTokenBudgetLogitsProcessor.""" + """Minimal reasoning config for ``ThinkingBudgetStateHolder`` tests.""" reasoning_start_token_ids = [THINK_START_TOKEN_ID] reasoning_end_token_ids = [THINK_END_TOKEN_ID] @@ -137,6 +135,18 @@ def _generate_fake_sampling_metadata( is_pin_memory=PIN_MEMORY_AVAILABLE, is_pooling_model=False, ) + num_spec = ( + vllm_config.speculative_config.num_speculative_tokens + if vllm_config.speculative_config + else 0 + ) + thinking_holder = maybe_create_thinking_budget_state_holder( + vllm_config.reasoning_config, + vllm_config.scheduler_config.max_num_seqs, + num_spec, + device, + PIN_MEMORY_AVAILABLE, + ) fake_sampling_metadata = SamplingMetadata( temperature=torch.full((batch_size,), 0.0), all_greedy=True, @@ -156,6 +166,7 @@ def _generate_fake_sampling_metadata( allowed_token_ids_mask=None, bad_words_token_ids={}, logitsprocs=logitsprocs, + thinking_budget_state_holder=thinking_holder, ) return fake_sampling_metadata @@ -187,7 +198,7 @@ def _sampling_params_from_logitproc(logitproc_type: LogitprocType) -> SamplingPa def _generate_mixed_logitsprocs_batch_params( reqs_per_logitproc: int, - logitsprocs_types: list[str], + logitsprocs_types: list[LogitprocType], ) -> list[LogitsProcsRequestParams]: """Define key params for a batch of requests with a different logitproc enabled per request. @@ -450,23 +461,21 @@ def _thinking_budget_validate( request_params: LogitsProcsRequestParams, step_idx: int, ) -> None: - """Validate thinking token budget processor behavior""" - # Get the ThinkingTokenBudgetLogitsProcessor instance - tb_processor: ThinkingTokenBudgetLogitsProcessor = next( - test_fakes.get_logitsprocs_by_cls(ThinkingTokenBudgetLogitsProcessor) - ) + """Validate ``ThinkingBudgetStateHolder`` thinking-budget behavior. - # Get current request state - state = tb_processor._state.get(batch_index) + State is keyed by **batch slot** (same index space as logits rows), matching + ``sync_batch`` / sampler integration (see PR #34668 discussion). + """ + holder = test_fakes.sampling_metadata.thinking_budget_state_holder + assert holder is not None + state = holder._state.get(batch_index) params = request_params.params - # Validate thinking token budget configuration if hasattr(params, "thinking_token_budget") and params.thinking_token_budget: - # State should exist for requests with thinking_token_budget if state is None: _raise_error_invalid( msg_suffix=( - f"Expected state for batch {batch_index} " + f"Expected holder state for batch slot {batch_index} " f"with thinking_token_budget={params.thinking_token_budget}" ), batch_index=batch_index, @@ -474,10 +483,8 @@ def _thinking_budget_validate( step_idx=step_idx, ) - # Validate budget matches what was set expected_budget = params.thinking_token_budget actual_budget = state["thinking_token_budget"] - if actual_budget != expected_budget: _raise_error_invalid( msg_suffix=( @@ -488,13 +495,9 @@ def _thinking_budget_validate( step_idx=step_idx, ) - # Check if we're in thinking mode and validate token counting output_tokens = request_params.out_tokens - - # Find if thinking has started in output tokens + start_tokens = holder.think_start_token_ids thinking_started = False - start_tokens = tb_processor.reasoning_start_token_ids - if len(start_tokens) > 0: for i in range(len(output_tokens) - len(start_tokens) + 1): if output_tokens[i : i + len(start_tokens)] == start_tokens: @@ -502,61 +505,42 @@ def _thinking_budget_validate( break if thinking_started: - # If budget is exceeded, validate end token forcing think_count = state["think_count"] budget = state["thinking_token_budget"] + if think_count >= budget and not state["in_end"]: + _raise_error_invalid( + msg_suffix=( + f"Budget exceeded ({think_count} >= {budget}) but " + "in_end is false" + ), + batch_index=batch_index, + request_params=request_params, + step_idx=step_idx, + ) - if think_count >= budget: - if not state["in_end"]: + end_tokens = holder.think_end_token_ids + if ( + think_count >= budget + and state["in_end"] + and len(end_tokens) > 0 + and holder.has_tracked_requests() + ): + expected_end_token_id = end_tokens[ + min(state["end_count"], len(end_tokens) - 1) + ] + # Holder bumps forced vocab positions to 1e9 (does not -inf others). + forced_logit = float(logits_new[batch_index, expected_end_token_id]) + if forced_logit < 1.0e8: _raise_error_invalid( msg_suffix=( - f"Budget exceeded ({think_count} >= " - f"{budget}) but not " - "forcing end tokens" + f"Expected forced end token {expected_end_token_id} " + f"with large logit, got {forced_logit}" ), batch_index=batch_index, request_params=request_params, step_idx=step_idx, ) - # Validate that only end tokens are allowed - end_tokens = tb_processor.reasoning_end_token_ids - if len(end_tokens) > 0: - expected_end_token_id = end_tokens[ - min(state["end_count"], len(end_tokens) - 1) - ] - - # Check logits masking - batch_logits = logits_new[batch_index] - for token_id in range(len(batch_logits)): - logit_value = batch_logits[token_id] - - if token_id == expected_end_token_id: - # End token should not be masked - if logit_value == -float("inf"): - _raise_error_invalid( - msg_suffix=( - f"End token {token_id} should not be " - "masked but is" - ), - batch_index=batch_index, - request_params=request_params, - step_idx=step_idx, - ) - else: - # All other tokens should be masked when forcing end - if logit_value != -float("inf"): - _raise_error_invalid( - msg_suffix=( - f"Token {token_id} should be masked " - f"when forcing end tokens, but " - f"logit={logit_value}" - ), - batch_index=batch_index, - request_params=request_params, - step_idx=step_idx, - ) - def _none_validate( test_fakes: LogitsprocsTestFakes, @@ -604,7 +588,7 @@ logitsprocs_test_mapping = { MinTokensLogitsProcessor: LogitsprocTestHelpers( gen_request_fxn=_min_tokens_params, eval_fxn=_min_tokens_validate ), - ThinkingTokenBudgetLogitsProcessor: LogitsprocTestHelpers( + STR_THINKING_BUDGET: LogitsprocTestHelpers( gen_request_fxn=_thinking_budget_params, eval_fxn=_thinking_budget_validate ), } @@ -614,20 +598,17 @@ def _get_test_cases() -> list[list[str]]: """Each test case is a set of logitsprocs""" logitsprocs_types = list(logitsprocs_test_mapping.keys()) - # Isolate ThinkingTokenBudgetLogitsProcessor from all other processors - # to avoid unexpected modification of logits interference - thinking_processor = ThinkingTokenBudgetLogitsProcessor + # Isolate thinking-budget handling from other processors to avoid cross-talk. + thinking_id: LogitprocType = STR_THINKING_BUDGET other_processors = [ - p - for p in logitsprocs_types - if p != STR_NO_LOGITPROC and p != thinking_processor + p for p in logitsprocs_types if p != STR_NO_LOGITPROC and p != thinking_id ] return ( [[STR_NO_LOGITPROC]] + [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors] + [other_processors] - + [[thinking_processor]] + + [[thinking_id]] ) @@ -802,12 +783,23 @@ def _assert_valid( ) +def _slot_outputs_for_metadata( + persistent_batch: list[LogitsProcsRequestParams], pad_len: int +) -> list[list[int]]: + """Per-batch-slot output token ids aligned with ``SamplingMetadata`` rows.""" + rows: list[list[int]] = [[] for _ in range(pad_len)] + for i, req in enumerate(persistent_batch): + if i < pad_len: + rows[i] = list(req.out_tokens) + return rows + + @create_new_process_for_each_test() @pytest.mark.parametrize("device", DEVICES) @pytest.mark.parametrize("reqs_per_logitproc", [REQS_PER_LOGITPROC]) @pytest.mark.parametrize("logitsprocs_under_test", _get_test_cases()) def test_logitsprocs( - device: str, reqs_per_logitproc: int, logitsprocs_under_test: list[str] + device: str, reqs_per_logitproc: int, logitsprocs_under_test: list[LogitprocType] ): random.seed(40) torch.set_default_device(device) @@ -855,9 +847,10 @@ def test_logitsprocs( # Apply fake batch update to logitsprocs fake_update_logitsprocs_state(test_fakes, batch_update) - # Emulate application of logits processors in engine + # Emulate application of logits processors + thinking holder (sampler order). slice_idxs = [req.workload_index for req in persistent_batch] - logits_w_lp = fake_apply_logitsprocs(test_fakes, slice_idxs).cpu() + slot_rows = _slot_outputs_for_metadata(persistent_batch, workload_size) + logits_w_lp = fake_apply_logitsprocs(test_fakes, slice_idxs, slot_rows).cpu() _assert_valid( batch_size=batch_size, @@ -869,3 +862,263 @@ def test_logitsprocs( ) step_idx += 1 + + +class MockReasoningNoEndTokens: + """Reasoning config with no end token ids (disables enforcement in holder).""" + + reasoning_start_token_ids = [THINK_START_TOKEN_ID] + reasoning_end_token_ids: list[int] = [] + + +def test_maybe_create_thinking_budget_holder_without_reasoning(): + cfg = VllmConfig() + assert cfg.reasoning_config is None + assert ( + maybe_create_thinking_budget_state_holder( + None, + cfg.scheduler_config.max_num_seqs, + 0, + torch.device("cpu"), + False, + ) + is None + ) + + +def test_thinking_budget_holder_has_tracked_after_sync_add(): + vc = VllmConfig() + vc.reasoning_config = MockReasoningConfig() + h = ThinkingBudgetStateHolder( + vc.reasoning_config, + vc.scheduler_config.max_num_seqs, + 0, + torch.device("cpu"), + False, + ) + assert not h.has_tracked_requests() + h.sync_batch( + BatchUpdate( + batch_size=1, + removed=(), + added=[ + ( + 0, + SamplingParams(thinking_token_budget=3), + None, + [THINK_START_TOKEN_ID], + ) + ], + moved=(), + ) + ) + assert h.has_tracked_requests() + assert h._state[0]["thinking_token_budget"] == 3 + + +def test_thinking_budget_holder_sync_remove_clears_state(): + vc = VllmConfig() + vc.reasoning_config = MockReasoningConfig() + h = ThinkingBudgetStateHolder( + vc.reasoning_config, + vc.scheduler_config.max_num_seqs, + 0, + torch.device("cpu"), + False, + ) + h.sync_batch( + BatchUpdate( + batch_size=1, + removed=(), + added=[ + ( + 0, + SamplingParams(thinking_token_budget=3), + None, + [], + ) + ], + moved=(), + ) + ) + assert h.has_tracked_requests() + h.sync_batch(BatchUpdate(batch_size=0, removed=(0,), added=(), moved=())) + assert not h.has_tracked_requests() + + +def test_thinking_budget_holder_sync_add_without_budget_drops_row(): + vc = VllmConfig() + vc.reasoning_config = MockReasoningConfig() + h = ThinkingBudgetStateHolder( + vc.reasoning_config, + vc.scheduler_config.max_num_seqs, + 0, + torch.device("cpu"), + False, + ) + h.sync_batch( + BatchUpdate( + batch_size=1, + removed=(), + added=[(0, SamplingParams(), None, [])], + moved=(), + ) + ) + assert not h.has_tracked_requests() + + +def test_thinking_budget_holder_swap_exchanges_state(): + vc = VllmConfig() + vc.reasoning_config = MockReasoningConfig() + h = ThinkingBudgetStateHolder( + vc.reasoning_config, + vc.scheduler_config.max_num_seqs, + 0, + torch.device("cpu"), + False, + ) + h.sync_batch( + BatchUpdate( + batch_size=2, + removed=(), + added=[ + ( + 0, + SamplingParams(thinking_token_budget=3), + None, + [], + ), + ( + 1, + SamplingParams(thinking_token_budget=7), + None, + [], + ), + ], + moved=(), + ) + ) + b0, b1 = h._state[0]["thinking_token_budget"], h._state[1]["thinking_token_budget"] + h.sync_batch( + BatchUpdate( + batch_size=2, + removed=(), + added=(), + moved=[(0, 1, MoveDirectionality.SWAP)], + ) + ) + assert h._state[0]["thinking_token_budget"] == b1 + assert h._state[1]["thinking_token_budget"] == b0 + + +def test_thinking_budget_holder_unidirectional_move(): + vc = VllmConfig() + vc.reasoning_config = MockReasoningConfig() + h = ThinkingBudgetStateHolder( + vc.reasoning_config, + vc.scheduler_config.max_num_seqs, + 0, + torch.device("cpu"), + False, + ) + h.sync_batch( + BatchUpdate( + batch_size=2, + removed=(), + added=[ + ( + 1, + SamplingParams(thinking_token_budget=4), + None, + [], + ), + ], + moved=(), + ) + ) + assert 1 in h._state and 0 not in h._state + h.sync_batch( + BatchUpdate( + batch_size=2, + removed=(), + added=(), + moved=[(1, 0, MoveDirectionality.UNIDIRECTIONAL)], + ) + ) + assert 0 in h._state and 1 not in h._state + assert h._state[0]["thinking_token_budget"] == 4 + + +def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins(): + vc = VllmConfig() + vc.reasoning_config = MockReasoningConfig() + h = ThinkingBudgetStateHolder( + vc.reasoning_config, + vc.scheduler_config.max_num_seqs, + 0, + torch.device("cpu"), + False, + ) + h.sync_batch( + BatchUpdate( + batch_size=1, + removed=(), + added=[ + ( + 0, + SamplingParams(thinking_token_budget=5), + None, + [THINK_START_TOKEN_ID], + ) + ], + moved=(), + ) + ) + out_lists = [[THINK_START_TOKEN_ID], [THINK_START_TOKEN_ID, 10, 11, 12, 13, 14]] + h.update_state( + out_lists, + None, + torch.tensor([0, 0], dtype=torch.long), + ) + assert h._state[0]["output_tok_ids"] == out_lists[1] + + +def test_thinking_budget_holder_spec_mode_tensor_layout(): + h = ThinkingBudgetStateHolder( + MockReasoningConfig(), + 8, + 2, + torch.device("cpu"), + False, + ) + assert h.in_spec_mode + assert h.mask.shape[0] == 8 * (2 + 1) + + +def test_thinking_budget_holder_empty_end_tokens_disables_row(): + vc = VllmConfig() + vc.reasoning_config = MockReasoningNoEndTokens() + h = ThinkingBudgetStateHolder( + vc.reasoning_config, + vc.scheduler_config.max_num_seqs, + 0, + torch.device("cpu"), + False, + ) + h.sync_batch( + BatchUpdate( + batch_size=1, + removed=(), + added=[ + ( + 0, + SamplingParams(thinking_token_budget=5), + None, + [THINK_START_TOKEN_ID], + ) + ], + moved=(), + ) + ) + h.update_state([[THINK_START_TOKEN_ID, 1]], None, None) + assert h._state[0]["thinking_token_budget"] == -1 diff --git a/tests/v1/sample/utils.py b/tests/v1/sample/utils.py index a0abb3b4c6..907be3614b 100644 --- a/tests/v1/sample/utils.py +++ b/tests/v1/sample/utils.py @@ -198,22 +198,43 @@ class LogitsprocsTestFakes(NamedTuple): def fake_update_logitsprocs_state( test_fakes: LogitsprocsTestFakes, - batch_update: BatchUpdate, + batch_update: BatchUpdate | None, ) -> None: """Imitate logits processors persistent batch state update in engine core""" for logitproc in test_fakes.get_logitsprocs(): logitproc.update_state(batch_update) + holder = test_fakes.sampling_metadata.thinking_budget_state_holder + if holder is not None: + holder.sync_batch(batch_update) def fake_apply_logitsprocs( test_fakes: LogitsprocsTestFakes, slice_indices: list[int], + slot_output_token_ids: list[list[int]] | None = None, ) -> torch.Tensor: - """Imitate application of logits processors in engine core""" + """Imitate application of logits processors in engine core. + + When ``thinking_budget_state_holder`` has tracked requests, this mirrors + :meth:`Sampler.apply_logits_processors` by refreshing per-slot + ``output_token_ids`` (if ``slot_output_token_ids`` is provided), then + ``update_state`` + ``apply_to_logits`` on the holder after built-in logits + processors. + """ logits = test_fakes.logits[torch.tensor(slice_indices, dtype=torch.long)].clone() for processor in test_fakes.get_logitsprocs(): logits = processor.apply(logits) + + md = test_fakes.sampling_metadata + holder = md.thinking_budget_state_holder + if holder is not None and holder.has_tracked_requests(): + if slot_output_token_ids is not None: + for i, toks in enumerate(slot_output_token_ids): + if i < len(md.output_token_ids): + md.output_token_ids[i] = list(toks) + holder.update_state(md.output_token_ids, md.spec_token_ids, None) + logits = holder.apply_to_logits(logits, False, md.spec_token_ids) return logits diff --git a/tests/v1/streaming_input/test_gpu_model_runner_streaming.py b/tests/v1/streaming_input/test_gpu_model_runner_streaming.py index 0ed7b6cb3e..946ca99507 100644 --- a/tests/v1/streaming_input/test_gpu_model_runner_streaming.py +++ b/tests/v1/streaming_input/test_gpu_model_runner_streaming.py @@ -39,7 +39,6 @@ def mock_model_runner_with_input_batch(): vocab_size=32000, block_sizes=[16], kernel_block_sizes=[16], - is_spec_decode=False, logitsprocs=None, is_pooling_model=False, ) diff --git a/vllm/v1/sample/logits_processor/__init__.py b/vllm/v1/sample/logits_processor/__init__.py index fb4a046fc0..2cb89e1ea9 100644 --- a/vllm/v1/sample/logits_processor/__init__.py +++ b/vllm/v1/sample/logits_processor/__init__.py @@ -18,7 +18,6 @@ from vllm.v1.sample.logits_processor.builtin import ( LogitBiasLogitsProcessor, MinPLogitsProcessor, MinTokensLogitsProcessor, - ThinkingTokenBudgetLogitsProcessor, process_dict_updates, ) from vllm.v1.sample.logits_processor.interface import ( @@ -51,7 +50,6 @@ BUILTIN_LOGITS_PROCESSORS: list[type[LogitsProcessor]] = [ MinTokensLogitsProcessor, LogitBiasLogitsProcessor, MinPLogitsProcessor, - ThinkingTokenBudgetLogitsProcessor, ] @@ -356,5 +354,4 @@ __all__ = [ "STR_POOLING_REJECTS_LOGITSPROCS", "LOGITSPROCS_GROUP", "AdapterLogitsProcessor", - "ThinkingTokenBudgetLogitsProcessor", ] diff --git a/vllm/v1/sample/logits_processor/builtin.py b/vllm/v1/sample/logits_processor/builtin.py index 1739452b44..11a52711d6 100644 --- a/vllm/v1/sample/logits_processor/builtin.py +++ b/vllm/v1/sample/logits_processor/builtin.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Callable, Sequence -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, TypeVar import numpy as np import torch @@ -291,267 +291,6 @@ class MinTokensLogitsProcessor(LogitsProcessor): return logits -class ThinkingTokenBudgetLogitsProcessor(LogitsProcessor): - """Limits the number of tokens allowed inside a 'thinking' section.""" - - def __init__( - self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool - ): - reasoning_config = vllm_config.reasoning_config - max_num_reqs = vllm_config.scheduler_config.max_num_seqs - - # Check if thinking is enabled - self.is_enabled = reasoning_config is not None and reasoning_config.enabled - - self.reasoning_start_token_ids = getattr( - reasoning_config, "reasoning_start_token_ids", [] - ) - self.reasoning_end_token_ids = getattr( - reasoning_config, "reasoning_end_token_ids", [] - ) - - self.pin_memory = is_pin_memory - self.device = device - # Per-request state tracking for thinking token management - # Key: request_index, Value: state dict containing: - # "in_think": bool - currently in thinking mode - # "in_end": bool - currently forcing end tokens output - # "check_count_down": int - steps remaining until next think - # start/end token parsing - # "think_count": int - number of thinking tokens generated - # "end_count": int - number of end tokens forced so far - # "thinking_token_budget": int - max allowed thinking tokens - # "output_tok_ids": list[int] - generated output tokens - # "prev_output_length": int - previous output length for - # incremental processing - self._state: dict[int, dict[str, Any]] = {} - - # Preallocate reusable tensors - self.mask = torch.zeros(max_num_reqs, dtype=torch.bool, device=device) - self.force_token_ids = torch.full( - (max_num_reqs,), -1, dtype=torch.long, device=device - ) - - @staticmethod - def _find_last_sequence_index(target_list: list[int], token_ids: list[int]) -> int: - """ - Returns the index of the last occurrence of token_ids in target_list. - - Args: - target_list (list[int]): The list of token IDs. - token_ids (list[int]): The sequence of token IDs to find. - """ - if not token_ids: - return -1 - for i in range(len(target_list) - len(token_ids), -1, -1): - if target_list[i : i + len(token_ids)] == token_ids: - return i - return -1 - - def _init_state_entry( - self, prompt_tok_ids: list[int] | None, thinking_token_budget: int - ) -> dict[str, Any]: - """Initializes the tracking state for a given sequence index.""" - if prompt_tok_ids is None: - last_start = -1 - last_end = -1 - in_think = False - think_count = 0 - else: - last_start = self._find_last_sequence_index( - prompt_tok_ids, self.reasoning_start_token_ids - ) - last_end = self._find_last_sequence_index( - prompt_tok_ids, self.reasoning_end_token_ids - ) - in_think = last_start > last_end - if in_think: - think_count = len(prompt_tok_ids) - ( - last_start + len(self.reasoning_start_token_ids) - ) - else: - think_count = 0 - - return { - "in_think": in_think, # Currently in thinking mode - "in_end": in_think and thinking_token_budget == 0, - "check_count_down": thinking_token_budget, - "think_count": think_count, # Number of tokens in thinking section - "end_count": 0, # Number of end tokens forced so far - "prompt_tok_ids": prompt_tok_ids, - "output_tok_ids": [], - "thinking_token_budget": thinking_token_budget, - "prev_output_length": 0, - # Track previous output length for incremental updates - } - - def _update_think_state(self, state: dict[str, Any]): - """Updates the state based on newly generated output tokens.""" - if not state.get("in_end", False) and state.get("check_count_down", 0) > 0: - state["check_count_down"] -= 1 - return - - output = state.get("output_tok_ids", []) - if not output: - return - - # Track previous output length for incremental processing - prev_length = state.get("prev_output_length", 0) - current_length = len(output) - - if current_length <= prev_length: - return - - # Process only newly added tokens - new_tokens = output[prev_length:] - state["prev_output_length"] = current_length - - # Check if new tokens contain think start or end sequences - start_len = len(self.reasoning_start_token_ids) - end_len = len(self.reasoning_end_token_ids) - - # Look for think sequences in recent tokens (including boundary) - # Check overlapping regions where sequences might span boundaries - check_start_idx = max(0, prev_length - max(start_len, end_len) + 1) - recent_tokens = output[check_start_idx:] - - # Find any think start/end sequences in recent tokens - recent_start_pos = self._find_last_sequence_index( - recent_tokens, self.reasoning_start_token_ids - ) - recent_end_pos = self._find_last_sequence_index( - recent_tokens, self.reasoning_end_token_ids - ) - - # Update state based on recent sequences - if not state["in_end"]: - if recent_start_pos >= 0 and recent_end_pos >= 0: - if recent_start_pos > recent_end_pos: - # Case: ......... - entering think mode - absolute_start_pos = check_start_idx + recent_start_pos - new_think_count = current_length - (absolute_start_pos + start_len) - state["in_think"] = True - state["think_count"] = new_think_count - else: - # Case: ......... - exiting think mode - state["in_think"] = False - state["think_count"] = 0 - elif recent_start_pos >= 0: - # Found think start - entering think mode - absolute_start_pos = check_start_idx + recent_start_pos - new_think_count = current_length - (absolute_start_pos + start_len) - state["in_think"] = True - state["think_count"] = new_think_count - elif recent_end_pos >= 0: - # Found think end - exiting think mode - state["in_think"] = False - state["think_count"] = 0 - elif state["in_think"]: - # Continue thinking mode, increment count by new tokens - state["think_count"] += len(new_tokens) - - # Set countdown based on current state - if state["in_think"]: - remaining_budget = max( - 0, state["thinking_token_budget"] - state["think_count"] - ) - state["check_count_down"] = max(0, remaining_budget - 1) - else: - state["check_count_down"] = state["thinking_token_budget"] - - # Check if need to transition to end mode - if ( - state["in_think"] - and state["think_count"] >= state["thinking_token_budget"] - ): - state["in_think"] = False - state["in_end"] = True - state["end_count"] = 0 - state["check_count_down"] = state["thinking_token_budget"] - else: - # In end mode - state["end_count"] += 1 - if state["end_count"] >= len(self.reasoning_end_token_ids): - state.update( - { - "in_end": False, - "end_count": 0, - "check_count_down": state["thinking_token_budget"], - } - ) - - def is_argmax_invariant(self) -> bool: - """This logits processor can change the outcome of - greedy sampling by forcing that the thinking section - ends after a certain number of tokens.""" - return False - - def update_state(self, batch_update: BatchUpdate | None): - if not self.is_enabled: - return - if batch_update: - for index, params, prompt_tok_ids, output_tok_ids in batch_update.added: - thinking_token_budget = params.thinking_token_budget - - if thinking_token_budget is not None: - self._state[index] = self._init_state_entry( - prompt_tok_ids, thinking_token_budget - ) - self._state[index]["output_tok_ids"] = output_tok_ids - else: - # Remove state if no thinking budget - self._state.pop(index, None) - - for index in batch_update.removed: - self._state.pop(index, {}) - - for i1, i2, direction in batch_update.moved: - if direction == MoveDirectionality.SWAP: - state1 = self._state.pop(i1, None) - state2 = self._state.pop(i2, None) - if state1 is not None: - self._state[i2] = state1 - if state2 is not None: - self._state[i1] = state2 - else: - state = self._state.pop(i1, None) - if state is not None: - self._state[i2] = state - - for state in self._state.values(): - self._update_think_state(state) - - def apply(self, logits: torch.Tensor) -> torch.Tensor: - if not self.is_enabled or not self._state: - return logits - - batch_size = logits.size(0) - self.mask[:batch_size] = False - - for i in range(batch_size): - state = self._state.get(i) - if state and state["in_end"]: - self.mask[i] = True - self.force_token_ids[i] = self.reasoning_end_token_ids[ - state["end_count"] - ] - - # Check in CPU first not to sync with GPU - has_active_thinking = any( - state.get("in_end", False) for state in self._state.values() - ) - - if has_active_thinking: - current_mask = self.mask[:batch_size] - active_indices = current_mask.nonzero(as_tuple=False).view(-1) - if len(active_indices) > 0: - force_tokens = self.force_token_ids[active_indices] - # Apply a large value for the end thinking token id index - logits[active_indices, force_tokens] = 1e9 - - return logits - - def process_dict_updates( req_entries: dict[int, T], batch_update: BatchUpdate | None, diff --git a/vllm/v1/sample/metadata.py b/vllm/v1/sample/metadata.py index 4682cde109..fa4ceac8e7 100644 --- a/vllm/v1/sample/metadata.py +++ b/vllm/v1/sample/metadata.py @@ -1,11 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + from dataclasses import dataclass import torch from vllm.v1.sample.logits_processor import LogitsProcessors +from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder @dataclass @@ -47,3 +50,6 @@ class SamplingMetadata: # Speculative token ids spec_token_ids: list[list[int]] | None = None + # When non-None, use ``holder.has_tracked_requests()`` to see if this batch applies + # thinking-token-budget logits (holder may exist with an empty tracking set). + thinking_budget_state_holder: ThinkingBudgetStateHolder | None = None diff --git a/vllm/v1/sample/rejection_sampler.py b/vllm/v1/sample/rejection_sampler.py index 2b63893c04..678654cb78 100644 --- a/vllm/v1/sample/rejection_sampler.py +++ b/vllm/v1/sample/rejection_sampler.py @@ -290,16 +290,24 @@ class RejectionSampler(nn.Module): any_penalties_or_bad_words = ( sampling_metadata.bad_words_token_ids or has_penalties ) + holder = sampling_metadata.thinking_budget_state_holder + needs_thinking = holder is not None and holder.has_tracked_requests() output_token_ids = sampling_metadata.output_token_ids - if any_penalties_or_bad_words: + if any_penalties_or_bad_words or needs_thinking: output_token_ids = self._combine_outputs_with_spec_tokens( output_token_ids, sampling_metadata.spec_token_ids, ) # Calculate indices of target logits. - if sampling_metadata.allowed_token_ids_mask is not None or has_penalties: + repeat_indices: torch.Tensor | None = None + need_repeat_indices = ( + sampling_metadata.allowed_token_ids_mask is not None + or has_penalties + or needs_thinking + ) + if need_repeat_indices: num_requests = len(metadata.num_draft_tokens) num_draft_tokens = torch.tensor(metadata.num_draft_tokens, device="cpu") original_indices = torch.arange(num_requests, device="cpu") @@ -327,7 +335,12 @@ class RejectionSampler(nn.Module): logits = processor.apply_with_spec_decode( logits, metadata.num_draft_tokens ) - + if holder is not None and holder.has_tracked_requests(): + logits = holder.apply_to_logits( + logits, + predict_bonus_token=False, + spec_token_ids=sampling_metadata.spec_token_ids, + ) return logits @staticmethod diff --git a/vllm/v1/sample/sampler.py b/vllm/v1/sample/sampler.py index 5341351352..a77eafba25 100644 --- a/vllm/v1/sample/sampler.py +++ b/vllm/v1/sample/sampler.py @@ -364,9 +364,13 @@ class Sampler(nn.Module): any_penalties_or_bad_words = ( bool(bad_words_token_ids) or not sampling_metadata.no_penalties ) + holder = sampling_metadata.thinking_budget_state_holder + needs_thinking_combine = holder is not None and holder.has_tracked_requests() output_token_ids = sampling_metadata.output_token_ids - if predict_bonus_token and any_penalties_or_bad_words: + if predict_bonus_token and ( + any_penalties_or_bad_words or needs_thinking_combine + ): # Combine base outputs with spec tokens when speculative decoding # is enabled. output_token_ids = self._combine_outputs_with_spec_tokens( @@ -388,6 +392,17 @@ class Sampler(nn.Module): # Apply penalties (e.g., freq_penalties). logits = self.apply_penalties(logits, sampling_metadata, output_token_ids) + if holder is not None and holder.has_tracked_requests(): + holder.update_state( + output_token_ids, + sampling_metadata.spec_token_ids, + repeat_indices=None, + ) + logits = holder.apply_to_logits( + logits, + predict_bonus_token, + sampling_metadata.spec_token_ids, + ) return logits @staticmethod diff --git a/vllm/v1/sample/thinking_budget_state.py b/vllm/v1/sample/thinking_budget_state.py new file mode 100644 index 0000000000..74599a1e8c --- /dev/null +++ b/vllm/v1/sample/thinking_budget_state.py @@ -0,0 +1,528 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Per-batch thinking token budget state; applied after penalties at sample time.""" + +from typing import TYPE_CHECKING, Any + +import torch + +from vllm.v1.sample.logits_processor.interface import ( + BatchUpdate, + MoveDirectionality, +) + +if TYPE_CHECKING: + from vllm.config.reasoning import ReasoningConfig + + +def maybe_create_thinking_budget_state_holder( + reasoning_config: "ReasoningConfig | None", + max_num_seqs: int, + num_spec_tokens: int, + device: torch.device, + is_pin_memory: bool, +) -> "ThinkingBudgetStateHolder | None": + if reasoning_config is None: + return None + return ThinkingBudgetStateHolder( + reasoning_config, max_num_seqs, num_spec_tokens, device, is_pin_memory + ) + + +class ThinkingBudgetStateHolder: + """Tracks thinking sections and forces end tokens when budget is exceeded.""" + + think_start_token_ids: list[int] + think_end_token_ids: list[int] + + def __init__( + self, + reasoning_config: "ReasoningConfig | None", + max_num_seqs: int, + num_spec_tokens: int, + device: torch.device, + is_pin_memory: bool, + ): + _ = is_pin_memory # API parity with logits processors + max_num_reqs = max_num_seqs + self.in_spec_mode = num_spec_tokens > 0 + self.num_spec_tokens = num_spec_tokens + + # No separate enable flag: a non-``None`` ``reasoning_config`` is the switch. + self.is_enabled = reasoning_config is not None + + if reasoning_config is None: + self.think_start_token_ids = [] + self.think_end_token_ids = [] + else: + rs = reasoning_config.reasoning_start_token_ids + re = reasoning_config.reasoning_end_token_ids + self.think_start_token_ids = rs if rs else [] + self.think_end_token_ids = re if re else [] + + self.device = device + self._state: dict[int, dict[str, Any]] = {} + self.cu_num_tokens: dict[int, int] = {} + + if self.num_spec_tokens > 0: + self.mask = torch.zeros( + max_num_reqs * (self.num_spec_tokens + 1), + dtype=torch.bool, + device=device, + ) + self.force_token_ids = torch.full( + (max_num_reqs * (self.num_spec_tokens + 1),), + -1, + dtype=torch.long, + device=device, + ) + else: + self.mask = torch.zeros(max_num_reqs, dtype=torch.bool, device=device) + self.force_token_ids = torch.full( + (max_num_reqs,), -1, dtype=torch.long, device=device + ) + + def has_tracked_requests(self) -> bool: + """True when ``sync_batch`` has state for a ``thinking_token_budget`` row. + + Used to decide whether sampling needs output-token rows and spec combining; + distinct from merely having a holder instance (reasoning may be on with no + budgeted requests in this batch). + """ + return bool(self._state) + + def sync_batch(self, batch_update: BatchUpdate | None) -> None: + """Add/remove/move per-request state only (no _update_think_state).""" + if not self.is_enabled or not batch_update: + return + for index in batch_update.removed: + self._state.pop(index, None) + + for index, params, prompt_tok_ids, output_tok_ids in batch_update.added: + thinking_token_budget = params.thinking_token_budget + if thinking_token_budget is not None: + self._state[index] = self._init_state_entry( + prompt_tok_ids, thinking_token_budget + ) + self._state[index]["output_tok_ids"] = output_tok_ids + self._state[index]["spec_token_ids"] = [] + else: + self._state.pop(index, None) + + for i1, i2, direction in batch_update.moved: + if direction == MoveDirectionality.SWAP: + state1 = self._state.get(i1) + state2 = self._state.get(i2) + if state1 is not None: + self._state[i2] = state1 + if state2 is not None: + self._state[i1] = state2 + else: + state = self._state.pop(i1, None) + if state is not None: + self._state[i2] = state + + def update_state( + self, + output_token_ids: list[list[int]], + spec_token_ids: list[list[int]] | None, + repeat_indices: torch.Tensor | None = None, + ) -> None: + """Refresh output/spec from sampling rows and recompute think state.""" + if not self.is_enabled or not self._state: + return + + spec_lists = spec_token_ids or [] + last_row_for_req: dict[int, int] | None = None + if repeat_indices is not None: + last_row_for_req = {} + rpt = repeat_indices.cpu().tolist() + for batch_row, req_i in enumerate(rpt): + last_row_for_req[req_i] = batch_row + + for seq_idx, state in list(self._state.items()): + if last_row_for_req is not None: + output_row: int | None = last_row_for_req.get(seq_idx) + if output_row is None or output_row >= len(output_token_ids): + continue + state["output_tok_ids"] = output_token_ids[output_row] + elif seq_idx >= len(output_token_ids): + continue + else: + state["output_tok_ids"] = output_token_ids[seq_idx] + if seq_idx < len(spec_lists): + state["spec_token_ids"] = list(spec_lists[seq_idx]) + else: + state["spec_token_ids"] = [] + state["in_spec_mode"] = self.in_spec_mode + state["force_index"] = [] + if len(state["output_tok_ids"]) > 0: + spec_len = len(state["spec_token_ids"]) + # Only strip draft suffix when there are spec tokens; ``[:-0]`` would + # clear the whole list (Python treats stop index 0 as "up to empty"). + if spec_len > 0 and len(state["output_tok_ids"]) >= spec_len: + state["output_tok_ids"] = state["output_tok_ids"][:-spec_len] + self._update_think_state(state) + + def apply_to_logits( + self, + logits: torch.Tensor, + predict_bonus_token: bool, + spec_token_ids: list[list[int]] | None, + ) -> torch.Tensor: + """Mask and bump logits for forced end-of-thinking tokens.""" + if not self.is_enabled or not self._state: + return logits + spec_lists = spec_token_ids or [] + return self._apply_forcing_to_logits(logits, predict_bonus_token, spec_lists) + + @staticmethod + def _find_last_sequence_index(target_list: list[int], token_ids: list[int]) -> int: + if not token_ids: + return -1 + for i in range(len(target_list) - len(token_ids), -1, -1): + if target_list[i : i + len(token_ids)] == token_ids: + return i + return -1 + + def _init_state_entry( + self, prompt_tok_ids: list[int] | None, thinking_token_budget: int + ) -> dict[str, Any]: + if prompt_tok_ids is None: + last_start = -1 + last_end = -1 + in_think = False + think_count = 0 + start_thinking = -1 + countdown = thinking_token_budget + continue_thinking = False + in_end = False + else: + start_thinking = -1 + countdown = thinking_token_budget + continue_thinking = False + in_end = False + last_start = self._find_last_sequence_index( + prompt_tok_ids, self.think_start_token_ids + ) + last_end = self._find_last_sequence_index( + prompt_tok_ids, self.think_end_token_ids + ) + in_think = last_start > last_end + # load metrics such as think count, start thinking + # if request is in thinking mode, already + if in_think: + think_count = len(prompt_tok_ids) - ( + last_start + len(self.think_start_token_ids) + ) + start_thinking = len(prompt_tok_ids) - think_count - 1 + countdown -= think_count + continue_thinking = True + # check if the token is exhausted within prompt + token_exhausted = thinking_token_budget - think_count + in_end = token_exhausted <= 0 + else: + think_count = 0 + + return { + "in_think": in_think, + "in_end": in_end, + "check_count_down": countdown, + "think_count": think_count, + "end_count": 0, + "prompt_tok_ids": prompt_tok_ids, + "output_tok_ids": [], + "thinking_token_budget": thinking_token_budget, + "prev_output_length": 0, + "spec_token_ids": [], + "force_index": [], + "start_thinking": start_thinking, + "end_thinking": -1, + "in_spec_mode": False, + "bonus_token_forced": False, + "continue_thinking": continue_thinking, + } + + def _update_think_state(self, state: dict[str, Any]) -> None: + if state.get("thinking_token_budget", -1) == -1: + return + if len(self.think_end_token_ids) == 0: + state["thinking_token_budget"] = -1 + state["in_end"] = False + state["force_index"] = [] + return + + if state["start_thinking"] == -1: + start_thinking = self._find_last_sequence_index( + state.get("output_tok_ids", []), self.think_start_token_ids + ) + state["start_thinking"] = start_thinking + if state["end_thinking"] == -1: + end_thinking = self._find_last_sequence_index( + state.get("output_tok_ids", []), self.think_end_token_ids + ) + state["end_thinking"] = end_thinking + + if state["start_thinking"] == -1: + return + + if state["continue_thinking"]: + sampled_tokens_from_previous_step = len( + state.get("output_tok_ids", []) + ) - state.get("prev_output_length", 0) + else: + if state["prev_output_length"] == 0: + sampled_tokens_from_previous_step = len( + state.get("output_tok_ids", []) + ) - len(self.think_start_token_ids) + else: + sampled_tokens_from_previous_step = ( + len(state.get("output_tok_ids", [])) - state["prev_output_length"] + ) + current_step_countdown = ( + state["check_count_down"] - sampled_tokens_from_previous_step + ) + predicted_countdown = current_step_countdown - len(state["spec_token_ids"]) - 1 + # We only proceed further if we have counted down the thinking budget + # to 0 or less and when we are in the "in think" mode. + if ( + not state.get("in_end", False) + and predicted_countdown >= 0 + and state["start_thinking"] > -1 + ): + state["check_count_down"] = current_step_countdown + state["prev_output_length"] = len(state.get("output_tok_ids", [])) + return + output = state.get("output_tok_ids", []) + if not output: + # When in_end was set at init (budget=0, prompt already in think), + # we must force the first generated token to be the end token; + # otherwise apply() sees in_end=True but force_index=[] and + # allows an extra thinking token. + if state.get("in_end", False): + state["force_index"] = [0] + return + + # Track previous output length for incremental processing + prev_length = state.get("prev_output_length", 0) + current_length = len(output) + + if current_length <= prev_length: + if state.get("in_end", False): + remaining_budget = state["thinking_token_budget"] - state["think_count"] + spec_len = len(state["spec_token_ids"]) + if spec_len > 0: + if 0 < remaining_budget < spec_len: + state["force_index"] = [remaining_budget] + elif remaining_budget <= 0: + state["force_index"] = [0] + else: + state["force_index"] = [spec_len] + else: + state["force_index"] = [0] + return + + state["prev_output_length"] = current_length + + start_len = len(self.think_start_token_ids) + absolute_start_pos = state["start_thinking"] + + if state["continue_thinking"] and state["end_thinking"] > -1: + absolute_end_pos = state["end_thinking"] + len( + state.get("prompt_tok_ids") or [] + ) + else: + absolute_end_pos = state["end_thinking"] + # Update state based on recent sequences + # This is the case where we are in end mode, but the rejection sampler + # rejected a token before the end token, + # so we need to go back to think mode and wait for the next end token + # eg with 999: [2,4,5,999] -> [3,-1,-1,-1] + if state["in_end"] and state["end_count"] == 0: + new_tokens = output[prev_length:] + stopping_thinking = ( + self.think_end_token_ids[state["end_count"]] in new_tokens + ) + if not stopping_thinking: + state["in_think"] = True + state["in_end"] = False + state["end_count"] = 0 + state["bonus_token_forced"] = False + + if not state["in_end"]: + if absolute_start_pos >= 0 and absolute_end_pos >= 0: + # Case: ......... - entering think mode + if absolute_start_pos > absolute_end_pos: + new_think_count = current_length - (absolute_start_pos + start_len) + state["in_think"] = True + state["think_count"] = new_think_count + else: + # Case: ......... - exiting think mode + state["in_think"] = False + state["think_count"] = 0 + + elif absolute_start_pos >= 0 and not state["continue_thinking"]: + # Found think start - entering think mode + new_think_count = current_length - (absolute_start_pos + start_len) + state["in_think"] = True + state["think_count"] = new_think_count + + elif absolute_end_pos >= 0: + # Found think end - exiting think mode + state["in_think"] = False + state["think_count"] = 0 + + elif state["in_think"]: + # Continue thinking mode, increment count by new tokens + prompt_tok_ids = state.get("prompt_tok_ids") or [] + think_tokens_in_prompt = len(prompt_tok_ids) - ( + absolute_start_pos + start_len + ) + state["think_count"] = ( + len(state["output_tok_ids"]) + think_tokens_in_prompt + ) + if state["in_think"]: + remaining_budget = max( + 0, state["thinking_token_budget"] - state["think_count"] + ) + state["check_count_down"] = remaining_budget + else: + state["check_count_down"] = state["thinking_token_budget"] + + total_thinking_tokens = ( + state["think_count"] + len(state["spec_token_ids"]) + 1 + ) + # Check if need to transition to end mode + # If we have more thinking tokens than the budget, + # we need to transition to end mode + if ( + state["in_think"] + and total_thinking_tokens > state["thinking_token_budget"] + ): + # Calculate force_index: position within spec_token_ids where + # forcing starts. If we're already over budget without spec + # tokens, force from position 0. Force from the position + # where budget is exceeded. + state["in_think"] = False + state["in_end"] = True + state["end_count"] = 0 + state["check_count_down"] = state["thinking_token_budget"] + remaining_budget = state["thinking_token_budget"] - state["think_count"] + spec_len = len(state["spec_token_ids"]) + if 0 < remaining_budget < spec_len: + state["force_index"] = [remaining_budget] + + elif remaining_budget <= 0: + state["force_index"] = [0] + + else: + # remaining_budget >= spec_len: all spec tokens are within + # budget; force the bonus token position + state["force_index"] = [len(state["spec_token_ids"])] + + else: + state["force_index"] = [] + if len(state["spec_token_ids"]) > 0: + for i, token_id in enumerate(state["spec_token_ids"]): + if state["end_count"] + 1 < len(self.think_end_token_ids): + if token_id == self.think_end_token_ids[state["end_count"] + 1]: + state["end_count"] += 1 + else: + state["end_count"] += 1 + state["force_index"] = [i] + break + else: + state["end_count"] += 1 + if len(state["force_index"]) == 0: + state["end_count"] += 1 + state["force_index"] = [len(state["spec_token_ids"])] + else: + state["end_count"] += 1 + state["force_index"] = [0] + if state["end_count"] >= len(self.think_end_token_ids): + state.update( + { + "in_end": False, + "end_count": 0, + "check_count_down": state["thinking_token_budget"], + } + ) + + def _apply_forcing_to_logits( + self, + logits: torch.Tensor, + predict_bonus_token: bool, + spec_token_ids_for_layout: list[list[int]], + ) -> torch.Tensor: + self.mask[:] = False + cumulative_total = 0 + self.cu_num_tokens.clear() + + n_layout = len(spec_token_ids_for_layout) + if self._state: + n_layout = max(n_layout, max(self._state.keys()) + 1) + + for index in range(n_layout): + self.cu_num_tokens[index] = cumulative_total + spec_tokens = ( + spec_token_ids_for_layout[index] + if index < len(spec_token_ids_for_layout) + else [] + ) + if self.in_spec_mode: + cumulative_total += len(spec_tokens) if not predict_bonus_token else 1 + else: + cumulative_total += 1 + + for seq_idx in sorted(self._state.keys()): + if seq_idx not in self.cu_num_tokens: + continue + state = self._state[seq_idx] + if state.get("in_end", False): + # logits processor in spec mode are called twice + # once for bonus token logits and + # second time for the target logits + # in case the force index is bonus token index + # we change the force index to 0 + if predict_bonus_token: + if state.get("force_index") and state["force_index"][0] < len( + state["spec_token_ids"] + ): + continue + else: + state["force_index"] = [0] + # continue enforcing the end thinking tokens + if state["end_count"] > 0: + state["bonus_token_forced"] = False + if state and not state["bonus_token_forced"]: + force_index = state.get("force_index", []) + if len(force_index) == 0: + continue + end_count = state.get("end_count", 0) + for force_idx in force_index: + if end_count < len(self.think_end_token_ids): + mask_idx = self.cu_num_tokens[seq_idx] + force_idx + if mask_idx < len(self.mask) and mask_idx < logits.shape[0]: + self.mask[mask_idx] = True + self.force_token_ids[mask_idx] = ( + self.think_end_token_ids[end_count] + ) + if predict_bonus_token: + if state["end_count"] > 0: + state["bonus_token_forced"] = False + state["force_index"] = [] + else: + state["bonus_token_forced"] = True + + has_active_thinking = any( + state.get("in_end", False) for state in self._state.values() + ) + + if has_active_thinking: + active_indices = self.mask.nonzero(as_tuple=False).view(-1) + + if len(active_indices) > 0: + force_tokens = self.force_token_ids[active_indices] + logits[active_indices, force_tokens] = 1e9 + + return logits diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index 89e63f3def..5930a1aaba 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -8,6 +8,7 @@ from typing import cast import numpy as np import torch +from vllm.config.reasoning import ReasoningConfig from vllm.lora.request import LoRARequest from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.pooling_params import PoolingParams @@ -22,6 +23,9 @@ from vllm.v1.sample.logits_processor import ( MoveDirectionality, ) from vllm.v1.sample.metadata import SamplingMetadata +from vllm.v1.sample.thinking_budget_state import ( + maybe_create_thinking_budget_state_holder, +) from vllm.v1.utils import copy_slice from vllm.v1.worker.block_table import MultiGroupBlockTable @@ -92,12 +96,20 @@ class InputBatch: max_num_blocks_per_req: list[int] | None = None, logitsprocs: LogitsProcessors | None = None, logitsprocs_need_output_token_ids: bool = False, - is_spec_decode: bool = False, + num_spec_tokens: int = 0, is_pooling_model: bool = False, cp_kv_cache_interleave_size: int = 1, + reasoning_config: ReasoningConfig | None = None, ): + self.thinking_budget_state_holder = maybe_create_thinking_budget_state_holder( + reasoning_config, + max_num_reqs, + num_spec_tokens, + device, + pin_memory, + ) + self.thinking_token_budget_reqs: set[str] = set() self.is_pooling_model = is_pooling_model - self.is_spec_decode = is_spec_decode self.max_num_reqs = max_num_reqs self.max_model_len = max_model_len self.max_num_batched_tokens = max_num_batched_tokens @@ -540,6 +552,7 @@ class InputBatch: # False means we don't fill with -inf. self.allowed_token_ids_mask_cpu_tensor[req_index].fill_(False) self.bad_words_token_ids.pop(req_index, None) + self.thinking_token_budget_reqs.discard(req_id) return req_index def swap_states(self, i1: int, i2: int) -> None: @@ -800,6 +813,8 @@ class InputBatch: # reset batch update tracking. # Update sampling metadata if batch state is changed. batch_update = self.batch_update_builder.get_and_reset(self.num_reqs) + if self.thinking_budget_state_holder is not None and batch_update: + self.thinking_budget_state_holder.sync_batch(batch_update) for logit_proc in self.logitsprocs.all: logit_proc.update_state(batch_update) if batch_update: @@ -853,10 +868,15 @@ class InputBatch: # Only set output_token_ids if required by the current requests' # sampling parameters. + holder = self.thinking_budget_state_holder + thinking_budget_tracks_reqs = ( + holder is not None and holder.has_tracked_requests() + ) needs_output_token_ids = ( not self.no_penalties or bool(self.bad_words_token_ids) or self.logitsprocs_need_output_token_ids + or not thinking_budget_tracks_reqs ) output_token_ids = ( cast(list[list[int]], self.req_output_token_ids) @@ -902,6 +922,7 @@ class InputBatch: allowed_token_ids_mask=allowed_token_ids_mask, bad_words_token_ids=self.bad_words_token_ids, logitsprocs=self.logitsprocs, + thinking_budget_state_holder=self.thinking_budget_state_holder, ) def get_pooling_params(self) -> list[PoolingParams]: @@ -1076,6 +1097,13 @@ class InputBatch: and len(self.repetition_penalties_reqs) == 0 ) + @property + def no_thinking_budget(self) -> bool: + return ( + self.thinking_budget_state_holder is None + or len(self.thinking_token_budget_reqs) == 0 + ) + @property def max_num_logprobs(self) -> int | None: return max(self.num_logprobs.values()) if self.num_logprobs else None diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index caf3bfdfc3..e2af34eecb 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -629,7 +629,7 @@ class GPUModelRunner( vocab_size=self.model_config.get_vocab_size(), block_sizes=[placeholder_block_size], kernel_block_sizes=[placeholder_block_size], - is_spec_decode=bool(self.vllm_config.speculative_config), + num_spec_tokens=self.num_spec_tokens, logitsprocs=build_logitsprocs( self.vllm_config, self.device, @@ -645,6 +645,7 @@ class GPUModelRunner( or self.vllm_config.reasoning_config is not None, is_pooling_model=self.is_pooling_model, cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size, + reasoning_config=self.vllm_config.reasoning_config, ) # Separate cuda stream for overlapping transfer of sampled token ids from @@ -6504,10 +6505,11 @@ class GPUModelRunner( block_sizes=block_sizes, kernel_block_sizes=kernel_block_sizes, max_num_blocks_per_req=max_num_blocks, - is_spec_decode=bool(self.vllm_config.speculative_config), + num_spec_tokens=self.num_spec_tokens, logitsprocs=self.input_batch.logitsprocs, logitsprocs_need_output_token_ids=self.input_batch.logitsprocs_need_output_token_ids, is_pooling_model=self.is_pooling_model, + reasoning_config=self.vllm_config.reasoning_config, ) assert self._init_block_sizes == block_sizes, (