diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index 9d16d2f710..9c42d91b38 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -314,6 +314,10 @@ class SchedulerOutput: # Number of spec tokens to schedule for the next step. num_spec_tokens_to_schedule: int = 0 + # Identity of this schedule() call (`Scheduler.current_step`). Used to + # retrieve the matching consumption-time draft-token snapshot. + scheduler_step: int = 0 + @classmethod def make_empty(cls) -> "SchedulerOutput": return cls( @@ -338,5 +342,10 @@ class GrammarOutput: # Per request, ordered as structured_output_request_ids: how many leading # drafts the bitmask constrained. `grammar_bitmask` fills every row after # the first -1 placeholder with the all-permissive `_full_mask`, so drafts + # from this index on must not be accepted. None means nothing to invalidate. + num_acceptable_drafts: list[int] | None = None + # Per request, ordered as structured_output_request_ids: how many leading + # drafts the bitmask constrained. `grammar_bitmask` fills every row after + # the first -1 placeholder with the all-permissive `_full_mask`, so drafts # from this index on must not be accepted. None invalidates every draft. num_acceptable_drafts: list[int] | None = None diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index d89279902f..905512eab3 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -78,7 +78,8 @@ from vllm.v1.core.sched import e2_alloc_log as _e2_alloc_log from vllm.v1.request import Request, RequestStatus, StreamingUpdate from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup from vllm.v1.spec_decode.metrics import SpecDecodingStats -from vllm.v1.structured_output import StructuredOutputManager +from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager +from vllm.v1.structured_output.utils import strip_speculative_padding from vllm.v1.structured_output.utils import strip_speculative_padding from vllm.v1.utils import record_function_or_nullcontext @@ -1529,6 +1530,7 @@ class Scheduler(SchedulerInterface): kv_connector_block_state=kv_connector_block_state, num_spec_tokens_to_schedule=num_spec_tokens_to_schedule, ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(), + scheduler_step=self.current_step, ) # NOTE(Kuntai): this function is designed for multiple purposes: diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index cc0a8beddd..f036207357 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -764,7 +764,9 @@ class EngineCore: # When draft tokens are used with structured output, validate them # before computing the grammar bitmask for the deferred request. if self.check_for_draft_tokens: - draft_token_ids = self.model_executor.take_draft_token_ids() + draft_token_ids = self.model_executor.take_draft_token_ids( + step_id=deferred_scheduler_output.scheduler_step + ) if draft_token_ids is not None: # Update the draft token ids in the scheduler output to # filter out the invalid spec tokens, which will be padded diff --git a/vllm/v1/executor/abstract.py b/vllm/v1/executor/abstract.py index f234e943b4..4795bf8613 100644 --- a/vllm/v1/executor/abstract.py +++ b/vllm/v1/executor/abstract.py @@ -265,8 +265,13 @@ class Executor(ABC): def execute_dummy_batch(self) -> None: self.collective_rpc("execute_dummy_batch") - def take_draft_token_ids(self) -> DraftTokenIds | None: - output: list[DraftTokenIds] = self.collective_rpc("take_draft_token_ids") + def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None: + # Omit step_id when unset so plugin workers with the documented + # no-argument take_draft_token_ids() keep working. + kwargs = {} if step_id is None else {"step_id": step_id} + output: list[DraftTokenIds] = self.collective_rpc( + "take_draft_token_ids", kwargs=kwargs + ) return output[0] def profile(self, is_start: bool = True, profile_prefix: str | None = None): diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index 24bd9e0d46..07263263a7 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -405,12 +405,16 @@ class MultiprocExecutor(Executor): def execute_dummy_batch(self) -> None: self.collective_rpc("execute_dummy_batch", unique_reply_rank=self.output_rank) - def take_draft_token_ids(self) -> DraftTokenIds | None: + def take_draft_token_ids( + self, step_id: int | None = None + ) -> DraftTokenIds | None: # OPTIMIZATION: Get output only from a single worker (output_rank) + kwargs = {} if step_id is None else {"step_id": step_id} return self.collective_rpc( "take_draft_token_ids", unique_reply_rank=self.output_rank, timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS, + kwargs=kwargs, ) def collective_rpc( # type: ignore[override] diff --git a/vllm/v1/executor/uniproc_executor.py b/vllm/v1/executor/uniproc_executor.py index da1f2c588b..dc798cdc5c 100644 --- a/vllm/v1/executor/uniproc_executor.py +++ b/vllm/v1/executor/uniproc_executor.py @@ -141,8 +141,13 @@ class UniProcExecutor(Executor): single_value=True, ) - def take_draft_token_ids(self) -> DraftTokenIds | None: - return self.collective_rpc("take_draft_token_ids", single_value=True) + def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None: + kwargs = {} if step_id is None else {"step_id": step_id} + return self.collective_rpc( + "take_draft_token_ids", + kwargs=kwargs, + single_value=True, + ) def check_health(self) -> None: # UniProcExecutor will always be healthy as long as diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index 5a0b95c17b..42297747cf 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -453,3 +453,16 @@ def choice_as_grammar(choice: list[str]) -> str: escaped_choices = (escape_ebnf_string(c) for c in choice) grammar = "root ::= " + " | ".join(f'"{c}"' for c in escaped_choices) return grammar + +def strip_speculative_padding(token_ids: list[int]) -> list[int]: + """Drop speculative-decoding padding from a token block. + + ngram and other speculative backends pad rejected draft positions with a + -1 sentinel. Structured-output grammars treat every entry as a real token + id, so the sentinels (and everything after the first one) are removed here, + before tokens reach any backend, rather than inside a single backend. + """ + for i, token_id in enumerate(token_ids): + if token_id < 0: + return token_ids[:i] + return token_ids diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 139d104d4d..8988f551d1 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -186,6 +186,39 @@ from vllm.v1.worker.workspace import lock_workspace, use_workspace_lane logger = init_logger(__name__) +def grammar_invalid_draft_positions( + input_batch: InputBatch, + grammar_req_ids: list[str], + num_acceptable_drafts: list[int] | None, + device: torch.device, +) -> torch.Tensor | None: + """Indices of `draft_sampled` entries whose bitmask row is permissive. + + Drafts from `num_acceptable_drafts` on were verified against rows carrying + `_full_mask`, so accepting one would sample with no grammar constraint. + Returns None when there is nothing to invalidate. + """ + if not grammar_req_ids or input_batch.num_draft_tokens == 0: + return None + cu_num_logits = input_batch.cu_num_logits_np.tolist() + req_id_to_idx = {req_id: i for i, req_id in enumerate(input_batch.req_ids)} + positions: list[int] = [] + for i, req_id in enumerate(grammar_req_ids): + req_idx = req_id_to_idx.get(req_id) + if req_idx is None: + continue + # Without the field (an older scheduler, or warmup) invalidate the whole + # window, which is the conservative choice. + num_acceptable = ( + num_acceptable_drafts[i] if num_acceptable_drafts is not None else 0 + ) + start = cu_num_logits[req_idx] + 1 + num_acceptable + positions.extend(range(start, cu_num_logits[req_idx + 1])) + if not positions: + return None + return torch.tensor(positions, dtype=torch.int64, device=device) + + class GPUModelRunner(LoRAModelRunnerMixin): def __init__(self, vllm_config: VllmConfig, device: torch.device): self.vllm_config = vllm_config @@ -288,7 +321,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.use_aux_hidden_state_outputs = True # Draft tokens propagation - for spec-dec + struct outputs. - self.draft_tokens_handler = DraftTokensHandler(self.device) + self.draft_tokens_handler = DraftTokensHandler( + self.device, + max_snapshots=max(8, vllm_config.max_concurrent_batches + 1), + ) self.pcp_manager: pcp.PCPManager | None = None @@ -1470,6 +1506,15 @@ class GPUModelRunner(LoRAModelRunnerMixin): total_num_logits, self.model_state.num_new_sampled_tokens_per_step, ) + if self.num_speculative_steps > 0: + # Snapshot the drafts that were just written into input_ids, tagged + # with this scheduler step. Proposal-time drafts (set later in + # sample_tokens) are for the next step and must not overwrite this. + self.draft_tokens_handler.snapshot_consumed_drafts( + scheduler_output.scheduler_step, + input_batch, + self.req_states.draft_tokens[input_batch.idx_mapping], + ) fast_prefill = None if self.fast_prefill is not None: @@ -1538,6 +1583,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): else None ), ) + invalid_draft_positions = None if self.pcp_manager is not None: input_batch = self.pcp_manager.partition_batch(input_batch, batch_desc) return input_batch @@ -1620,6 +1666,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): grammar_output.structured_output_request_ids, grammar_output.num_acceptable_drafts, ) + invalid_draft_positions = grammar_invalid_draft_positions( + input_batch, + grammar_output.structured_output_request_ids, + grammar_output.num_acceptable_drafts, + self.device, + ) sampler_output: SamplerOutput | None if input_batch.num_reqs == 0: @@ -1638,6 +1690,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): input_batch, # Draft logits are needed for probabilistic rejection sampling. self.speculator.draft_logits, + invalid_draft_positions, invalid_drafts, ) @@ -2278,8 +2331,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): return async_output - def take_draft_token_ids(self) -> DraftTokenIds | None: - return self.draft_tokens_handler.get_draft_tokens() + def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None: + return self.draft_tokens_handler.get_draft_tokens(step_id=step_id) @torch.inference_mode() @step_eplb_after() diff --git a/vllm/v1/worker/gpu/spec_decode/utils.py b/vllm/v1/worker/gpu/spec_decode/utils.py index f73acb2bda..65ad1603ca 100644 --- a/vllm/v1/worker/gpu/spec_decode/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/utils.py @@ -1,5 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections import deque +from dataclasses import dataclass + import numpy as np import torch @@ -10,6 +13,10 @@ from vllm.v1.outputs import DraftTokenIds from vllm.v1.worker.gpu.async_utils import async_copy_to_np from vllm.v1.worker.gpu.input_batch import InputBatch +# Keep enough consumption-time snapshots to cover async PP in-flight +# batches (`max_concurrent_batches` is `pp_size + 1`) with headroom. +_DEFAULT_MAX_SNAPSHOTS = 8 + logger = init_logger(__name__) @@ -26,48 +33,135 @@ def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig: return load_config -class DraftTokensHandler: - def __init__(self, device: torch.device | None = None): - self.device = device - self.copy_stream = torch.cuda.Stream(device) - # Blocking (sleep) event to avoid busy-polling the CUDA driver lock. - self.copy_event = torch.cuda.Event(blocking=True) +@dataclass +class _DraftSnapshot: + step_id: int | None + req_ids: list[str] + num_draft_tokens: int + draft_tokens_np: np.ndarray | None + copy_event: torch.cuda.Event | None - self.req_ids: list[str] = [] - self.draft_tokens_np: np.ndarray | None = None - self.num_draft_tokens: int = 0 + def to_draft_token_ids(self) -> DraftTokenIds: + if self.draft_tokens_np is not None: + if self.copy_event is not None: + self.copy_event.synchronize() + draft_token_ids = self.draft_tokens_np.tolist() + else: + draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids] + return DraftTokenIds(self.req_ids, draft_token_ids) + + +class DraftTokensHandler: + def __init__( + self, + device: torch.device | None = None, + max_snapshots: int = _DEFAULT_MAX_SNAPSHOTS, + ): + self.device = device + self.copy_stream: torch.cuda.Stream | None = None + if device is not None and device.type == "cuda": + self.copy_stream = torch.cuda.Stream(device) + + self._max_snapshots = max(1, max_snapshots) + # Consumption-time snapshots tagged with scheduler step identity. + self._consumed: deque[_DraftSnapshot] = deque(maxlen=self._max_snapshots) + # Latest proposal-time drafts for the non-async post_step path. + self._proposed: _DraftSnapshot | None = None + + def _copy_drafts( + self, draft_tokens: torch.Tensor + ) -> tuple[np.ndarray, torch.cuda.Event | None]: + if self.copy_stream is None: + return draft_tokens.detach().cpu().numpy(), None + + current_stream = torch.cuda.current_stream(self.device) + self.copy_stream.wait_stream(current_stream) + copy_event = torch.cuda.Event(blocking=True) + with torch.cuda.stream(self.copy_stream): + draft_tokens_np = async_copy_to_np(draft_tokens) + # draft_tokens is a temporary allocation on the main stream and + # read here on copy_stream; without record_stream, the caching + # allocator may reuse its memory before the async copy executes. + draft_tokens.record_stream(self.copy_stream) + copy_event.record() + return draft_tokens_np, copy_event + + def snapshot_consumed_drafts( + self, + step_id: int, + input_batch: InputBatch, + draft_tokens: torch.Tensor, + ) -> None: + """Record drafts that prepare_inputs fed into input_ids for this step. + + Tagged with the scheduler step so the engine can retrieve the matching + snapshot later by equality. Batches without structured-output requests + are not stored, so they cannot evict a useful snapshot. + """ + if not input_batch.has_structured_output_reqs: + return + + draft_tokens_np, copy_event = self._copy_drafts(draft_tokens) + self._consumed.append( + _DraftSnapshot( + step_id=step_id, + req_ids=list(input_batch.req_ids), + num_draft_tokens=draft_tokens.shape[1], + draft_tokens_np=draft_tokens_np, + copy_event=copy_event, + ) + ) def set_draft_tokens( self, input_batch: InputBatch, draft_tokens: torch.Tensor ) -> None: - self.req_ids = input_batch.req_ids - self.num_draft_tokens = draft_tokens.shape[1] + req_ids = list(input_batch.req_ids) + num_draft_tokens = draft_tokens.shape[1] if not input_batch.has_structured_output_reqs: # No draft token validation needs to be performed by # the scheduler for this batch. - self.draft_tokens_np = None + self._proposed = _DraftSnapshot( + step_id=None, + req_ids=req_ids, + num_draft_tokens=num_draft_tokens, + draft_tokens_np=None, + copy_event=None, + ) return # For spec decoding + structured outputs, we must transfer the # draft tokens back to the scheduler for grammar validation. - current_stream = torch.cuda.current_stream(self.device) - self.copy_stream.wait_stream(current_stream) - with torch.cuda.stream(self.copy_stream): - self.draft_tokens_np = async_copy_to_np(draft_tokens) - # draft_tokens is a temporary allocation on the main stream and read here on - # copy_stream; without record_stream, the caching allocator may reuse its - # memory before the async copy executes. - draft_tokens.record_stream(self.copy_stream) - self.copy_event.record() + draft_tokens_np, copy_event = self._copy_drafts(draft_tokens) + self._proposed = _DraftSnapshot( + step_id=None, + req_ids=req_ids, + num_draft_tokens=num_draft_tokens, + draft_tokens_np=draft_tokens_np, + copy_event=copy_event, + ) - def get_draft_tokens(self) -> DraftTokenIds | None: - if self.draft_tokens_np is not None: - self.copy_event.synchronize() - draft_token_ids = self.draft_tokens_np.tolist() - else: - # This case only happens when async scheduling is disabled. - draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids] - return DraftTokenIds(self.req_ids, draft_token_ids) + def get_draft_tokens(self, step_id: int | None = None) -> DraftTokenIds | None: + """Return draft tokens for grammar validation. + + If ``step_id`` is set, look up the consumption-time snapshot for that + scheduler step by equality. A miss returns None so placeholders stay + -1 and fail-closed invalidation can pin acceptance via is_valid_draft. + If ``step_id`` is None, return the latest proposal-time drafts (the + non-async post_step path). + """ + if step_id is not None: + for snapshot in reversed(self._consumed): + if snapshot.step_id == step_id: + return snapshot.to_draft_token_ids() + logger.debug( + "Draft-token snapshot miss for scheduler step %s", + step_id, + ) + return None + + if self._proposed is None: + return DraftTokenIds([], []) + return self._proposed.to_draft_token_ids() def get_parallel_drafting_token_id(hf_config) -> int: diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 047fdc5ff0..03b11d8909 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -4821,7 +4821,8 @@ class GPUModelRunner( bt.block_table.gpu.copy_(saved_gpu) bt.block_table.cpu.copy_(saved_cpu) - def take_draft_token_ids(self) -> DraftTokenIds | None: + def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None: + del step_id # V2-only identity; V1 keeps a single proposal slot. if not self.num_spec_tokens or not self._draft_token_req_ids: return None draft_token_ids, req_ids = self._get_draft_token_ids_cpu() diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 81e864ec1e..946dffd881 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -1473,8 +1473,8 @@ class Worker(WorkerBase): return self.model_runner.pool() # type: ignore return None - def take_draft_token_ids(self) -> DraftTokenIds | None: - return self.model_runner.take_draft_token_ids() + def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None: + return self.model_runner.take_draft_token_ids(step_id=step_id) def profile(self, is_start: bool = True, profile_prefix: str | None = None): # Check if profiling is enabled diff --git a/vllm/v1/worker/worker_base.py b/vllm/v1/worker/worker_base.py index 95a0c3475d..b24d529d67 100644 --- a/vllm/v1/worker/worker_base.py +++ b/vllm/v1/worker/worker_base.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import inspect from collections.abc import Callable from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar @@ -24,18 +25,34 @@ from vllm.v1.kv_cache_interface import KVCacheSpec if TYPE_CHECKING: from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput - from vllm.v1.outputs import AsyncModelRunnerOutput, ModelRunnerOutput + from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput else: SchedulerOutput = object GrammarOutput = object AsyncModelRunnerOutput = object ModelRunnerOutput = object + DraftTokenIds = object logger = init_logger(__name__) _R = TypeVar("_R") +def _callable_accepts_kwarg(func: Callable[..., Any], name: str) -> bool: + """Return True if ``func`` can be called with ``name`` as a keyword.""" + try: + params = inspect.signature(func).parameters + except (TypeError, ValueError): + return False + param = params.get(name) + if param is not None: + return param.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()) + + class CompilationTimes(NamedTuple): language_model: float encoder: float @@ -352,6 +369,20 @@ class WorkerWrapperBase: def __getattr__(self, attr: str): return getattr(self.worker, attr) + def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None: + """Forward draft-token lookup, dropping ``step_id`` if unsupported. + + Plugin workers are documented to implement no-argument + ``take_draft_token_ids()``. Passing ``step_id`` to those methods would + raise TypeError; fall back to the legacy call instead. + """ + method = getattr(self.worker, "take_draft_token_ids", None) + if method is None: + return None + if step_id is not None and _callable_accepts_kwarg(method, "step_id"): + return method(step_id=step_id) + return method() + def _apply_mm_cache(self, scheduler_output: SchedulerOutput) -> None: mm_cache = self.mm_receiver_cache if mm_cache is None: