[Spec Decode] Structured output under concurrency on a PP shape

Ports upstream vLLM PRs #54442 and #56802 (issue #54437): never sample from an
unmasked row, and key draft-token hand-off snapshots by scheduler step so an
alternating PP microbatch cannot overwrite the slot.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 19:43:58 +07:00
co-authored by Hinata
parent 03c8ebae34
commit 17c1c70447
12 changed files with 262 additions and 43 deletions
+9
View File
@@ -314,6 +314,10 @@ class SchedulerOutput:
# Number of spec tokens to schedule for the next step.
num_spec_tokens_to_schedule: int = 0
# Identity of this schedule() call (`Scheduler.current_step`). Used to
# retrieve the matching consumption-time draft-token snapshot.
scheduler_step: int = 0
@classmethod
def make_empty(cls) -> "SchedulerOutput":
return cls(
@@ -338,5 +342,10 @@ class GrammarOutput:
# Per request, ordered as structured_output_request_ids: how many leading
# drafts the bitmask constrained. `grammar_bitmask` fills every row after
# the first -1 placeholder with the all-permissive `_full_mask`, so drafts
# from this index on must not be accepted. None means nothing to invalidate.
num_acceptable_drafts: list[int] | None = None
# Per request, ordered as structured_output_request_ids: how many leading
# drafts the bitmask constrained. `grammar_bitmask` fills every row after
# the first -1 placeholder with the all-permissive `_full_mask`, so drafts
# from this index on must not be accepted. None invalidates every draft.
num_acceptable_drafts: list[int] | None = None
+3 -1
View File
@@ -78,7 +78,8 @@ from vllm.v1.core.sched import e2_alloc_log as _e2_alloc_log
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
from vllm.v1.spec_decode.metrics import SpecDecodingStats
from vllm.v1.structured_output import StructuredOutputManager
from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager
from vllm.v1.structured_output.utils import strip_speculative_padding
from vllm.v1.structured_output.utils import strip_speculative_padding
from vllm.v1.utils import record_function_or_nullcontext
@@ -1529,6 +1530,7 @@ class Scheduler(SchedulerInterface):
kv_connector_block_state=kv_connector_block_state,
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(),
scheduler_step=self.current_step,
)
# NOTE(Kuntai): this function is designed for multiple purposes:
+3 -1
View File
@@ -764,7 +764,9 @@ class EngineCore:
# When draft tokens are used with structured output, validate them
# before computing the grammar bitmask for the deferred request.
if self.check_for_draft_tokens:
draft_token_ids = self.model_executor.take_draft_token_ids()
draft_token_ids = self.model_executor.take_draft_token_ids(
step_id=deferred_scheduler_output.scheduler_step
)
if draft_token_ids is not None:
# Update the draft token ids in the scheduler output to
# filter out the invalid spec tokens, which will be padded
+7 -2
View File
@@ -265,8 +265,13 @@ class Executor(ABC):
def execute_dummy_batch(self) -> None:
self.collective_rpc("execute_dummy_batch")
def take_draft_token_ids(self) -> DraftTokenIds | None:
output: list[DraftTokenIds] = self.collective_rpc("take_draft_token_ids")
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
# Omit step_id when unset so plugin workers with the documented
# no-argument take_draft_token_ids() keep working.
kwargs = {} if step_id is None else {"step_id": step_id}
output: list[DraftTokenIds] = self.collective_rpc(
"take_draft_token_ids", kwargs=kwargs
)
return output[0]
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
+5 -1
View File
@@ -405,12 +405,16 @@ class MultiprocExecutor(Executor):
def execute_dummy_batch(self) -> None:
self.collective_rpc("execute_dummy_batch", unique_reply_rank=self.output_rank)
def take_draft_token_ids(self) -> DraftTokenIds | None:
def take_draft_token_ids(
self, step_id: int | None = None
) -> DraftTokenIds | None:
# OPTIMIZATION: Get output only from a single worker (output_rank)
kwargs = {} if step_id is None else {"step_id": step_id}
return self.collective_rpc(
"take_draft_token_ids",
unique_reply_rank=self.output_rank,
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
kwargs=kwargs,
)
def collective_rpc( # type: ignore[override]
+7 -2
View File
@@ -141,8 +141,13 @@ class UniProcExecutor(Executor):
single_value=True,
)
def take_draft_token_ids(self) -> DraftTokenIds | None:
return self.collective_rpc("take_draft_token_ids", single_value=True)
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
kwargs = {} if step_id is None else {"step_id": step_id}
return self.collective_rpc(
"take_draft_token_ids",
kwargs=kwargs,
single_value=True,
)
def check_health(self) -> None:
# UniProcExecutor will always be healthy as long as
+13
View File
@@ -453,3 +453,16 @@ def choice_as_grammar(choice: list[str]) -> str:
escaped_choices = (escape_ebnf_string(c) for c in choice)
grammar = "root ::= " + " | ".join(f'"{c}"' for c in escaped_choices)
return grammar
def strip_speculative_padding(token_ids: list[int]) -> list[int]:
"""Drop speculative-decoding padding from a token block.
ngram and other speculative backends pad rejected draft positions with a
-1 sentinel. Structured-output grammars treat every entry as a real token
id, so the sentinels (and everything after the first one) are removed here,
before tokens reach any backend, rather than inside a single backend.
"""
for i, token_id in enumerate(token_ids):
if token_id < 0:
return token_ids[:i]
return token_ids
+56 -3
View File
@@ -186,6 +186,39 @@ from vllm.v1.worker.workspace import lock_workspace, use_workspace_lane
logger = init_logger(__name__)
def grammar_invalid_draft_positions(
input_batch: InputBatch,
grammar_req_ids: list[str],
num_acceptable_drafts: list[int] | None,
device: torch.device,
) -> torch.Tensor | None:
"""Indices of `draft_sampled` entries whose bitmask row is permissive.
Drafts from `num_acceptable_drafts` on were verified against rows carrying
`_full_mask`, so accepting one would sample with no grammar constraint.
Returns None when there is nothing to invalidate.
"""
if not grammar_req_ids or input_batch.num_draft_tokens == 0:
return None
cu_num_logits = input_batch.cu_num_logits_np.tolist()
req_id_to_idx = {req_id: i for i, req_id in enumerate(input_batch.req_ids)}
positions: list[int] = []
for i, req_id in enumerate(grammar_req_ids):
req_idx = req_id_to_idx.get(req_id)
if req_idx is None:
continue
# Without the field (an older scheduler, or warmup) invalidate the whole
# window, which is the conservative choice.
num_acceptable = (
num_acceptable_drafts[i] if num_acceptable_drafts is not None else 0
)
start = cu_num_logits[req_idx] + 1 + num_acceptable
positions.extend(range(start, cu_num_logits[req_idx + 1]))
if not positions:
return None
return torch.tensor(positions, dtype=torch.int64, device=device)
class GPUModelRunner(LoRAModelRunnerMixin):
def __init__(self, vllm_config: VllmConfig, device: torch.device):
self.vllm_config = vllm_config
@@ -288,7 +321,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.use_aux_hidden_state_outputs = True
# Draft tokens propagation - for spec-dec + struct outputs.
self.draft_tokens_handler = DraftTokensHandler(self.device)
self.draft_tokens_handler = DraftTokensHandler(
self.device,
max_snapshots=max(8, vllm_config.max_concurrent_batches + 1),
)
self.pcp_manager: pcp.PCPManager | None = None
@@ -1470,6 +1506,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
total_num_logits,
self.model_state.num_new_sampled_tokens_per_step,
)
if self.num_speculative_steps > 0:
# Snapshot the drafts that were just written into input_ids, tagged
# with this scheduler step. Proposal-time drafts (set later in
# sample_tokens) are for the next step and must not overwrite this.
self.draft_tokens_handler.snapshot_consumed_drafts(
scheduler_output.scheduler_step,
input_batch,
self.req_states.draft_tokens[input_batch.idx_mapping],
)
fast_prefill = None
if self.fast_prefill is not None:
@@ -1538,6 +1583,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
else None
),
)
invalid_draft_positions = None
if self.pcp_manager is not None:
input_batch = self.pcp_manager.partition_batch(input_batch, batch_desc)
return input_batch
@@ -1620,6 +1666,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
grammar_output.structured_output_request_ids,
grammar_output.num_acceptable_drafts,
)
invalid_draft_positions = grammar_invalid_draft_positions(
input_batch,
grammar_output.structured_output_request_ids,
grammar_output.num_acceptable_drafts,
self.device,
)
sampler_output: SamplerOutput | None
if input_batch.num_reqs == 0:
@@ -1638,6 +1690,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch,
# Draft logits are needed for probabilistic rejection sampling.
self.speculator.draft_logits,
invalid_draft_positions,
invalid_drafts,
)
@@ -2278,8 +2331,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return async_output
def take_draft_token_ids(self) -> DraftTokenIds | None:
return self.draft_tokens_handler.get_draft_tokens()
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
return self.draft_tokens_handler.get_draft_tokens(step_id=step_id)
@torch.inference_mode()
@step_eplb_after()
+123 -29
View File
@@ -1,5 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import deque
from dataclasses import dataclass
import numpy as np
import torch
@@ -10,6 +13,10 @@ from vllm.v1.outputs import DraftTokenIds
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
from vllm.v1.worker.gpu.input_batch import InputBatch
# Keep enough consumption-time snapshots to cover async PP in-flight
# batches (`max_concurrent_batches` is `pp_size + 1`) with headroom.
_DEFAULT_MAX_SNAPSHOTS = 8
logger = init_logger(__name__)
@@ -26,48 +33,135 @@ def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig:
return load_config
class DraftTokensHandler:
def __init__(self, device: torch.device | None = None):
self.device = device
self.copy_stream = torch.cuda.Stream(device)
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.copy_event = torch.cuda.Event(blocking=True)
@dataclass
class _DraftSnapshot:
step_id: int | None
req_ids: list[str]
num_draft_tokens: int
draft_tokens_np: np.ndarray | None
copy_event: torch.cuda.Event | None
self.req_ids: list[str] = []
self.draft_tokens_np: np.ndarray | None = None
self.num_draft_tokens: int = 0
def to_draft_token_ids(self) -> DraftTokenIds:
if self.draft_tokens_np is not None:
if self.copy_event is not None:
self.copy_event.synchronize()
draft_token_ids = self.draft_tokens_np.tolist()
else:
draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids]
return DraftTokenIds(self.req_ids, draft_token_ids)
class DraftTokensHandler:
def __init__(
self,
device: torch.device | None = None,
max_snapshots: int = _DEFAULT_MAX_SNAPSHOTS,
):
self.device = device
self.copy_stream: torch.cuda.Stream | None = None
if device is not None and device.type == "cuda":
self.copy_stream = torch.cuda.Stream(device)
self._max_snapshots = max(1, max_snapshots)
# Consumption-time snapshots tagged with scheduler step identity.
self._consumed: deque[_DraftSnapshot] = deque(maxlen=self._max_snapshots)
# Latest proposal-time drafts for the non-async post_step path.
self._proposed: _DraftSnapshot | None = None
def _copy_drafts(
self, draft_tokens: torch.Tensor
) -> tuple[np.ndarray, torch.cuda.Event | None]:
if self.copy_stream is None:
return draft_tokens.detach().cpu().numpy(), None
current_stream = torch.cuda.current_stream(self.device)
self.copy_stream.wait_stream(current_stream)
copy_event = torch.cuda.Event(blocking=True)
with torch.cuda.stream(self.copy_stream):
draft_tokens_np = async_copy_to_np(draft_tokens)
# draft_tokens is a temporary allocation on the main stream and
# read here on copy_stream; without record_stream, the caching
# allocator may reuse its memory before the async copy executes.
draft_tokens.record_stream(self.copy_stream)
copy_event.record()
return draft_tokens_np, copy_event
def snapshot_consumed_drafts(
self,
step_id: int,
input_batch: InputBatch,
draft_tokens: torch.Tensor,
) -> None:
"""Record drafts that prepare_inputs fed into input_ids for this step.
Tagged with the scheduler step so the engine can retrieve the matching
snapshot later by equality. Batches without structured-output requests
are not stored, so they cannot evict a useful snapshot.
"""
if not input_batch.has_structured_output_reqs:
return
draft_tokens_np, copy_event = self._copy_drafts(draft_tokens)
self._consumed.append(
_DraftSnapshot(
step_id=step_id,
req_ids=list(input_batch.req_ids),
num_draft_tokens=draft_tokens.shape[1],
draft_tokens_np=draft_tokens_np,
copy_event=copy_event,
)
)
def set_draft_tokens(
self, input_batch: InputBatch, draft_tokens: torch.Tensor
) -> None:
self.req_ids = input_batch.req_ids
self.num_draft_tokens = draft_tokens.shape[1]
req_ids = list(input_batch.req_ids)
num_draft_tokens = draft_tokens.shape[1]
if not input_batch.has_structured_output_reqs:
# No draft token validation needs to be performed by
# the scheduler for this batch.
self.draft_tokens_np = None
self._proposed = _DraftSnapshot(
step_id=None,
req_ids=req_ids,
num_draft_tokens=num_draft_tokens,
draft_tokens_np=None,
copy_event=None,
)
return
# For spec decoding + structured outputs, we must transfer the
# draft tokens back to the scheduler for grammar validation.
current_stream = torch.cuda.current_stream(self.device)
self.copy_stream.wait_stream(current_stream)
with torch.cuda.stream(self.copy_stream):
self.draft_tokens_np = async_copy_to_np(draft_tokens)
# draft_tokens is a temporary allocation on the main stream and read here on
# copy_stream; without record_stream, the caching allocator may reuse its
# memory before the async copy executes.
draft_tokens.record_stream(self.copy_stream)
self.copy_event.record()
draft_tokens_np, copy_event = self._copy_drafts(draft_tokens)
self._proposed = _DraftSnapshot(
step_id=None,
req_ids=req_ids,
num_draft_tokens=num_draft_tokens,
draft_tokens_np=draft_tokens_np,
copy_event=copy_event,
)
def get_draft_tokens(self) -> DraftTokenIds | None:
if self.draft_tokens_np is not None:
self.copy_event.synchronize()
draft_token_ids = self.draft_tokens_np.tolist()
else:
# This case only happens when async scheduling is disabled.
draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids]
return DraftTokenIds(self.req_ids, draft_token_ids)
def get_draft_tokens(self, step_id: int | None = None) -> DraftTokenIds | None:
"""Return draft tokens for grammar validation.
If ``step_id`` is set, look up the consumption-time snapshot for that
scheduler step by equality. A miss returns None so placeholders stay
-1 and fail-closed invalidation can pin acceptance via is_valid_draft.
If ``step_id`` is None, return the latest proposal-time drafts (the
non-async post_step path).
"""
if step_id is not None:
for snapshot in reversed(self._consumed):
if snapshot.step_id == step_id:
return snapshot.to_draft_token_ids()
logger.debug(
"Draft-token snapshot miss for scheduler step %s",
step_id,
)
return None
if self._proposed is None:
return DraftTokenIds([], [])
return self._proposed.to_draft_token_ids()
def get_parallel_drafting_token_id(hf_config) -> int:
+2 -1
View File
@@ -4821,7 +4821,8 @@ class GPUModelRunner(
bt.block_table.gpu.copy_(saved_gpu)
bt.block_table.cpu.copy_(saved_cpu)
def take_draft_token_ids(self) -> DraftTokenIds | None:
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
del step_id # V2-only identity; V1 keeps a single proposal slot.
if not self.num_spec_tokens or not self._draft_token_req_ids:
return None
draft_token_ids, req_ids = self._get_draft_token_ids_cpu()
+2 -2
View File
@@ -1473,8 +1473,8 @@ class Worker(WorkerBase):
return self.model_runner.pool() # type: ignore
return None
def take_draft_token_ids(self) -> DraftTokenIds | None:
return self.model_runner.take_draft_token_ids()
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
return self.model_runner.take_draft_token_ids(step_id=step_id)
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
# Check if profiling is enabled
+32 -1
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar
@@ -24,18 +25,34 @@ from vllm.v1.kv_cache_interface import KVCacheSpec
if TYPE_CHECKING:
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.outputs import AsyncModelRunnerOutput, ModelRunnerOutput
from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput
else:
SchedulerOutput = object
GrammarOutput = object
AsyncModelRunnerOutput = object
ModelRunnerOutput = object
DraftTokenIds = object
logger = init_logger(__name__)
_R = TypeVar("_R")
def _callable_accepts_kwarg(func: Callable[..., Any], name: str) -> bool:
"""Return True if ``func`` can be called with ``name`` as a keyword."""
try:
params = inspect.signature(func).parameters
except (TypeError, ValueError):
return False
param = params.get(name)
if param is not None:
return param.kind in (
inspect.Parameter.POSITIONAL_OR_KEYWORD,
inspect.Parameter.KEYWORD_ONLY,
)
return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
class CompilationTimes(NamedTuple):
language_model: float
encoder: float
@@ -352,6 +369,20 @@ class WorkerWrapperBase:
def __getattr__(self, attr: str):
return getattr(self.worker, attr)
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
"""Forward draft-token lookup, dropping ``step_id`` if unsupported.
Plugin workers are documented to implement no-argument
``take_draft_token_ids()``. Passing ``step_id`` to those methods would
raise TypeError; fall back to the legacy call instead.
"""
method = getattr(self.worker, "take_draft_token_ids", None)
if method is None:
return None
if step_id is not None and _callable_accepts_kwarg(method, "step_id"):
return method(step_id=step_id)
return method()
def _apply_mm_cache(self, scheduler_output: SchedulerOutput) -> None:
mm_cache = self.mm_receiver_cache
if mm_cache is None: