[Spec Decode] Structured output under concurrency on a PP shape
Ports upstream vLLM PRs #54442 and #56802 (issue #54437): never sample from an unmasked row, and key draft-token hand-off snapshots by scheduler step so an alternating PP microbatch cannot overwrite the slot. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -314,6 +314,10 @@ class SchedulerOutput:
|
||||
# Number of spec tokens to schedule for the next step.
|
||||
num_spec_tokens_to_schedule: int = 0
|
||||
|
||||
# Identity of this schedule() call (`Scheduler.current_step`). Used to
|
||||
# retrieve the matching consumption-time draft-token snapshot.
|
||||
scheduler_step: int = 0
|
||||
|
||||
@classmethod
|
||||
def make_empty(cls) -> "SchedulerOutput":
|
||||
return cls(
|
||||
@@ -338,5 +342,10 @@ class GrammarOutput:
|
||||
# Per request, ordered as structured_output_request_ids: how many leading
|
||||
# drafts the bitmask constrained. `grammar_bitmask` fills every row after
|
||||
# the first -1 placeholder with the all-permissive `_full_mask`, so drafts
|
||||
# from this index on must not be accepted. None means nothing to invalidate.
|
||||
num_acceptable_drafts: list[int] | None = None
|
||||
# Per request, ordered as structured_output_request_ids: how many leading
|
||||
# drafts the bitmask constrained. `grammar_bitmask` fills every row after
|
||||
# the first -1 placeholder with the all-permissive `_full_mask`, so drafts
|
||||
# from this index on must not be accepted. None invalidates every draft.
|
||||
num_acceptable_drafts: list[int] | None = None
|
||||
|
||||
@@ -78,7 +78,8 @@ from vllm.v1.core.sched import e2_alloc_log as _e2_alloc_log
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
from vllm.v1.spec_decode.metrics import SpecDecodingStats
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager
|
||||
from vllm.v1.structured_output.utils import strip_speculative_padding
|
||||
from vllm.v1.structured_output.utils import strip_speculative_padding
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
|
||||
@@ -1529,6 +1530,7 @@ class Scheduler(SchedulerInterface):
|
||||
kv_connector_block_state=kv_connector_block_state,
|
||||
num_spec_tokens_to_schedule=num_spec_tokens_to_schedule,
|
||||
ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(),
|
||||
scheduler_step=self.current_step,
|
||||
)
|
||||
|
||||
# NOTE(Kuntai): this function is designed for multiple purposes:
|
||||
|
||||
@@ -764,7 +764,9 @@ class EngineCore:
|
||||
# When draft tokens are used with structured output, validate them
|
||||
# before computing the grammar bitmask for the deferred request.
|
||||
if self.check_for_draft_tokens:
|
||||
draft_token_ids = self.model_executor.take_draft_token_ids()
|
||||
draft_token_ids = self.model_executor.take_draft_token_ids(
|
||||
step_id=deferred_scheduler_output.scheduler_step
|
||||
)
|
||||
if draft_token_ids is not None:
|
||||
# Update the draft token ids in the scheduler output to
|
||||
# filter out the invalid spec tokens, which will be padded
|
||||
|
||||
@@ -265,8 +265,13 @@ class Executor(ABC):
|
||||
def execute_dummy_batch(self) -> None:
|
||||
self.collective_rpc("execute_dummy_batch")
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
output: list[DraftTokenIds] = self.collective_rpc("take_draft_token_ids")
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
# Omit step_id when unset so plugin workers with the documented
|
||||
# no-argument take_draft_token_ids() keep working.
|
||||
kwargs = {} if step_id is None else {"step_id": step_id}
|
||||
output: list[DraftTokenIds] = self.collective_rpc(
|
||||
"take_draft_token_ids", kwargs=kwargs
|
||||
)
|
||||
return output[0]
|
||||
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||
|
||||
@@ -405,12 +405,16 @@ class MultiprocExecutor(Executor):
|
||||
def execute_dummy_batch(self) -> None:
|
||||
self.collective_rpc("execute_dummy_batch", unique_reply_rank=self.output_rank)
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
def take_draft_token_ids(
|
||||
self, step_id: int | None = None
|
||||
) -> DraftTokenIds | None:
|
||||
# OPTIMIZATION: Get output only from a single worker (output_rank)
|
||||
kwargs = {} if step_id is None else {"step_id": step_id}
|
||||
return self.collective_rpc(
|
||||
"take_draft_token_ids",
|
||||
unique_reply_rank=self.output_rank,
|
||||
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
def collective_rpc( # type: ignore[override]
|
||||
|
||||
@@ -141,8 +141,13 @@ class UniProcExecutor(Executor):
|
||||
single_value=True,
|
||||
)
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
return self.collective_rpc("take_draft_token_ids", single_value=True)
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
kwargs = {} if step_id is None else {"step_id": step_id}
|
||||
return self.collective_rpc(
|
||||
"take_draft_token_ids",
|
||||
kwargs=kwargs,
|
||||
single_value=True,
|
||||
)
|
||||
|
||||
def check_health(self) -> None:
|
||||
# UniProcExecutor will always be healthy as long as
|
||||
|
||||
@@ -453,3 +453,16 @@ def choice_as_grammar(choice: list[str]) -> str:
|
||||
escaped_choices = (escape_ebnf_string(c) for c in choice)
|
||||
grammar = "root ::= " + " | ".join(f'"{c}"' for c in escaped_choices)
|
||||
return grammar
|
||||
|
||||
def strip_speculative_padding(token_ids: list[int]) -> list[int]:
|
||||
"""Drop speculative-decoding padding from a token block.
|
||||
|
||||
ngram and other speculative backends pad rejected draft positions with a
|
||||
-1 sentinel. Structured-output grammars treat every entry as a real token
|
||||
id, so the sentinels (and everything after the first one) are removed here,
|
||||
before tokens reach any backend, rather than inside a single backend.
|
||||
"""
|
||||
for i, token_id in enumerate(token_ids):
|
||||
if token_id < 0:
|
||||
return token_ids[:i]
|
||||
return token_ids
|
||||
|
||||
@@ -186,6 +186,39 @@ from vllm.v1.worker.workspace import lock_workspace, use_workspace_lane
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def grammar_invalid_draft_positions(
|
||||
input_batch: InputBatch,
|
||||
grammar_req_ids: list[str],
|
||||
num_acceptable_drafts: list[int] | None,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
"""Indices of `draft_sampled` entries whose bitmask row is permissive.
|
||||
|
||||
Drafts from `num_acceptable_drafts` on were verified against rows carrying
|
||||
`_full_mask`, so accepting one would sample with no grammar constraint.
|
||||
Returns None when there is nothing to invalidate.
|
||||
"""
|
||||
if not grammar_req_ids or input_batch.num_draft_tokens == 0:
|
||||
return None
|
||||
cu_num_logits = input_batch.cu_num_logits_np.tolist()
|
||||
req_id_to_idx = {req_id: i for i, req_id in enumerate(input_batch.req_ids)}
|
||||
positions: list[int] = []
|
||||
for i, req_id in enumerate(grammar_req_ids):
|
||||
req_idx = req_id_to_idx.get(req_id)
|
||||
if req_idx is None:
|
||||
continue
|
||||
# Without the field (an older scheduler, or warmup) invalidate the whole
|
||||
# window, which is the conservative choice.
|
||||
num_acceptable = (
|
||||
num_acceptable_drafts[i] if num_acceptable_drafts is not None else 0
|
||||
)
|
||||
start = cu_num_logits[req_idx] + 1 + num_acceptable
|
||||
positions.extend(range(start, cu_num_logits[req_idx + 1]))
|
||||
if not positions:
|
||||
return None
|
||||
return torch.tensor(positions, dtype=torch.int64, device=device)
|
||||
|
||||
|
||||
class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
self.vllm_config = vllm_config
|
||||
@@ -288,7 +321,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.use_aux_hidden_state_outputs = True
|
||||
|
||||
# Draft tokens propagation - for spec-dec + struct outputs.
|
||||
self.draft_tokens_handler = DraftTokensHandler(self.device)
|
||||
self.draft_tokens_handler = DraftTokensHandler(
|
||||
self.device,
|
||||
max_snapshots=max(8, vllm_config.max_concurrent_batches + 1),
|
||||
)
|
||||
|
||||
self.pcp_manager: pcp.PCPManager | None = None
|
||||
|
||||
@@ -1470,6 +1506,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
total_num_logits,
|
||||
self.model_state.num_new_sampled_tokens_per_step,
|
||||
)
|
||||
if self.num_speculative_steps > 0:
|
||||
# Snapshot the drafts that were just written into input_ids, tagged
|
||||
# with this scheduler step. Proposal-time drafts (set later in
|
||||
# sample_tokens) are for the next step and must not overwrite this.
|
||||
self.draft_tokens_handler.snapshot_consumed_drafts(
|
||||
scheduler_output.scheduler_step,
|
||||
input_batch,
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping],
|
||||
)
|
||||
|
||||
fast_prefill = None
|
||||
if self.fast_prefill is not None:
|
||||
@@ -1538,6 +1583,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
else None
|
||||
),
|
||||
)
|
||||
invalid_draft_positions = None
|
||||
if self.pcp_manager is not None:
|
||||
input_batch = self.pcp_manager.partition_batch(input_batch, batch_desc)
|
||||
return input_batch
|
||||
@@ -1620,6 +1666,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
grammar_output.structured_output_request_ids,
|
||||
grammar_output.num_acceptable_drafts,
|
||||
)
|
||||
invalid_draft_positions = grammar_invalid_draft_positions(
|
||||
input_batch,
|
||||
grammar_output.structured_output_request_ids,
|
||||
grammar_output.num_acceptable_drafts,
|
||||
self.device,
|
||||
)
|
||||
|
||||
sampler_output: SamplerOutput | None
|
||||
if input_batch.num_reqs == 0:
|
||||
@@ -1638,6 +1690,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
input_batch,
|
||||
# Draft logits are needed for probabilistic rejection sampling.
|
||||
self.speculator.draft_logits,
|
||||
invalid_draft_positions,
|
||||
invalid_drafts,
|
||||
)
|
||||
|
||||
@@ -2278,8 +2331,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
return async_output
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
return self.draft_tokens_handler.get_draft_tokens()
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
return self.draft_tokens_handler.get_draft_tokens(step_id=step_id)
|
||||
|
||||
@torch.inference_mode()
|
||||
@step_eplb_after()
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
@@ -10,6 +13,10 @@ from vllm.v1.outputs import DraftTokenIds
|
||||
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
# Keep enough consumption-time snapshots to cover async PP in-flight
|
||||
# batches (`max_concurrent_batches` is `pp_size + 1`) with headroom.
|
||||
_DEFAULT_MAX_SNAPSHOTS = 8
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -26,48 +33,135 @@ def get_pp_safe_draft_load_config(load_config: LoadConfig) -> LoadConfig:
|
||||
return load_config
|
||||
|
||||
|
||||
class DraftTokensHandler:
|
||||
def __init__(self, device: torch.device | None = None):
|
||||
self.device = device
|
||||
self.copy_stream = torch.cuda.Stream(device)
|
||||
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
|
||||
self.copy_event = torch.cuda.Event(blocking=True)
|
||||
@dataclass
|
||||
class _DraftSnapshot:
|
||||
step_id: int | None
|
||||
req_ids: list[str]
|
||||
num_draft_tokens: int
|
||||
draft_tokens_np: np.ndarray | None
|
||||
copy_event: torch.cuda.Event | None
|
||||
|
||||
self.req_ids: list[str] = []
|
||||
self.draft_tokens_np: np.ndarray | None = None
|
||||
self.num_draft_tokens: int = 0
|
||||
def to_draft_token_ids(self) -> DraftTokenIds:
|
||||
if self.draft_tokens_np is not None:
|
||||
if self.copy_event is not None:
|
||||
self.copy_event.synchronize()
|
||||
draft_token_ids = self.draft_tokens_np.tolist()
|
||||
else:
|
||||
draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids]
|
||||
return DraftTokenIds(self.req_ids, draft_token_ids)
|
||||
|
||||
|
||||
class DraftTokensHandler:
|
||||
def __init__(
|
||||
self,
|
||||
device: torch.device | None = None,
|
||||
max_snapshots: int = _DEFAULT_MAX_SNAPSHOTS,
|
||||
):
|
||||
self.device = device
|
||||
self.copy_stream: torch.cuda.Stream | None = None
|
||||
if device is not None and device.type == "cuda":
|
||||
self.copy_stream = torch.cuda.Stream(device)
|
||||
|
||||
self._max_snapshots = max(1, max_snapshots)
|
||||
# Consumption-time snapshots tagged with scheduler step identity.
|
||||
self._consumed: deque[_DraftSnapshot] = deque(maxlen=self._max_snapshots)
|
||||
# Latest proposal-time drafts for the non-async post_step path.
|
||||
self._proposed: _DraftSnapshot | None = None
|
||||
|
||||
def _copy_drafts(
|
||||
self, draft_tokens: torch.Tensor
|
||||
) -> tuple[np.ndarray, torch.cuda.Event | None]:
|
||||
if self.copy_stream is None:
|
||||
return draft_tokens.detach().cpu().numpy(), None
|
||||
|
||||
current_stream = torch.cuda.current_stream(self.device)
|
||||
self.copy_stream.wait_stream(current_stream)
|
||||
copy_event = torch.cuda.Event(blocking=True)
|
||||
with torch.cuda.stream(self.copy_stream):
|
||||
draft_tokens_np = async_copy_to_np(draft_tokens)
|
||||
# draft_tokens is a temporary allocation on the main stream and
|
||||
# read here on copy_stream; without record_stream, the caching
|
||||
# allocator may reuse its memory before the async copy executes.
|
||||
draft_tokens.record_stream(self.copy_stream)
|
||||
copy_event.record()
|
||||
return draft_tokens_np, copy_event
|
||||
|
||||
def snapshot_consumed_drafts(
|
||||
self,
|
||||
step_id: int,
|
||||
input_batch: InputBatch,
|
||||
draft_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
"""Record drafts that prepare_inputs fed into input_ids for this step.
|
||||
|
||||
Tagged with the scheduler step so the engine can retrieve the matching
|
||||
snapshot later by equality. Batches without structured-output requests
|
||||
are not stored, so they cannot evict a useful snapshot.
|
||||
"""
|
||||
if not input_batch.has_structured_output_reqs:
|
||||
return
|
||||
|
||||
draft_tokens_np, copy_event = self._copy_drafts(draft_tokens)
|
||||
self._consumed.append(
|
||||
_DraftSnapshot(
|
||||
step_id=step_id,
|
||||
req_ids=list(input_batch.req_ids),
|
||||
num_draft_tokens=draft_tokens.shape[1],
|
||||
draft_tokens_np=draft_tokens_np,
|
||||
copy_event=copy_event,
|
||||
)
|
||||
)
|
||||
|
||||
def set_draft_tokens(
|
||||
self, input_batch: InputBatch, draft_tokens: torch.Tensor
|
||||
) -> None:
|
||||
self.req_ids = input_batch.req_ids
|
||||
self.num_draft_tokens = draft_tokens.shape[1]
|
||||
req_ids = list(input_batch.req_ids)
|
||||
num_draft_tokens = draft_tokens.shape[1]
|
||||
if not input_batch.has_structured_output_reqs:
|
||||
# No draft token validation needs to be performed by
|
||||
# the scheduler for this batch.
|
||||
self.draft_tokens_np = None
|
||||
self._proposed = _DraftSnapshot(
|
||||
step_id=None,
|
||||
req_ids=req_ids,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
draft_tokens_np=None,
|
||||
copy_event=None,
|
||||
)
|
||||
return
|
||||
|
||||
# For spec decoding + structured outputs, we must transfer the
|
||||
# draft tokens back to the scheduler for grammar validation.
|
||||
current_stream = torch.cuda.current_stream(self.device)
|
||||
self.copy_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(self.copy_stream):
|
||||
self.draft_tokens_np = async_copy_to_np(draft_tokens)
|
||||
# draft_tokens is a temporary allocation on the main stream and read here on
|
||||
# copy_stream; without record_stream, the caching allocator may reuse its
|
||||
# memory before the async copy executes.
|
||||
draft_tokens.record_stream(self.copy_stream)
|
||||
self.copy_event.record()
|
||||
draft_tokens_np, copy_event = self._copy_drafts(draft_tokens)
|
||||
self._proposed = _DraftSnapshot(
|
||||
step_id=None,
|
||||
req_ids=req_ids,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
draft_tokens_np=draft_tokens_np,
|
||||
copy_event=copy_event,
|
||||
)
|
||||
|
||||
def get_draft_tokens(self) -> DraftTokenIds | None:
|
||||
if self.draft_tokens_np is not None:
|
||||
self.copy_event.synchronize()
|
||||
draft_token_ids = self.draft_tokens_np.tolist()
|
||||
else:
|
||||
# This case only happens when async scheduling is disabled.
|
||||
draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids]
|
||||
return DraftTokenIds(self.req_ids, draft_token_ids)
|
||||
def get_draft_tokens(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
"""Return draft tokens for grammar validation.
|
||||
|
||||
If ``step_id`` is set, look up the consumption-time snapshot for that
|
||||
scheduler step by equality. A miss returns None so placeholders stay
|
||||
-1 and fail-closed invalidation can pin acceptance via is_valid_draft.
|
||||
If ``step_id`` is None, return the latest proposal-time drafts (the
|
||||
non-async post_step path).
|
||||
"""
|
||||
if step_id is not None:
|
||||
for snapshot in reversed(self._consumed):
|
||||
if snapshot.step_id == step_id:
|
||||
return snapshot.to_draft_token_ids()
|
||||
logger.debug(
|
||||
"Draft-token snapshot miss for scheduler step %s",
|
||||
step_id,
|
||||
)
|
||||
return None
|
||||
|
||||
if self._proposed is None:
|
||||
return DraftTokenIds([], [])
|
||||
return self._proposed.to_draft_token_ids()
|
||||
|
||||
|
||||
def get_parallel_drafting_token_id(hf_config) -> int:
|
||||
|
||||
@@ -4821,7 +4821,8 @@ class GPUModelRunner(
|
||||
bt.block_table.gpu.copy_(saved_gpu)
|
||||
bt.block_table.cpu.copy_(saved_cpu)
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
del step_id # V2-only identity; V1 keeps a single proposal slot.
|
||||
if not self.num_spec_tokens or not self._draft_token_req_ids:
|
||||
return None
|
||||
draft_token_ids, req_ids = self._get_draft_token_ids_cpu()
|
||||
|
||||
@@ -1473,8 +1473,8 @@ class Worker(WorkerBase):
|
||||
return self.model_runner.pool() # type: ignore
|
||||
return None
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
return self.model_runner.take_draft_token_ids()
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
return self.model_runner.take_draft_token_ids(step_id=step_id)
|
||||
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||
# Check if profiling is enabled
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, TypeVar
|
||||
|
||||
@@ -24,18 +25,34 @@ from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput, ModelRunnerOutput
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput
|
||||
else:
|
||||
SchedulerOutput = object
|
||||
GrammarOutput = object
|
||||
AsyncModelRunnerOutput = object
|
||||
ModelRunnerOutput = object
|
||||
DraftTokenIds = object
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_R = TypeVar("_R")
|
||||
|
||||
|
||||
def _callable_accepts_kwarg(func: Callable[..., Any], name: str) -> bool:
|
||||
"""Return True if ``func`` can be called with ``name`` as a keyword."""
|
||||
try:
|
||||
params = inspect.signature(func).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
param = params.get(name)
|
||||
if param is not None:
|
||||
return param.kind in (
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
inspect.Parameter.KEYWORD_ONLY,
|
||||
)
|
||||
return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
|
||||
|
||||
|
||||
class CompilationTimes(NamedTuple):
|
||||
language_model: float
|
||||
encoder: float
|
||||
@@ -352,6 +369,20 @@ class WorkerWrapperBase:
|
||||
def __getattr__(self, attr: str):
|
||||
return getattr(self.worker, attr)
|
||||
|
||||
def take_draft_token_ids(self, step_id: int | None = None) -> DraftTokenIds | None:
|
||||
"""Forward draft-token lookup, dropping ``step_id`` if unsupported.
|
||||
|
||||
Plugin workers are documented to implement no-argument
|
||||
``take_draft_token_ids()``. Passing ``step_id`` to those methods would
|
||||
raise TypeError; fall back to the legacy call instead.
|
||||
"""
|
||||
method = getattr(self.worker, "take_draft_token_ids", None)
|
||||
if method is None:
|
||||
return None
|
||||
if step_id is not None and _callable_accepts_kwarg(method, "step_id"):
|
||||
return method(step_id=step_id)
|
||||
return method()
|
||||
|
||||
def _apply_mm_cache(self, scheduler_output: SchedulerOutput) -> None:
|
||||
mm_cache = self.mm_receiver_cache
|
||||
if mm_cache is None:
|
||||
|
||||
Reference in New Issue
Block a user