[Spec Decode] DSpark confidence-scheduled verification (#47808)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-08-12 12:27:48 -07:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5 Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill OpenAI Codex
parent fe889ac925
commit 7f7a32cfec
40 changed files with 1643 additions and 111 deletions
+94
View File
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
from collections.abc import Iterator
from dataclasses import dataclass, field
from typing import TYPE_CHECKING
import numpy as np
import torch
@@ -17,6 +20,97 @@ from vllm.v1.outputs import (
from vllm.v1.worker.gpu.sample.output import SamplerOutput
from vllm.v1.worker.utils import raise_if_nan_logits
if TYPE_CHECKING:
from vllm.v1.worker.gpu.input_batch import InputBatch
@dataclass(frozen=True)
class StepTimingSample:
forward_ms: float
drafter_ms: float
num_target_tokens: int
num_reqs: int
full_cudagraph: bool
def _timing_event() -> torch.cuda.Event:
return torch.cuda.Event(enable_timing=True)
@dataclass
class StepTimingEvents:
forward_start: torch.cuda.Event = field(default_factory=_timing_event)
forward_end: torch.cuda.Event = field(default_factory=_timing_event)
drafter_start: torch.cuda.Event = field(default_factory=_timing_event)
drafter_end: torch.cuda.Event = field(default_factory=_timing_event)
class StepTimingCollector:
"""Times the steps run inside ``collect``; record calls no-op outside it.
Every step gets its own events, so the steps queue back-to-back and the
block resolves them all behind one sync on the way out.
"""
def __init__(self):
self._collecting = False
self._step: StepTimingEvents | None = None
self._batch = (False, 0, 0)
self._timed: list[tuple[StepTimingEvents, tuple[bool, int, int]]] = []
@contextlib.contextmanager
def collect(self) -> Iterator[list[StepTimingSample]]:
"""Time every step run in this block.
The yielded list holds one sample per timed step once the block exits;
it stays empty inside the block, where the timings are still on device.
"""
samples: list[StepTimingSample] = []
self._collecting = True
try:
yield samples
finally:
self._collecting = False
timed, self._timed, self._step = self._timed, [], None
if not timed:
return
# Same stream, issue order: once the last step is done, so are the rest.
timed[-1][0].drafter_end.synchronize()
samples.extend(
StepTimingSample(
events.forward_start.elapsed_time(events.forward_end),
events.drafter_start.elapsed_time(events.drafter_end),
num_target_tokens,
num_reqs,
full_cudagraph,
)
for events, (full_cudagraph, num_target_tokens, num_reqs) in timed
)
def record_batch(self, input_batch: "InputBatch", full_cudagraph: bool) -> None:
"""Costs from different execution modes must not share a cost curve."""
self._batch = (full_cudagraph, input_batch.num_tokens, input_batch.num_reqs)
def forward_start(self) -> None:
if self._collecting:
self._step = StepTimingEvents()
self._step.forward_start.record()
def forward_end(self) -> None:
if self._step is not None:
self._step.forward_end.record()
def drafter_start(self) -> None:
if self._step is not None:
self._step.drafter_start.record()
def drafter_end(self) -> None:
"""Ends the step: only steps that reach here have a draft cost."""
if self._step is not None:
self._step.drafter_end.record()
self._timed.append((self._step, self._batch))
self._step = None
class AsyncOutput(AsyncModelRunnerOutput):
def __init__(