[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -17,6 +20,97 @@ from vllm.v1.outputs import (
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.utils import raise_if_nan_logits
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StepTimingSample:
|
||||
forward_ms: float
|
||||
drafter_ms: float
|
||||
num_target_tokens: int
|
||||
num_reqs: int
|
||||
full_cudagraph: bool
|
||||
|
||||
|
||||
def _timing_event() -> torch.cuda.Event:
|
||||
return torch.cuda.Event(enable_timing=True)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StepTimingEvents:
|
||||
forward_start: torch.cuda.Event = field(default_factory=_timing_event)
|
||||
forward_end: torch.cuda.Event = field(default_factory=_timing_event)
|
||||
drafter_start: torch.cuda.Event = field(default_factory=_timing_event)
|
||||
drafter_end: torch.cuda.Event = field(default_factory=_timing_event)
|
||||
|
||||
|
||||
class StepTimingCollector:
|
||||
"""Times the steps run inside ``collect``; record calls no-op outside it.
|
||||
|
||||
Every step gets its own events, so the steps queue back-to-back and the
|
||||
block resolves them all behind one sync on the way out.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._collecting = False
|
||||
self._step: StepTimingEvents | None = None
|
||||
self._batch = (False, 0, 0)
|
||||
self._timed: list[tuple[StepTimingEvents, tuple[bool, int, int]]] = []
|
||||
|
||||
@contextlib.contextmanager
|
||||
def collect(self) -> Iterator[list[StepTimingSample]]:
|
||||
"""Time every step run in this block.
|
||||
|
||||
The yielded list holds one sample per timed step once the block exits;
|
||||
it stays empty inside the block, where the timings are still on device.
|
||||
"""
|
||||
samples: list[StepTimingSample] = []
|
||||
self._collecting = True
|
||||
try:
|
||||
yield samples
|
||||
finally:
|
||||
self._collecting = False
|
||||
timed, self._timed, self._step = self._timed, [], None
|
||||
if not timed:
|
||||
return
|
||||
# Same stream, issue order: once the last step is done, so are the rest.
|
||||
timed[-1][0].drafter_end.synchronize()
|
||||
samples.extend(
|
||||
StepTimingSample(
|
||||
events.forward_start.elapsed_time(events.forward_end),
|
||||
events.drafter_start.elapsed_time(events.drafter_end),
|
||||
num_target_tokens,
|
||||
num_reqs,
|
||||
full_cudagraph,
|
||||
)
|
||||
for events, (full_cudagraph, num_target_tokens, num_reqs) in timed
|
||||
)
|
||||
|
||||
def record_batch(self, input_batch: "InputBatch", full_cudagraph: bool) -> None:
|
||||
"""Costs from different execution modes must not share a cost curve."""
|
||||
self._batch = (full_cudagraph, input_batch.num_tokens, input_batch.num_reqs)
|
||||
|
||||
def forward_start(self) -> None:
|
||||
if self._collecting:
|
||||
self._step = StepTimingEvents()
|
||||
self._step.forward_start.record()
|
||||
|
||||
def forward_end(self) -> None:
|
||||
if self._step is not None:
|
||||
self._step.forward_end.record()
|
||||
|
||||
def drafter_start(self) -> None:
|
||||
if self._step is not None:
|
||||
self._step.drafter_start.record()
|
||||
|
||||
def drafter_end(self) -> None:
|
||||
"""Ends the step: only steps that reach here have a draft cost."""
|
||||
if self._step is not None:
|
||||
self._step.drafter_end.record()
|
||||
self._timed.append((self._step, self._batch))
|
||||
self._step = None
|
||||
|
||||
|
||||
class AsyncOutput(AsyncModelRunnerOutput):
|
||||
def __init__(
|
||||
|
||||
Reference in New Issue
Block a user