[Bug Fix][Model Runner V2][Spec Decode] Warmup & capture with different attention states for speculator prefill (#44253)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-06-03 13:32:40 -07:00
committed by GitHub
co-authored by Woosuk Kwon
parent 2b237c7a41
commit 91945b6e4a
4 changed files with 59 additions and 34 deletions
+46 -25
View File
@@ -3,7 +3,7 @@
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any, NamedTuple
from typing import Any, NamedTuple, Protocol
import torch
import torch.nn as nn
@@ -37,11 +37,16 @@ from vllm.v1.worker.utils import AttentionGroup
logger = init_logger(__name__)
class CapturedAttentionState(NamedTuple):
class AttentionState(NamedTuple):
attn_metadata: dict[str, Any] | None
slot_mappings: dict[str, torch.Tensor]
class AttentionStatePair(NamedTuple):
warmup: AttentionState
captured: AttentionState
@dataclass(frozen=True)
class BatchExecutionDescriptor:
"""Describes the shape of the batch and CG mode to run; this is used to make shape
@@ -53,6 +58,18 @@ class BatchExecutionDescriptor:
uniform_token_count: int | None = None
class CreateForwardFn(Protocol):
"""Factory that prepares inputs (OUTSIDE the graph) and returns a tuple of
(forward_fn, attn_state). Called with warmup=True for the warmup pass and
warmup=False for the captured pass."""
def __call__(
self,
desc: BatchExecutionDescriptor,
warmup: bool,
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: ...
def _is_compatible(
desc: BatchExecutionDescriptor,
num_reqs: int,
@@ -198,21 +215,21 @@ class CudaGraphManager:
@torch.inference_mode()
def capture(
self,
create_forward_fn: Callable[
[BatchExecutionDescriptor],
tuple[Callable[[CUDAGraphMode], None], CapturedAttentionState],
],
create_forward_fn: CreateForwardFn,
progress_bar_desc: str = "Capturing CUDA graphs",
) -> dict[BatchExecutionDescriptor, CapturedAttentionState]:
) -> dict[BatchExecutionDescriptor, AttentionStatePair]:
"""Capture CUDA graphs.
Args:
create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and
returns a tuple of (forward_fn, captured_attn_state).
returns a tuple of (forward_fn, attn_state). For FULL cudagraph
mode, it is invoked once with warmup=True for the warmup pass,
and again with warmup=False for the captured pass. For attention
backends that perform lazy metadata initialization (e.g. FlashMLA),
FULL cudagraph capture requires distinct metadatas for warmup and
capture.
"""
captured_attn_states: dict[
BatchExecutionDescriptor, CapturedAttentionState
] = {}
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair] = {}
with graph_capture(device=self.device):
# Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger
# activations so FULL activations should fit in already allocated
@@ -226,7 +243,7 @@ class CudaGraphManager:
descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})")
for desc in descs:
# Prepare inputs and get forward function
forward_fn, attn_state = create_forward_fn(desc)
forward_fn, warmup_attn_state = create_forward_fn(desc, warmup=True)
# Warmup
forward_fn(CUDAGraphMode.NONE)
@@ -236,15 +253,18 @@ class CudaGraphManager:
"CG Capture: mode=%s, batch_desc=%s", desc.cg_mode.name, desc
)
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
captured_attn_states[desc] = attn_state
attn_states[desc] = AttentionStatePair(
warmup_attn_state, warmup_attn_state
)
forward_fn(CUDAGraphMode.PIECEWISE)
else:
# Capture with fresh attention state. The warmup
# attention state is discarded because some backends
# (e.g. FlashMLA) perform lazy initializations that
# must be captured in the graph.
forward_fn, attn_state = create_forward_fn(desc)
captured_attn_states[desc] = attn_state
# Capture with fresh attention state.
forward_fn, capture_attn_state = create_forward_fn(
desc, warmup=False
)
attn_states[desc] = AttentionStatePair(
warmup_attn_state, capture_attn_state
)
assert desc not in self.graphs, (
f"Graph already captured for {desc}"
)
@@ -262,7 +282,7 @@ class CudaGraphManager:
self.graphs[desc] = graph
compilation_counter.num_cudagraph_captured += 1
self._graphs_captured = True
return captured_attn_states
return attn_states
def dispatch(
self,
@@ -337,7 +357,7 @@ class ModelCudaGraphManager(CudaGraphManager):
has_lora: bool = False,
use_aux_hidden_state_outputs: bool = False,
progress_bar_desc: str = "Capturing CUDA graphs",
) -> dict[BatchExecutionDescriptor, CapturedAttentionState]:
) -> dict[BatchExecutionDescriptor, AttentionStatePair]:
"""Capture CUDA graphs for model forward pass."""
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
if self.use_breakable_cg:
@@ -345,9 +365,10 @@ class ModelCudaGraphManager(CudaGraphManager):
def create_forward_fn(
desc: BatchExecutionDescriptor,
warmup: bool,
) -> tuple[
Callable[[CUDAGraphMode], None],
CapturedAttentionState,
AttentionState,
]:
num_tokens = desc.num_tokens
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
@@ -435,7 +456,7 @@ class ModelCudaGraphManager(CudaGraphManager):
for k, v in intermediate_tensors.tensors.items():
self.intermediate_tensors[k][:num_tokens] = v
return forward_fn, CapturedAttentionState(attn_metadata, slot_mappings)
return forward_fn, AttentionState(attn_metadata, slot_mappings)
return super().capture(create_forward_fn, progress_bar_desc)
@@ -464,7 +485,7 @@ def prepare_inputs_to_capture(
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
skip_attn: bool = False,
) -> CapturedAttentionState:
) -> AttentionState:
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
slot_mappings = block_tables.get_dummy_slot_mappings(num_tokens)
@@ -495,4 +516,4 @@ def prepare_inputs_to_capture(
kv_cache_config,
for_capture=True,
)
return CapturedAttentionState(attn_metadata, slot_mappings_by_layer)
return AttentionState(attn_metadata, slot_mappings_by_layer)
+2 -2
View File
@@ -678,7 +678,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
start_free_gpu_memory = torch.cuda.mem_get_info()[0]
with self.maybe_setup_dummy_loras(self.lora_config):
captured_attn_states = self.cudagraph_manager.capture(
attn_states = self.cudagraph_manager.capture(
self.model,
self.model_state,
self.input_buffers,
@@ -690,7 +690,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
)
if self.speculator is not None:
self.speculator.capture(captured_attn_states)
self.speculator.capture(attn_states)
end_time = time.perf_counter()
end_free_gpu_memory = torch.cuda.mem_get_info()[0]
@@ -8,8 +8,9 @@ from vllm.config.compilation import CUDAGraphMode
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionState,
AttentionStatePair,
BatchExecutionDescriptor,
CapturedAttentionState,
CudaGraphManager,
prepare_inputs_to_capture,
)
@@ -25,12 +26,13 @@ class PrefillEagleCudaGraphManager(CudaGraphManager):
def capture(
self,
forward_fn: Callable,
full_cg_attn_states: dict[BatchExecutionDescriptor, CapturedAttentionState],
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
progress_bar_desc: str = "Capturing CUDA graphs",
) -> None:
def create_forward_fn(
desc: BatchExecutionDescriptor,
) -> tuple[Callable[[CUDAGraphMode], None], CapturedAttentionState]:
warmup: bool,
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
num_tokens = desc.num_tokens
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
num_tokens_across_dp = (
@@ -38,7 +40,8 @@ class PrefillEagleCudaGraphManager(CudaGraphManager):
if self.dp_size > 1
else None
)
attn_state = full_cg_attn_states[desc]
attn_state_pair = attn_states[desc]
attn_state = attn_state_pair.warmup if warmup else attn_state_pair.captured
attn_metadata, slot_mappings = attn_state
fwd = lambda cg_mode: forward_fn(
num_reqs,
@@ -69,7 +72,8 @@ class DecodeEagleCudaGraphManager(CudaGraphManager):
) -> None:
def create_forward_fn(
desc: BatchExecutionDescriptor,
) -> tuple[Callable[[CUDAGraphMode], None], CapturedAttentionState]:
warmup: bool,
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
num_tokens = desc.num_tokens
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
num_tokens_across_dp = (
@@ -20,8 +20,8 @@ from vllm.v1.worker.gpu.attn_utils import (
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionStatePair,
BatchExecutionDescriptor,
CapturedAttentionState,
get_uniform_token_count,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
@@ -426,7 +426,7 @@ class EagleSpeculator:
def capture(
self,
attn_states: dict[BatchExecutionDescriptor, CapturedAttentionState],
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
) -> None:
logger.info("Capturing model for Eagle speculator...")
# Reset indices to zeros to prevent stale values from prior