diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 0648de2985..dff6047ecb 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -3,7 +3,7 @@ from collections import defaultdict from collections.abc import Callable from dataclasses import dataclass -from typing import Any, NamedTuple +from typing import Any, NamedTuple, Protocol import torch import torch.nn as nn @@ -37,11 +37,16 @@ from vllm.v1.worker.utils import AttentionGroup logger = init_logger(__name__) -class CapturedAttentionState(NamedTuple): +class AttentionState(NamedTuple): attn_metadata: dict[str, Any] | None slot_mappings: dict[str, torch.Tensor] +class AttentionStatePair(NamedTuple): + warmup: AttentionState + captured: AttentionState + + @dataclass(frozen=True) class BatchExecutionDescriptor: """Describes the shape of the batch and CG mode to run; this is used to make shape @@ -53,6 +58,18 @@ class BatchExecutionDescriptor: uniform_token_count: int | None = None +class CreateForwardFn(Protocol): + """Factory that prepares inputs (OUTSIDE the graph) and returns a tuple of + (forward_fn, attn_state). Called with warmup=True for the warmup pass and + warmup=False for the captured pass.""" + + def __call__( + self, + desc: BatchExecutionDescriptor, + warmup: bool, + ) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: ... + + def _is_compatible( desc: BatchExecutionDescriptor, num_reqs: int, @@ -198,21 +215,21 @@ class CudaGraphManager: @torch.inference_mode() def capture( self, - create_forward_fn: Callable[ - [BatchExecutionDescriptor], - tuple[Callable[[CUDAGraphMode], None], CapturedAttentionState], - ], + create_forward_fn: CreateForwardFn, progress_bar_desc: str = "Capturing CUDA graphs", - ) -> dict[BatchExecutionDescriptor, CapturedAttentionState]: + ) -> dict[BatchExecutionDescriptor, AttentionStatePair]: """Capture CUDA graphs. Args: create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and - returns a tuple of (forward_fn, captured_attn_state). + returns a tuple of (forward_fn, attn_state). For FULL cudagraph + mode, it is invoked once with warmup=True for the warmup pass, + and again with warmup=False for the captured pass. For attention + backends that perform lazy metadata initialization (e.g. FlashMLA), + FULL cudagraph capture requires distinct metadatas for warmup and + capture. """ - captured_attn_states: dict[ - BatchExecutionDescriptor, CapturedAttentionState - ] = {} + attn_states: dict[BatchExecutionDescriptor, AttentionStatePair] = {} with graph_capture(device=self.device): # Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger # activations so FULL activations should fit in already allocated @@ -226,7 +243,7 @@ class CudaGraphManager: descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})") for desc in descs: # Prepare inputs and get forward function - forward_fn, attn_state = create_forward_fn(desc) + forward_fn, warmup_attn_state = create_forward_fn(desc, warmup=True) # Warmup forward_fn(CUDAGraphMode.NONE) @@ -236,15 +253,18 @@ class CudaGraphManager: "CG Capture: mode=%s, batch_desc=%s", desc.cg_mode.name, desc ) if desc.cg_mode == CUDAGraphMode.PIECEWISE: - captured_attn_states[desc] = attn_state + attn_states[desc] = AttentionStatePair( + warmup_attn_state, warmup_attn_state + ) forward_fn(CUDAGraphMode.PIECEWISE) else: - # Capture with fresh attention state. The warmup - # attention state is discarded because some backends - # (e.g. FlashMLA) perform lazy initializations that - # must be captured in the graph. - forward_fn, attn_state = create_forward_fn(desc) - captured_attn_states[desc] = attn_state + # Capture with fresh attention state. + forward_fn, capture_attn_state = create_forward_fn( + desc, warmup=False + ) + attn_states[desc] = AttentionStatePair( + warmup_attn_state, capture_attn_state + ) assert desc not in self.graphs, ( f"Graph already captured for {desc}" ) @@ -262,7 +282,7 @@ class CudaGraphManager: self.graphs[desc] = graph compilation_counter.num_cudagraph_captured += 1 self._graphs_captured = True - return captured_attn_states + return attn_states def dispatch( self, @@ -337,7 +357,7 @@ class ModelCudaGraphManager(CudaGraphManager): has_lora: bool = False, use_aux_hidden_state_outputs: bool = False, progress_bar_desc: str = "Capturing CUDA graphs", - ) -> dict[BatchExecutionDescriptor, CapturedAttentionState]: + ) -> dict[BatchExecutionDescriptor, AttentionStatePair]: """Capture CUDA graphs for model forward pass.""" self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs if self.use_breakable_cg: @@ -345,9 +365,10 @@ class ModelCudaGraphManager(CudaGraphManager): def create_forward_fn( desc: BatchExecutionDescriptor, + warmup: bool, ) -> tuple[ Callable[[CUDAGraphMode], None], - CapturedAttentionState, + AttentionState, ]: num_tokens = desc.num_tokens num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs) @@ -435,7 +456,7 @@ class ModelCudaGraphManager(CudaGraphManager): for k, v in intermediate_tensors.tensors.items(): self.intermediate_tensors[k][:num_tokens] = v - return forward_fn, CapturedAttentionState(attn_metadata, slot_mappings) + return forward_fn, AttentionState(attn_metadata, slot_mappings) return super().capture(create_forward_fn, progress_bar_desc) @@ -464,7 +485,7 @@ def prepare_inputs_to_capture( attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, skip_attn: bool = False, -) -> CapturedAttentionState: +) -> AttentionState: input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers) input_block_tables = block_tables.get_dummy_block_tables(num_reqs) slot_mappings = block_tables.get_dummy_slot_mappings(num_tokens) @@ -495,4 +516,4 @@ def prepare_inputs_to_capture( kv_cache_config, for_capture=True, ) - return CapturedAttentionState(attn_metadata, slot_mappings_by_layer) + return AttentionState(attn_metadata, slot_mappings_by_layer) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 2e3133822f..be0460c64a 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -678,7 +678,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): start_free_gpu_memory = torch.cuda.mem_get_info()[0] with self.maybe_setup_dummy_loras(self.lora_config): - captured_attn_states = self.cudagraph_manager.capture( + attn_states = self.cudagraph_manager.capture( self.model, self.model_state, self.input_buffers, @@ -690,7 +690,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs, ) if self.speculator is not None: - self.speculator.capture(captured_attn_states) + self.speculator.capture(attn_states) end_time = time.perf_counter() end_free_gpu_memory = torch.cuda.mem_get_info()[0] diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py b/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py index 300a57ec70..2c400498be 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py @@ -8,8 +8,9 @@ from vllm.config.compilation import CUDAGraphMode from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.cudagraph_utils import ( + AttentionState, + AttentionStatePair, BatchExecutionDescriptor, - CapturedAttentionState, CudaGraphManager, prepare_inputs_to_capture, ) @@ -25,12 +26,13 @@ class PrefillEagleCudaGraphManager(CudaGraphManager): def capture( self, forward_fn: Callable, - full_cg_attn_states: dict[BatchExecutionDescriptor, CapturedAttentionState], + attn_states: dict[BatchExecutionDescriptor, AttentionStatePair], progress_bar_desc: str = "Capturing CUDA graphs", ) -> None: def create_forward_fn( desc: BatchExecutionDescriptor, - ) -> tuple[Callable[[CUDAGraphMode], None], CapturedAttentionState]: + warmup: bool, + ) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: num_tokens = desc.num_tokens num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs) num_tokens_across_dp = ( @@ -38,7 +40,8 @@ class PrefillEagleCudaGraphManager(CudaGraphManager): if self.dp_size > 1 else None ) - attn_state = full_cg_attn_states[desc] + attn_state_pair = attn_states[desc] + attn_state = attn_state_pair.warmup if warmup else attn_state_pair.captured attn_metadata, slot_mappings = attn_state fwd = lambda cg_mode: forward_fn( num_reqs, @@ -69,7 +72,8 @@ class DecodeEagleCudaGraphManager(CudaGraphManager): ) -> None: def create_forward_fn( desc: BatchExecutionDescriptor, - ) -> tuple[Callable[[CUDAGraphMode], None], CapturedAttentionState]: + warmup: bool, + ) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: num_tokens = desc.num_tokens num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs) num_tokens_across_dp = ( diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py index 1a1ae1f63e..3f88c89d72 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py @@ -20,8 +20,8 @@ from vllm.v1.worker.gpu.attn_utils import ( ) from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.cudagraph_utils import ( + AttentionStatePair, BatchExecutionDescriptor, - CapturedAttentionState, get_uniform_token_count, ) from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp @@ -426,7 +426,7 @@ class EagleSpeculator: def capture( self, - attn_states: dict[BatchExecutionDescriptor, CapturedAttentionState], + attn_states: dict[BatchExecutionDescriptor, AttentionStatePair], ) -> None: logger.info("Capturing model for Eagle speculator...") # Reset indices to zeros to prevent stale values from prior