From 26587f9519e22a5c4549ead7595ad9ca3229c4fd Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Mon, 13 Jul 2026 17:39:15 +0100 Subject: [PATCH] [BugFix][ModelRunner V2] Fix stale attn metadata in speculator prefill cudagraph capture (#48261) Signed-off-by: Nick Hill --- vllm/v1/worker/gpu/cudagraph_utils.py | 52 ++++++----------- vllm/v1/worker/gpu/model_runner.py | 10 +++- .../autoregressive/cudagraph_utils.py | 56 ++++--------------- .../spec_decode/autoregressive/speculator.py | 26 +++++---- .../gpu/spec_decode/dflash/cudagraph.py | 5 +- .../gpu/spec_decode/dflash/speculator.py | 13 ++++- vllm/v1/worker/gpu/spec_decode/speculator.py | 18 +++--- 7 files changed, 72 insertions(+), 108 deletions(-) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 9b93d17035..9898fadc25 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -44,11 +44,6 @@ class AttentionState(NamedTuple): slot_mappings: dict[str, torch.Tensor] -class AttentionStatePair(NamedTuple): - warmup: AttentionState - captured: AttentionState - - @dataclass(frozen=True) class BatchExecutionDescriptor: """Describes the shape of the batch and CG mode to run; this is used to make shape @@ -62,15 +57,15 @@ class BatchExecutionDescriptor: class CreateForwardFn(Protocol): - """Factory that prepares inputs (OUTSIDE the graph) and returns a tuple of - (forward_fn, attn_state). Called with warmup=True for the warmup pass and - warmup=False for the captured pass.""" + """Factory that prepares inputs (OUTSIDE the graph) and returns a + forward_fn. Called with warmup=True for the warmup pass and warmup=False + for the captured pass.""" def __call__( self, desc: BatchExecutionDescriptor, warmup: bool, - ) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: ... + ) -> Callable[[CUDAGraphMode], None]: ... def _is_compatible( @@ -301,19 +296,18 @@ class CudaGraphManager: self, create_forward_fn: CreateForwardFn, progress_bar_desc: str = "Capturing CUDA graphs", - ) -> dict[BatchExecutionDescriptor, AttentionStatePair]: + ) -> None: """Capture CUDA graphs. Args: create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and - returns a tuple of (forward_fn, attn_state). For FULL cudagraph - mode, it is invoked once with warmup=True for the warmup pass, - and again with warmup=False for the captured pass. For attention - backends that perform lazy metadata initialization (e.g. FlashMLA), - FULL cudagraph capture requires distinct metadatas for warmup and - capture. + returns a forward_fn. For FULL cudagraph mode, it is invoked + once with warmup=True for the warmup pass, and again with + warmup=False for the captured pass. For attention backends + that perform lazy metadata initialization (e.g. FlashMLA), + FULL cudagraph capture requires distinct metadatas for warmup + and capture. """ - attn_states: dict[BatchExecutionDescriptor, AttentionStatePair] = {} with graph_capture(device=self.device): # Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger # activations so FULL activations should fit in already allocated @@ -327,7 +321,7 @@ class CudaGraphManager: descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})") for desc in descs: # Prepare inputs and get forward function - forward_fn, warmup_attn_state = create_forward_fn(desc, warmup=True) + forward_fn = create_forward_fn(desc, warmup=True) # Warmup forward_fn(CUDAGraphMode.NONE) @@ -337,18 +331,10 @@ class CudaGraphManager: "CG Capture: mode=%s, batch_desc=%s", desc.cg_mode.name, desc ) if desc.cg_mode == CUDAGraphMode.PIECEWISE: - attn_states[desc] = AttentionStatePair( - warmup_attn_state, warmup_attn_state - ) forward_fn(CUDAGraphMode.PIECEWISE) else: # Capture with fresh attention state. - forward_fn, capture_attn_state = create_forward_fn( - desc, warmup=False - ) - attn_states[desc] = AttentionStatePair( - warmup_attn_state, capture_attn_state - ) + forward_fn = create_forward_fn(desc, warmup=False) assert desc not in self.graphs, ( f"Graph already captured for {desc}" ) @@ -366,7 +352,6 @@ class CudaGraphManager: self.graphs[desc] = graph compilation_counter.num_cudagraph_captured += 1 self._graphs_captured = True - return attn_states def dispatch( self, @@ -461,7 +446,7 @@ class ModelCudaGraphManager(CudaGraphManager): use_aux_hidden_state_outputs: bool = False, lora_capture_hook: Callable[[int, int, int], None] | None = None, progress_bar_desc: str = "Capturing CUDA graphs", - ) -> dict[BatchExecutionDescriptor, AttentionStatePair]: + ) -> None: """Capture CUDA graphs for model forward pass.""" self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs if self.use_breakable_cg: @@ -470,10 +455,7 @@ class ModelCudaGraphManager(CudaGraphManager): def create_forward_fn( desc: BatchExecutionDescriptor, warmup: bool, - ) -> tuple[ - Callable[[CUDAGraphMode], None], - AttentionState, - ]: + ) -> Callable[[CUDAGraphMode], None]: num_tokens = desc.num_tokens num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs) @@ -571,9 +553,9 @@ class ModelCudaGraphManager(CudaGraphManager): for k, v in intermediate_tensors.tensors.items(): self.intermediate_tensors[k][:num_tokens] = v - return forward_fn, AttentionState(attn_metadata, slot_mappings) + return forward_fn - return super().capture(create_forward_fn, progress_bar_desc) + super().capture(create_forward_fn, progress_bar_desc) def run_fullgraph( self, desc: BatchExecutionDescriptor diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index d2e1e0524f..1dee0d1b95 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -479,7 +479,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): if isinstance(self.speculator, DraftModelSpeculator): # HACK(woosuk) self.speculator.set_attn( - self.model_state, self.kv_cache_config, self.block_tables + self.model_state, + self.kv_cache_config, + self.block_tables, + self.input_buffers, + self.attn_groups, ) self.kv_caches: list[torch.Tensor] = [] @@ -706,7 +710,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): start_free_gpu_memory = torch.accelerator.get_memory_info()[0] with self.maybe_setup_dummy_loras(self.lora_config): - attn_states = self.cudagraph_manager.capture( + self.cudagraph_manager.capture( self.model, self.model_state, self.input_buffers, @@ -719,7 +723,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): lora_capture_hook=create_lora_capture_hook(self.lora_config, self), ) if self.speculator is not None: - self.speculator.capture(attn_states) + self.speculator.capture() end_time = time.perf_counter() end_free_gpu_memory = torch.accelerator.get_memory_info()[0] diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py index 15ab7430c9..b30712b5a5 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py @@ -8,8 +8,6 @@ from vllm.config.compilation import CUDAGraphMode from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.cudagraph_utils import ( - AttentionState, - AttentionStatePair, BatchExecutionDescriptor, CudaGraphManager, prepare_inputs_to_capture, @@ -19,45 +17,15 @@ from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.utils import AttentionGroup -class PrefillSpeculatorCudaGraphManager(CudaGraphManager): - """CudaGraphManager for draft prefill, using pre-built attention states - from the target model's capture.""" +class SpeculatorCudaGraphManager(CudaGraphManager): + """CudaGraphManager for draft prefill and decode. - def capture( - self, - forward_fn: Callable, - attn_states: dict[BatchExecutionDescriptor, AttentionStatePair], - progress_bar_desc: str = "Capturing CUDA graphs", - ) -> None: - def create_forward_fn( - desc: BatchExecutionDescriptor, - warmup: bool, - ) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: - num_tokens = desc.num_tokens - num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs) - num_tokens_across_dp = ( - torch.full((self.dp_size,), num_tokens, dtype=torch.int32, device="cpu") - if self.dp_size > 1 - else None - ) - attn_state_pair = attn_states[desc] - attn_state = attn_state_pair.warmup if warmup else attn_state_pair.captured - attn_metadata, slot_mappings = attn_state - fwd = lambda cg_mode: forward_fn( - num_reqs, - num_tokens, - attn_metadata, - slot_mappings, - num_tokens_across_dp, - cg_mode, - ) - return fwd, attn_state - - super().capture(create_forward_fn, progress_bar_desc) - - -class DecodeSpeculatorCudaGraphManager(CudaGraphManager): - """CudaGraphManager for draft decode, building its own attention metadata.""" + Builds fresh dummy inputs and attention metadata for every warmup and + capture pass so that the contents of the shared persistent buffers + (e.g. query_start_loc, seq_lens, FA3 scheduler metadata) always match + the batch descriptor being captured. Reusing metadata built during an + earlier capture would execute kernels with stale buffer contents. + """ def capture( self, @@ -72,7 +40,7 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager): def create_forward_fn( desc: BatchExecutionDescriptor, warmup: bool, - ) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: + ) -> Callable[[CUDAGraphMode], None]: num_tokens = desc.num_tokens num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs) num_tokens_across_dp = ( @@ -80,7 +48,7 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager): if self.dp_size > 1 else None ) - attn_state = prepare_inputs_to_capture( + attn_metadata, slot_mappings = prepare_inputs_to_capture( num_reqs, num_tokens, model_state, @@ -90,9 +58,8 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager): kv_cache_config, skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE), ) - attn_metadata, slot_mappings = attn_state - fwd = lambda cg_mode: forward_fn( + return lambda cg_mode: forward_fn( num_reqs, num_tokens, attn_metadata, @@ -100,6 +67,5 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager): num_tokens_across_dp, cg_mode, ) - return fwd, attn_state super().capture(create_forward_fn, progress_bar_desc) diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index 747fb3a390..3cb478348a 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -12,15 +12,13 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.triton_utils import tl, triton from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.cudagraph_utils import ( - AttentionStatePair, BatchExecutionDescriptor, get_uniform_token_count, ) from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import ( - DecodeSpeculatorCudaGraphManager, - PrefillSpeculatorCudaGraphManager, + SpeculatorCudaGraphManager, ) from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator @@ -47,8 +45,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device ) - self.prefill_cudagraph_manager: PrefillSpeculatorCudaGraphManager | None = None - self.decode_cudagraph_manager: DecodeSpeculatorCudaGraphManager | None = None + self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None + self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None @property def advance_draft_positions(self) -> bool: @@ -62,7 +60,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: # Initialize cudagraph manager for draft prefill (draft position 0). - self.prefill_cudagraph_manager = PrefillSpeculatorCudaGraphManager( + self.prefill_cudagraph_manager = SpeculatorCudaGraphManager( self.vllm_config, self.device, cudagraph_mode, @@ -76,17 +74,14 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): cudagraph_mode = CUDAGraphMode.NONE # Initialize cudagraph manager for draft decodes (draft positions > 0). - self.decode_cudagraph_manager = DecodeSpeculatorCudaGraphManager( + self.decode_cudagraph_manager = SpeculatorCudaGraphManager( self.vllm_config, self.device, cudagraph_mode, decode_query_len=1, ) - def capture( - self, - attn_states: dict[BatchExecutionDescriptor, AttentionStatePair], - ) -> None: + def capture(self) -> None: logger.info("Capturing model for speculator...") # Reset indices to zeros to prevent stale values from prior # dummy runs to cause out-of-bounds indexing during capture. @@ -97,12 +92,19 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): # For FULL graphs, the entire routine is recorded as one graph. # For PIECEWISE, only the model's compiled regions are captured # and the rest (compute_logits, gumbel_sample) runs eagerly. + # Draft prefill reuses the target model's attention metadata at + # runtime, so capture builds its dummy metadata through the target + # model runner's builders and buffers. assert self.prefill_cudagraph_manager is not None if self.prefill_cudagraph_manager.use_breakable_cg: self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model) self.prefill_cudagraph_manager.capture( self._prefill, - attn_states, + self.model_state, + self.target_input_buffers, + self.block_tables, + self.target_attn_groups, + self.kv_cache_config, progress_bar_desc="Capturing prefill CUDA graphs", ) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py index a4b4033caf..e59202c879 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py @@ -77,7 +77,7 @@ class DFlashCudaGraphManager(CudaGraphManager): def create_forward_fn( desc: BatchExecutionDescriptor, warmup: bool, - ) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: + ) -> Callable[[CUDAGraphMode], None]: num_tokens = desc.num_tokens num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs) num_tokens_across_dp = ( @@ -98,7 +98,7 @@ class DFlashCudaGraphManager(CudaGraphManager): ) attn_metadata, slot_mappings = attn_state - fwd = lambda cg_mode: forward_fn( + return lambda cg_mode: forward_fn( num_reqs, num_tokens, attn_metadata, @@ -106,6 +106,5 @@ class DFlashCudaGraphManager(CudaGraphManager): num_tokens_across_dp, cg_mode, ) - return fwd, attn_state super().capture(create_forward_fn, progress_bar_desc) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index f9ffd3135d..6fd99d55fc 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -25,6 +25,7 @@ from vllm.v1.worker.gpu.spec_decode.dflash.utils import ( ) from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator from vllm.v1.worker.gpu.spec_decode.utils import get_parallel_drafting_token_id +from vllm.v1.worker.utils import AttentionGroup logger = init_logger(__name__) @@ -97,7 +98,7 @@ class DFlashSpeculator(DraftModelSpeculator): decode_query_len=self.num_query_per_req, ) - def capture(self, attn_states: dict | None = None) -> None: + def capture(self) -> None: logger.info("Capturing model for %s speculator...", self._speculator_name) # Reset sampling indices to zero to prevent stale values from prior # dummy runs from being baked into the captured graph. @@ -128,8 +129,16 @@ class DFlashSpeculator(DraftModelSpeculator): model_state: ModelState, kv_cache_config: KVCacheConfig, block_tables: BlockTables, + target_input_buffers: InputBuffers, + target_attn_groups: list[list[AttentionGroup]], ) -> None: - super().set_attn(model_state, kv_cache_config, block_tables) + super().set_attn( + model_state, + kv_cache_config, + block_tables, + target_input_buffers, + target_attn_groups, + ) self.draft_kv_cache_group_ids = [ gid for gid, g in enumerate(self.attn_groups) if g diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 4b7d7fef41..36be473221 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -18,13 +18,10 @@ from vllm.v1.worker.gpu.attn_utils import ( init_attn_backend, ) from vllm.v1.worker.gpu.block_table import BlockTables -from vllm.v1.worker.gpu.cudagraph_utils import ( - AttentionStatePair, - BatchExecutionDescriptor, -) from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample +from vllm.v1.worker.utils import AttentionGroup logger = init_logger(__name__) @@ -35,10 +32,7 @@ class BaseSpeculator(ABC): pass @abstractmethod - def capture( - self, - attn_states: dict[BatchExecutionDescriptor, AttentionStatePair], - ) -> None: + def capture(self) -> None: pass @abstractmethod @@ -186,6 +180,8 @@ class DraftModelSpeculator(BaseSpeculator): model_state: ModelState, kv_cache_config: KVCacheConfig, block_tables: BlockTables, + target_input_buffers: InputBuffers, + target_attn_groups: list[list[AttentionGroup]], ) -> None: self.model_state = model_state self.kv_cache_config = kv_cache_config @@ -196,6 +192,12 @@ class DraftModelSpeculator(BaseSpeculator): active_layer_names=self.draft_attn_layer_names, ) self.block_tables = block_tables + # The target model runner's buffers and attention groups. Draft + # prefill reuses the target model's attention metadata, so its + # cudagraph capture must build dummy metadata through the same + # builders and buffers. + self.target_input_buffers = target_input_buffers + self.target_attn_groups = target_attn_groups def _build_draft_attn_metadata( self,