From 494845e79f8ae40f356902ba8ac181bf545603b5 Mon Sep 17 00:00:00 2001 From: vllm-agent Date: Thu, 23 Jul 2026 09:51:33 -0700 Subject: [PATCH] Revert "[MRV2] Always build attn metadata at capture time" (#49364) (#49451) Co-authored-by: vllm-agent CI bot --- vllm/v1/worker/gpu/cudagraph_utils.py | 51 +++++++------------ .../autoregressive/cudagraph_utils.py | 5 +- 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 3a174cba80..26f331fe84 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -498,7 +498,10 @@ class ModelCudaGraphManager(CudaGraphManager): block_tables, attn_groups, kv_cache_config, - full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL, + skip_attn=( + desc.cg_mode == CUDAGraphMode.PIECEWISE + and not self.use_breakable_cg + ), ) # Capture with dummy rows marked as padding. @@ -507,6 +510,7 @@ class ModelCudaGraphManager(CudaGraphManager): def forward_fn(cg_mode: CUDAGraphMode) -> None: batch_descriptor = None if cg_mode == CUDAGraphMode.PIECEWISE: + assert (attn_metadata is not None) == self.use_breakable_cg batch_descriptor = BatchDescriptor( num_tokens=num_tokens, has_lora=has_lora, @@ -589,7 +593,7 @@ def prepare_inputs_to_capture( block_tables: BlockTables, attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, - full_cudagraph: bool, + skip_attn: bool = False, ) -> AttentionState: input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers) input_block_tables = block_tables.get_dummy_block_tables(num_reqs) @@ -610,36 +614,15 @@ def prepare_inputs_to_capture( ) input_batch.dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_reqs] - # NOTE(woosuk): Attention metadata is required not just by standard attention - # kernels, but also by specialized attention-like operations (e.g., Inkling's sconv, - # DSV4 compressor), which maintain their own states and require special metadata - # such as block tables. - # During CUDA graph capture: - # - For FULL CUDA graphs: We set for_capture=True so that both attention and - # attention-like ops produce capturable metadata compatible with CUDA graphs. - # - For PIECEWISE CUDA graphs: We still build attention metadata, but set - # for_capture=False. This is because: - # * Attention-like ops (such as sconv or DSV4 compressor) may not be used as - # breakpoints in PIECEWISE CUDA graphs, so we must generate their attention - # metadata so they can execute and be captured during graph capture. - # * Standard attention ops that are treated as breakpoints will be executed - # eagerly at capture time (not included in the graph itself), and for these, - # setting for_capture=False is essential. Some attention backends - # (like linear attention) cannot generate capturable metadata for prefill, - # so for_capture=False ensures they execute without issue. - # * We assume that attention-like operations intended for capture will still - # produce capturable metadata, even when for_capture=False. While this - # assumption is brittle, it currently works in practice. - # In summary: We always generate attention metadata for both FULL and PIECEWISE - # CUDA graphs, setting for_capture=True for FULL graphs, and for_capture=False - # for PIECEWISE graphs, to ensure correct execution and capture. - attn_metadata = model_state.prepare_attn( - input_batch, - CUDAGraphMode.NONE, - input_block_tables, - slot_mappings, - attn_groups, - kv_cache_config, - for_capture=full_cudagraph, - ) + attn_metadata = None + if not skip_attn: + attn_metadata = model_state.prepare_attn( + input_batch, + CUDAGraphMode.NONE, + input_block_tables, + slot_mappings, + attn_groups, + kv_cache_config, + for_capture=True, + ) return AttentionState(attn_metadata, slot_mappings_by_layer) diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py index ef3b6e2ed5..19919043c8 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/cudagraph_utils.py @@ -56,7 +56,10 @@ class SpeculatorCudaGraphManager(CudaGraphManager): block_tables, attn_groups, kv_cache_config, - full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL, + skip_attn=( + desc.cg_mode == CUDAGraphMode.PIECEWISE + and not self.use_breakable_cg + ), ) return lambda cg_mode: forward_fn(