[MRV2] Always build attn metadata at capture time (#49364) (#49995)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Nick Hill
2026-07-27 17:25:17 +00:00
committed by GitHub
co-authored by Woosuk Kwon
parent 15d65f8669
commit 8112b6c997
2 changed files with 35 additions and 21 deletions
+34 -17
View File
@@ -498,10 +498,7 @@ class ModelCudaGraphManager(CudaGraphManager):
block_tables, block_tables,
attn_groups, attn_groups,
kv_cache_config, kv_cache_config,
skip_attn=( full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL,
desc.cg_mode == CUDAGraphMode.PIECEWISE
and not self.use_breakable_cg
),
) )
# Capture with dummy rows marked as padding. # Capture with dummy rows marked as padding.
@@ -510,7 +507,6 @@ class ModelCudaGraphManager(CudaGraphManager):
def forward_fn(cg_mode: CUDAGraphMode) -> None: def forward_fn(cg_mode: CUDAGraphMode) -> None:
batch_descriptor = None batch_descriptor = None
if cg_mode == CUDAGraphMode.PIECEWISE: if cg_mode == CUDAGraphMode.PIECEWISE:
assert (attn_metadata is not None) == self.use_breakable_cg
batch_descriptor = BatchDescriptor( batch_descriptor = BatchDescriptor(
num_tokens=num_tokens, num_tokens=num_tokens,
has_lora=has_lora, has_lora=has_lora,
@@ -593,7 +589,7 @@ def prepare_inputs_to_capture(
block_tables: BlockTables, block_tables: BlockTables,
attn_groups: list[list[AttentionGroup]], attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig, kv_cache_config: KVCacheConfig,
skip_attn: bool = False, full_cudagraph: bool,
) -> AttentionState: ) -> AttentionState:
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers) input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
input_block_tables = block_tables.get_dummy_block_tables(num_reqs) input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
@@ -614,15 +610,36 @@ def prepare_inputs_to_capture(
) )
input_batch.dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_reqs] input_batch.dcp_local_seq_lens = input_buffers.dcp_local_seq_lens[:num_reqs]
attn_metadata = None # NOTE(woosuk): Attention metadata is required not just by standard attention
if not skip_attn: # kernels, but also by specialized attention-like operations (e.g., Inkling's sconv,
attn_metadata = model_state.prepare_attn( # DSV4 compressor), which maintain their own states and require special metadata
input_batch, # such as block tables.
CUDAGraphMode.NONE, # During CUDA graph capture:
input_block_tables, # - For FULL CUDA graphs: We set for_capture=True so that both attention and
slot_mappings, # attention-like ops produce capturable metadata compatible with CUDA graphs.
attn_groups, # - For PIECEWISE CUDA graphs: We still build attention metadata, but set
kv_cache_config, # for_capture=False. This is because:
for_capture=True, # * Attention-like ops (such as sconv or DSV4 compressor) may not be used as
) # breakpoints in PIECEWISE CUDA graphs, so we must generate their attention
# metadata so they can execute and be captured during graph capture.
# * Standard attention ops that are treated as breakpoints will be executed
# eagerly at capture time (not included in the graph itself), and for these,
# setting for_capture=False is essential. Some attention backends
# (like linear attention) cannot generate capturable metadata for prefill,
# so for_capture=False ensures they execute without issue.
# * We assume that attention-like operations intended for capture will still
# produce capturable metadata, even when for_capture=False. While this
# assumption is brittle, it currently works in practice.
# In summary: We always generate attention metadata for both FULL and PIECEWISE
# CUDA graphs, setting for_capture=True for FULL graphs, and for_capture=False
# for PIECEWISE graphs, to ensure correct execution and capture.
attn_metadata = model_state.prepare_attn(
input_batch,
CUDAGraphMode.NONE,
input_block_tables,
slot_mappings,
attn_groups,
kv_cache_config,
for_capture=full_cudagraph,
)
return AttentionState(attn_metadata, slot_mappings_by_layer) return AttentionState(attn_metadata, slot_mappings_by_layer)
@@ -56,10 +56,7 @@ class SpeculatorCudaGraphManager(CudaGraphManager):
block_tables, block_tables,
attn_groups, attn_groups,
kv_cache_config, kv_cache_config,
skip_attn=( full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL,
desc.cg_mode == CUDAGraphMode.PIECEWISE
and not self.use_breakable_cg
),
) )
return lambda cg_mode: forward_fn( return lambda cg_mode: forward_fn(