[BugFix][ModelRunner V2] Fix stale attn metadata in speculator prefill cudagraph capture (#48261)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-07-13 09:39:15 -07:00
committed by GitHub
parent 93e3bc8f30
commit 26587f9519
7 changed files with 72 additions and 108 deletions
+17 -35
View File
@@ -44,11 +44,6 @@ class AttentionState(NamedTuple):
slot_mappings: dict[str, torch.Tensor]
class AttentionStatePair(NamedTuple):
warmup: AttentionState
captured: AttentionState
@dataclass(frozen=True)
class BatchExecutionDescriptor:
"""Describes the shape of the batch and CG mode to run; this is used to make shape
@@ -62,15 +57,15 @@ class BatchExecutionDescriptor:
class CreateForwardFn(Protocol):
"""Factory that prepares inputs (OUTSIDE the graph) and returns a tuple of
(forward_fn, attn_state). Called with warmup=True for the warmup pass and
warmup=False for the captured pass."""
"""Factory that prepares inputs (OUTSIDE the graph) and returns a
forward_fn. Called with warmup=True for the warmup pass and warmup=False
for the captured pass."""
def __call__(
self,
desc: BatchExecutionDescriptor,
warmup: bool,
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: ...
) -> Callable[[CUDAGraphMode], None]: ...
def _is_compatible(
@@ -301,19 +296,18 @@ class CudaGraphManager:
self,
create_forward_fn: CreateForwardFn,
progress_bar_desc: str = "Capturing CUDA graphs",
) -> dict[BatchExecutionDescriptor, AttentionStatePair]:
) -> None:
"""Capture CUDA graphs.
Args:
create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and
returns a tuple of (forward_fn, attn_state). For FULL cudagraph
mode, it is invoked once with warmup=True for the warmup pass,
and again with warmup=False for the captured pass. For attention
backends that perform lazy metadata initialization (e.g. FlashMLA),
FULL cudagraph capture requires distinct metadatas for warmup and
capture.
returns a forward_fn. For FULL cudagraph mode, it is invoked
once with warmup=True for the warmup pass, and again with
warmup=False for the captured pass. For attention backends
that perform lazy metadata initialization (e.g. FlashMLA),
FULL cudagraph capture requires distinct metadatas for warmup
and capture.
"""
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair] = {}
with graph_capture(device=self.device):
# Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger
# activations so FULL activations should fit in already allocated
@@ -327,7 +321,7 @@ class CudaGraphManager:
descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})")
for desc in descs:
# Prepare inputs and get forward function
forward_fn, warmup_attn_state = create_forward_fn(desc, warmup=True)
forward_fn = create_forward_fn(desc, warmup=True)
# Warmup
forward_fn(CUDAGraphMode.NONE)
@@ -337,18 +331,10 @@ class CudaGraphManager:
"CG Capture: mode=%s, batch_desc=%s", desc.cg_mode.name, desc
)
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
attn_states[desc] = AttentionStatePair(
warmup_attn_state, warmup_attn_state
)
forward_fn(CUDAGraphMode.PIECEWISE)
else:
# Capture with fresh attention state.
forward_fn, capture_attn_state = create_forward_fn(
desc, warmup=False
)
attn_states[desc] = AttentionStatePair(
warmup_attn_state, capture_attn_state
)
forward_fn = create_forward_fn(desc, warmup=False)
assert desc not in self.graphs, (
f"Graph already captured for {desc}"
)
@@ -366,7 +352,6 @@ class CudaGraphManager:
self.graphs[desc] = graph
compilation_counter.num_cudagraph_captured += 1
self._graphs_captured = True
return attn_states
def dispatch(
self,
@@ -461,7 +446,7 @@ class ModelCudaGraphManager(CudaGraphManager):
use_aux_hidden_state_outputs: bool = False,
lora_capture_hook: Callable[[int, int, int], None] | None = None,
progress_bar_desc: str = "Capturing CUDA graphs",
) -> dict[BatchExecutionDescriptor, AttentionStatePair]:
) -> None:
"""Capture CUDA graphs for model forward pass."""
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
if self.use_breakable_cg:
@@ -470,10 +455,7 @@ class ModelCudaGraphManager(CudaGraphManager):
def create_forward_fn(
desc: BatchExecutionDescriptor,
warmup: bool,
) -> tuple[
Callable[[CUDAGraphMode], None],
AttentionState,
]:
) -> Callable[[CUDAGraphMode], None]:
num_tokens = desc.num_tokens
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
@@ -571,9 +553,9 @@ class ModelCudaGraphManager(CudaGraphManager):
for k, v in intermediate_tensors.tensors.items():
self.intermediate_tensors[k][:num_tokens] = v
return forward_fn, AttentionState(attn_metadata, slot_mappings)
return forward_fn
return super().capture(create_forward_fn, progress_bar_desc)
super().capture(create_forward_fn, progress_bar_desc)
def run_fullgraph(
self, desc: BatchExecutionDescriptor
+7 -3
View File
@@ -479,7 +479,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if isinstance(self.speculator, DraftModelSpeculator):
# HACK(woosuk)
self.speculator.set_attn(
self.model_state, self.kv_cache_config, self.block_tables
self.model_state,
self.kv_cache_config,
self.block_tables,
self.input_buffers,
self.attn_groups,
)
self.kv_caches: list[torch.Tensor] = []
@@ -706,7 +710,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
with self.maybe_setup_dummy_loras(self.lora_config):
attn_states = self.cudagraph_manager.capture(
self.cudagraph_manager.capture(
self.model,
self.model_state,
self.input_buffers,
@@ -719,7 +723,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
)
if self.speculator is not None:
self.speculator.capture(attn_states)
self.speculator.capture()
end_time = time.perf_counter()
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
@@ -8,8 +8,6 @@ from vllm.config.compilation import CUDAGraphMode
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionState,
AttentionStatePair,
BatchExecutionDescriptor,
CudaGraphManager,
prepare_inputs_to_capture,
@@ -19,45 +17,15 @@ from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.utils import AttentionGroup
class PrefillSpeculatorCudaGraphManager(CudaGraphManager):
"""CudaGraphManager for draft prefill, using pre-built attention states
from the target model's capture."""
class SpeculatorCudaGraphManager(CudaGraphManager):
"""CudaGraphManager for draft prefill and decode.
def capture(
self,
forward_fn: Callable,
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
progress_bar_desc: str = "Capturing CUDA graphs",
) -> None:
def create_forward_fn(
desc: BatchExecutionDescriptor,
warmup: bool,
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
num_tokens = desc.num_tokens
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
num_tokens_across_dp = (
torch.full((self.dp_size,), num_tokens, dtype=torch.int32, device="cpu")
if self.dp_size > 1
else None
)
attn_state_pair = attn_states[desc]
attn_state = attn_state_pair.warmup if warmup else attn_state_pair.captured
attn_metadata, slot_mappings = attn_state
fwd = lambda cg_mode: forward_fn(
num_reqs,
num_tokens,
attn_metadata,
slot_mappings,
num_tokens_across_dp,
cg_mode,
)
return fwd, attn_state
super().capture(create_forward_fn, progress_bar_desc)
class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
"""CudaGraphManager for draft decode, building its own attention metadata."""
Builds fresh dummy inputs and attention metadata for every warmup and
capture pass so that the contents of the shared persistent buffers
(e.g. query_start_loc, seq_lens, FA3 scheduler metadata) always match
the batch descriptor being captured. Reusing metadata built during an
earlier capture would execute kernels with stale buffer contents.
"""
def capture(
self,
@@ -72,7 +40,7 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
def create_forward_fn(
desc: BatchExecutionDescriptor,
warmup: bool,
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
) -> Callable[[CUDAGraphMode], None]:
num_tokens = desc.num_tokens
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
num_tokens_across_dp = (
@@ -80,7 +48,7 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
if self.dp_size > 1
else None
)
attn_state = prepare_inputs_to_capture(
attn_metadata, slot_mappings = prepare_inputs_to_capture(
num_reqs,
num_tokens,
model_state,
@@ -90,9 +58,8 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
kv_cache_config,
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
)
attn_metadata, slot_mappings = attn_state
fwd = lambda cg_mode: forward_fn(
return lambda cg_mode: forward_fn(
num_reqs,
num_tokens,
attn_metadata,
@@ -100,6 +67,5 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
num_tokens_across_dp,
cg_mode,
)
return fwd, attn_state
super().capture(create_forward_fn, progress_bar_desc)
@@ -12,15 +12,13 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionStatePair,
BatchExecutionDescriptor,
get_uniform_token_count,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
DecodeSpeculatorCudaGraphManager,
PrefillSpeculatorCudaGraphManager,
SpeculatorCudaGraphManager,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
@@ -47,8 +45,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
)
self.prefill_cudagraph_manager: PrefillSpeculatorCudaGraphManager | None = None
self.decode_cudagraph_manager: DecodeSpeculatorCudaGraphManager | None = None
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
@property
def advance_draft_positions(self) -> bool:
@@ -62,7 +60,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
# Initialize cudagraph manager for draft prefill (draft position 0).
self.prefill_cudagraph_manager = PrefillSpeculatorCudaGraphManager(
self.prefill_cudagraph_manager = SpeculatorCudaGraphManager(
self.vllm_config,
self.device,
cudagraph_mode,
@@ -76,17 +74,14 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
cudagraph_mode = CUDAGraphMode.NONE
# Initialize cudagraph manager for draft decodes (draft positions > 0).
self.decode_cudagraph_manager = DecodeSpeculatorCudaGraphManager(
self.decode_cudagraph_manager = SpeculatorCudaGraphManager(
self.vllm_config,
self.device,
cudagraph_mode,
decode_query_len=1,
)
def capture(
self,
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
) -> None:
def capture(self) -> None:
logger.info("Capturing model for speculator...")
# Reset indices to zeros to prevent stale values from prior
# dummy runs to cause out-of-bounds indexing during capture.
@@ -97,12 +92,19 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
# For FULL graphs, the entire routine is recorded as one graph.
# For PIECEWISE, only the model's compiled regions are captured
# and the rest (compute_logits, gumbel_sample) runs eagerly.
# Draft prefill reuses the target model's attention metadata at
# runtime, so capture builds its dummy metadata through the target
# model runner's builders and buffers.
assert self.prefill_cudagraph_manager is not None
if self.prefill_cudagraph_manager.use_breakable_cg:
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
self.prefill_cudagraph_manager.capture(
self._prefill,
attn_states,
self.model_state,
self.target_input_buffers,
self.block_tables,
self.target_attn_groups,
self.kv_cache_config,
progress_bar_desc="Capturing prefill CUDA graphs",
)
@@ -77,7 +77,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
def create_forward_fn(
desc: BatchExecutionDescriptor,
warmup: bool,
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
) -> Callable[[CUDAGraphMode], None]:
num_tokens = desc.num_tokens
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
num_tokens_across_dp = (
@@ -98,7 +98,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
)
attn_metadata, slot_mappings = attn_state
fwd = lambda cg_mode: forward_fn(
return lambda cg_mode: forward_fn(
num_reqs,
num_tokens,
attn_metadata,
@@ -106,6 +106,5 @@ class DFlashCudaGraphManager(CudaGraphManager):
num_tokens_across_dp,
cg_mode,
)
return fwd, attn_state
super().capture(create_forward_fn, progress_bar_desc)
@@ -25,6 +25,7 @@ from vllm.v1.worker.gpu.spec_decode.dflash.utils import (
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.utils import get_parallel_drafting_token_id
from vllm.v1.worker.utils import AttentionGroup
logger = init_logger(__name__)
@@ -97,7 +98,7 @@ class DFlashSpeculator(DraftModelSpeculator):
decode_query_len=self.num_query_per_req,
)
def capture(self, attn_states: dict | None = None) -> None:
def capture(self) -> None:
logger.info("Capturing model for %s speculator...", self._speculator_name)
# Reset sampling indices to zero to prevent stale values from prior
# dummy runs from being baked into the captured graph.
@@ -128,8 +129,16 @@ class DFlashSpeculator(DraftModelSpeculator):
model_state: ModelState,
kv_cache_config: KVCacheConfig,
block_tables: BlockTables,
target_input_buffers: InputBuffers,
target_attn_groups: list[list[AttentionGroup]],
) -> None:
super().set_attn(model_state, kv_cache_config, block_tables)
super().set_attn(
model_state,
kv_cache_config,
block_tables,
target_input_buffers,
target_attn_groups,
)
self.draft_kv_cache_group_ids = [
gid for gid, g in enumerate(self.attn_groups) if g
+10 -8
View File
@@ -18,13 +18,10 @@ from vllm.v1.worker.gpu.attn_utils import (
init_attn_backend,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cudagraph_utils import (
AttentionStatePair,
BatchExecutionDescriptor,
)
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.utils import AttentionGroup
logger = init_logger(__name__)
@@ -35,10 +32,7 @@ class BaseSpeculator(ABC):
pass
@abstractmethod
def capture(
self,
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
) -> None:
def capture(self) -> None:
pass
@abstractmethod
@@ -186,6 +180,8 @@ class DraftModelSpeculator(BaseSpeculator):
model_state: ModelState,
kv_cache_config: KVCacheConfig,
block_tables: BlockTables,
target_input_buffers: InputBuffers,
target_attn_groups: list[list[AttentionGroup]],
) -> None:
self.model_state = model_state
self.kv_cache_config = kv_cache_config
@@ -196,6 +192,12 @@ class DraftModelSpeculator(BaseSpeculator):
active_layer_names=self.draft_attn_layer_names,
)
self.block_tables = block_tables
# The target model runner's buffers and attention groups. Draft
# prefill reuses the target model's attention metadata, so its
# cudagraph capture must build dummy metadata through the same
# builders and buffers.
self.target_input_buffers = target_input_buffers
self.target_attn_groups = target_attn_groups
def _build_draft_attn_metadata(
self,