[BugFix][ModelRunner V2] Fix stale attn metadata in speculator prefill cudagraph capture (#48261)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -44,11 +44,6 @@ class AttentionState(NamedTuple):
|
||||
slot_mappings: dict[str, torch.Tensor]
|
||||
|
||||
|
||||
class AttentionStatePair(NamedTuple):
|
||||
warmup: AttentionState
|
||||
captured: AttentionState
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BatchExecutionDescriptor:
|
||||
"""Describes the shape of the batch and CG mode to run; this is used to make shape
|
||||
@@ -62,15 +57,15 @@ class BatchExecutionDescriptor:
|
||||
|
||||
|
||||
class CreateForwardFn(Protocol):
|
||||
"""Factory that prepares inputs (OUTSIDE the graph) and returns a tuple of
|
||||
(forward_fn, attn_state). Called with warmup=True for the warmup pass and
|
||||
warmup=False for the captured pass."""
|
||||
"""Factory that prepares inputs (OUTSIDE the graph) and returns a
|
||||
forward_fn. Called with warmup=True for the warmup pass and warmup=False
|
||||
for the captured pass."""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
desc: BatchExecutionDescriptor,
|
||||
warmup: bool,
|
||||
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: ...
|
||||
) -> Callable[[CUDAGraphMode], None]: ...
|
||||
|
||||
|
||||
def _is_compatible(
|
||||
@@ -301,19 +296,18 @@ class CudaGraphManager:
|
||||
self,
|
||||
create_forward_fn: CreateForwardFn,
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> dict[BatchExecutionDescriptor, AttentionStatePair]:
|
||||
) -> None:
|
||||
"""Capture CUDA graphs.
|
||||
|
||||
Args:
|
||||
create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and
|
||||
returns a tuple of (forward_fn, attn_state). For FULL cudagraph
|
||||
mode, it is invoked once with warmup=True for the warmup pass,
|
||||
and again with warmup=False for the captured pass. For attention
|
||||
backends that perform lazy metadata initialization (e.g. FlashMLA),
|
||||
FULL cudagraph capture requires distinct metadatas for warmup and
|
||||
capture.
|
||||
returns a forward_fn. For FULL cudagraph mode, it is invoked
|
||||
once with warmup=True for the warmup pass, and again with
|
||||
warmup=False for the captured pass. For attention backends
|
||||
that perform lazy metadata initialization (e.g. FlashMLA),
|
||||
FULL cudagraph capture requires distinct metadatas for warmup
|
||||
and capture.
|
||||
"""
|
||||
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair] = {}
|
||||
with graph_capture(device=self.device):
|
||||
# Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger
|
||||
# activations so FULL activations should fit in already allocated
|
||||
@@ -327,7 +321,7 @@ class CudaGraphManager:
|
||||
descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})")
|
||||
for desc in descs:
|
||||
# Prepare inputs and get forward function
|
||||
forward_fn, warmup_attn_state = create_forward_fn(desc, warmup=True)
|
||||
forward_fn = create_forward_fn(desc, warmup=True)
|
||||
|
||||
# Warmup
|
||||
forward_fn(CUDAGraphMode.NONE)
|
||||
@@ -337,18 +331,10 @@ class CudaGraphManager:
|
||||
"CG Capture: mode=%s, batch_desc=%s", desc.cg_mode.name, desc
|
||||
)
|
||||
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
attn_states[desc] = AttentionStatePair(
|
||||
warmup_attn_state, warmup_attn_state
|
||||
)
|
||||
forward_fn(CUDAGraphMode.PIECEWISE)
|
||||
else:
|
||||
# Capture with fresh attention state.
|
||||
forward_fn, capture_attn_state = create_forward_fn(
|
||||
desc, warmup=False
|
||||
)
|
||||
attn_states[desc] = AttentionStatePair(
|
||||
warmup_attn_state, capture_attn_state
|
||||
)
|
||||
forward_fn = create_forward_fn(desc, warmup=False)
|
||||
assert desc not in self.graphs, (
|
||||
f"Graph already captured for {desc}"
|
||||
)
|
||||
@@ -366,7 +352,6 @@ class CudaGraphManager:
|
||||
self.graphs[desc] = graph
|
||||
compilation_counter.num_cudagraph_captured += 1
|
||||
self._graphs_captured = True
|
||||
return attn_states
|
||||
|
||||
def dispatch(
|
||||
self,
|
||||
@@ -461,7 +446,7 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
use_aux_hidden_state_outputs: bool = False,
|
||||
lora_capture_hook: Callable[[int, int, int], None] | None = None,
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> dict[BatchExecutionDescriptor, AttentionStatePair]:
|
||||
) -> None:
|
||||
"""Capture CUDA graphs for model forward pass."""
|
||||
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
|
||||
if self.use_breakable_cg:
|
||||
@@ -470,10 +455,7 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
warmup: bool,
|
||||
) -> tuple[
|
||||
Callable[[CUDAGraphMode], None],
|
||||
AttentionState,
|
||||
]:
|
||||
) -> Callable[[CUDAGraphMode], None]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
|
||||
@@ -571,9 +553,9 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
for k, v in intermediate_tensors.tensors.items():
|
||||
self.intermediate_tensors[k][:num_tokens] = v
|
||||
|
||||
return forward_fn, AttentionState(attn_metadata, slot_mappings)
|
||||
return forward_fn
|
||||
|
||||
return super().capture(create_forward_fn, progress_bar_desc)
|
||||
super().capture(create_forward_fn, progress_bar_desc)
|
||||
|
||||
def run_fullgraph(
|
||||
self, desc: BatchExecutionDescriptor
|
||||
|
||||
@@ -479,7 +479,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
# HACK(woosuk)
|
||||
self.speculator.set_attn(
|
||||
self.model_state, self.kv_cache_config, self.block_tables
|
||||
self.model_state,
|
||||
self.kv_cache_config,
|
||||
self.block_tables,
|
||||
self.input_buffers,
|
||||
self.attn_groups,
|
||||
)
|
||||
|
||||
self.kv_caches: list[torch.Tensor] = []
|
||||
@@ -706,7 +710,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
with self.maybe_setup_dummy_loras(self.lora_config):
|
||||
attn_states = self.cudagraph_manager.capture(
|
||||
self.cudagraph_manager.capture(
|
||||
self.model,
|
||||
self.model_state,
|
||||
self.input_buffers,
|
||||
@@ -719,7 +723,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.capture(attn_states)
|
||||
self.speculator.capture()
|
||||
|
||||
end_time = time.perf_counter()
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
@@ -8,8 +8,6 @@ from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionState,
|
||||
AttentionStatePair,
|
||||
BatchExecutionDescriptor,
|
||||
CudaGraphManager,
|
||||
prepare_inputs_to_capture,
|
||||
@@ -19,45 +17,15 @@ from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
class PrefillSpeculatorCudaGraphManager(CudaGraphManager):
|
||||
"""CudaGraphManager for draft prefill, using pre-built attention states
|
||||
from the target model's capture."""
|
||||
class SpeculatorCudaGraphManager(CudaGraphManager):
|
||||
"""CudaGraphManager for draft prefill and decode.
|
||||
|
||||
def capture(
|
||||
self,
|
||||
forward_fn: Callable,
|
||||
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> None:
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
warmup: bool,
|
||||
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
num_tokens_across_dp = (
|
||||
torch.full((self.dp_size,), num_tokens, dtype=torch.int32, device="cpu")
|
||||
if self.dp_size > 1
|
||||
else None
|
||||
)
|
||||
attn_state_pair = attn_states[desc]
|
||||
attn_state = attn_state_pair.warmup if warmup else attn_state_pair.captured
|
||||
attn_metadata, slot_mappings = attn_state
|
||||
fwd = lambda cg_mode: forward_fn(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
cg_mode,
|
||||
)
|
||||
return fwd, attn_state
|
||||
|
||||
super().capture(create_forward_fn, progress_bar_desc)
|
||||
|
||||
|
||||
class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
|
||||
"""CudaGraphManager for draft decode, building its own attention metadata."""
|
||||
Builds fresh dummy inputs and attention metadata for every warmup and
|
||||
capture pass so that the contents of the shared persistent buffers
|
||||
(e.g. query_start_loc, seq_lens, FA3 scheduler metadata) always match
|
||||
the batch descriptor being captured. Reusing metadata built during an
|
||||
earlier capture would execute kernels with stale buffer contents.
|
||||
"""
|
||||
|
||||
def capture(
|
||||
self,
|
||||
@@ -72,7 +40,7 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
warmup: bool,
|
||||
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
|
||||
) -> Callable[[CUDAGraphMode], None]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
num_tokens_across_dp = (
|
||||
@@ -80,7 +48,7 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
|
||||
if self.dp_size > 1
|
||||
else None
|
||||
)
|
||||
attn_state = prepare_inputs_to_capture(
|
||||
attn_metadata, slot_mappings = prepare_inputs_to_capture(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
model_state,
|
||||
@@ -90,9 +58,8 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
|
||||
kv_cache_config,
|
||||
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
||||
)
|
||||
attn_metadata, slot_mappings = attn_state
|
||||
|
||||
fwd = lambda cg_mode: forward_fn(
|
||||
return lambda cg_mode: forward_fn(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
attn_metadata,
|
||||
@@ -100,6 +67,5 @@ class DecodeSpeculatorCudaGraphManager(CudaGraphManager):
|
||||
num_tokens_across_dp,
|
||||
cg_mode,
|
||||
)
|
||||
return fwd, attn_state
|
||||
|
||||
super().capture(create_forward_fn, progress_bar_desc)
|
||||
|
||||
@@ -12,15 +12,13 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionStatePair,
|
||||
BatchExecutionDescriptor,
|
||||
get_uniform_token_count,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
|
||||
DecodeSpeculatorCudaGraphManager,
|
||||
PrefillSpeculatorCudaGraphManager,
|
||||
SpeculatorCudaGraphManager,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
|
||||
@@ -47,8 +45,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
self.max_num_tokens, self.hidden_size, dtype=self.dtype, device=device
|
||||
)
|
||||
|
||||
self.prefill_cudagraph_manager: PrefillSpeculatorCudaGraphManager | None = None
|
||||
self.decode_cudagraph_manager: DecodeSpeculatorCudaGraphManager | None = None
|
||||
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
|
||||
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
|
||||
|
||||
@property
|
||||
def advance_draft_positions(self) -> bool:
|
||||
@@ -62,7 +60,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
# Initialize cudagraph manager for draft prefill (draft position 0).
|
||||
self.prefill_cudagraph_manager = PrefillSpeculatorCudaGraphManager(
|
||||
self.prefill_cudagraph_manager = SpeculatorCudaGraphManager(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
@@ -76,17 +74,14 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
# Initialize cudagraph manager for draft decodes (draft positions > 0).
|
||||
self.decode_cudagraph_manager = DecodeSpeculatorCudaGraphManager(
|
||||
self.decode_cudagraph_manager = SpeculatorCudaGraphManager(
|
||||
self.vllm_config,
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
decode_query_len=1,
|
||||
)
|
||||
|
||||
def capture(
|
||||
self,
|
||||
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
|
||||
) -> None:
|
||||
def capture(self) -> None:
|
||||
logger.info("Capturing model for speculator...")
|
||||
# Reset indices to zeros to prevent stale values from prior
|
||||
# dummy runs to cause out-of-bounds indexing during capture.
|
||||
@@ -97,12 +92,19 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
# For FULL graphs, the entire routine is recorded as one graph.
|
||||
# For PIECEWISE, only the model's compiled regions are captured
|
||||
# and the rest (compute_logits, gumbel_sample) runs eagerly.
|
||||
# Draft prefill reuses the target model's attention metadata at
|
||||
# runtime, so capture builds its dummy metadata through the target
|
||||
# model runner's builders and buffers.
|
||||
assert self.prefill_cudagraph_manager is not None
|
||||
if self.prefill_cudagraph_manager.use_breakable_cg:
|
||||
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
|
||||
self.prefill_cudagraph_manager.capture(
|
||||
self._prefill,
|
||||
attn_states,
|
||||
self.model_state,
|
||||
self.target_input_buffers,
|
||||
self.block_tables,
|
||||
self.target_attn_groups,
|
||||
self.kv_cache_config,
|
||||
progress_bar_desc="Capturing prefill CUDA graphs",
|
||||
)
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
warmup: bool,
|
||||
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
|
||||
) -> Callable[[CUDAGraphMode], None]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
num_tokens_across_dp = (
|
||||
@@ -98,7 +98,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
)
|
||||
attn_metadata, slot_mappings = attn_state
|
||||
|
||||
fwd = lambda cg_mode: forward_fn(
|
||||
return lambda cg_mode: forward_fn(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
attn_metadata,
|
||||
@@ -106,6 +106,5 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
num_tokens_across_dp,
|
||||
cg_mode,
|
||||
)
|
||||
return fwd, attn_state
|
||||
|
||||
super().capture(create_forward_fn, progress_bar_desc)
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.v1.worker.gpu.spec_decode.dflash.utils import (
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_parallel_drafting_token_id
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -97,7 +98,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
decode_query_len=self.num_query_per_req,
|
||||
)
|
||||
|
||||
def capture(self, attn_states: dict | None = None) -> None:
|
||||
def capture(self) -> None:
|
||||
logger.info("Capturing model for %s speculator...", self._speculator_name)
|
||||
# Reset sampling indices to zero to prevent stale values from prior
|
||||
# dummy runs from being baked into the captured graph.
|
||||
@@ -128,8 +129,16 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
model_state: ModelState,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
block_tables: BlockTables,
|
||||
target_input_buffers: InputBuffers,
|
||||
target_attn_groups: list[list[AttentionGroup]],
|
||||
) -> None:
|
||||
super().set_attn(model_state, kv_cache_config, block_tables)
|
||||
super().set_attn(
|
||||
model_state,
|
||||
kv_cache_config,
|
||||
block_tables,
|
||||
target_input_buffers,
|
||||
target_attn_groups,
|
||||
)
|
||||
|
||||
self.draft_kv_cache_group_ids = [
|
||||
gid for gid, g in enumerate(self.attn_groups) if g
|
||||
|
||||
@@ -18,13 +18,10 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
init_attn_backend,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionStatePair,
|
||||
BatchExecutionDescriptor,
|
||||
)
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -35,10 +32,7 @@ class BaseSpeculator(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def capture(
|
||||
self,
|
||||
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
|
||||
) -> None:
|
||||
def capture(self) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
@@ -186,6 +180,8 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
model_state: ModelState,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
block_tables: BlockTables,
|
||||
target_input_buffers: InputBuffers,
|
||||
target_attn_groups: list[list[AttentionGroup]],
|
||||
) -> None:
|
||||
self.model_state = model_state
|
||||
self.kv_cache_config = kv_cache_config
|
||||
@@ -196,6 +192,12 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
active_layer_names=self.draft_attn_layer_names,
|
||||
)
|
||||
self.block_tables = block_tables
|
||||
# The target model runner's buffers and attention groups. Draft
|
||||
# prefill reuses the target model's attention metadata, so its
|
||||
# cudagraph capture must build dummy metadata through the same
|
||||
# builders and buffers.
|
||||
self.target_input_buffers = target_input_buffers
|
||||
self.target_attn_groups = target_attn_groups
|
||||
|
||||
def _build_draft_attn_metadata(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user