[Model Runner V2] Skip attention metadata rebuild before draft prefill (#40410)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -32,6 +32,11 @@ from vllm.v1.worker.utils import AttentionGroup
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CapturedAttentionState(NamedTuple):
|
||||
attn_metadata: dict[str, Any]
|
||||
slot_mappings: dict[str, torch.Tensor]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BatchExecutionDescriptor:
|
||||
"""Describes the shape of the batch and CG mode to run; this is used to make shape
|
||||
@@ -177,16 +182,20 @@ class CudaGraphManager:
|
||||
def capture(
|
||||
self,
|
||||
create_forward_fn: Callable[
|
||||
[BatchExecutionDescriptor], Callable[[CUDAGraphMode], None]
|
||||
[BatchExecutionDescriptor],
|
||||
tuple[Callable[[CUDAGraphMode], None], CapturedAttentionState],
|
||||
],
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> None:
|
||||
) -> dict[BatchExecutionDescriptor, CapturedAttentionState]:
|
||||
"""Capture CUDA graphs.
|
||||
|
||||
Args:
|
||||
create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and
|
||||
returns a function that runs forward with a given CUDAGraphMode.
|
||||
returns a tuple of (forward_fn, captured_attn_state).
|
||||
"""
|
||||
captured_attn_states: dict[
|
||||
BatchExecutionDescriptor, CapturedAttentionState
|
||||
] = {}
|
||||
with graph_capture(device=self.device):
|
||||
# Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger
|
||||
# activations so FULL activations should fit in already allocated
|
||||
@@ -200,7 +209,8 @@ class CudaGraphManager:
|
||||
descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})")
|
||||
for desc in descs:
|
||||
# Prepare inputs and get forward function
|
||||
forward_fn = create_forward_fn(desc)
|
||||
forward_fn, attn_state = create_forward_fn(desc)
|
||||
captured_attn_states[desc] = attn_state
|
||||
|
||||
# Warmup
|
||||
forward_fn(CUDAGraphMode.NONE)
|
||||
@@ -228,6 +238,7 @@ class CudaGraphManager:
|
||||
get_offloader().join_after_forward()
|
||||
self.graphs[desc] = graph
|
||||
self._graphs_captured = True
|
||||
return captured_attn_states
|
||||
|
||||
def dispatch(
|
||||
self,
|
||||
@@ -289,13 +300,16 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
has_lora: bool = False,
|
||||
use_aux_hidden_state_outputs: bool = False,
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> None:
|
||||
) -> dict[BatchExecutionDescriptor, CapturedAttentionState]:
|
||||
"""Capture CUDA graphs for model forward pass."""
|
||||
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
|
||||
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
) -> Callable[[CUDAGraphMode], None]:
|
||||
) -> tuple[
|
||||
Callable[[CUDAGraphMode], None],
|
||||
CapturedAttentionState,
|
||||
]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
num_tokens_across_dp = (
|
||||
@@ -375,9 +389,9 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
for k, v in intermediate_tensors.tensors.items():
|
||||
self.intermediate_tensors[k][:num_tokens] = v
|
||||
|
||||
return forward_fn
|
||||
return forward_fn, CapturedAttentionState(attn_metadata, slot_mappings)
|
||||
|
||||
super().capture(create_forward_fn, progress_bar_desc)
|
||||
return super().capture(create_forward_fn, progress_bar_desc)
|
||||
|
||||
def run_fullgraph(
|
||||
self, desc: BatchExecutionDescriptor
|
||||
@@ -403,7 +417,7 @@ def prepare_inputs_to_capture(
|
||||
block_tables: BlockTables,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> tuple[dict[str, Any], dict[str, torch.Tensor]]:
|
||||
) -> CapturedAttentionState:
|
||||
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
|
||||
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
|
||||
slot_mappings = block_tables.get_dummy_slot_mappings(num_tokens)
|
||||
@@ -432,4 +446,4 @@ def prepare_inputs_to_capture(
|
||||
kv_cache_config,
|
||||
for_capture=True,
|
||||
)
|
||||
return attn_metadata, slot_mappings_by_layer
|
||||
return CapturedAttentionState(attn_metadata, slot_mappings_by_layer)
|
||||
|
||||
Reference in New Issue
Block a user