[Bug Fix][Model Runner V2][Spec Decode] Warmup & capture with different attention states for speculator prefill (#44253)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Woosuk Kwon
parent
2b237c7a41
commit
91945b6e4a
@@ -3,7 +3,7 @@
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, NamedTuple
|
||||
from typing import Any, NamedTuple, Protocol
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -37,11 +37,16 @@ from vllm.v1.worker.utils import AttentionGroup
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CapturedAttentionState(NamedTuple):
|
||||
class AttentionState(NamedTuple):
|
||||
attn_metadata: dict[str, Any] | None
|
||||
slot_mappings: dict[str, torch.Tensor]
|
||||
|
||||
|
||||
class AttentionStatePair(NamedTuple):
|
||||
warmup: AttentionState
|
||||
captured: AttentionState
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BatchExecutionDescriptor:
|
||||
"""Describes the shape of the batch and CG mode to run; this is used to make shape
|
||||
@@ -53,6 +58,18 @@ class BatchExecutionDescriptor:
|
||||
uniform_token_count: int | None = None
|
||||
|
||||
|
||||
class CreateForwardFn(Protocol):
|
||||
"""Factory that prepares inputs (OUTSIDE the graph) and returns a tuple of
|
||||
(forward_fn, attn_state). Called with warmup=True for the warmup pass and
|
||||
warmup=False for the captured pass."""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
desc: BatchExecutionDescriptor,
|
||||
warmup: bool,
|
||||
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]: ...
|
||||
|
||||
|
||||
def _is_compatible(
|
||||
desc: BatchExecutionDescriptor,
|
||||
num_reqs: int,
|
||||
@@ -198,21 +215,21 @@ class CudaGraphManager:
|
||||
@torch.inference_mode()
|
||||
def capture(
|
||||
self,
|
||||
create_forward_fn: Callable[
|
||||
[BatchExecutionDescriptor],
|
||||
tuple[Callable[[CUDAGraphMode], None], CapturedAttentionState],
|
||||
],
|
||||
create_forward_fn: CreateForwardFn,
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> dict[BatchExecutionDescriptor, CapturedAttentionState]:
|
||||
) -> dict[BatchExecutionDescriptor, AttentionStatePair]:
|
||||
"""Capture CUDA graphs.
|
||||
|
||||
Args:
|
||||
create_forward_fn: Factory that prepares inputs (OUTSIDE graph) and
|
||||
returns a tuple of (forward_fn, captured_attn_state).
|
||||
returns a tuple of (forward_fn, attn_state). For FULL cudagraph
|
||||
mode, it is invoked once with warmup=True for the warmup pass,
|
||||
and again with warmup=False for the captured pass. For attention
|
||||
backends that perform lazy metadata initialization (e.g. FlashMLA),
|
||||
FULL cudagraph capture requires distinct metadatas for warmup and
|
||||
capture.
|
||||
"""
|
||||
captured_attn_states: dict[
|
||||
BatchExecutionDescriptor, CapturedAttentionState
|
||||
] = {}
|
||||
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair] = {}
|
||||
with graph_capture(device=self.device):
|
||||
# Capture in order: PIECEWISE first, then FULL. PIECEWISE has larger
|
||||
# activations so FULL activations should fit in already allocated
|
||||
@@ -226,7 +243,7 @@ class CudaGraphManager:
|
||||
descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})")
|
||||
for desc in descs:
|
||||
# Prepare inputs and get forward function
|
||||
forward_fn, attn_state = create_forward_fn(desc)
|
||||
forward_fn, warmup_attn_state = create_forward_fn(desc, warmup=True)
|
||||
|
||||
# Warmup
|
||||
forward_fn(CUDAGraphMode.NONE)
|
||||
@@ -236,15 +253,18 @@ class CudaGraphManager:
|
||||
"CG Capture: mode=%s, batch_desc=%s", desc.cg_mode.name, desc
|
||||
)
|
||||
if desc.cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
captured_attn_states[desc] = attn_state
|
||||
attn_states[desc] = AttentionStatePair(
|
||||
warmup_attn_state, warmup_attn_state
|
||||
)
|
||||
forward_fn(CUDAGraphMode.PIECEWISE)
|
||||
else:
|
||||
# Capture with fresh attention state. The warmup
|
||||
# attention state is discarded because some backends
|
||||
# (e.g. FlashMLA) perform lazy initializations that
|
||||
# must be captured in the graph.
|
||||
forward_fn, attn_state = create_forward_fn(desc)
|
||||
captured_attn_states[desc] = attn_state
|
||||
# Capture with fresh attention state.
|
||||
forward_fn, capture_attn_state = create_forward_fn(
|
||||
desc, warmup=False
|
||||
)
|
||||
attn_states[desc] = AttentionStatePair(
|
||||
warmup_attn_state, capture_attn_state
|
||||
)
|
||||
assert desc not in self.graphs, (
|
||||
f"Graph already captured for {desc}"
|
||||
)
|
||||
@@ -262,7 +282,7 @@ class CudaGraphManager:
|
||||
self.graphs[desc] = graph
|
||||
compilation_counter.num_cudagraph_captured += 1
|
||||
self._graphs_captured = True
|
||||
return captured_attn_states
|
||||
return attn_states
|
||||
|
||||
def dispatch(
|
||||
self,
|
||||
@@ -337,7 +357,7 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
has_lora: bool = False,
|
||||
use_aux_hidden_state_outputs: bool = False,
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> dict[BatchExecutionDescriptor, CapturedAttentionState]:
|
||||
) -> dict[BatchExecutionDescriptor, AttentionStatePair]:
|
||||
"""Capture CUDA graphs for model forward pass."""
|
||||
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
|
||||
if self.use_breakable_cg:
|
||||
@@ -345,9 +365,10 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
warmup: bool,
|
||||
) -> tuple[
|
||||
Callable[[CUDAGraphMode], None],
|
||||
CapturedAttentionState,
|
||||
AttentionState,
|
||||
]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
@@ -435,7 +456,7 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
for k, v in intermediate_tensors.tensors.items():
|
||||
self.intermediate_tensors[k][:num_tokens] = v
|
||||
|
||||
return forward_fn, CapturedAttentionState(attn_metadata, slot_mappings)
|
||||
return forward_fn, AttentionState(attn_metadata, slot_mappings)
|
||||
|
||||
return super().capture(create_forward_fn, progress_bar_desc)
|
||||
|
||||
@@ -464,7 +485,7 @@ def prepare_inputs_to_capture(
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
skip_attn: bool = False,
|
||||
) -> CapturedAttentionState:
|
||||
) -> AttentionState:
|
||||
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
|
||||
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
|
||||
slot_mappings = block_tables.get_dummy_slot_mappings(num_tokens)
|
||||
@@ -495,4 +516,4 @@ def prepare_inputs_to_capture(
|
||||
kv_cache_config,
|
||||
for_capture=True,
|
||||
)
|
||||
return CapturedAttentionState(attn_metadata, slot_mappings_by_layer)
|
||||
return AttentionState(attn_metadata, slot_mappings_by_layer)
|
||||
|
||||
@@ -678,7 +678,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
start_free_gpu_memory = torch.cuda.mem_get_info()[0]
|
||||
|
||||
with self.maybe_setup_dummy_loras(self.lora_config):
|
||||
captured_attn_states = self.cudagraph_manager.capture(
|
||||
attn_states = self.cudagraph_manager.capture(
|
||||
self.model,
|
||||
self.model_state,
|
||||
self.input_buffers,
|
||||
@@ -690,7 +690,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.capture(captured_attn_states)
|
||||
self.speculator.capture(attn_states)
|
||||
|
||||
end_time = time.perf_counter()
|
||||
end_free_gpu_memory = torch.cuda.mem_get_info()[0]
|
||||
|
||||
@@ -8,8 +8,9 @@ from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionState,
|
||||
AttentionStatePair,
|
||||
BatchExecutionDescriptor,
|
||||
CapturedAttentionState,
|
||||
CudaGraphManager,
|
||||
prepare_inputs_to_capture,
|
||||
)
|
||||
@@ -25,12 +26,13 @@ class PrefillEagleCudaGraphManager(CudaGraphManager):
|
||||
def capture(
|
||||
self,
|
||||
forward_fn: Callable,
|
||||
full_cg_attn_states: dict[BatchExecutionDescriptor, CapturedAttentionState],
|
||||
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> None:
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
) -> tuple[Callable[[CUDAGraphMode], None], CapturedAttentionState]:
|
||||
warmup: bool,
|
||||
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
num_tokens_across_dp = (
|
||||
@@ -38,7 +40,8 @@ class PrefillEagleCudaGraphManager(CudaGraphManager):
|
||||
if self.dp_size > 1
|
||||
else None
|
||||
)
|
||||
attn_state = full_cg_attn_states[desc]
|
||||
attn_state_pair = attn_states[desc]
|
||||
attn_state = attn_state_pair.warmup if warmup else attn_state_pair.captured
|
||||
attn_metadata, slot_mappings = attn_state
|
||||
fwd = lambda cg_mode: forward_fn(
|
||||
num_reqs,
|
||||
@@ -69,7 +72,8 @@ class DecodeEagleCudaGraphManager(CudaGraphManager):
|
||||
) -> None:
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
) -> tuple[Callable[[CUDAGraphMode], None], CapturedAttentionState]:
|
||||
warmup: bool,
|
||||
) -> tuple[Callable[[CUDAGraphMode], None], AttentionState]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
num_tokens_across_dp = (
|
||||
|
||||
@@ -20,8 +20,8 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
AttentionStatePair,
|
||||
BatchExecutionDescriptor,
|
||||
CapturedAttentionState,
|
||||
get_uniform_token_count,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
@@ -426,7 +426,7 @@ class EagleSpeculator:
|
||||
|
||||
def capture(
|
||||
self,
|
||||
attn_states: dict[BatchExecutionDescriptor, CapturedAttentionState],
|
||||
attn_states: dict[BatchExecutionDescriptor, AttentionStatePair],
|
||||
) -> None:
|
||||
logger.info("Capturing model for Eagle speculator...")
|
||||
# Reset indices to zeros to prevent stale values from prior
|
||||
|
||||
Reference in New Issue
Block a user