[MRV2] Extensible CG dispatch rework (#35959)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -1,181 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
capture_graphs,
|
||||
get_cudagraph_sizes,
|
||||
BatchExecutionDescriptor,
|
||||
CudaGraphManager,
|
||||
prepare_inputs_to_capture,
|
||||
)
|
||||
from vllm.v1.worker.gpu.dp_utils import make_num_tokens_across_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
class EagleCudaGraphManager:
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
self.vllm_config = vllm_config
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
self.device = device
|
||||
class EagleCudaGraphManager(CudaGraphManager):
|
||||
"""CudaGraphManager for Eagle speculative decoding (FULL mode only)."""
|
||||
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
self.max_num_reqs = self.scheduler_config.max_num_seqs
|
||||
self.max_num_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
self.dp_size = vllm_config.parallel_config.data_parallel_size
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
assert self.compilation_config is not None
|
||||
|
||||
# NOTE(woosuk): For Eagle, we only use CUDA graphs for decode.
|
||||
self.cudagraph_mode = self.compilation_config.cudagraph_mode.decode_mode()
|
||||
|
||||
# only need to capture uniform decode cudagraph sizes (the 2nd return value)
|
||||
_, self.cudagraph_sizes = get_cudagraph_sizes(
|
||||
self.compilation_config.cudagraph_capture_sizes,
|
||||
self.max_num_reqs,
|
||||
self.max_num_tokens,
|
||||
self.cudagraph_mode,
|
||||
uniform_decode_query_len=1,
|
||||
uniform_decode_cudagraph=True,
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
draft_tokens: torch.Tensor,
|
||||
):
|
||||
assert not cudagraph_mode.has_mode(CUDAGraphMode.PIECEWISE), (
|
||||
"EagleCudaGraphManager does not support PIECEWISE mode yet"
|
||||
)
|
||||
# Eagle always uses uniform decode with query_len=1
|
||||
super().__init__(vllm_config, device, cudagraph_mode, decode_query_len=1)
|
||||
self.draft_tokens = draft_tokens
|
||||
|
||||
self.graphs: dict[int, torch.cuda.CUDAGraph] = {}
|
||||
self.pool = None
|
||||
if self.cudagraph_mode != CUDAGraphMode.NONE:
|
||||
# Use a dedicated pool for Eagle to avoid memory overlap with the main
|
||||
# model's cudagraph. The base class uses a shared global pool, but Eagle's
|
||||
# internal allocations (e.g., gumbel_sample temporaries) can conflict with
|
||||
# the main model's allocations when sharing the same pool.
|
||||
if cudagraph_mode:
|
||||
self.pool = torch.cuda.graph_pool_handle()
|
||||
|
||||
def get_cudagraph_size(self, num_tokens: int) -> int | None:
|
||||
return self.cudagraph_sizes.get(num_tokens)
|
||||
|
||||
def get_cudagraph_runtime_mode(
|
||||
self, num_tokens: int
|
||||
) -> tuple[CUDAGraphMode, int | None]:
|
||||
cudagraph_size = self.get_cudagraph_size(num_tokens)
|
||||
if cudagraph_size is None:
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
else:
|
||||
cudagraph_mode = self.cudagraph_mode
|
||||
|
||||
if (
|
||||
cudagraph_mode == CUDAGraphMode.FULL
|
||||
and cudagraph_size is not None
|
||||
and cudagraph_size not in self.graphs
|
||||
):
|
||||
# If graph wasn't captured yet, fall back to eager.
|
||||
# This might happen when the dummy run is called before capture.
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
cudagraph_size = None
|
||||
return cudagraph_mode, cudagraph_size
|
||||
|
||||
def capture_graph(
|
||||
self,
|
||||
num_tokens: int,
|
||||
capture_cg_mode: CUDAGraphMode,
|
||||
generate_fn: Callable,
|
||||
model_state: ModelState,
|
||||
input_buffers: InputBuffers,
|
||||
block_tables: BlockTables,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
assert capture_cg_mode in [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL], (
|
||||
f"Invalid capture_cudagraph_mode for capture: {capture_cg_mode}"
|
||||
)
|
||||
if capture_cg_mode == CUDAGraphMode.PIECEWISE:
|
||||
capture_fn = self._capture_piecewise_graph
|
||||
else:
|
||||
capture_fn = self._capture_full_graph
|
||||
|
||||
num_reqs = min(num_tokens, self.max_num_reqs)
|
||||
attn_metadata, slot_mappings = prepare_inputs_to_capture(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
model_state,
|
||||
input_buffers,
|
||||
block_tables,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
)
|
||||
num_tokens_across_dp = make_num_tokens_across_dp(self.dp_size, num_tokens)
|
||||
|
||||
# Warm up.
|
||||
generate_fn(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
CUDAGraphMode.NONE,
|
||||
)
|
||||
|
||||
# Capture the graph.
|
||||
capture_fn(
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
generate_fn=generate_fn,
|
||||
attn_metadata=attn_metadata,
|
||||
slot_mappings=slot_mappings,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
)
|
||||
|
||||
def _capture_full_graph(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
generate_fn: Callable,
|
||||
attn_metadata: dict[str, Any],
|
||||
slot_mappings: dict[str, torch.Tensor],
|
||||
num_tokens_across_dp: torch.Tensor,
|
||||
) -> None:
|
||||
assert num_tokens not in self.graphs
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
|
||||
# Sync offloader's copy stream before capture.
|
||||
# Ensure any pre-capture prefetches from offloader are complete.
|
||||
get_offloader().sync_prev_onload()
|
||||
|
||||
with torch.cuda.graph(graph, self.pool):
|
||||
generate_fn(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
CUDAGraphMode.NONE,
|
||||
)
|
||||
# Join offloader's copy stream after forward to avoid unjoined
|
||||
# stream error. The last layer's start_prefetch forks copy_stream,
|
||||
# but wait_prefetch only happens in the next forward pass.
|
||||
get_offloader().join_after_forward()
|
||||
self.graphs[num_tokens] = graph
|
||||
|
||||
def _capture_piecewise_graph(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
generate_fn: Callable,
|
||||
attn_metadata: dict[str, Any],
|
||||
slot_mappings: dict[str, torch.Tensor],
|
||||
num_tokens_across_dp: torch.Tensor,
|
||||
) -> None:
|
||||
generate_fn(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
CUDAGraphMode.PIECEWISE,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def capture(
|
||||
self,
|
||||
generate_fn: Callable,
|
||||
@@ -184,31 +50,42 @@ class EagleCudaGraphManager:
|
||||
block_tables: BlockTables,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> None:
|
||||
if self.cudagraph_mode == CUDAGraphMode.NONE:
|
||||
return
|
||||
"""Capture CUDA graphs for Eagle speculative decoding (FULL mode only)."""
|
||||
|
||||
capture_graphs(
|
||||
self.cudagraph_sizes,
|
||||
self.device,
|
||||
self.capture_graph,
|
||||
capture_cudagraph_mode=self.cudagraph_mode,
|
||||
desc=f"Capturing eagle CUDA graphs ({self.cudagraph_mode.name})",
|
||||
generate_fn=generate_fn,
|
||||
model_state=model_state,
|
||||
input_buffers=input_buffers,
|
||||
block_tables=block_tables,
|
||||
attn_groups=attn_groups,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
def create_forward_fn(
|
||||
desc: BatchExecutionDescriptor,
|
||||
) -> Callable[[CUDAGraphMode], None]:
|
||||
num_tokens = desc.num_tokens
|
||||
num_reqs = desc.num_reqs or min(num_tokens, self.max_num_reqs)
|
||||
num_tokens_across_dp = (
|
||||
torch.full((self.dp_size,), num_tokens, dtype=torch.int32, device="cpu")
|
||||
if self.dp_size > 1
|
||||
else None
|
||||
)
|
||||
attn_metadata, slot_mappings = prepare_inputs_to_capture(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
model_state,
|
||||
input_buffers,
|
||||
block_tables,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
)
|
||||
|
||||
def run_fullgraph(self, num_tokens: int) -> None:
|
||||
assert num_tokens in self.graphs
|
||||
# Sync offloader before replay - needed when transitioning from
|
||||
# eager/piecewise to full cudagraph (e.g., prefill → decode).
|
||||
# The previous eager iteration's start_prefetch may have queued
|
||||
# H2D copies on copy_stream that the graph's captured events
|
||||
# cannot see. Without this, replay could overwrite static buffers
|
||||
# while those copies are still in flight.
|
||||
get_offloader().sync_prev_onload()
|
||||
self.graphs[num_tokens].replay()
|
||||
return lambda cg_mode: generate_fn(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
attn_metadata,
|
||||
slot_mappings,
|
||||
num_tokens_across_dp,
|
||||
cg_mode,
|
||||
)
|
||||
|
||||
super().capture(create_forward_fn, progress_bar_desc)
|
||||
|
||||
def run_fullgraph(self, desc: BatchExecutionDescriptor) -> torch.Tensor:
|
||||
"""Replay a captured FULL cudagraph and return draft tokens."""
|
||||
super().run_fullgraph(desc)
|
||||
return self.draft_tokens
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_slot_mappings_by_layer,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.dp_utils import get_cudagraph_and_dp_padding
|
||||
from vllm.v1.worker.gpu.dp_utils import sync_cudagraph_and_dp_padding
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
@@ -75,7 +75,16 @@ class EagleSpeculator:
|
||||
device=device,
|
||||
)
|
||||
|
||||
self.cudagraph_manager = EagleCudaGraphManager(vllm_config, device)
|
||||
# currently we don't support PIECEWISE for Eagle.
|
||||
cudagraph_mode = vllm_config.compilation_config.cudagraph_mode
|
||||
if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL:
|
||||
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
||||
else:
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
self.cudagraph_manager = EagleCudaGraphManager(
|
||||
vllm_config, device, cudagraph_mode, self.draft_tokens
|
||||
)
|
||||
|
||||
def load_model(self, target_model: nn.Module) -> None:
|
||||
self.model = load_eagle_model(target_model, self.vllm_config)
|
||||
@@ -171,7 +180,7 @@ class EagleSpeculator:
|
||||
)
|
||||
if attn_metadata is not None:
|
||||
self.block_tables.compute_slot_mappings(
|
||||
idx_mapping, query_start_loc, pos
|
||||
idx_mapping, query_start_loc, pos, num_tokens_padded
|
||||
)
|
||||
|
||||
def capture_model(self) -> None:
|
||||
@@ -185,6 +194,7 @@ class EagleSpeculator:
|
||||
self.block_tables,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
progress_bar_desc="Capturing eagle CUDA graphs",
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -251,6 +261,7 @@ class EagleSpeculator:
|
||||
logits = self.model.compute_logits(sample_hidden_states)
|
||||
|
||||
num_reqs = input_batch.num_reqs
|
||||
num_reqs_padded = input_batch.num_reqs_after_padding
|
||||
# NOTE(woosuk): For draft sampling, we only consider the temperature
|
||||
# and ignore the other sampling parameters such as top_k and top_p,
|
||||
# for simplicity and performance.
|
||||
@@ -292,48 +303,52 @@ class EagleSpeculator:
|
||||
self.max_num_reqs,
|
||||
)
|
||||
|
||||
if not (dummy_run and skip_attn_for_dummy_run):
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
||||
slot_mappings = self.block_tables.compute_slot_mappings(
|
||||
idx_mapping, query_start_loc, pos
|
||||
)
|
||||
# Get batch descriptor and sync across DP ranks.
|
||||
# Eagle uses FULL-only mode, dispatch with uniform_token_count=1 for decode
|
||||
|
||||
cudagraph_mode, cudagraph_size = (
|
||||
self.cudagraph_manager.get_cudagraph_runtime_mode(num_reqs)
|
||||
)
|
||||
num_tokens_padded, num_tokens_across_dp, synced_cudagraph_mode = (
|
||||
get_cudagraph_and_dp_padding(
|
||||
batch_desc = self.cudagraph_manager.dispatch(num_reqs, num_reqs, 1)
|
||||
num_tokens_across_dp = None
|
||||
|
||||
if self.dp_size > 1:
|
||||
batch_desc, num_tokens_across_dp = sync_cudagraph_and_dp_padding(
|
||||
self.cudagraph_manager,
|
||||
batch_desc,
|
||||
num_reqs,
|
||||
cudagraph_size,
|
||||
cudagraph_mode.value,
|
||||
num_reqs,
|
||||
1, # uniform_token_count
|
||||
self.dp_size,
|
||||
self.dp_rank,
|
||||
)
|
||||
)
|
||||
cudagraph_mode = CUDAGraphMode(synced_cudagraph_mode)
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
# Run full CUDA graph.
|
||||
self.cudagraph_manager.run_fullgraph(num_tokens_padded)
|
||||
return self.draft_tokens[:num_reqs]
|
||||
|
||||
if not (dummy_run and skip_attn_for_dummy_run):
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs + 1]
|
||||
slot_mappings = self.block_tables.compute_slot_mappings(
|
||||
idx_mapping, query_start_loc, pos, batch_desc.num_tokens
|
||||
)
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
return self.cudagraph_manager.run_fullgraph(batch_desc)[:num_reqs]
|
||||
|
||||
# Run eager or piecewise CUDA graph.
|
||||
attn_metadata_updated = None
|
||||
slot_mappings_updated = None
|
||||
if not (dummy_run and skip_attn_for_dummy_run):
|
||||
query_start_loc_cpu = torch.arange(
|
||||
num_reqs + 1, dtype=torch.int32, device="cpu"
|
||||
num_reqs_padded + 1, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
block_tables = [x[:num_reqs] for x in self.block_tables.input_block_tables]
|
||||
block_tables = [
|
||||
x[:num_reqs_padded] for x in self.block_tables.input_block_tables
|
||||
]
|
||||
|
||||
# FIXME(woosuk): This is UNSAFE!!
|
||||
attn_metadata_updated = build_attn_metadata(
|
||||
attn_groups=self.attn_groups,
|
||||
num_reqs=num_reqs,
|
||||
num_tokens=num_reqs,
|
||||
num_reqs=num_reqs_padded,
|
||||
num_tokens=num_reqs_padded,
|
||||
query_start_loc_gpu=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
max_query_len=1,
|
||||
seq_lens=self.input_buffers.seq_lens[:num_reqs],
|
||||
seq_lens=self.input_buffers.seq_lens[:num_reqs_padded],
|
||||
max_seq_len=self.max_model_len,
|
||||
block_tables=block_tables,
|
||||
slot_mappings=slot_mappings,
|
||||
@@ -345,11 +360,11 @@ class EagleSpeculator:
|
||||
|
||||
self.generate_draft(
|
||||
num_reqs,
|
||||
num_tokens_padded,
|
||||
batch_desc.num_tokens,
|
||||
attn_metadata_updated,
|
||||
slot_mappings_updated,
|
||||
num_tokens_across_dp=num_tokens_across_dp,
|
||||
cudagraph_runtime_mode=cudagraph_mode,
|
||||
cudagraph_runtime_mode=batch_desc.cg_mode,
|
||||
)
|
||||
return self.draft_tokens[:num_reqs]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user