[MRV2] Support breakable CUDA graph (#44050)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-05-30 09:40:52 -07:00
committed by GitHub
parent e1105064b2
commit 27fa5aa3b9
3 changed files with 59 additions and 5 deletions
+35 -3
View File
@@ -9,6 +9,10 @@ import torch
import torch.nn as nn
from tqdm import tqdm
from vllm.compilation.breakable_cudagraph import (
BreakableCUDAGraphWrapper,
is_breakable_cudagraph_enabled,
)
from vllm.compilation.counter import compilation_counter
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
@@ -116,6 +120,13 @@ class CudaGraphManager:
)
self._init_candidates()
# Breakable CUDA graph (PW CUDA graph without torch.compile)
self.use_breakable_cg = (
is_breakable_cudagraph_enabled()
and self.cudagraph_mode.has_piecewise_cudagraphs()
)
self.breakable_cg_runner: BreakableCUDAGraphWrapper | None = None
def _init_candidates(self) -> None:
"""Build priority-ordered candidate lists for each token count."""
capture_sizes = self.compilation_config.cudagraph_capture_sizes
@@ -283,6 +294,20 @@ class CudaGraphManager:
get_offloader().sync_prev_onload()
self.graphs[desc].replay()
def init_breakable_cg_runner(self, model: nn.Module) -> None:
if self.breakable_cg_runner is None:
self.breakable_cg_runner = BreakableCUDAGraphWrapper(
model, self.vllm_config
)
self.breakable_cg_runner.graph_pool = self.pool
def run_pw_graph(self, model: nn.Module, model_inputs: dict[str, Any]) -> Any:
if not self.use_breakable_cg:
# Default: Use torch-compiled piecewise cudagraph.
return model(**model_inputs)
assert self.breakable_cg_runner is not None
return self.breakable_cg_runner(**model_inputs)
class ModelCudaGraphManager(CudaGraphManager):
"""CudaGraphManager with model-specific capture and hidden state management."""
@@ -316,6 +341,8 @@ class ModelCudaGraphManager(CudaGraphManager):
) -> dict[BatchExecutionDescriptor, CapturedAttentionState]:
"""Capture CUDA graphs for model forward pass."""
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
if self.use_breakable_cg:
self.init_breakable_cg_runner(model)
def create_forward_fn(
desc: BatchExecutionDescriptor,
@@ -370,11 +397,16 @@ class ModelCudaGraphManager(CudaGraphManager):
slot_mapping=slot_mappings,
batch_descriptor=batch_descriptor,
):
model_output = model(**model_inputs)
if cg_mode == CUDAGraphMode.PIECEWISE:
# PIECEWISE graph (compiled PW or breakable, chosen inside
# run_pw_graph).
model_output = self.run_pw_graph(model, model_inputs)
else:
model_output = model(**model_inputs)
if cg_mode == CUDAGraphMode.PIECEWISE:
# PW CUDA graph internally handles the model outputs.
# No need to keep track of the hidden states.
# PW CUDA graph (compiled or breakable) internally handles the
# model outputs. No need to keep track of the hidden states.
return None
if self.is_last_pp_rank:
+11 -1
View File
@@ -1189,7 +1189,17 @@ class GPUModelRunner(LoRAModelRunnerMixin):
skip_compiled=skip_compiled,
):
self.kv_connector.pre_forward(scheduler_output)
model_output = self.model(**model_inputs)
if batch_desc.cg_mode == CUDAGraphMode.PIECEWISE:
# Run the PIECEWISE graph (compiled PW cudagraph or breakable
# cudagraph, chosen inside run_pw_graph). cg_mode is only
# PIECEWISE after the cudagraph manager exists.
assert self.cudagraph_manager is not None
model_output = self.cudagraph_manager.run_pw_graph(
self.model, model_inputs
)
else:
# Eager (NONE): call the raw model directly.
model_output = self.model(**model_inputs)
if self.is_last_pp_rank:
if self.use_aux_hidden_state_outputs:
@@ -215,12 +215,22 @@ class EagleSpeculator:
)
inputs_embeds = self.inputs_embeds[:num_tokens]
ret_hidden_states = self.model(
model_inputs = dict(
input_ids=self.input_buffers.input_ids[:num_tokens],
positions=self.input_buffers.positions[:num_tokens],
hidden_states=self.hidden_states[:num_tokens],
inputs_embeds=inputs_embeds,
)
if cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE:
# Draft prefill with PIECEWISE cudagraph (compiled PW or breakable),
# chosen inside run_pw_graph.
assert self.prefill_cudagraph_manager is not None
ret_hidden_states = self.prefill_cudagraph_manager.run_pw_graph(
self.model, model_inputs
)
else:
# Eager (NONE): call the raw model directly.
ret_hidden_states = self.model(**model_inputs)
if self.method == "mtp":
last_hidden_states = ret_hidden_states
hidden_states = ret_hidden_states
@@ -431,6 +441,8 @@ class EagleSpeculator:
# For PIECEWISE, only the model's compiled regions are captured
# and the rest (compute_logits, gumbel_sample) runs eagerly.
assert self.prefill_cudagraph_manager is not None
if self.prefill_cudagraph_manager.use_breakable_cg:
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
self.prefill_cudagraph_manager.capture(
self.prefill,
attn_states,