diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index c7a7ffe442..384f119243 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -9,6 +9,10 @@ import torch import torch.nn as nn from tqdm import tqdm +from vllm.compilation.breakable_cudagraph import ( + BreakableCUDAGraphWrapper, + is_breakable_cudagraph_enabled, +) from vllm.compilation.counter import compilation_counter from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode @@ -116,6 +120,13 @@ class CudaGraphManager: ) self._init_candidates() + # Breakable CUDA graph (PW CUDA graph without torch.compile) + self.use_breakable_cg = ( + is_breakable_cudagraph_enabled() + and self.cudagraph_mode.has_piecewise_cudagraphs() + ) + self.breakable_cg_runner: BreakableCUDAGraphWrapper | None = None + def _init_candidates(self) -> None: """Build priority-ordered candidate lists for each token count.""" capture_sizes = self.compilation_config.cudagraph_capture_sizes @@ -283,6 +294,20 @@ class CudaGraphManager: get_offloader().sync_prev_onload() self.graphs[desc].replay() + def init_breakable_cg_runner(self, model: nn.Module) -> None: + if self.breakable_cg_runner is None: + self.breakable_cg_runner = BreakableCUDAGraphWrapper( + model, self.vllm_config + ) + self.breakable_cg_runner.graph_pool = self.pool + + def run_pw_graph(self, model: nn.Module, model_inputs: dict[str, Any]) -> Any: + if not self.use_breakable_cg: + # Default: Use torch-compiled piecewise cudagraph. + return model(**model_inputs) + assert self.breakable_cg_runner is not None + return self.breakable_cg_runner(**model_inputs) + class ModelCudaGraphManager(CudaGraphManager): """CudaGraphManager with model-specific capture and hidden state management.""" @@ -316,6 +341,8 @@ class ModelCudaGraphManager(CudaGraphManager): ) -> dict[BatchExecutionDescriptor, CapturedAttentionState]: """Capture CUDA graphs for model forward pass.""" self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs + if self.use_breakable_cg: + self.init_breakable_cg_runner(model) def create_forward_fn( desc: BatchExecutionDescriptor, @@ -370,11 +397,16 @@ class ModelCudaGraphManager(CudaGraphManager): slot_mapping=slot_mappings, batch_descriptor=batch_descriptor, ): - model_output = model(**model_inputs) + if cg_mode == CUDAGraphMode.PIECEWISE: + # PIECEWISE graph (compiled PW or breakable, chosen inside + # run_pw_graph). + model_output = self.run_pw_graph(model, model_inputs) + else: + model_output = model(**model_inputs) if cg_mode == CUDAGraphMode.PIECEWISE: - # PW CUDA graph internally handles the model outputs. - # No need to keep track of the hidden states. + # PW CUDA graph (compiled or breakable) internally handles the + # model outputs. No need to keep track of the hidden states. return None if self.is_last_pp_rank: diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 5cba66e5c9..519395e90b 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1189,7 +1189,17 @@ class GPUModelRunner(LoRAModelRunnerMixin): skip_compiled=skip_compiled, ): self.kv_connector.pre_forward(scheduler_output) - model_output = self.model(**model_inputs) + if batch_desc.cg_mode == CUDAGraphMode.PIECEWISE: + # Run the PIECEWISE graph (compiled PW cudagraph or breakable + # cudagraph, chosen inside run_pw_graph). cg_mode is only + # PIECEWISE after the cudagraph manager exists. + assert self.cudagraph_manager is not None + model_output = self.cudagraph_manager.run_pw_graph( + self.model, model_inputs + ) + else: + # Eager (NONE): call the raw model directly. + model_output = self.model(**model_inputs) if self.is_last_pp_rank: if self.use_aux_hidden_state_outputs: diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py index 6ae3fe793b..d5095add1e 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py @@ -215,12 +215,22 @@ class EagleSpeculator: ) inputs_embeds = self.inputs_embeds[:num_tokens] - ret_hidden_states = self.model( + model_inputs = dict( input_ids=self.input_buffers.input_ids[:num_tokens], positions=self.input_buffers.positions[:num_tokens], hidden_states=self.hidden_states[:num_tokens], inputs_embeds=inputs_embeds, ) + if cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE: + # Draft prefill with PIECEWISE cudagraph (compiled PW or breakable), + # chosen inside run_pw_graph. + assert self.prefill_cudagraph_manager is not None + ret_hidden_states = self.prefill_cudagraph_manager.run_pw_graph( + self.model, model_inputs + ) + else: + # Eager (NONE): call the raw model directly. + ret_hidden_states = self.model(**model_inputs) if self.method == "mtp": last_hidden_states = ret_hidden_states hidden_states = ret_hidden_states @@ -431,6 +441,8 @@ class EagleSpeculator: # For PIECEWISE, only the model's compiled regions are captured # and the rest (compute_logits, gumbel_sample) runs eagerly. assert self.prefill_cudagraph_manager is not None + if self.prefill_cudagraph_manager.use_breakable_cg: + self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model) self.prefill_cudagraph_manager.capture( self.prefill, attn_states,