diff --git a/vllm/v1/worker/encoder_cudagraph.py b/vllm/v1/worker/encoder_cudagraph.py index 0b3543da1e..9f61b704cb 100644 --- a/vllm/v1/worker/encoder_cudagraph.py +++ b/vllm/v1/worker/encoder_cudagraph.py @@ -203,6 +203,10 @@ class EncoderCudaGraphManager: """Check if a modality is supported by this manager.""" return modality in self.config.modalities + def is_captured(self) -> bool: + """Return whether a CUDA graph pool is active.""" + return self.graph_pool is not None + def clear(self) -> None: """Release captured encoder CUDA graphs and the manager-local pool.""" for graph_set in self.budget_graphs.values(): diff --git a/vllm/v1/worker/gpu/mm/encoder_runner.py b/vllm/v1/worker/gpu/mm/encoder_runner.py index 92bd435342..a1c79a8eb7 100644 --- a/vllm/v1/worker/gpu/mm/encoder_runner.py +++ b/vllm/v1/worker/gpu/mm/encoder_runner.py @@ -4,6 +4,7 @@ import threading import time from collections.abc import Collection from contextlib import contextmanager +from typing import TYPE_CHECKING import numpy as np import torch @@ -24,6 +25,9 @@ from vllm.v1.worker.utils import ( sanity_check_mm_encoder_outputs, ) +if TYPE_CHECKING: + from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager + logger = init_logger(__name__) @@ -36,6 +40,7 @@ class EncoderRunner: encoder_cache: EncoderCache, dtype: torch.dtype, device: torch.device, + cudagraph_manager: "EncoderCudaGraphManager | None" = None, enable_timing: bool = False, ): self.model = model @@ -45,6 +50,7 @@ class EncoderRunner: self.dtype = dtype self.device = device self.is_realtime = supports_realtime(model) + self.cudagraph_manager = cudagraph_manager self.enable_timing = enable_timing self.encoder_timing_registry: dict[str, EncoderTimingStats] = {} self._timing_lock = threading.Lock() @@ -53,6 +59,25 @@ class EncoderRunner: max_num_tokens, hidden_size, dtype=dtype, device=device ) + def has_cudagraph(self) -> bool: + return self.cudagraph_manager is not None + + @torch.inference_mode() + def capture(self) -> None: + manager = self.cudagraph_manager + assert manager is not None + + from vllm.distributed.parallel_state import graph_capture + from vllm.platforms import current_platform + + with graph_capture(device=self.device): + manager.capture(graph_pool=current_platform.graph_pool_handle()) + torch.accelerator.synchronize() + + def clear(self) -> None: + if self.cudagraph_manager is not None: + self.cudagraph_manager.clear() + def prepare_mm_inputs( self, scheduled_encoder_inputs: dict[str, list[int]] ) -> tuple[list[str], list[tuple[str, MultiModalKwargsItem]]]: @@ -118,7 +143,19 @@ class EncoderRunner: for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs( mm_kwargs, device=self.device, pin_memory=PIN_MEMORY ): - batch_outputs = self.model.embed_multimodal(**mm_kwargs_batch) + cg_manager = self.cudagraph_manager + cudagraph_output = ( + cg_manager.execute(mm_kwargs_batch) + if cg_manager is not None + and cg_manager.is_captured() + and cg_manager.supports_modality(modality) + else None + ) + batch_outputs = ( + cudagraph_output + if cudagraph_output is not None + else self.model.embed_multimodal(**mm_kwargs_batch) + ) sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items) encoder_outputs.extend(batch_outputs) return encoder_outputs diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 4b53defa47..220d112e92 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -842,10 +842,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): return 0 assert self.cudagraph_manager is not None - if not self.cudagraph_manager.needs_capture(): + capture_encoder = ( + self.model_state.supports_mm_inputs + and self.model_state.encoder_runner.has_cudagraph() + ) + capture_decoder = self.cudagraph_manager.needs_capture() + if not capture_encoder and not capture_decoder: logger.warning( - "Skipping CUDA graph capture. To turn on CUDA graph capture, " - "ensure `cudagraph_mode` was not manually set to `NONE`" + "Skipping encoder and decoder CUDA graph capture. To enable " + "encoder capture, ensure `cudagraph_mm_encoder` is enabled; " + "to enable decoder capture, ensure `cudagraph_mode` is not `NONE`." ) return 0 @@ -857,27 +863,31 @@ class GPUModelRunner(LoRAModelRunnerMixin): start_free_gpu_memory = torch.accelerator.get_memory_info()[0] with self.maybe_setup_dummy_loras(self.lora_config): - self.cudagraph_manager.capture( - self.model, - self.model_state, - self.input_buffers, - self.intermediate_tensors, - self.block_tables, - self.attn_groups, - self.kv_cache_config, - has_lora=self.lora_config is not None, - use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs, - lora_capture_hook=create_lora_capture_hook(self.lora_config, self), - ) - if self.speculator is not None: - self.speculator.capture() - if self.adaptive_verification is not None: - with self.step_timing.collect() as timings: - for batch in self.adaptive_verification.batches_to_profile( - self.cudagraph_manager.captured_token_counts() - ): - self._dummy_run(**batch) - self.adaptive_verification.set_initial_cost_curves(timings) + if capture_encoder: + self.model_state.encoder_runner.capture() + + if capture_decoder: + self.cudagraph_manager.capture( + self.model, + self.model_state, + self.input_buffers, + self.intermediate_tensors, + self.block_tables, + self.attn_groups, + self.kv_cache_config, + has_lora=self.lora_config is not None, + use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs, + lora_capture_hook=create_lora_capture_hook(self.lora_config, self), + ) + if self.speculator is not None: + self.speculator.capture() + if self.adaptive_verification is not None: + with self.step_timing.collect() as timings: + for batch in self.adaptive_verification.batches_to_profile( + self.cudagraph_manager.captured_token_counts() + ): + self._dummy_run(**batch) + self.adaptive_verification.set_initial_cost_curves(timings) end_time = time.perf_counter() end_free_gpu_memory = torch.accelerator.get_memory_info()[0] @@ -1885,6 +1895,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.attn_groups.clear() if hasattr(self, "kv_cache_config"): del self.kv_cache_config + if hasattr(self, "model_state") and self.model_state.supports_mm_inputs: + self.model_state.encoder_runner.clear() free_before_shutdown(self.vllm_config) if hasattr(self, "model_state"): del self.model_state diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index 722b709bb5..980ec1dd8e 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -1,17 +1,22 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from abc import ABC, abstractmethod -from typing import Any +from typing import Any, cast import torch import torch.nn as nn from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode +from vllm.model_executor.models.interfaces import ( + SupportsEncoderCudaGraph, + supports_encoder_cudagraph, +) from vllm.tasks import GenerationTask from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.core.sched.output import NewRequestData from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner @@ -59,6 +64,22 @@ class ModelState(ABC): self.supports_mm_inputs = encoder_cache is not None if encoder_cache is not None: + enable_encoder_cuda_graph = ( + not self.model_config.enforce_eager + and vllm_config.compilation_config.cudagraph_mm_encoder + and supports_encoder_cudagraph(model) + ) + cudagraph_manager = ( + EncoderCudaGraphManager( + vllm_config=vllm_config, + device=device, + dtype=self.dtype, + model=cast(SupportsEncoderCudaGraph, model), + ) + if enable_encoder_cuda_graph + else None + ) + self.encoder_cache = encoder_cache observability_config = vllm_config.observability_config self.encoder_runner = EncoderRunner( @@ -68,6 +89,7 @@ class ModelState(ABC): encoder_cache=encoder_cache, dtype=self.dtype, device=self.device, + cudagraph_manager=cudagraph_manager, enable_timing=bool( observability_config and observability_config.enable_mm_processor_stats