[MRV2][Multimodal] Enable encoder cuda graph for model runner v2 (#49852)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Isotr0py
2026-08-14 09:33:21 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent b8165e5e58
commit 66728feb1f
4 changed files with 101 additions and 26 deletions
+4
View File
@@ -203,6 +203,10 @@ class EncoderCudaGraphManager:
"""Check if a modality is supported by this manager."""
return modality in self.config.modalities
def is_captured(self) -> bool:
"""Return whether a CUDA graph pool is active."""
return self.graph_pool is not None
def clear(self) -> None:
"""Release captured encoder CUDA graphs and the manager-local pool."""
for graph_set in self.budget_graphs.values():
+38 -1
View File
@@ -4,6 +4,7 @@ import threading
import time
from collections.abc import Collection
from contextlib import contextmanager
from typing import TYPE_CHECKING
import numpy as np
import torch
@@ -24,6 +25,9 @@ from vllm.v1.worker.utils import (
sanity_check_mm_encoder_outputs,
)
if TYPE_CHECKING:
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
logger = init_logger(__name__)
@@ -36,6 +40,7 @@ class EncoderRunner:
encoder_cache: EncoderCache,
dtype: torch.dtype,
device: torch.device,
cudagraph_manager: "EncoderCudaGraphManager | None" = None,
enable_timing: bool = False,
):
self.model = model
@@ -45,6 +50,7 @@ class EncoderRunner:
self.dtype = dtype
self.device = device
self.is_realtime = supports_realtime(model)
self.cudagraph_manager = cudagraph_manager
self.enable_timing = enable_timing
self.encoder_timing_registry: dict[str, EncoderTimingStats] = {}
self._timing_lock = threading.Lock()
@@ -53,6 +59,25 @@ class EncoderRunner:
max_num_tokens, hidden_size, dtype=dtype, device=device
)
def has_cudagraph(self) -> bool:
return self.cudagraph_manager is not None
@torch.inference_mode()
def capture(self) -> None:
manager = self.cudagraph_manager
assert manager is not None
from vllm.distributed.parallel_state import graph_capture
from vllm.platforms import current_platform
with graph_capture(device=self.device):
manager.capture(graph_pool=current_platform.graph_pool_handle())
torch.accelerator.synchronize()
def clear(self) -> None:
if self.cudagraph_manager is not None:
self.cudagraph_manager.clear()
def prepare_mm_inputs(
self, scheduled_encoder_inputs: dict[str, list[int]]
) -> tuple[list[str], list[tuple[str, MultiModalKwargsItem]]]:
@@ -118,7 +143,19 @@ class EncoderRunner:
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
mm_kwargs, device=self.device, pin_memory=PIN_MEMORY
):
batch_outputs = self.model.embed_multimodal(**mm_kwargs_batch)
cg_manager = self.cudagraph_manager
cudagraph_output = (
cg_manager.execute(mm_kwargs_batch)
if cg_manager is not None
and cg_manager.is_captured()
and cg_manager.supports_modality(modality)
else None
)
batch_outputs = (
cudagraph_output
if cudagraph_output is not None
else self.model.embed_multimodal(**mm_kwargs_batch)
)
sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
encoder_outputs.extend(batch_outputs)
return encoder_outputs
+36 -24
View File
@@ -842,10 +842,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return 0
assert self.cudagraph_manager is not None
if not self.cudagraph_manager.needs_capture():
capture_encoder = (
self.model_state.supports_mm_inputs
and self.model_state.encoder_runner.has_cudagraph()
)
capture_decoder = self.cudagraph_manager.needs_capture()
if not capture_encoder and not capture_decoder:
logger.warning(
"Skipping CUDA graph capture. To turn on CUDA graph capture, "
"ensure `cudagraph_mode` was not manually set to `NONE`"
"Skipping encoder and decoder CUDA graph capture. To enable "
"encoder capture, ensure `cudagraph_mm_encoder` is enabled; "
"to enable decoder capture, ensure `cudagraph_mode` is not `NONE`."
)
return 0
@@ -857,27 +863,31 @@ class GPUModelRunner(LoRAModelRunnerMixin):
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
with self.maybe_setup_dummy_loras(self.lora_config):
self.cudagraph_manager.capture(
self.model,
self.model_state,
self.input_buffers,
self.intermediate_tensors,
self.block_tables,
self.attn_groups,
self.kv_cache_config,
has_lora=self.lora_config is not None,
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
)
if self.speculator is not None:
self.speculator.capture()
if self.adaptive_verification is not None:
with self.step_timing.collect() as timings:
for batch in self.adaptive_verification.batches_to_profile(
self.cudagraph_manager.captured_token_counts()
):
self._dummy_run(**batch)
self.adaptive_verification.set_initial_cost_curves(timings)
if capture_encoder:
self.model_state.encoder_runner.capture()
if capture_decoder:
self.cudagraph_manager.capture(
self.model,
self.model_state,
self.input_buffers,
self.intermediate_tensors,
self.block_tables,
self.attn_groups,
self.kv_cache_config,
has_lora=self.lora_config is not None,
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
)
if self.speculator is not None:
self.speculator.capture()
if self.adaptive_verification is not None:
with self.step_timing.collect() as timings:
for batch in self.adaptive_verification.batches_to_profile(
self.cudagraph_manager.captured_token_counts()
):
self._dummy_run(**batch)
self.adaptive_verification.set_initial_cost_curves(timings)
end_time = time.perf_counter()
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
@@ -1885,6 +1895,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.attn_groups.clear()
if hasattr(self, "kv_cache_config"):
del self.kv_cache_config
if hasattr(self, "model_state") and self.model_state.supports_mm_inputs:
self.model_state.encoder_runner.clear()
free_before_shutdown(self.vllm_config)
if hasattr(self, "model_state"):
del self.model_state
+23 -1
View File
@@ -1,17 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from typing import Any
from typing import Any, cast
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.model_executor.models.interfaces import (
SupportsEncoderCudaGraph,
supports_encoder_cudagraph,
)
from vllm.tasks import GenerationTask
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
@@ -59,6 +64,22 @@ class ModelState(ABC):
self.supports_mm_inputs = encoder_cache is not None
if encoder_cache is not None:
enable_encoder_cuda_graph = (
not self.model_config.enforce_eager
and vllm_config.compilation_config.cudagraph_mm_encoder
and supports_encoder_cudagraph(model)
)
cudagraph_manager = (
EncoderCudaGraphManager(
vllm_config=vllm_config,
device=device,
dtype=self.dtype,
model=cast(SupportsEncoderCudaGraph, model),
)
if enable_encoder_cuda_graph
else None
)
self.encoder_cache = encoder_cache
observability_config = vllm_config.observability_config
self.encoder_runner = EncoderRunner(
@@ -68,6 +89,7 @@ class ModelState(ABC):
encoder_cache=encoder_cache,
dtype=self.dtype,
device=self.device,
cudagraph_manager=cudagraph_manager,
enable_timing=bool(
observability_config
and observability_config.enable_mm_processor_stats