[MRV2][Multimodal] Enable encoder cuda graph for model runner v2 (#49852)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Isotr0py
2026-08-14 09:33:21 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent b8165e5e58
commit 66728feb1f
4 changed files with 101 additions and 26 deletions
+38 -1
View File
@@ -4,6 +4,7 @@ import threading
import time
from collections.abc import Collection
from contextlib import contextmanager
from typing import TYPE_CHECKING
import numpy as np
import torch
@@ -24,6 +25,9 @@ from vllm.v1.worker.utils import (
sanity_check_mm_encoder_outputs,
)
if TYPE_CHECKING:
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
logger = init_logger(__name__)
@@ -36,6 +40,7 @@ class EncoderRunner:
encoder_cache: EncoderCache,
dtype: torch.dtype,
device: torch.device,
cudagraph_manager: "EncoderCudaGraphManager | None" = None,
enable_timing: bool = False,
):
self.model = model
@@ -45,6 +50,7 @@ class EncoderRunner:
self.dtype = dtype
self.device = device
self.is_realtime = supports_realtime(model)
self.cudagraph_manager = cudagraph_manager
self.enable_timing = enable_timing
self.encoder_timing_registry: dict[str, EncoderTimingStats] = {}
self._timing_lock = threading.Lock()
@@ -53,6 +59,25 @@ class EncoderRunner:
max_num_tokens, hidden_size, dtype=dtype, device=device
)
def has_cudagraph(self) -> bool:
return self.cudagraph_manager is not None
@torch.inference_mode()
def capture(self) -> None:
manager = self.cudagraph_manager
assert manager is not None
from vllm.distributed.parallel_state import graph_capture
from vllm.platforms import current_platform
with graph_capture(device=self.device):
manager.capture(graph_pool=current_platform.graph_pool_handle())
torch.accelerator.synchronize()
def clear(self) -> None:
if self.cudagraph_manager is not None:
self.cudagraph_manager.clear()
def prepare_mm_inputs(
self, scheduled_encoder_inputs: dict[str, list[int]]
) -> tuple[list[str], list[tuple[str, MultiModalKwargsItem]]]:
@@ -118,7 +143,19 @@ class EncoderRunner:
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
mm_kwargs, device=self.device, pin_memory=PIN_MEMORY
):
batch_outputs = self.model.embed_multimodal(**mm_kwargs_batch)
cg_manager = self.cudagraph_manager
cudagraph_output = (
cg_manager.execute(mm_kwargs_batch)
if cg_manager is not None
and cg_manager.is_captured()
and cg_manager.supports_modality(modality)
else None
)
batch_outputs = (
cudagraph_output
if cudagraph_output is not None
else self.model.embed_multimodal(**mm_kwargs_batch)
)
sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
encoder_outputs.extend(batch_outputs)
return encoder_outputs