[MRV2][Multimodal] Enable encoder cuda graph for model runner v2 (#49852)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -4,6 +4,7 @@ import threading
|
||||
import time
|
||||
from collections.abc import Collection
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -24,6 +25,9 @@ from vllm.v1.worker.utils import (
|
||||
sanity_check_mm_encoder_outputs,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -36,6 +40,7 @@ class EncoderRunner:
|
||||
encoder_cache: EncoderCache,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
cudagraph_manager: "EncoderCudaGraphManager | None" = None,
|
||||
enable_timing: bool = False,
|
||||
):
|
||||
self.model = model
|
||||
@@ -45,6 +50,7 @@ class EncoderRunner:
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
self.is_realtime = supports_realtime(model)
|
||||
self.cudagraph_manager = cudagraph_manager
|
||||
self.enable_timing = enable_timing
|
||||
self.encoder_timing_registry: dict[str, EncoderTimingStats] = {}
|
||||
self._timing_lock = threading.Lock()
|
||||
@@ -53,6 +59,25 @@ class EncoderRunner:
|
||||
max_num_tokens, hidden_size, dtype=dtype, device=device
|
||||
)
|
||||
|
||||
def has_cudagraph(self) -> bool:
|
||||
return self.cudagraph_manager is not None
|
||||
|
||||
@torch.inference_mode()
|
||||
def capture(self) -> None:
|
||||
manager = self.cudagraph_manager
|
||||
assert manager is not None
|
||||
|
||||
from vllm.distributed.parallel_state import graph_capture
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
with graph_capture(device=self.device):
|
||||
manager.capture(graph_pool=current_platform.graph_pool_handle())
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def clear(self) -> None:
|
||||
if self.cudagraph_manager is not None:
|
||||
self.cudagraph_manager.clear()
|
||||
|
||||
def prepare_mm_inputs(
|
||||
self, scheduled_encoder_inputs: dict[str, list[int]]
|
||||
) -> tuple[list[str], list[tuple[str, MultiModalKwargsItem]]]:
|
||||
@@ -118,7 +143,19 @@ class EncoderRunner:
|
||||
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
|
||||
mm_kwargs, device=self.device, pin_memory=PIN_MEMORY
|
||||
):
|
||||
batch_outputs = self.model.embed_multimodal(**mm_kwargs_batch)
|
||||
cg_manager = self.cudagraph_manager
|
||||
cudagraph_output = (
|
||||
cg_manager.execute(mm_kwargs_batch)
|
||||
if cg_manager is not None
|
||||
and cg_manager.is_captured()
|
||||
and cg_manager.supports_modality(modality)
|
||||
else None
|
||||
)
|
||||
batch_outputs = (
|
||||
cudagraph_output
|
||||
if cudagraph_output is not None
|
||||
else self.model.embed_multimodal(**mm_kwargs_batch)
|
||||
)
|
||||
sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
|
||||
encoder_outputs.extend(batch_outputs)
|
||||
return encoder_outputs
|
||||
|
||||
Reference in New Issue
Block a user