[MRV2][Multimodal] Enable encoder cuda graph for model runner v2 (#49852)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -203,6 +203,10 @@ class EncoderCudaGraphManager:
|
||||
"""Check if a modality is supported by this manager."""
|
||||
return modality in self.config.modalities
|
||||
|
||||
def is_captured(self) -> bool:
|
||||
"""Return whether a CUDA graph pool is active."""
|
||||
return self.graph_pool is not None
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Release captured encoder CUDA graphs and the manager-local pool."""
|
||||
for graph_set in self.budget_graphs.values():
|
||||
|
||||
@@ -4,6 +4,7 @@ import threading
|
||||
import time
|
||||
from collections.abc import Collection
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -24,6 +25,9 @@ from vllm.v1.worker.utils import (
|
||||
sanity_check_mm_encoder_outputs,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -36,6 +40,7 @@ class EncoderRunner:
|
||||
encoder_cache: EncoderCache,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
cudagraph_manager: "EncoderCudaGraphManager | None" = None,
|
||||
enable_timing: bool = False,
|
||||
):
|
||||
self.model = model
|
||||
@@ -45,6 +50,7 @@ class EncoderRunner:
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
self.is_realtime = supports_realtime(model)
|
||||
self.cudagraph_manager = cudagraph_manager
|
||||
self.enable_timing = enable_timing
|
||||
self.encoder_timing_registry: dict[str, EncoderTimingStats] = {}
|
||||
self._timing_lock = threading.Lock()
|
||||
@@ -53,6 +59,25 @@ class EncoderRunner:
|
||||
max_num_tokens, hidden_size, dtype=dtype, device=device
|
||||
)
|
||||
|
||||
def has_cudagraph(self) -> bool:
|
||||
return self.cudagraph_manager is not None
|
||||
|
||||
@torch.inference_mode()
|
||||
def capture(self) -> None:
|
||||
manager = self.cudagraph_manager
|
||||
assert manager is not None
|
||||
|
||||
from vllm.distributed.parallel_state import graph_capture
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
with graph_capture(device=self.device):
|
||||
manager.capture(graph_pool=current_platform.graph_pool_handle())
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
def clear(self) -> None:
|
||||
if self.cudagraph_manager is not None:
|
||||
self.cudagraph_manager.clear()
|
||||
|
||||
def prepare_mm_inputs(
|
||||
self, scheduled_encoder_inputs: dict[str, list[int]]
|
||||
) -> tuple[list[str], list[tuple[str, MultiModalKwargsItem]]]:
|
||||
@@ -118,7 +143,19 @@ class EncoderRunner:
|
||||
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
|
||||
mm_kwargs, device=self.device, pin_memory=PIN_MEMORY
|
||||
):
|
||||
batch_outputs = self.model.embed_multimodal(**mm_kwargs_batch)
|
||||
cg_manager = self.cudagraph_manager
|
||||
cudagraph_output = (
|
||||
cg_manager.execute(mm_kwargs_batch)
|
||||
if cg_manager is not None
|
||||
and cg_manager.is_captured()
|
||||
and cg_manager.supports_modality(modality)
|
||||
else None
|
||||
)
|
||||
batch_outputs = (
|
||||
cudagraph_output
|
||||
if cudagraph_output is not None
|
||||
else self.model.embed_multimodal(**mm_kwargs_batch)
|
||||
)
|
||||
sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
|
||||
encoder_outputs.extend(batch_outputs)
|
||||
return encoder_outputs
|
||||
|
||||
@@ -842,10 +842,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return 0
|
||||
|
||||
assert self.cudagraph_manager is not None
|
||||
if not self.cudagraph_manager.needs_capture():
|
||||
capture_encoder = (
|
||||
self.model_state.supports_mm_inputs
|
||||
and self.model_state.encoder_runner.has_cudagraph()
|
||||
)
|
||||
capture_decoder = self.cudagraph_manager.needs_capture()
|
||||
if not capture_encoder and not capture_decoder:
|
||||
logger.warning(
|
||||
"Skipping CUDA graph capture. To turn on CUDA graph capture, "
|
||||
"ensure `cudagraph_mode` was not manually set to `NONE`"
|
||||
"Skipping encoder and decoder CUDA graph capture. To enable "
|
||||
"encoder capture, ensure `cudagraph_mm_encoder` is enabled; "
|
||||
"to enable decoder capture, ensure `cudagraph_mode` is not `NONE`."
|
||||
)
|
||||
return 0
|
||||
|
||||
@@ -857,27 +863,31 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
with self.maybe_setup_dummy_loras(self.lora_config):
|
||||
self.cudagraph_manager.capture(
|
||||
self.model,
|
||||
self.model_state,
|
||||
self.input_buffers,
|
||||
self.intermediate_tensors,
|
||||
self.block_tables,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
has_lora=self.lora_config is not None,
|
||||
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
|
||||
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.capture()
|
||||
if self.adaptive_verification is not None:
|
||||
with self.step_timing.collect() as timings:
|
||||
for batch in self.adaptive_verification.batches_to_profile(
|
||||
self.cudagraph_manager.captured_token_counts()
|
||||
):
|
||||
self._dummy_run(**batch)
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
if capture_encoder:
|
||||
self.model_state.encoder_runner.capture()
|
||||
|
||||
if capture_decoder:
|
||||
self.cudagraph_manager.capture(
|
||||
self.model,
|
||||
self.model_state,
|
||||
self.input_buffers,
|
||||
self.intermediate_tensors,
|
||||
self.block_tables,
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
has_lora=self.lora_config is not None,
|
||||
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
|
||||
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.capture()
|
||||
if self.adaptive_verification is not None:
|
||||
with self.step_timing.collect() as timings:
|
||||
for batch in self.adaptive_verification.batches_to_profile(
|
||||
self.cudagraph_manager.captured_token_counts()
|
||||
):
|
||||
self._dummy_run(**batch)
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
|
||||
end_time = time.perf_counter()
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
@@ -1885,6 +1895,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.attn_groups.clear()
|
||||
if hasattr(self, "kv_cache_config"):
|
||||
del self.kv_cache_config
|
||||
if hasattr(self, "model_state") and self.model_state.supports_mm_inputs:
|
||||
self.model_state.encoder_runner.clear()
|
||||
free_before_shutdown(self.vllm_config)
|
||||
if hasattr(self, "model_state"):
|
||||
del self.model_state
|
||||
|
||||
@@ -1,17 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsEncoderCudaGraph,
|
||||
supports_encoder_cudagraph,
|
||||
)
|
||||
from vllm.tasks import GenerationTask
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner
|
||||
@@ -59,6 +64,22 @@ class ModelState(ABC):
|
||||
|
||||
self.supports_mm_inputs = encoder_cache is not None
|
||||
if encoder_cache is not None:
|
||||
enable_encoder_cuda_graph = (
|
||||
not self.model_config.enforce_eager
|
||||
and vllm_config.compilation_config.cudagraph_mm_encoder
|
||||
and supports_encoder_cudagraph(model)
|
||||
)
|
||||
cudagraph_manager = (
|
||||
EncoderCudaGraphManager(
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
dtype=self.dtype,
|
||||
model=cast(SupportsEncoderCudaGraph, model),
|
||||
)
|
||||
if enable_encoder_cuda_graph
|
||||
else None
|
||||
)
|
||||
|
||||
self.encoder_cache = encoder_cache
|
||||
observability_config = vllm_config.observability_config
|
||||
self.encoder_runner = EncoderRunner(
|
||||
@@ -68,6 +89,7 @@ class ModelState(ABC):
|
||||
encoder_cache=encoder_cache,
|
||||
dtype=self.dtype,
|
||||
device=self.device,
|
||||
cudagraph_manager=cudagraph_manager,
|
||||
enable_timing=bool(
|
||||
observability_config
|
||||
and observability_config.enable_mm_processor_stats
|
||||
|
||||
Reference in New Issue
Block a user