[MM][V2] Enable encoder-only ViT CUDA graph capture (#56922)

Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
jiangkuaixue123
2026-09-17 16:12:17 +08:00
committed by GitHub
co-authored by Codex
parent 4e5ffda19d
commit de24e51908
7 changed files with 362 additions and 5 deletions
+32 -3
View File
@@ -4,9 +4,10 @@
An encoder-only instance -- `--mm-encoder-only`, or the producer side of
encoder-cache disaggregation -- encodes the multi-modal items and publishes the
embeddings. It runs no language model: no KV cache, no sampler, no CUDA graphs.
embeddings. It runs no language model: no KV cache, sampler, or decoder graphs.
"""
import time
from collections.abc import Iterator
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any
@@ -15,7 +16,9 @@ import torch
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.logger import init_logger
from vllm.sequence import IntermediateTensors
from vllm.utils.gc_utils import freeze_gc_for_cudagraph_capture
from vllm.v1.kv_cache_interface import KVCacheSpec
from vllm.v1.outputs import (
ModelRunnerOutput,
@@ -24,10 +27,13 @@ from vllm.v1.outputs import (
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
from vllm.v1.worker.workspace import lock_workspace
if TYPE_CHECKING:
from vllm.v1.core.sched.output import SchedulerOutput
logger = init_logger(__name__)
class MMEncoderModelRunner(GPUModelRunner):
"""Encoder-only variant of the V2 GPU model runner."""
@@ -67,8 +73,31 @@ class MMEncoderModelRunner(GPUModelRunner):
def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
return {}
@torch.inference_mode()
def capture_model(self, *, profile_only: bool = False) -> int:
return 0
encoder_runner = self.model_state.encoder_runner
if not encoder_runner.has_cudagraph():
return 0
start_time = time.perf_counter()
with freeze_gc_for_cudagraph_capture():
torch.accelerator.empty_cache()
start_free_memory = torch.accelerator.get_memory_info()[0]
encoder_runner.capture()
graph_memory = start_free_memory - torch.accelerator.get_memory_info()[0]
if not profile_only:
lock_workspace()
manager = encoder_runner.cudagraph_manager
assert manager is not None
logger.info(
"Encoder graph capturing finished in %.0f secs, took %.2f GiB (%d graphs)",
time.perf_counter() - start_time,
graph_memory / (1 << 30),
manager.get_num_graphs_to_capture(),
)
return graph_memory
def needs_cudagraph_capture(self) -> bool:
return False
@@ -114,7 +143,7 @@ class MMEncoderModelRunner(GPUModelRunner):
batch_req_state, _ = self.gather_batch_req_state(scheduler_output, False)
assert batch_req_state is not None
# No CUDA graph, and no DP peer to agree a padded shape with.
# No decoder graph or DP padding; encoder graphs run independently.
batch_desc = BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.NONE,
num_tokens=batch_req_state.num_tokens,