[Feature] ViT Full CUDA Graph (#35963)

Signed-off-by: Baorun Mu <bmu@nvidia.com>
This commit is contained in:
Baorun (Lauren) Mu
2026-03-23 13:01:10 +08:00
committed by GitHub
parent 1f0d210641
commit f85e479e66
7 changed files with 1584 additions and 31 deletions
+48 -1
View File
@@ -207,6 +207,7 @@ from .utils import (
if TYPE_CHECKING:
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.spec_decode.ngram_proposer import NgramProposer
from vllm.v1.worker.gpu.mm.encoder_cudagraph import EncoderCudaGraphManager
logger = init_logger(__name__)
@@ -499,6 +500,9 @@ class GPUModelRunner(
self.encoder_cache: dict[str, torch.Tensor] = {}
self.late_interaction_runner = LateInteractionRunner()
# Encoder CUDA graph manager (initialized after model load if enabled)
self.encoder_cudagraph_manager: EncoderCudaGraphManager | None = None
self.use_aux_hidden_state_outputs = False
# Set up speculative decoding.
# NOTE(Jiayi): currently we put the entire draft model on
@@ -2664,7 +2668,19 @@ class GPUModelRunner(
with self.timed_encoder_operation(
should_time, mm_lora_refs, current_item_idx, num_items
):
batch_outputs = model.embed_multimodal(**mm_kwargs_batch)
cudagraph_output = None
if (
self.encoder_cudagraph_manager is not None
and self.encoder_cudagraph_manager.supports_modality(modality)
):
cudagraph_output = self.encoder_cudagraph_manager.execute(
mm_kwargs_batch,
)
if cudagraph_output is not None:
batch_outputs = cudagraph_output
else:
batch_outputs = model.embed_multimodal(**mm_kwargs_batch)
sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
encoder_outputs.extend(batch_outputs)
@@ -5715,6 +5731,33 @@ class GPUModelRunner(
)
return 0
# Initialize encoder CUDA graph manager if enabled.
# Use get_model() to unwrap CUDAGraphWrapper/UBatchWrapper,
# because @runtime_checkable Protocol isinstance() checks do not
# work through __getattr__ forwarding.
if (
self.compilation_config.cudagraph_mm_encoder
and self.supports_mm_inputs
and self.encoder_cudagraph_manager is None
):
from vllm.model_executor.models.interfaces import (
SupportsEncoderCudaGraph,
supports_encoder_cudagraph,
)
from vllm.v1.worker.gpu.mm.encoder_cudagraph import (
EncoderCudaGraphManager,
)
raw_model = self.get_model()
if supports_encoder_cudagraph(raw_model):
self.encoder_cudagraph_manager = EncoderCudaGraphManager(
vllm_config=self.vllm_config,
device=self.device,
dtype=self.dtype,
model=cast(SupportsEncoderCudaGraph, raw_model),
)
logger.info("Initialized EncoderCudaGraphManager for vision encoder")
compilation_counter.num_gpu_runner_capture_triggers += 1
start_time = time.perf_counter()
@@ -5738,6 +5781,10 @@ class GPUModelRunner(
)
torch.accelerator.synchronize()
# Capture encoder CUDA graphs if enabled
if self.encoder_cudagraph_manager is not None:
self.encoder_cudagraph_manager.capture()
torch.accelerator.synchronize()
end_free_gpu_memory = torch.cuda.mem_get_info()[0]