[MM][Perf][CG] Support ViT full CUDA graph for Qwen3-VL video inference (#38061)

Signed-off-by: shen-shanshan <467638484@qq.com>
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Shanshan Shen
2026-04-14 16:49:32 +08:00
committed by GitHub
co-authored by Roger Wang
parent c0ecaed950
commit 80118853f4
7 changed files with 582 additions and 67 deletions
+10 -1
View File
@@ -1524,6 +1524,13 @@ class SupportsEncoderCudaGraph(Protocol):
def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig": ...
def get_input_modality(
self,
mm_kwargs: dict[str, Any],
) -> str:
"""Return the modality of the inputs."""
...
def get_encoder_cudagraph_budget_range(
self,
vllm_config: "VllmConfig",
@@ -1536,7 +1543,7 @@ class SupportsEncoderCudaGraph(Protocol):
(e.g. max_num_batched_tokens)
Used when ``encoder_cudagraph_token_budgets`` and/or
``encoder_cudagraph_max_images_per_batch`` are not explicitly
``encoder_cudagraph_max_vision_items_per_batch`` are not explicitly
specified by the user.
"""
...
@@ -1590,6 +1597,7 @@ class SupportsEncoderCudaGraph(Protocol):
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
) -> "EncoderCudaGraphCaptureInputs":
@@ -1600,6 +1608,7 @@ class SupportsEncoderCudaGraph(Protocol):
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
) -> "EncoderCudaGraphReplayBuffers":
"""Compute buffer values from actual batch inputs for replay."""
...