From 936e0b79aa93008bb90e4cc190ea8adb9296b1df Mon Sep 17 00:00:00 2001 From: Shanshan Shen <467638484@qq.com> Date: Tue, 21 Apr 2026 22:47:53 +0800 Subject: [PATCH] [MM][CG] Optimize default `max_frames_per_batch` auto-infer for ViT CUDA graph video inference (#40445) Signed-off-by: shen-shanshan <467638484@qq.com> --- docs/design/cuda_graphs_multimodal.md | 3 ++- vllm/config/compilation.py | 15 +++++++++------ vllm/model_executor/models/interfaces.py | 6 ++++++ vllm/model_executor/models/qwen3_vl.py | 10 ++++++++++ vllm/multimodal/registry.py | 3 +++ vllm/v1/worker/encoder_cudagraph.py | 24 ++++++++++++++++-------- 6 files changed, 46 insertions(+), 15 deletions(-) diff --git a/docs/design/cuda_graphs_multimodal.md b/docs/design/cuda_graphs_multimodal.md index f9c77d89f0..e32010232e 100644 --- a/docs/design/cuda_graphs_multimodal.md +++ b/docs/design/cuda_graphs_multimodal.md @@ -76,6 +76,7 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra * `encoder_cudagraph_forward(...)` — forward pass using precomputed buffers (called during capture and replay). * `encoder_eager_forward(...)` — fallback eager forward when no graph fits. * `get_input_modality(...)` - return the modality of the inputs. +* `get_max_frames_per_video()` - return model-specific max frames per video. !!! note The `SupportsEncoderCudaGraph` protocol is designed to be model-agnostic. New vision encoder models can opt-in by implementing the protocol methods without modifying the manager. @@ -96,7 +97,7 @@ Three fields in `CompilationConfig` control encoder CUDA Graphs: * `cudagraph_mm_encoder` (`bool`, default `False`) — enable CUDA Graph capture for multimodal encoder. When enabled, captures the full encoder forward as a CUDA Graph for each token budget level. * `encoder_cudagraph_token_budgets` (`list[int]`, default `[]`) — token budget levels for capture. If empty (default), auto-inferred from model architecture as power-of-2 levels. User-provided values override auto-inference. * `encoder_cudagraph_max_vision_items_per_batch` (`int`, default `0`) — maximum number of images/videos per batch during capture. If 0 (default), auto-inferred as `max_budget // min_budget`. -* `encoder_cudagraph_max_frames_per_batch` (`int`, default `0`) — maximum number of video frames per batch during capture. If 0 (default), auto-inferred as `encoder_cudagraph_max_vision_items_per_batch * 2` (to be optimized). +* `encoder_cudagraph_max_frames_per_batch` (`int`, default `None`) — maximum number of video frames per batch during capture. If `None` (default), auto-inferred as `encoder_cudagraph_max_vision_items_per_batch * max_frames_per_video` (`max_frames_per_video` is a model-specific value according to its `processing_info`). If we limit the video count per prompt to `0`, it will also be set to `0` (i.e., fall back to image-only mode). ## Usage guide diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index df86f579a1..f7483db52a 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -535,13 +535,15 @@ class CompilationConfig: model's budget range. User-provided positive value overrides auto-inference.""" - encoder_cudagraph_max_frames_per_batch: int = 0 + encoder_cudagraph_max_frames_per_batch: int | None = None """Maximum total video frames per batch for encoder CUDA graph capture. Controls the cu_seqlens buffer size (one entry per attention sequence, - i.e. one per video frame). If 0 (default), auto-inferred per budget - level as token_budget (tight bound: packing guarantees - sum(T_i) <= token_budget). Positive value overrides auto-inference - and applies to all budget levels.""" + i.e. one per video frame). + If None (default), auto-inferred as encoder_cudagraph_max_vision_items_per_batch + * max_frames_per_video (model-specific value according to processing_info). + Positive value overrides auto-inference and applies to all budget levels. + If we limit the video count per prompt to `0`, it will also be set to `0` + (i.e., fall back to image-only mode).""" # Inductor capture compile_sizes: list[int | str] | None = None @@ -993,11 +995,12 @@ class CompilationConfig: ) if ( self.cudagraph_mm_encoder + and self.encoder_cudagraph_max_frames_per_batch is not None and self.encoder_cudagraph_max_frames_per_batch < 0 ): raise ValueError( "encoder_cudagraph_max_frames_per_batch must be " - "non-negative (0 = auto-infer)" + "non-negative (None = auto-infer)" ) if self.backend == "": diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index c24798e084..3ee8ac23d7 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -1531,6 +1531,12 @@ class SupportsEncoderCudaGraph(Protocol): """Return the modality of the inputs.""" ... + def get_max_frames_per_video( + self, + ) -> int: + """Return model-specific max frames per video.""" + ... + def get_encoder_cudagraph_budget_range( self, vllm_config: "VllmConfig", diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index fee0b937b7..7f110a4c2d 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -1640,6 +1640,7 @@ class Qwen3VLForConditionalGeneration( multimodal_config = vllm_config.model_config.multimodal_config self.config = config + self.model_config = vllm_config.model_config self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config) self.multimodal_config = multimodal_config self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" @@ -1783,6 +1784,15 @@ class Qwen3VLForConditionalGeneration( return "image" return "video" + def get_max_frames_per_video(self) -> int: + mm_registry = MULTIMODAL_REGISTRY + info = mm_registry.get_processing_info(self.model_config) + max_frames_per_video = info.get_num_frames_with_most_features( + seq_len=self.model_config.max_model_len, + mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")}, + ) + return max_frames_per_video + def get_encoder_cudagraph_budget_range( self, vllm_config, diff --git a/vllm/multimodal/registry.py b/vllm/multimodal/registry.py index fa414a5928..cac170399d 100644 --- a/vllm/multimodal/registry.py +++ b/vllm/multimodal/registry.py @@ -193,6 +193,9 @@ class MultiModalRegistry: ctx = self._create_processing_ctx(model_config, tokenizer) return factories.info(ctx) + def get_processing_info(self, model_config: "ModelConfig") -> BaseProcessingInfo: + return self._create_processing_info(model_config, tokenizer=None) + def create_processor( self, model_config: "ModelConfig", diff --git a/vllm/v1/worker/encoder_cudagraph.py b/vllm/v1/worker/encoder_cudagraph.py index 65ad8e2cec..bb805a5d6c 100644 --- a/vllm/v1/worker/encoder_cudagraph.py +++ b/vllm/v1/worker/encoder_cudagraph.py @@ -67,13 +67,15 @@ class EncoderCudaGraphManager: comp_config = vllm_config.compilation_config user_budgets = comp_config.encoder_cudagraph_token_budgets - user_max_mm_items = comp_config.encoder_cudagraph_max_vision_items_per_batch + user_max_vision_items = comp_config.encoder_cudagraph_max_vision_items_per_batch user_max_frames = comp_config.encoder_cudagraph_max_frames_per_batch - if user_budgets and user_max_mm_items > 0: + multimodal_config = vllm_config.model_config.multimodal_config + + if user_budgets and user_max_vision_items > 0: # Fully user-specified self.token_budgets = sorted(user_budgets) - self.max_batch_size = user_max_mm_items + self.max_batch_size = user_max_vision_items else: # Auto-infer missing values from model min_budget, max_budget = model.get_encoder_cudagraph_budget_range( @@ -85,14 +87,20 @@ class EncoderCudaGraphManager: else self._generate_budgets(min_budget, max_budget) ) self.max_batch_size = ( - user_max_mm_items if user_max_mm_items > 0 else max_budget // min_budget + user_max_vision_items + if user_max_vision_items > 0 + else max_budget // min_budget ) - if user_max_frames > 0: + assert multimodal_config is not None + if multimodal_config.get_limit_per_prompt("video") == 0: + self.max_frames_per_batch = 0 + elif user_max_frames is not None: self.max_frames_per_batch = user_max_frames else: - # TODO(shen-shanshan): optimize this auto-infer for max_frames_per_batch. - self.max_frames_per_batch = self.max_batch_size * 2 + # Set it to the model-specific value according to its `processing_info`. + max_frames_per_video = self.model.get_max_frames_per_video() + self.max_frames_per_batch = self.max_batch_size * max_frames_per_video mm_config = vllm_config.model_config.multimodal_config self.use_dp = ( @@ -111,7 +119,7 @@ class EncoderCudaGraphManager: "budgets=%s, max_batch_size=%d, max_frames_per_batch=%s, use_dp=%s", self.token_budgets, self.max_batch_size, - self.max_frames_per_batch if self.max_frames_per_batch > 0 else "auto", + self.max_frames_per_batch, self.use_dp, )