[MM][CG] Optimize default max_frames_per_batch auto-infer for ViT CUDA graph video inference (#40445)
Signed-off-by: shen-shanshan <467638484@qq.com>
This commit is contained in:
@@ -1531,6 +1531,12 @@ class SupportsEncoderCudaGraph(Protocol):
|
||||
"""Return the modality of the inputs."""
|
||||
...
|
||||
|
||||
def get_max_frames_per_video(
|
||||
self,
|
||||
) -> int:
|
||||
"""Return model-specific max frames per video."""
|
||||
...
|
||||
|
||||
def get_encoder_cudagraph_budget_range(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
|
||||
@@ -1640,6 +1640,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
|
||||
self.multimodal_config = multimodal_config
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
@@ -1783,6 +1784,15 @@ class Qwen3VLForConditionalGeneration(
|
||||
return "image"
|
||||
return "video"
|
||||
|
||||
def get_max_frames_per_video(self) -> int:
|
||||
mm_registry = MULTIMODAL_REGISTRY
|
||||
info = mm_registry.get_processing_info(self.model_config)
|
||||
max_frames_per_video = info.get_num_frames_with_most_features(
|
||||
seq_len=self.model_config.max_model_len,
|
||||
mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
|
||||
)
|
||||
return max_frames_per_video
|
||||
|
||||
def get_encoder_cudagraph_budget_range(
|
||||
self,
|
||||
vllm_config,
|
||||
|
||||
Reference in New Issue
Block a user