[Multimodal] Simplify ViT CUDA graph interfaces (#41234)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-05-22 22:31:00 +08:00
committed by GitHub
parent fb21d8b4f9
commit f0feb15e7f
9 changed files with 151 additions and 149 deletions
+14 -12
View File
@@ -21,6 +21,7 @@ from vllm.model_executor.models.utils import scatter_output_slices
from vllm.model_executor.models.vision import get_load_balance_assignment
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphConfig,
EncoderItemSpec,
)
logger = init_logger(__name__)
@@ -141,8 +142,8 @@ class EncoderCudaGraphManager:
elif user_max_frames is not None:
self.max_frames_per_batch = user_max_frames
else:
# Set it to the model-specific value according to its `processing_info`.
max_frames_per_video = self.model.get_max_frames_per_video()
# Set it to the model-specific value from config.
max_frames_per_video = self.config.max_frames_per_video
self.max_frames_per_batch = self.max_batch_size * max_frames_per_video
mm_config = vllm_config.model_config.multimodal_config
@@ -250,12 +251,13 @@ class EncoderCudaGraphManager:
return budget
return None
def _get_item_specs(self, mm_kwargs: dict[str, Any]) -> list[EncoderItemSpec]:
"""Get item specs from the model."""
return self.model.get_encoder_cudagraph_item_specs(mm_kwargs)
def _get_per_item_out_tokens(self, mm_kwargs: dict[str, Any]) -> list[int]:
"""Get per-item output token counts as plain ints."""
return [
int(t)
for t in self.model.get_encoder_cudagraph_per_item_output_tokens(mm_kwargs)
]
return [spec.output_tokens for spec in self._get_item_specs(mm_kwargs)]
def _run_budget_graph(
self,
@@ -274,7 +276,7 @@ class EncoderCudaGraphManager:
Returns:
Encoder outputs, or None if graph not captured.
"""
num_items = self.model.get_encoder_cudagraph_num_items(mm_kwargs)
num_items = len(self._get_item_specs(mm_kwargs))
if token_budget not in self.budget_graphs:
self.graph_misses += num_items
return None
@@ -332,10 +334,11 @@ class EncoderCudaGraphManager:
always satisfy total_tokens <= max_budget and therefore
always find a valid budget (no miss).
"""
num_items = self.model.get_encoder_cudagraph_num_items(mm_kwargs)
item_specs = self._get_item_specs(mm_kwargs)
num_items = len(item_specs)
max_budget = self.token_budgets[-1]
per_item_out_tokens = self._get_per_item_out_tokens(mm_kwargs)
per_item_out_tokens = [spec.output_tokens for spec in item_specs]
# Sort ascending by output token count (smallest first)
sorted_indices = sorted(range(num_items), key=lambda i: per_item_out_tokens[i])
@@ -459,9 +462,8 @@ class EncoderCudaGraphManager:
tp_size = get_tensor_model_parallel_world_size()
current_rank = get_tensor_model_parallel_rank()
per_item_input_sizes = self.model.get_encoder_cudagraph_per_item_input_sizes(
mm_kwargs
)
item_specs = self._get_item_specs(mm_kwargs)
per_item_input_sizes = [spec.input_size for spec in item_specs]
(image_rank_assignment, images_per_rank, input_patches_per_rank) = (
get_load_balance_assignment(per_item_input_sizes, tp_size)