[MM][Perf][CG] Support ViT full CUDA graph for Kimi-VL (#41992)

Signed-off-by: oguz <oguzhankir17@gmail.com>
This commit is contained in:
Oğuzhan KIR
2026-06-17 12:14:01 +00:00
committed by GitHub
parent e28e8c8782
commit fa85ead2f3
5 changed files with 504 additions and 45 deletions
+195 -2
View File
@@ -56,7 +56,11 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP
from vllm.model_executor.models.interfaces import (
SupportsEncoderCudaGraph,
SupportsMultiModal,
SupportsPP,
)
from vllm.model_executor.models.moonvit import MoonVitPretrainedModel
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -79,6 +83,7 @@ from vllm.multimodal.processing import (
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.kimi_vl import KimiVLConfig, MoonViTConfig
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
from .vision import is_vit_use_data_parallel, run_dp_sharded_mrope_vision_model
@@ -287,7 +292,9 @@ class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]):
info=KimiVLProcessingInfo,
dummy_inputs=KimiVLDummyInputsBuilder,
)
class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
class KimiVLForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsEncoderCudaGraph, SupportsPP
):
supports_encoder_tp_data = True
@classmethod
@@ -340,6 +347,192 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
self.media_placeholder: int = self.config.media_placeholder_token_id
self.model_config = model_config
# -- SupportsEncoderCudaGraph protocol methods --
def get_encoder_cudagraph_config(self):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphConfig,
)
return EncoderCudaGraphConfig(
modalities=["image"],
buffer_keys=[
"pixel_values",
"pos_embeds",
"rope_freqs_cis",
"cu_seqlens",
"max_seqlen",
"merge_gather_idx",
],
out_hidden_size=self.hidden_size,
)
def get_encoder_cudagraph_budget_range(
self,
vllm_config,
) -> tuple[int, int]:
# Min: estimated smallest possible encoder input.
# 224x224 image with patch_size=14 -> 16x16 patches, then merge
# kernel (2,2) -> 8x8 = 64 output tokens.
min_budget = 64
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
self.model_config.max_model_len,
)
return (min_budget, max_budget)
def _get_grid_hws(
self,
mm_kwargs: dict[str, Any],
) -> list[tuple[int, int]]:
grid_hws = mm_kwargs["image_grid_hws"]
if not isinstance(grid_hws, list):
grid_hws = grid_hws.tolist()
return grid_hws
def get_encoder_cudagraph_item_specs(
self,
mm_kwargs: dict[str, Any],
):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
kh, kw = self.config.vision_config.merge_kernel_size
return [
EncoderItemSpec(
input_size=h * w,
output_tokens=(h // kh) * (w // kw),
)
for h, w in self._get_grid_hws(mm_kwargs)
]
def select_encoder_cudagraph_items(
self,
mm_kwargs: dict[str, Any],
indices: list[int],
) -> dict[str, Any]:
grid_hws = self._get_grid_hws(mm_kwargs)
pixel_values = mm_kwargs["pixel_values"]
if len(indices) == 0:
return {
"pixel_values": pixel_values[:0],
"image_grid_hws": pixel_values.new_zeros((0, 2), dtype=torch.long),
}
patches_per_item = [h * w for h, w in grid_hws]
cum_patches = [0]
for p in patches_per_item:
cum_patches.append(cum_patches[-1] + p)
selected_pv = torch.cat(
[pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices]
)
selected_grid = torch.tensor(
[grid_hws[i] for i in indices],
dtype=torch.long,
device=pixel_values.device,
)
return {
"pixel_values": selected_pv,
"image_grid_hws": selected_grid,
}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
)
kh, kw = self.config.vision_config.merge_kernel_size
# Ceil so the buffer fits the worst case of one item using the full
# budget. Floor under-allocates when budget is not a multiple of
# max_batch_size.
per_mm_item_output = (token_budget + max_batch_size - 1) // max_batch_size
# Shape the synthetic grid so neither dimension exceeds Rope2DPosEmb's
# precomputed range. Pack as wide a row as fits, then add rows.
rope = self.vision_tower.encoder.rope_2d
max_wo = rope.max_width // kw
wo = min(per_mm_item_output, max_wo)
ho = (per_mm_item_output + wo - 1) // wo
assert ho * kh <= rope.max_height, (
f"per_mm_item_output={per_mm_item_output} exceeds RoPE grid capacity "
f"(max {(rope.max_height // kh) * (rope.max_width // kw)} tokens)"
)
grid_hws_list = [(ho * kh, wo * kw) for _ in range(max_batch_size)]
patch_size = self.config.vision_config.patch_size
if isinstance(patch_size, int):
patch_size = (patch_size, patch_size)
total_patches = sum(h * w for h, w in grid_hws_list)
in_channels = 3
dummy_pixel_values = torch.randn(
total_patches,
in_channels,
patch_size[0],
patch_size[1],
device=device,
dtype=dtype,
)
buffers = self.vision_tower.prepare_encoder_metadata(
grid_hws_list,
max_batch_size=max_batch_size,
max_seqlen_override=token_budget,
device=device,
)
values = buffers | {"pixel_values": dummy_pixel_values}
return EncoderCudaGraphCaptureInputs(values=values)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
):
grid_hws_list = self._get_grid_hws(mm_kwargs)
buffers = self.vision_tower.prepare_encoder_metadata(
grid_hws_list,
max_batch_size=max_batch_size,
device=mm_kwargs["pixel_values"].device,
)
values = buffers | {"pixel_values": mm_kwargs["pixel_values"]}
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self,
values: dict[str, torch.Tensor],
path: str = "default",
) -> torch.Tensor:
pixel_values = values.pop("pixel_values")
metadata = values
image_features = self.vision_tower(
pixel_values, grid_hw=None, encoder_metadata=metadata
)
return self.multi_modal_projector(image_features)
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
path: str = "default",
) -> torch.Tensor:
pixel_values = mm_kwargs["pixel_values"]
image_grid_hws = mm_kwargs["image_grid_hws"]
image_features = self.vision_tower(pixel_values, image_grid_hws)
return self.multi_modal_projector(torch.cat(image_features))
def _parse_and_validate_image_input(
self, **kwargs: object
) -> KimiVLImageInputs | None: