[MM][CG] Support ViT full CUDA graph for Kimi-K2.5 (#50929)

Signed-off-by: lkchen <anthropic@lkchen.net>
Co-authored-by: lkchen <anthropic@lkchen.net>
This commit is contained in:
Linkun
2026-08-04 17:15:56 +00:00
committed by GitHub
co-authored by lkchen
parent 7635a9002b
commit 7cab4368f2
+247 -1
View File
@@ -8,7 +8,7 @@ Kimi-K2.5 extends Kimi-K2 with vision support.
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from typing import Annotated, Any, Literal
from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal
import torch
from torch import nn
@@ -25,10 +25,19 @@ from vllm.model_executor.layers.quantization.compressed_tensors import (
from vllm.model_executor.models.interfaces import (
SupportsEagle,
SupportsEagle3,
SupportsEncoderCudaGraph,
SupportsMultiModal,
SupportsPP,
SupportsQuant,
)
if TYPE_CHECKING:
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
EncoderCudaGraphConfig,
EncoderCudaGraphReplayBuffers,
EncoderItemSpec,
)
from vllm.model_executor.models.kimi_k25_vit import (
KimiK25MultiModalProjector,
MoonViT3dPretrainedModel,
@@ -294,6 +303,7 @@ class KimiK25ForConditionalGeneration(
SupportsQuant,
SupportsEagle,
SupportsEagle3,
SupportsEncoderCudaGraph,
):
"""Kimi-K2.5 model for conditional generation.
@@ -303,6 +313,7 @@ class KimiK25ForConditionalGeneration(
"""
supports_encoder_tp_data = True
supports_encoder_cudagraph: ClassVar[Literal[True]] = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -470,6 +481,241 @@ class KimiK25ForConditionalGeneration(
def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:
return self.language_model.get_eagle3_aux_hidden_state_layers()
# ------------------------------------------------------------------ #
# SupportsEncoderCudaGraph protocol #
# Image-only (t == 1). Video chunks (t > 1) fall back to eager. #
# ------------------------------------------------------------------ #
@property
def _encoder_cudagraph_pad_totals(self) -> dict[int, int]:
"""Row count of each captured buffer set, keyed by cu_seqlens ptr."""
totals = self.__dict__.get("_encoder_cg_pad_totals")
if totals is None:
totals = {}
self.__dict__["_encoder_cg_pad_totals"] = totals
return totals
def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig":
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
pad_totals = self._encoder_cudagraph_pad_totals
def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None:
# Varlen attention requires cu_seqlens[-1] to equal the number of
# rows actually passed in. The captured buffers are sized for the
# full token budget, so a smaller real batch has to be completed
# with one trailing padding sequence; declaring fewer rows than the
# buffer holds is undefined behaviour and returns NaN on FlashAttn.
total = pad_totals.get(dst.data_ptr())
n = min(src.shape[0], dst.shape[0])
dst[:n].copy_(src[:n])
dst[n:] = total if total is not None else src[-1]
return EncoderCudaGraphConfig(
modalities=["vision_chunk"],
buffer_keys=[
"pixel_values",
"pos_embeds",
"rope_freqs_cis",
"cu_seqlens",
"max_seqlen",
"merge_gather_idx",
],
out_hidden_size=self.config.text_config.hidden_size,
padding_logics={"cu_seqlens": pad_cu_seqlens},
)
def get_encoder_cudagraph_budget_range(
self,
vllm_config: VllmConfig,
) -> tuple[int, int]:
# Min: 64 output tokens (e.g. ~128×128 image with patch 14 + merge 2×2).
min_budget = 64
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
vllm_config.model_config.max_model_len,
)
return (min_budget, max_budget)
def _get_grid_thw_list(
self,
mm_kwargs: dict[str, Any],
) -> list[list[int]]:
grid_thws = mm_kwargs["grid_thws"]
if isinstance(grid_thws, torch.Tensor):
return [[int(x) for x in row] for row in grid_thws.tolist()]
return [[int(x) for x in row] for row in grid_thws]
def get_encoder_cudagraph_item_specs(
self,
mm_kwargs: dict[str, Any],
) -> list["EncoderItemSpec"]:
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
kh, kw = self.vision_tower.merge_kernel_size
specs = []
for t, h, w in self._get_grid_thw_list(mm_kwargs):
if t != 1:
# Video chunks not supported in encoder CUDA graph;
# sentinel forces eager fallback via the manager.
specs.append(EncoderItemSpec(input_size=t * h * w, output_tokens=2**30))
else:
specs.append(
EncoderItemSpec(
input_size=h * w,
output_tokens=(h // kh) * (w // kw),
)
)
return specs
def select_encoder_cudagraph_items(
self,
mm_kwargs: dict[str, Any],
indices: list[int],
) -> dict[str, Any]:
grid_thw_list = self._get_grid_thw_list(mm_kwargs)
pixel_values = mm_kwargs["pixel_values"]
if len(indices) == 0:
return {
"pixel_values": pixel_values[:0],
"grid_thws": pixel_values.new_zeros((0, 3), dtype=torch.long),
}
patch_counts = [t * h * w for t, h, w in grid_thw_list]
cum = [0]
for pc in patch_counts:
cum.append(cum[-1] + pc)
selected_pv = torch.cat(
[pixel_values[cum[i] : cum[i + 1]] for i in indices], dim=0
)
selected_grid = torch.tensor(
[grid_thw_list[i] for i in indices],
dtype=torch.long,
device=pixel_values.device,
)
return {"pixel_values": selected_pv, "grid_thws": selected_grid}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
) -> "EncoderCudaGraphCaptureInputs":
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphCaptureInputs
kh, kw = self.vision_tower.merge_kernel_size
# Output tokens per item in the dummy grid (ceiling so total >= budget).
per_item_out = (token_budget + max_batch_size - 1) // max_batch_size
# Fit within RoPE precomputed max dimensions.
rope = self.vision_tower.encoder.rope_2d
max_wo = rope.max_width // kw
wo = min(per_item_out, max_wo)
ho = (per_item_out + wo - 1) // wo
assert ho * kh <= rope.max_height, (
f"per_item_out={per_item_out} exceeds RoPE grid capacity "
f"(max {(rope.max_height // kh) * (rope.max_width // kw)} tokens)"
)
grid_thw_list = [[1, ho * kh, wo * kw] for _ in range(max_batch_size)]
ps = self.vision_tower.patch_size
if isinstance(ps, int):
ps = (ps, ps)
total_patches = max_batch_size * ho * kh * wo * kw
dummy_pixel_values = torch.zeros(
total_patches, 3, ps[0], ps[1], device=device, dtype=dtype
)
# max_seqlen must cover the worst case: one item consuming the full
# budget, i.e. token_budget * kh * kw patches.
# max_batch_size + 1 leaves a spare cu_seqlens slot so replay can append
# a padding sequence covering rows the real batch does not fill.
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
grid_thw_list,
max_batch_size=max_batch_size + 1,
max_seqlen_override=token_budget * kh * kw,
device=device,
)
values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values}
values.update({k: v for k, v in metadata.items() if v is not None})
cu_seqlens = values.get("cu_seqlens")
if cu_seqlens is not None:
self._encoder_cudagraph_pad_totals[cu_seqlens.data_ptr()] = total_patches
return EncoderCudaGraphCaptureInputs(values=values)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
) -> "EncoderCudaGraphReplayBuffers":
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
grid_thw_list = self._get_grid_thw_list(mm_kwargs)
pixel_values = mm_kwargs["pixel_values"]
# Unpadded: pad_cu_seqlens completes the tail with the padding sequence
# so cu_seqlens[-1] matches the captured buffer's row count.
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
grid_thw_list,
max_batch_size=None,
device=pixel_values.device,
)
values: dict[str, torch.Tensor | None] = {"pixel_values": pixel_values}
values.update(metadata)
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self,
inputs: dict[str, torch.Tensor],
path: str = "default",
) -> torch.Tensor:
pixel_values = inputs.pop("pixel_values")
# Remaining keys (pos_embeds, rope_freqs_cis, cu_seqlens, max_seqlen,
# merge_gather_idx, sequence_lengths) are consumed as encoder_metadata.
encoder_metadata = inputs
# Fast path: uses precomputed pos_embeds + merge_gather_idx.
vt_output = self.vision_tower(
pixel_values, grid_thws=None, encoder_metadata=encoder_metadata
)
# vt_output: (total_output_tokens, kh*kw, vit_hidden_dim)
proj_dtype = next(self.mm_projector.parameters()).dtype
if vt_output.dtype != proj_dtype:
vt_output = vt_output.to(proj_dtype)
projected = self.mm_projector(vt_output)
return projected.view(-1, self.config.text_config.hidden_size)
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
path: str = "default",
) -> torch.Tensor:
media_input = self._parse_and_validate_media_input(**mm_kwargs)
if media_input is None:
proj_dtype = next(self.mm_projector.parameters()).dtype
return torch.zeros(
0,
self.config.text_config.hidden_size,
device=self.device,
dtype=proj_dtype,
)
embeddings = self._process_media_input(media_input)
return torch.cat(embeddings, dim=0)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)