[MM][CG] Support ViT full CUDA graph for Kimi-K2.5 (#50929)

Signed-off-by: lkchen <anthropic@lkchen.net>
Co-authored-by: lkchen <anthropic@lkchen.net>
This commit is contained in:
Linkun
2026-08-04 17:15:56 +00:00
committed by GitHub
co-authored by lkchen
parent 7635a9002b
commit 7cab4368f2
+247 -1
View File
@@ -8,7 +8,7 @@ Kimi-K2.5 extends Kimi-K2 with vision support.
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass from dataclasses import dataclass
from typing import Annotated, Any, Literal from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal
import torch import torch
from torch import nn from torch import nn
@@ -25,10 +25,19 @@ from vllm.model_executor.layers.quantization.compressed_tensors import (
from vllm.model_executor.models.interfaces import ( from vllm.model_executor.models.interfaces import (
SupportsEagle, SupportsEagle,
SupportsEagle3, SupportsEagle3,
SupportsEncoderCudaGraph,
SupportsMultiModal, SupportsMultiModal,
SupportsPP, SupportsPP,
SupportsQuant, SupportsQuant,
) )
if TYPE_CHECKING:
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
EncoderCudaGraphConfig,
EncoderCudaGraphReplayBuffers,
EncoderItemSpec,
)
from vllm.model_executor.models.kimi_k25_vit import ( from vllm.model_executor.models.kimi_k25_vit import (
KimiK25MultiModalProjector, KimiK25MultiModalProjector,
MoonViT3dPretrainedModel, MoonViT3dPretrainedModel,
@@ -294,6 +303,7 @@ class KimiK25ForConditionalGeneration(
SupportsQuant, SupportsQuant,
SupportsEagle, SupportsEagle,
SupportsEagle3, SupportsEagle3,
SupportsEncoderCudaGraph,
): ):
"""Kimi-K2.5 model for conditional generation. """Kimi-K2.5 model for conditional generation.
@@ -303,6 +313,7 @@ class KimiK25ForConditionalGeneration(
""" """
supports_encoder_tp_data = True supports_encoder_tp_data = True
supports_encoder_cudagraph: ClassVar[Literal[True]] = True
hf_to_vllm_mapper = WeightsMapper( hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={ orig_to_new_prefix={
@@ -470,6 +481,241 @@ class KimiK25ForConditionalGeneration(
def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]: def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]:
return self.language_model.get_eagle3_aux_hidden_state_layers() return self.language_model.get_eagle3_aux_hidden_state_layers()
# ------------------------------------------------------------------ #
# SupportsEncoderCudaGraph protocol #
# Image-only (t == 1). Video chunks (t > 1) fall back to eager. #
# ------------------------------------------------------------------ #
@property
def _encoder_cudagraph_pad_totals(self) -> dict[int, int]:
"""Row count of each captured buffer set, keyed by cu_seqlens ptr."""
totals = self.__dict__.get("_encoder_cg_pad_totals")
if totals is None:
totals = {}
self.__dict__["_encoder_cg_pad_totals"] = totals
return totals
def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig":
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
pad_totals = self._encoder_cudagraph_pad_totals
def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None:
# Varlen attention requires cu_seqlens[-1] to equal the number of
# rows actually passed in. The captured buffers are sized for the
# full token budget, so a smaller real batch has to be completed
# with one trailing padding sequence; declaring fewer rows than the
# buffer holds is undefined behaviour and returns NaN on FlashAttn.
total = pad_totals.get(dst.data_ptr())
n = min(src.shape[0], dst.shape[0])
dst[:n].copy_(src[:n])
dst[n:] = total if total is not None else src[-1]
return EncoderCudaGraphConfig(
modalities=["vision_chunk"],
buffer_keys=[
"pixel_values",
"pos_embeds",
"rope_freqs_cis",
"cu_seqlens",
"max_seqlen",
"merge_gather_idx",
],
out_hidden_size=self.config.text_config.hidden_size,
padding_logics={"cu_seqlens": pad_cu_seqlens},
)
def get_encoder_cudagraph_budget_range(
self,
vllm_config: VllmConfig,
) -> tuple[int, int]:
# Min: 64 output tokens (e.g. ~128×128 image with patch 14 + merge 2×2).
min_budget = 64
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
vllm_config.model_config.max_model_len,
)
return (min_budget, max_budget)
def _get_grid_thw_list(
self,
mm_kwargs: dict[str, Any],
) -> list[list[int]]:
grid_thws = mm_kwargs["grid_thws"]
if isinstance(grid_thws, torch.Tensor):
return [[int(x) for x in row] for row in grid_thws.tolist()]
return [[int(x) for x in row] for row in grid_thws]
def get_encoder_cudagraph_item_specs(
self,
mm_kwargs: dict[str, Any],
) -> list["EncoderItemSpec"]:
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
kh, kw = self.vision_tower.merge_kernel_size
specs = []
for t, h, w in self._get_grid_thw_list(mm_kwargs):
if t != 1:
# Video chunks not supported in encoder CUDA graph;
# sentinel forces eager fallback via the manager.
specs.append(EncoderItemSpec(input_size=t * h * w, output_tokens=2**30))
else:
specs.append(
EncoderItemSpec(
input_size=h * w,
output_tokens=(h // kh) * (w // kw),
)
)
return specs
def select_encoder_cudagraph_items(
self,
mm_kwargs: dict[str, Any],
indices: list[int],
) -> dict[str, Any]:
grid_thw_list = self._get_grid_thw_list(mm_kwargs)
pixel_values = mm_kwargs["pixel_values"]
if len(indices) == 0:
return {
"pixel_values": pixel_values[:0],
"grid_thws": pixel_values.new_zeros((0, 3), dtype=torch.long),
}
patch_counts = [t * h * w for t, h, w in grid_thw_list]
cum = [0]
for pc in patch_counts:
cum.append(cum[-1] + pc)
selected_pv = torch.cat(
[pixel_values[cum[i] : cum[i + 1]] for i in indices], dim=0
)
selected_grid = torch.tensor(
[grid_thw_list[i] for i in indices],
dtype=torch.long,
device=pixel_values.device,
)
return {"pixel_values": selected_pv, "grid_thws": selected_grid}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
) -> "EncoderCudaGraphCaptureInputs":
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphCaptureInputs
kh, kw = self.vision_tower.merge_kernel_size
# Output tokens per item in the dummy grid (ceiling so total >= budget).
per_item_out = (token_budget + max_batch_size - 1) // max_batch_size
# Fit within RoPE precomputed max dimensions.
rope = self.vision_tower.encoder.rope_2d
max_wo = rope.max_width // kw
wo = min(per_item_out, max_wo)
ho = (per_item_out + wo - 1) // wo
assert ho * kh <= rope.max_height, (
f"per_item_out={per_item_out} exceeds RoPE grid capacity "
f"(max {(rope.max_height // kh) * (rope.max_width // kw)} tokens)"
)
grid_thw_list = [[1, ho * kh, wo * kw] for _ in range(max_batch_size)]
ps = self.vision_tower.patch_size
if isinstance(ps, int):
ps = (ps, ps)
total_patches = max_batch_size * ho * kh * wo * kw
dummy_pixel_values = torch.zeros(
total_patches, 3, ps[0], ps[1], device=device, dtype=dtype
)
# max_seqlen must cover the worst case: one item consuming the full
# budget, i.e. token_budget * kh * kw patches.
# max_batch_size + 1 leaves a spare cu_seqlens slot so replay can append
# a padding sequence covering rows the real batch does not fill.
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
grid_thw_list,
max_batch_size=max_batch_size + 1,
max_seqlen_override=token_budget * kh * kw,
device=device,
)
values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values}
values.update({k: v for k, v in metadata.items() if v is not None})
cu_seqlens = values.get("cu_seqlens")
if cu_seqlens is not None:
self._encoder_cudagraph_pad_totals[cu_seqlens.data_ptr()] = total_patches
return EncoderCudaGraphCaptureInputs(values=values)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
) -> "EncoderCudaGraphReplayBuffers":
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
grid_thw_list = self._get_grid_thw_list(mm_kwargs)
pixel_values = mm_kwargs["pixel_values"]
# Unpadded: pad_cu_seqlens completes the tail with the padding sequence
# so cu_seqlens[-1] matches the captured buffer's row count.
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
grid_thw_list,
max_batch_size=None,
device=pixel_values.device,
)
values: dict[str, torch.Tensor | None] = {"pixel_values": pixel_values}
values.update(metadata)
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self,
inputs: dict[str, torch.Tensor],
path: str = "default",
) -> torch.Tensor:
pixel_values = inputs.pop("pixel_values")
# Remaining keys (pos_embeds, rope_freqs_cis, cu_seqlens, max_seqlen,
# merge_gather_idx, sequence_lengths) are consumed as encoder_metadata.
encoder_metadata = inputs
# Fast path: uses precomputed pos_embeds + merge_gather_idx.
vt_output = self.vision_tower(
pixel_values, grid_thws=None, encoder_metadata=encoder_metadata
)
# vt_output: (total_output_tokens, kh*kw, vit_hidden_dim)
proj_dtype = next(self.mm_projector.parameters()).dtype
if vt_output.dtype != proj_dtype:
vt_output = vt_output.to(proj_dtype)
projected = self.mm_projector(vt_output)
return projected.view(-1, self.config.text_config.hidden_size)
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
path: str = "default",
) -> torch.Tensor:
media_input = self._parse_and_validate_media_input(**mm_kwargs)
if media_input is None:
proj_dtype = next(self.mm_projector.parameters()).dtype
return torch.zeros(
0,
self.config.text_config.hidden_size,
device=self.device,
dtype=proj_dtype,
)
embeddings = self._process_media_input(media_input)
return torch.cat(embeddings, dim=0)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
loader = AutoWeightsLoader(self) loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)