diff --git a/vllm/model_executor/models/kimi_k25.py b/vllm/model_executor/models/kimi_k25.py index 79b6e272bb..b5c1c925d1 100644 --- a/vllm/model_executor/models/kimi_k25.py +++ b/vllm/model_executor/models/kimi_k25.py @@ -8,7 +8,7 @@ Kimi-K2.5 extends Kimi-K2 with vision support. from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass -from typing import Annotated, Any, Literal +from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal import torch from torch import nn @@ -25,10 +25,19 @@ from vllm.model_executor.layers.quantization.compressed_tensors import ( from vllm.model_executor.models.interfaces import ( SupportsEagle, SupportsEagle3, + SupportsEncoderCudaGraph, SupportsMultiModal, SupportsPP, SupportsQuant, ) + +if TYPE_CHECKING: + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphCaptureInputs, + EncoderCudaGraphConfig, + EncoderCudaGraphReplayBuffers, + EncoderItemSpec, + ) from vllm.model_executor.models.kimi_k25_vit import ( KimiK25MultiModalProjector, MoonViT3dPretrainedModel, @@ -294,6 +303,7 @@ class KimiK25ForConditionalGeneration( SupportsQuant, SupportsEagle, SupportsEagle3, + SupportsEncoderCudaGraph, ): """Kimi-K2.5 model for conditional generation. @@ -303,6 +313,7 @@ class KimiK25ForConditionalGeneration( """ supports_encoder_tp_data = True + supports_encoder_cudagraph: ClassVar[Literal[True]] = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ @@ -470,6 +481,241 @@ class KimiK25ForConditionalGeneration( def get_eagle3_aux_hidden_state_layers(self) -> tuple[int, ...]: return self.language_model.get_eagle3_aux_hidden_state_layers() + # ------------------------------------------------------------------ # + # SupportsEncoderCudaGraph protocol # + # Image-only (t == 1). Video chunks (t > 1) fall back to eager. # + # ------------------------------------------------------------------ # + + @property + def _encoder_cudagraph_pad_totals(self) -> dict[int, int]: + """Row count of each captured buffer set, keyed by cu_seqlens ptr.""" + totals = self.__dict__.get("_encoder_cg_pad_totals") + if totals is None: + totals = {} + self.__dict__["_encoder_cg_pad_totals"] = totals + return totals + + def get_encoder_cudagraph_config(self) -> "EncoderCudaGraphConfig": + from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig + + pad_totals = self._encoder_cudagraph_pad_totals + + def pad_cu_seqlens(dst: torch.Tensor, src: torch.Tensor) -> None: + # Varlen attention requires cu_seqlens[-1] to equal the number of + # rows actually passed in. The captured buffers are sized for the + # full token budget, so a smaller real batch has to be completed + # with one trailing padding sequence; declaring fewer rows than the + # buffer holds is undefined behaviour and returns NaN on FlashAttn. + total = pad_totals.get(dst.data_ptr()) + n = min(src.shape[0], dst.shape[0]) + dst[:n].copy_(src[:n]) + dst[n:] = total if total is not None else src[-1] + + return EncoderCudaGraphConfig( + modalities=["vision_chunk"], + buffer_keys=[ + "pixel_values", + "pos_embeds", + "rope_freqs_cis", + "cu_seqlens", + "max_seqlen", + "merge_gather_idx", + ], + out_hidden_size=self.config.text_config.hidden_size, + padding_logics={"cu_seqlens": pad_cu_seqlens}, + ) + + def get_encoder_cudagraph_budget_range( + self, + vllm_config: VllmConfig, + ) -> tuple[int, int]: + # Min: 64 output tokens (e.g. ~128×128 image with patch 14 + merge 2×2). + min_budget = 64 + max_budget = min( + vllm_config.scheduler_config.max_num_batched_tokens, + vllm_config.model_config.max_model_len, + ) + return (min_budget, max_budget) + + def _get_grid_thw_list( + self, + mm_kwargs: dict[str, Any], + ) -> list[list[int]]: + grid_thws = mm_kwargs["grid_thws"] + if isinstance(grid_thws, torch.Tensor): + return [[int(x) for x in row] for row in grid_thws.tolist()] + return [[int(x) for x in row] for row in grid_thws] + + def get_encoder_cudagraph_item_specs( + self, + mm_kwargs: dict[str, Any], + ) -> list["EncoderItemSpec"]: + from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec + + kh, kw = self.vision_tower.merge_kernel_size + specs = [] + for t, h, w in self._get_grid_thw_list(mm_kwargs): + if t != 1: + # Video chunks not supported in encoder CUDA graph; + # sentinel forces eager fallback via the manager. + specs.append(EncoderItemSpec(input_size=t * h * w, output_tokens=2**30)) + else: + specs.append( + EncoderItemSpec( + input_size=h * w, + output_tokens=(h // kh) * (w // kw), + ) + ) + return specs + + def select_encoder_cudagraph_items( + self, + mm_kwargs: dict[str, Any], + indices: list[int], + ) -> dict[str, Any]: + grid_thw_list = self._get_grid_thw_list(mm_kwargs) + pixel_values = mm_kwargs["pixel_values"] + + if len(indices) == 0: + return { + "pixel_values": pixel_values[:0], + "grid_thws": pixel_values.new_zeros((0, 3), dtype=torch.long), + } + + patch_counts = [t * h * w for t, h, w in grid_thw_list] + cum = [0] + for pc in patch_counts: + cum.append(cum[-1] + pc) + + selected_pv = torch.cat( + [pixel_values[cum[i] : cum[i + 1]] for i in indices], dim=0 + ) + selected_grid = torch.tensor( + [grid_thw_list[i] for i in indices], + dtype=torch.long, + device=pixel_values.device, + ) + return {"pixel_values": selected_pv, "grid_thws": selected_grid} + + def prepare_encoder_cudagraph_capture_inputs( + self, + token_budget: int, + max_batch_size: int, + max_frames_per_batch: int, + device: torch.device, + dtype: torch.dtype, + path: str = "default", + ) -> "EncoderCudaGraphCaptureInputs": + from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphCaptureInputs + + kh, kw = self.vision_tower.merge_kernel_size + # Output tokens per item in the dummy grid (ceiling so total >= budget). + per_item_out = (token_budget + max_batch_size - 1) // max_batch_size + + # Fit within RoPE precomputed max dimensions. + rope = self.vision_tower.encoder.rope_2d + max_wo = rope.max_width // kw + wo = min(per_item_out, max_wo) + ho = (per_item_out + wo - 1) // wo + assert ho * kh <= rope.max_height, ( + f"per_item_out={per_item_out} exceeds RoPE grid capacity " + f"(max {(rope.max_height // kh) * (rope.max_width // kw)} tokens)" + ) + + grid_thw_list = [[1, ho * kh, wo * kw] for _ in range(max_batch_size)] + + ps = self.vision_tower.patch_size + if isinstance(ps, int): + ps = (ps, ps) + total_patches = max_batch_size * ho * kh * wo * kw + dummy_pixel_values = torch.zeros( + total_patches, 3, ps[0], ps[1], device=device, dtype=dtype + ) + + # max_seqlen must cover the worst case: one item consuming the full + # budget, i.e. token_budget * kh * kw patches. + # max_batch_size + 1 leaves a spare cu_seqlens slot so replay can append + # a padding sequence covering rows the real batch does not fill. + metadata = self.vision_tower.prepare_encoder_cudagraph_metadata( + grid_thw_list, + max_batch_size=max_batch_size + 1, + max_seqlen_override=token_budget * kh * kw, + device=device, + ) + + values: dict[str, torch.Tensor] = {"pixel_values": dummy_pixel_values} + values.update({k: v for k, v in metadata.items() if v is not None}) + + cu_seqlens = values.get("cu_seqlens") + if cu_seqlens is not None: + self._encoder_cudagraph_pad_totals[cu_seqlens.data_ptr()] = total_patches + + return EncoderCudaGraphCaptureInputs(values=values) + + def prepare_encoder_cudagraph_replay_buffers( + self, + mm_kwargs: dict[str, Any], + max_batch_size: int, + max_frames_per_batch: int, + path: str = "default", + ) -> "EncoderCudaGraphReplayBuffers": + from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers + + grid_thw_list = self._get_grid_thw_list(mm_kwargs) + pixel_values = mm_kwargs["pixel_values"] + + # Unpadded: pad_cu_seqlens completes the tail with the padding sequence + # so cu_seqlens[-1] matches the captured buffer's row count. + metadata = self.vision_tower.prepare_encoder_cudagraph_metadata( + grid_thw_list, + max_batch_size=None, + device=pixel_values.device, + ) + + values: dict[str, torch.Tensor | None] = {"pixel_values": pixel_values} + values.update(metadata) + return EncoderCudaGraphReplayBuffers(values=values) + + def encoder_cudagraph_forward( + self, + inputs: dict[str, torch.Tensor], + path: str = "default", + ) -> torch.Tensor: + pixel_values = inputs.pop("pixel_values") + # Remaining keys (pos_embeds, rope_freqs_cis, cu_seqlens, max_seqlen, + # merge_gather_idx, sequence_lengths) are consumed as encoder_metadata. + encoder_metadata = inputs + + # Fast path: uses precomputed pos_embeds + merge_gather_idx. + vt_output = self.vision_tower( + pixel_values, grid_thws=None, encoder_metadata=encoder_metadata + ) + # vt_output: (total_output_tokens, kh*kw, vit_hidden_dim) + + proj_dtype = next(self.mm_projector.parameters()).dtype + if vt_output.dtype != proj_dtype: + vt_output = vt_output.to(proj_dtype) + + projected = self.mm_projector(vt_output) + return projected.view(-1, self.config.text_config.hidden_size) + + def encoder_eager_forward( + self, + mm_kwargs: dict[str, Any], + path: str = "default", + ) -> torch.Tensor: + media_input = self._parse_and_validate_media_input(**mm_kwargs) + if media_input is None: + proj_dtype = next(self.mm_projector.parameters()).dtype + return torch.zeros( + 0, + self.config.text_config.hidden_size, + device=self.device, + dtype=proj_dtype, + ) + embeddings = self._process_media_input(media_input) + return torch.cat(embeddings, dim=0) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)