[Model] Extend device-side mm normalization to Kimi K2.5 / K3 (#59278)

Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
cjackal
2026-10-05 21:28:13 +08:00
committed by GitHub
parent ff53f32409
commit 0e468adb43
11 changed files with 130 additions and 15 deletions
+3 -2
View File
@@ -9,6 +9,7 @@ import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors import (
compressed_tensors,
@@ -65,6 +66,7 @@ class KimiK3ForConditionalGeneration(
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
supports_encoder_tp_data = True
supports_mm_device_do_normalize = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -103,6 +105,7 @@ class KimiK3ForConditionalGeneration(
self.vision_tower = MoonViT3dPretrainedModel(
config.vision_config,
quant_config=self._maybe_ignore_quant_config(quant_config),
input_norm=build_mm_input_norm(vllm_config.model_config),
prefix=maybe_prefix(prefix, "vision_tower"),
)
if self._maybe_ignore_quant_config(quant_config) is not None:
@@ -163,8 +166,6 @@ class KimiK3ForConditionalGeneration(
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
)
target_dtype = next(self.vision_tower.parameters()).dtype
pixel_values = pixel_values.to(target_dtype)
assert isinstance(grid_thws, torch.Tensor), (
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
)
@@ -30,6 +30,9 @@ from vllm.multimodal.processing import (
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
from vllm.transformers_utils.processor import cached_get_image_processor
from vllm.transformers_utils.processors.kimi_k3 import KimiK3Processor
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
KimiK25FusedVisionProcessor,
)
logger = init_logger(__name__)
@@ -106,6 +109,7 @@ class KimiK3ProcessingInfo(BaseProcessingInfo):
self.ctx.model_config.model,
revision=self.ctx.model_config.revision,
trust_remote_code=self.ctx.model_config.trust_remote_code,
processor_cls_overrides=KimiK25FusedVisionProcessor,
)
# Resolve token ID from the tokenizer because transformers v5
+6 -7
View File
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
fused_grouped_topk,
)
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -1736,6 +1737,7 @@ class KimiK3ForConditionalGeneration(
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
supports_encoder_tp_data = True
supports_mm_device_do_normalize = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -1775,6 +1777,7 @@ class KimiK3ForConditionalGeneration(
self.vision_tower = MoonViT3dPretrainedModel(
config.vision_config,
quant_config=self._maybe_ignore_quant_config(quant_config),
input_norm=build_mm_input_norm(vllm_config.model_config),
prefix=maybe_prefix(prefix, "vision_tower"),
)
if is_meta_module(self.vision_tower):
@@ -1964,13 +1967,13 @@ class KimiK3ForConditionalGeneration(
if isinstance(patch_size, int):
patch_size = (patch_size, patch_size)
total_patches = sum(t * h * w for t, h, w in grid_thws)
pixel_values = torch.randn(
pixel_values = torch.zeros(
total_patches,
3,
patch_size[0],
patch_size[1],
device=device,
dtype=dtype,
dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype,
)
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
grid_thws,
@@ -2028,9 +2031,7 @@ class KimiK3ForConditionalGeneration(
path: str = "default",
) -> torch.Tensor:
image_features = self.vision_tower(
self._get_pixel_values(mm_kwargs).to(
next(self.vision_tower.parameters()).dtype
),
self._get_pixel_values(mm_kwargs),
self._get_grid_thws(mm_kwargs),
)
return self._project_encoder_features(torch.cat(image_features))
@@ -2063,8 +2064,6 @@ class KimiK3ForConditionalGeneration(
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
)
target_dtype = next(self.vision_tower.parameters()).dtype
pixel_values = pixel_values.to(target_dtype)
assert isinstance(grid_thws, torch.Tensor), (
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
)