[Model] Extend device-side mm normalization to Kimi K2.5 / K3 (#59278)

Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
cjackal
2026-10-05 21:28:13 +08:00
committed by GitHub
parent ff53f32409
commit 0e468adb43
11 changed files with 130 additions and 15 deletions
+9 -4
View File
@@ -17,6 +17,7 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors import (
compressed_tensors,
@@ -306,6 +307,7 @@ class KimiK25ForConditionalGeneration(
supports_encoder_tp_data = True
supports_encoder_cudagraph: ClassVar[Literal[True]] = True
supports_mm_device_do_normalize = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -350,6 +352,7 @@ class KimiK25ForConditionalGeneration(
self.vision_tower = MoonViT3dPretrainedModel(
config.vision_config,
quant_config=self._maybe_ignore_quant_config(quant_config),
input_norm=build_mm_input_norm(vllm_config.model_config),
prefix=maybe_prefix(prefix, "vision_tower"),
)
if self._maybe_ignore_quant_config(quant_config) is not None:
@@ -404,9 +407,6 @@ class KimiK25ForConditionalGeneration(
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
)
# The batch dimension of pixel_values has been flattened into shape[0]
target_dtype = next(self.vision_tower.parameters()).dtype
pixel_values = pixel_values.to(target_dtype)
assert isinstance(grid_thws, torch.Tensor), (
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
)
@@ -623,7 +623,12 @@ class KimiK25ForConditionalGeneration(
ps = (ps, ps)
total_patches = max_batch_size * ho * kh * wo * kw
dummy_pixel_values = torch.zeros(
total_patches, 3, ps[0], ps[1], device=device, dtype=dtype
total_patches,
3,
ps[0],
ps[1],
device=device,
dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype,
)
# max_seqlen must cover the worst case: one item consuming the full
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -187,6 +188,7 @@ class MoonVision3dPatchEmbed(nn.Module):
pos_emb_type: str = "divided_fixed",
patch_embed_proj_bias: bool = True,
pos_emb_interpolation_mode: str = "bicubic",
input_norm: nn.Module | None = None,
):
super().__init__()
assert isinstance(patch_size, int | Sequence), (
@@ -206,6 +208,7 @@ class MoonVision3dPatchEmbed(nn.Module):
stride=patch_size,
bias=patch_embed_proj_bias,
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
if pos_emb_type == "divided_fixed":
self.pos_emb = Learnable2DInterpPosEmbDivided_fixed(
@@ -225,6 +228,7 @@ class MoonVision3dPatchEmbed(nn.Module):
*,
pos_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
x = self.input_norm(x.flatten(1), self.proj.weight.dtype).view_as(x)
# forward_native dispatches this non-overlapping patch projection to GEMM.
x = self.proj.forward_native(x).view(x.size(0), self.proj.out_channels)
if pos_embeds is not None:
@@ -703,6 +707,7 @@ class MoonViT3dPretrainedModel(nn.Module):
self,
config: KimiK25VisionConfig,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
):
super().__init__()
@@ -723,6 +728,7 @@ class MoonViT3dPretrainedModel(nn.Module):
pos_emb_interpolation_mode=getattr(
config, "pos_emb_interpolation_mode", "bicubic"
),
input_norm=input_norm,
)
self.encoder = MoonViT3dEncoder(