[Model] Extend device-side mm normalization to Kimi K2.5 / K3 (#59278)

Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
cjackal
2026-10-05 21:28:13 +08:00
committed by GitHub
parent ff53f32409
commit 0e468adb43
11 changed files with 130 additions and 15 deletions
+2
View File
@@ -85,6 +85,8 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`:
| `glm-ocr` | `GlmOcrForConditionalGeneration` | `zai-org/GLM-OCR` |
| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` |
| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` |
| `kimi-k2.5` | `KimiK25ForConditionalGeneration` | `moonshotai/Kimi-K2.5`, etc. |
| `kimi-k3` | `KimiK3ForConditionalGeneration` | `moonshotai/Kimi-K3` |
| `llama-nemotron-vl-embed` | `LlamaNemotronVLForEmbedding` | `nvidia/llama-nemotron-embed-vl-1b-v2` |
| `llama-nemotron-vl-rerank` | `LlamaNemotronVLForSequenceClassification` | `nvidia/llama-nemotron-rerank-vl-1b-v2` |
| `mistral3` | `Mistral3ForConditionalGeneration` | `mistralai/Mistral-Small-3.2-24B-Instruct-2506`, etc. |
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
from PIL import Image
from transformers import AutoImageProcessor
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
KimiK25FusedVisionProcessor,
)
from ...utils import build_model_context
_MODEL_ID = "moonshotai/Kimi-K3"
@pytest.fixture(scope="module")
def image_processors():
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
k25_fused = processor.info.image_processor
assert isinstance(k25_fused, KimiK25FusedVisionProcessor)
k3_hf = AutoImageProcessor.from_pretrained(_MODEL_ID, trust_remote_code=True)
return k25_fused, k3_hf
def _noise_image(width: int, height: int) -> Image.Image:
rng = np.random.default_rng(0)
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
@pytest.mark.parametrize(
("width", "height"),
[(310, 470), (7300, 30), (13, 9)],
)
def test_fused_image_processor_matches_hf(image_processors, width: int, height: int):
fused, hf = image_processors
image = _noise_image(width, height)
expected = hf.preprocess([{"type": "image", "image": image}], return_tensors="pt")
actual = fused.preprocess([{"type": "image", "image": image}], return_tensors="pt")
torch.testing.assert_close(actual["grid_thws"], expected["grid_thws"])
torch.testing.assert_close(actual["pixel_values"], expected["pixel_values"])
assert fused.media_tokens_calculator(
{"type": "image", "image": image}
) == hf.media_tokens_calculator({"type": "image", "image": image})
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
device = current_platform.device_type
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 2})
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (310, 470), color=(17, 89, 231)),
Image.new("RGB", (480, 320), color=(201, 13, 127)),
]
prompt = processor.info.get_hf_config().image_placeholder * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert raw_values.dtype == torch.uint8
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
)
@@ -152,6 +152,7 @@ def _load_norm_params(model_config: ModelConfig) -> NormParams:
for key in ("do_normalize", "do_rescale", "image_mean", "image_std")
):
config = model_config.hf_image_processor_config
config = config.get("media_proc_cfg", config)
has_norm_params = "image_mean" in config and "image_std" in config
do_normalize = bool(config.get("do_normalize", has_norm_params))
+9 -4
View File
@@ -17,6 +17,7 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors import (
compressed_tensors,
@@ -306,6 +307,7 @@ class KimiK25ForConditionalGeneration(
supports_encoder_tp_data = True
supports_encoder_cudagraph: ClassVar[Literal[True]] = True
supports_mm_device_do_normalize = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -350,6 +352,7 @@ class KimiK25ForConditionalGeneration(
self.vision_tower = MoonViT3dPretrainedModel(
config.vision_config,
quant_config=self._maybe_ignore_quant_config(quant_config),
input_norm=build_mm_input_norm(vllm_config.model_config),
prefix=maybe_prefix(prefix, "vision_tower"),
)
if self._maybe_ignore_quant_config(quant_config) is not None:
@@ -404,9 +407,6 @@ class KimiK25ForConditionalGeneration(
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
)
# The batch dimension of pixel_values has been flattened into shape[0]
target_dtype = next(self.vision_tower.parameters()).dtype
pixel_values = pixel_values.to(target_dtype)
assert isinstance(grid_thws, torch.Tensor), (
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
)
@@ -623,7 +623,12 @@ class KimiK25ForConditionalGeneration(
ps = (ps, ps)
total_patches = max_batch_size * ho * kh * wo * kw
dummy_pixel_values = torch.zeros(
total_patches, 3, ps[0], ps[1], device=device, dtype=dtype
total_patches,
3,
ps[0],
ps[1],
device=device,
dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype,
)
# max_seqlen must cover the worst case: one item consuming the full
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -187,6 +188,7 @@ class MoonVision3dPatchEmbed(nn.Module):
pos_emb_type: str = "divided_fixed",
patch_embed_proj_bias: bool = True,
pos_emb_interpolation_mode: str = "bicubic",
input_norm: nn.Module | None = None,
):
super().__init__()
assert isinstance(patch_size, int | Sequence), (
@@ -206,6 +208,7 @@ class MoonVision3dPatchEmbed(nn.Module):
stride=patch_size,
bias=patch_embed_proj_bias,
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
if pos_emb_type == "divided_fixed":
self.pos_emb = Learnable2DInterpPosEmbDivided_fixed(
@@ -225,6 +228,7 @@ class MoonVision3dPatchEmbed(nn.Module):
*,
pos_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
x = self.input_norm(x.flatten(1), self.proj.weight.dtype).view_as(x)
# forward_native dispatches this non-overlapping patch projection to GEMM.
x = self.proj.forward_native(x).view(x.size(0), self.proj.out_channels)
if pos_embeds is not None:
@@ -703,6 +707,7 @@ class MoonViT3dPretrainedModel(nn.Module):
self,
config: KimiK25VisionConfig,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
):
super().__init__()
@@ -723,6 +728,7 @@ class MoonViT3dPretrainedModel(nn.Module):
pos_emb_interpolation_mode=getattr(
config, "pos_emb_interpolation_mode", "bicubic"
),
input_norm=input_norm,
)
self.encoder = MoonViT3dEncoder(
+3 -2
View File
@@ -9,6 +9,7 @@ import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.compressed_tensors import (
compressed_tensors,
@@ -65,6 +66,7 @@ class KimiK3ForConditionalGeneration(
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
supports_encoder_tp_data = True
supports_mm_device_do_normalize = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -103,6 +105,7 @@ class KimiK3ForConditionalGeneration(
self.vision_tower = MoonViT3dPretrainedModel(
config.vision_config,
quant_config=self._maybe_ignore_quant_config(quant_config),
input_norm=build_mm_input_norm(vllm_config.model_config),
prefix=maybe_prefix(prefix, "vision_tower"),
)
if self._maybe_ignore_quant_config(quant_config) is not None:
@@ -163,8 +166,6 @@ class KimiK3ForConditionalGeneration(
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
)
target_dtype = next(self.vision_tower.parameters()).dtype
pixel_values = pixel_values.to(target_dtype)
assert isinstance(grid_thws, torch.Tensor), (
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
)
@@ -30,6 +30,9 @@ from vllm.multimodal.processing import (
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
from vllm.transformers_utils.processor import cached_get_image_processor
from vllm.transformers_utils.processors.kimi_k3 import KimiK3Processor
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
KimiK25FusedVisionProcessor,
)
logger = init_logger(__name__)
@@ -106,6 +109,7 @@ class KimiK3ProcessingInfo(BaseProcessingInfo):
self.ctx.model_config.model,
revision=self.ctx.model_config.revision,
trust_remote_code=self.ctx.model_config.trust_remote_code,
processor_cls_overrides=KimiK25FusedVisionProcessor,
)
# Resolve token ID from the tokenizer because transformers v5
+6 -7
View File
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
fused_grouped_topk,
)
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -1736,6 +1737,7 @@ class KimiK3ForConditionalGeneration(
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
supports_encoder_tp_data = True
supports_mm_device_do_normalize = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
@@ -1775,6 +1777,7 @@ class KimiK3ForConditionalGeneration(
self.vision_tower = MoonViT3dPretrainedModel(
config.vision_config,
quant_config=self._maybe_ignore_quant_config(quant_config),
input_norm=build_mm_input_norm(vllm_config.model_config),
prefix=maybe_prefix(prefix, "vision_tower"),
)
if is_meta_module(self.vision_tower):
@@ -1964,13 +1967,13 @@ class KimiK3ForConditionalGeneration(
if isinstance(patch_size, int):
patch_size = (patch_size, patch_size)
total_patches = sum(t * h * w for t, h, w in grid_thws)
pixel_values = torch.randn(
pixel_values = torch.zeros(
total_patches,
3,
patch_size[0],
patch_size[1],
device=device,
dtype=dtype,
dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype,
)
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
grid_thws,
@@ -2028,9 +2031,7 @@ class KimiK3ForConditionalGeneration(
path: str = "default",
) -> torch.Tensor:
image_features = self.vision_tower(
self._get_pixel_values(mm_kwargs).to(
next(self.vision_tower.parameters()).dtype
),
self._get_pixel_values(mm_kwargs),
self._get_grid_thws(mm_kwargs),
)
return self._project_encoder_features(torch.cat(image_features))
@@ -2063,8 +2064,6 @@ class KimiK3ForConditionalGeneration(
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
)
target_dtype = next(self.vision_tower.parameters()).dtype
pixel_values = pixel_values.to(target_dtype)
assert isinstance(grid_thws, torch.Tensor), (
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
)
@@ -26,6 +26,8 @@ class KimiK25Processor(ProcessorMixin):
text: str | list[str] | None = None,
vision_chunks: list[VisionChunk] | None = None,
return_tensors: str | TensorType | None = None,
do_rescale: bool = True,
do_normalize: bool = True,
**kwargs,
) -> BatchFeature:
"""Args:
@@ -49,6 +51,8 @@ class KimiK25Processor(ProcessorMixin):
mm_inputs = self.image_processor.preprocess(
vision_chunks,
return_tensors=return_tensors,
do_rescale=do_rescale,
do_normalize=do_normalize,
)
else:
mm_inputs = {}
@@ -18,6 +18,8 @@ from vllm.multimodal.image import open_image
from vllm.utils.import_utils import is_numba_available
from vllm.utils.jit_monitor import numba_workqueue_threading_layer
_IDENTITY_LUT = np.tile(np.arange(256, dtype=np.uint8)[:, None], (1, 3))
if is_numba_available():
from numba import njit, prange
@@ -239,7 +241,12 @@ class KimiK25FusedVisionProcessor(BaseImageProcessor):
self,
medias: list[dict[str, Any]],
return_tensors: str | TensorType | None = None,
do_rescale: bool = True,
do_normalize: bool = True,
) -> BatchFeature:
assert do_rescale == do_normalize
device_normalize = not do_normalize
if not isinstance(medias, list):
medias = [medias]
if not medias:
@@ -297,8 +304,9 @@ class KimiK25FusedVisionProcessor(BaseImageProcessor):
)
total_patches += num_patches
lut = _IDENTITY_LUT if device_normalize else self.normalize_lut
pixel_values_np = np.empty(
(total_patches, 3, patch_size, patch_size), dtype=np.float32
(total_patches, 3, patch_size, patch_size), dtype=lut.dtype
)
out_offset = 0
with numba_workqueue_threading_layer():
@@ -319,7 +327,7 @@ class KimiK25FusedVisionProcessor(BaseImageProcessor):
padded_height,
padded_width,
patch_size,
self.normalize_lut,
lut,
)
out_offset += num_patches
@@ -35,6 +35,8 @@ class KimiK3Processor(ProcessorMixin):
text: str | list[str] | None = None,
images: object | list[object] | None = None,
return_tensors: str | TensorType | None = None,
do_rescale: bool = True,
do_normalize: bool = True,
**kwargs,
) -> BatchFeature:
if images is not None:
@@ -44,6 +46,8 @@ class KimiK3Processor(ProcessorMixin):
mm_inputs = self.image_processor.preprocess(
medias,
return_tensors=return_tensors,
do_rescale=do_rescale,
do_normalize=do_normalize,
)
else:
mm_inputs = {}