diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index 67b1a7742b..a267aee38e 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -85,6 +85,8 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`: | `glm-ocr` | `GlmOcrForConditionalGeneration` | `zai-org/GLM-OCR` | | `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` | | `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` | +| `kimi-k2.5` | `KimiK25ForConditionalGeneration` | `moonshotai/Kimi-K2.5`, etc. | +| `kimi-k3` | `KimiK3ForConditionalGeneration` | `moonshotai/Kimi-K3` | | `llama-nemotron-vl-embed` | `LlamaNemotronVLForEmbedding` | `nvidia/llama-nemotron-embed-vl-1b-v2` | | `llama-nemotron-vl-rerank` | `LlamaNemotronVLForSequenceClassification` | `nvidia/llama-nemotron-rerank-vl-1b-v2` | | `mistral3` | `Mistral3ForConditionalGeneration` | `mistralai/Mistral-Small-3.2-24B-Instruct-2506`, etc. | diff --git a/tests/models/multimodal/processing/test_kimi_k3.py b/tests/models/multimodal/processing/test_kimi_k3.py new file mode 100644 index 0000000000..62c99b24f1 --- /dev/null +++ b/tests/models/multimodal/processing/test_kimi_k3.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import numpy as np +import pytest +import torch +from PIL import Image +from transformers import AutoImageProcessor + +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.platforms import current_platform +from vllm.transformers_utils.processors.kimi_k25_vision_fused import ( + KimiK25FusedVisionProcessor, +) + +from ...utils import build_model_context + +_MODEL_ID = "moonshotai/Kimi-K3" + + +@pytest.fixture(scope="module") +def image_processors(): + ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1}) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + k25_fused = processor.info.image_processor + assert isinstance(k25_fused, KimiK25FusedVisionProcessor) + k3_hf = AutoImageProcessor.from_pretrained(_MODEL_ID, trust_remote_code=True) + return k25_fused, k3_hf + + +def _noise_image(width: int, height: int) -> Image.Image: + rng = np.random.default_rng(0) + return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8)) + + +@pytest.mark.parametrize( + ("width", "height"), + [(310, 470), (7300, 30), (13, 9)], +) +def test_fused_image_processor_matches_hf(image_processors, width: int, height: int): + fused, hf = image_processors + image = _noise_image(width, height) + expected = hf.preprocess([{"type": "image", "image": image}], return_tensors="pt") + actual = fused.preprocess([{"type": "image", "image": image}], return_tensors="pt") + + torch.testing.assert_close(actual["grid_thws"], expected["grid_thws"]) + torch.testing.assert_close(actual["pixel_values"], expected["pixel_values"]) + assert fused.media_tokens_calculator( + {"type": "image", "image": image} + ) == hf.media_tokens_calculator({"type": "image", "image": image}) + + +@pytest.mark.usefixtures("default_vllm_config") +def test_mm_device_do_normalize(): + device = current_platform.device_type + ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 2}) + assert ctx.model_config.multimodal_config.mm_device_do_normalize + + ctx.model_config.multimodal_config.mm_device_do_normalize = False + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + images = [ + Image.new("RGB", (310, 470), color=(17, 89, 231)), + Image.new("RGB", (480, 320), color=(201, 13, 127)), + ] + prompt = processor.info.get_hf_config().image_placeholder * len(images) + mm_items = processor.info.parse_mm_data({"image": images}) + + normalized_inputs = processor(prompt, mm_items=mm_items) + normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"] + + ctx.model_config.multimodal_config.mm_device_do_normalize = True + raw_inputs = processor(prompt, mm_items=mm_items) + raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"] + assert raw_values.dtype == torch.uint8 + + input_norm = build_mm_input_norm(ctx.model_config).to(device) + output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype) + torch.testing.assert_close( + output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5 + ) diff --git a/vllm/model_executor/layers/fusion/mm_input_norm.py b/vllm/model_executor/layers/fusion/mm_input_norm.py index d125f8d2d2..3dc6974c84 100644 --- a/vllm/model_executor/layers/fusion/mm_input_norm.py +++ b/vllm/model_executor/layers/fusion/mm_input_norm.py @@ -152,6 +152,7 @@ def _load_norm_params(model_config: ModelConfig) -> NormParams: for key in ("do_normalize", "do_rescale", "image_mean", "image_std") ): config = model_config.hf_image_processor_config + config = config.get("media_proc_cfg", config) has_norm_params = "image_mean" in config and "image_std" in config do_normalize = bool(config.get("do_normalize", has_norm_params)) diff --git a/vllm/model_executor/models/kimi_k25.py b/vllm/model_executor/models/kimi_k25.py index 2426ec61ef..d66bed3642 100644 --- a/vllm/model_executor/models/kimi_k25.py +++ b/vllm/model_executor/models/kimi_k25.py @@ -17,6 +17,7 @@ from vllm.config import VllmConfig from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization.compressed_tensors import ( compressed_tensors, @@ -306,6 +307,7 @@ class KimiK25ForConditionalGeneration( supports_encoder_tp_data = True supports_encoder_cudagraph: ClassVar[Literal[True]] = True + supports_mm_device_do_normalize = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ @@ -350,6 +352,7 @@ class KimiK25ForConditionalGeneration( self.vision_tower = MoonViT3dPretrainedModel( config.vision_config, quant_config=self._maybe_ignore_quant_config(quant_config), + input_norm=build_mm_input_norm(vllm_config.model_config), prefix=maybe_prefix(prefix, "vision_tower"), ) if self._maybe_ignore_quant_config(quant_config) is not None: @@ -404,9 +407,6 @@ class KimiK25ForConditionalGeneration( pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:] ) - # The batch dimension of pixel_values has been flattened into shape[0] - target_dtype = next(self.vision_tower.parameters()).dtype - pixel_values = pixel_values.to(target_dtype) assert isinstance(grid_thws, torch.Tensor), ( f"expect grid_thws to be a tensor, got {type(grid_thws)}" ) @@ -623,7 +623,12 @@ class KimiK25ForConditionalGeneration( ps = (ps, ps) total_patches = max_batch_size * ho * kh * wo * kw dummy_pixel_values = torch.zeros( - total_patches, 3, ps[0], ps[1], device=device, dtype=dtype + total_patches, + 3, + ps[0], + ps[1], + device=device, + dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype, ) # max_seqlen must cover the worst case: one item consuming the full diff --git a/vllm/model_executor/models/kimi_k25_vit.py b/vllm/model_executor/models/kimi_k25_vit.py index ee8adb6a74..44e8498d21 100644 --- a/vllm/model_executor/models/kimi_k25_vit.py +++ b/vllm/model_executor/models/kimi_k25_vit.py @@ -22,6 +22,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention from vllm.model_executor.layers.conv import Conv2dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, @@ -187,6 +188,7 @@ class MoonVision3dPatchEmbed(nn.Module): pos_emb_type: str = "divided_fixed", patch_embed_proj_bias: bool = True, pos_emb_interpolation_mode: str = "bicubic", + input_norm: nn.Module | None = None, ): super().__init__() assert isinstance(patch_size, int | Sequence), ( @@ -206,6 +208,7 @@ class MoonVision3dPatchEmbed(nn.Module): stride=patch_size, bias=patch_embed_proj_bias, ) + self.input_norm = input_norm if input_norm is not None else IdentityInputNorm() if pos_emb_type == "divided_fixed": self.pos_emb = Learnable2DInterpPosEmbDivided_fixed( @@ -225,6 +228,7 @@ class MoonVision3dPatchEmbed(nn.Module): *, pos_embeds: torch.Tensor | None = None, ) -> torch.Tensor: + x = self.input_norm(x.flatten(1), self.proj.weight.dtype).view_as(x) # forward_native dispatches this non-overlapping patch projection to GEMM. x = self.proj.forward_native(x).view(x.size(0), self.proj.out_channels) if pos_embeds is not None: @@ -703,6 +707,7 @@ class MoonViT3dPretrainedModel(nn.Module): self, config: KimiK25VisionConfig, quant_config: QuantizationConfig | None = None, + input_norm: nn.Module | None = None, prefix: str = "", ): super().__init__() @@ -723,6 +728,7 @@ class MoonViT3dPretrainedModel(nn.Module): pos_emb_interpolation_mode=getattr( config, "pos_emb_interpolation_mode", "bicubic" ), + input_norm=input_norm, ) self.encoder = MoonViT3dEncoder( diff --git a/vllm/models/kimi_k3/amd/model.py b/vllm/models/kimi_k3/amd/model.py index ec49891ddb..227f6dba24 100644 --- a/vllm/models/kimi_k3/amd/model.py +++ b/vllm/models/kimi_k3/amd/model.py @@ -9,6 +9,7 @@ import torch from torch import nn from vllm.config import VllmConfig +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization.compressed_tensors import ( compressed_tensors, @@ -65,6 +66,7 @@ class KimiK3ForConditionalGeneration( """Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text.""" supports_encoder_tp_data = True + supports_mm_device_do_normalize = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ @@ -103,6 +105,7 @@ class KimiK3ForConditionalGeneration( self.vision_tower = MoonViT3dPretrainedModel( config.vision_config, quant_config=self._maybe_ignore_quant_config(quant_config), + input_norm=build_mm_input_norm(vllm_config.model_config), prefix=maybe_prefix(prefix, "vision_tower"), ) if self._maybe_ignore_quant_config(quant_config) is not None: @@ -163,8 +166,6 @@ class KimiK3ForConditionalGeneration( pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:] ) - target_dtype = next(self.vision_tower.parameters()).dtype - pixel_values = pixel_values.to(target_dtype) assert isinstance(grid_thws, torch.Tensor), ( f"expect grid_thws to be a tensor, got {type(grid_thws)}" ) diff --git a/vllm/models/kimi_k3/common/mm_preprocess.py b/vllm/models/kimi_k3/common/mm_preprocess.py index dfd90fe9a9..f9eb142031 100644 --- a/vllm/models/kimi_k3/common/mm_preprocess.py +++ b/vllm/models/kimi_k3/common/mm_preprocess.py @@ -30,6 +30,9 @@ from vllm.multimodal.processing import ( from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config from vllm.transformers_utils.processor import cached_get_image_processor from vllm.transformers_utils.processors.kimi_k3 import KimiK3Processor +from vllm.transformers_utils.processors.kimi_k25_vision_fused import ( + KimiK25FusedVisionProcessor, +) logger = init_logger(__name__) @@ -106,6 +109,7 @@ class KimiK3ProcessingInfo(BaseProcessingInfo): self.ctx.model_config.model, revision=self.ctx.model_config.revision, trust_remote_code=self.ctx.model_config.trust_remote_code, + processor_cls_overrides=KimiK25FusedVisionProcessor, ) # Resolve token ID from the tokenizer because transformers v5 diff --git a/vllm/models/kimi_k3/nvidia/model.py b/vllm/models/kimi_k3/nvidia/model.py index c1818a444b..cc1ca609a6 100644 --- a/vllm/models/kimi_k3/nvidia/model.py +++ b/vllm/models/kimi_k3/nvidia/model.py @@ -30,6 +30,7 @@ from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import ( fused_grouped_topk, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -1736,6 +1737,7 @@ class KimiK3ForConditionalGeneration( """Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text.""" supports_encoder_tp_data = True + supports_mm_device_do_normalize = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ @@ -1775,6 +1777,7 @@ class KimiK3ForConditionalGeneration( self.vision_tower = MoonViT3dPretrainedModel( config.vision_config, quant_config=self._maybe_ignore_quant_config(quant_config), + input_norm=build_mm_input_norm(vllm_config.model_config), prefix=maybe_prefix(prefix, "vision_tower"), ) if is_meta_module(self.vision_tower): @@ -1964,13 +1967,13 @@ class KimiK3ForConditionalGeneration( if isinstance(patch_size, int): patch_size = (patch_size, patch_size) total_patches = sum(t * h * w for t, h, w in grid_thws) - pixel_values = torch.randn( + pixel_values = torch.zeros( total_patches, 3, patch_size[0], patch_size[1], device=device, - dtype=dtype, + dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype, ) metadata = self.vision_tower.prepare_encoder_cudagraph_metadata( grid_thws, @@ -2028,9 +2031,7 @@ class KimiK3ForConditionalGeneration( path: str = "default", ) -> torch.Tensor: image_features = self.vision_tower( - self._get_pixel_values(mm_kwargs).to( - next(self.vision_tower.parameters()).dtype - ), + self._get_pixel_values(mm_kwargs), self._get_grid_thws(mm_kwargs), ) return self._project_encoder_features(torch.cat(image_features)) @@ -2063,8 +2064,6 @@ class KimiK3ForConditionalGeneration( pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:] ) - target_dtype = next(self.vision_tower.parameters()).dtype - pixel_values = pixel_values.to(target_dtype) assert isinstance(grid_thws, torch.Tensor), ( f"expect grid_thws to be a tensor, got {type(grid_thws)}" ) diff --git a/vllm/transformers_utils/processors/kimi_k25.py b/vllm/transformers_utils/processors/kimi_k25.py index 13aed85de4..800dd3a7ed 100644 --- a/vllm/transformers_utils/processors/kimi_k25.py +++ b/vllm/transformers_utils/processors/kimi_k25.py @@ -26,6 +26,8 @@ class KimiK25Processor(ProcessorMixin): text: str | list[str] | None = None, vision_chunks: list[VisionChunk] | None = None, return_tensors: str | TensorType | None = None, + do_rescale: bool = True, + do_normalize: bool = True, **kwargs, ) -> BatchFeature: """Args: @@ -49,6 +51,8 @@ class KimiK25Processor(ProcessorMixin): mm_inputs = self.image_processor.preprocess( vision_chunks, return_tensors=return_tensors, + do_rescale=do_rescale, + do_normalize=do_normalize, ) else: mm_inputs = {} diff --git a/vllm/transformers_utils/processors/kimi_k25_vision_fused.py b/vllm/transformers_utils/processors/kimi_k25_vision_fused.py index 3d1a96e888..254ffe013e 100644 --- a/vllm/transformers_utils/processors/kimi_k25_vision_fused.py +++ b/vllm/transformers_utils/processors/kimi_k25_vision_fused.py @@ -18,6 +18,8 @@ from vllm.multimodal.image import open_image from vllm.utils.import_utils import is_numba_available from vllm.utils.jit_monitor import numba_workqueue_threading_layer +_IDENTITY_LUT = np.tile(np.arange(256, dtype=np.uint8)[:, None], (1, 3)) + if is_numba_available(): from numba import njit, prange @@ -239,7 +241,12 @@ class KimiK25FusedVisionProcessor(BaseImageProcessor): self, medias: list[dict[str, Any]], return_tensors: str | TensorType | None = None, + do_rescale: bool = True, + do_normalize: bool = True, ) -> BatchFeature: + assert do_rescale == do_normalize + device_normalize = not do_normalize + if not isinstance(medias, list): medias = [medias] if not medias: @@ -297,8 +304,9 @@ class KimiK25FusedVisionProcessor(BaseImageProcessor): ) total_patches += num_patches + lut = _IDENTITY_LUT if device_normalize else self.normalize_lut pixel_values_np = np.empty( - (total_patches, 3, patch_size, patch_size), dtype=np.float32 + (total_patches, 3, patch_size, patch_size), dtype=lut.dtype ) out_offset = 0 with numba_workqueue_threading_layer(): @@ -319,7 +327,7 @@ class KimiK25FusedVisionProcessor(BaseImageProcessor): padded_height, padded_width, patch_size, - self.normalize_lut, + lut, ) out_offset += num_patches diff --git a/vllm/transformers_utils/processors/kimi_k3.py b/vllm/transformers_utils/processors/kimi_k3.py index 73af85eb09..e726a9c38a 100644 --- a/vllm/transformers_utils/processors/kimi_k3.py +++ b/vllm/transformers_utils/processors/kimi_k3.py @@ -35,6 +35,8 @@ class KimiK3Processor(ProcessorMixin): text: str | list[str] | None = None, images: object | list[object] | None = None, return_tensors: str | TensorType | None = None, + do_rescale: bool = True, + do_normalize: bool = True, **kwargs, ) -> BatchFeature: if images is not None: @@ -44,6 +46,8 @@ class KimiK3Processor(ProcessorMixin): mm_inputs = self.image_processor.preprocess( medias, return_tensors=return_tensors, + do_rescale=do_rescale, + do_normalize=do_normalize, ) else: mm_inputs = {}