[Model] Extend device-side mm normalization to Kimi K2.5 / K3 (#59278)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
@@ -85,6 +85,8 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`:
|
||||
| `glm-ocr` | `GlmOcrForConditionalGeneration` | `zai-org/GLM-OCR` |
|
||||
| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` |
|
||||
| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` |
|
||||
| `kimi-k2.5` | `KimiK25ForConditionalGeneration` | `moonshotai/Kimi-K2.5`, etc. |
|
||||
| `kimi-k3` | `KimiK3ForConditionalGeneration` | `moonshotai/Kimi-K3` |
|
||||
| `llama-nemotron-vl-embed` | `LlamaNemotronVLForEmbedding` | `nvidia/llama-nemotron-embed-vl-1b-v2` |
|
||||
| `llama-nemotron-vl-rerank` | `LlamaNemotronVLForSequenceClassification` | `nvidia/llama-nemotron-rerank-vl-1b-v2` |
|
||||
| `mistral3` | `Mistral3ForConditionalGeneration` | `mistralai/Mistral-Small-3.2-24B-Instruct-2506`, etc. |
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoImageProcessor
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
|
||||
KimiK25FusedVisionProcessor,
|
||||
)
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
_MODEL_ID = "moonshotai/Kimi-K3"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image_processors():
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
k25_fused = processor.info.image_processor
|
||||
assert isinstance(k25_fused, KimiK25FusedVisionProcessor)
|
||||
k3_hf = AutoImageProcessor.from_pretrained(_MODEL_ID, trust_remote_code=True)
|
||||
return k25_fused, k3_hf
|
||||
|
||||
|
||||
def _noise_image(width: int, height: int) -> Image.Image:
|
||||
rng = np.random.default_rng(0)
|
||||
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("width", "height"),
|
||||
[(310, 470), (7300, 30), (13, 9)],
|
||||
)
|
||||
def test_fused_image_processor_matches_hf(image_processors, width: int, height: int):
|
||||
fused, hf = image_processors
|
||||
image = _noise_image(width, height)
|
||||
expected = hf.preprocess([{"type": "image", "image": image}], return_tensors="pt")
|
||||
actual = fused.preprocess([{"type": "image", "image": image}], return_tensors="pt")
|
||||
|
||||
torch.testing.assert_close(actual["grid_thws"], expected["grid_thws"])
|
||||
torch.testing.assert_close(actual["pixel_values"], expected["pixel_values"])
|
||||
assert fused.media_tokens_calculator(
|
||||
{"type": "image", "image": image}
|
||||
) == hf.media_tokens_calculator({"type": "image", "image": image})
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_mm_device_do_normalize():
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 2})
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
images = [
|
||||
Image.new("RGB", (310, 470), color=(17, 89, 231)),
|
||||
Image.new("RGB", (480, 320), color=(201, 13, 127)),
|
||||
]
|
||||
prompt = processor.info.get_hf_config().image_placeholder * len(images)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
|
||||
normalized_inputs = processor(prompt, mm_items=mm_items)
|
||||
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
raw_inputs = processor(prompt, mm_items=mm_items)
|
||||
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
assert raw_values.dtype == torch.uint8
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
|
||||
)
|
||||
@@ -152,6 +152,7 @@ def _load_norm_params(model_config: ModelConfig) -> NormParams:
|
||||
for key in ("do_normalize", "do_rescale", "image_mean", "image_std")
|
||||
):
|
||||
config = model_config.hf_image_processor_config
|
||||
config = config.get("media_proc_cfg", config)
|
||||
|
||||
has_norm_params = "image_mean" in config and "image_std" in config
|
||||
do_normalize = bool(config.get("do_normalize", has_norm_params))
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors import (
|
||||
compressed_tensors,
|
||||
@@ -306,6 +307,7 @@ class KimiK25ForConditionalGeneration(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_encoder_cudagraph: ClassVar[Literal[True]] = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
@@ -350,6 +352,7 @@ class KimiK25ForConditionalGeneration(
|
||||
self.vision_tower = MoonViT3dPretrainedModel(
|
||||
config.vision_config,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
input_norm=build_mm_input_norm(vllm_config.model_config),
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
if self._maybe_ignore_quant_config(quant_config) is not None:
|
||||
@@ -404,9 +407,6 @@ class KimiK25ForConditionalGeneration(
|
||||
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
|
||||
)
|
||||
|
||||
# The batch dimension of pixel_values has been flattened into shape[0]
|
||||
target_dtype = next(self.vision_tower.parameters()).dtype
|
||||
pixel_values = pixel_values.to(target_dtype)
|
||||
assert isinstance(grid_thws, torch.Tensor), (
|
||||
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
|
||||
)
|
||||
@@ -623,7 +623,12 @@ class KimiK25ForConditionalGeneration(
|
||||
ps = (ps, ps)
|
||||
total_patches = max_batch_size * ho * kh * wo * kw
|
||||
dummy_pixel_values = torch.zeros(
|
||||
total_patches, 3, ps[0], ps[1], device=device, dtype=dtype
|
||||
total_patches,
|
||||
3,
|
||||
ps[0],
|
||||
ps[1],
|
||||
device=device,
|
||||
dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype,
|
||||
)
|
||||
|
||||
# max_seqlen must cover the worst case: one item consuming the full
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
@@ -187,6 +188,7 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
pos_emb_type: str = "divided_fixed",
|
||||
patch_embed_proj_bias: bool = True,
|
||||
pos_emb_interpolation_mode: str = "bicubic",
|
||||
input_norm: nn.Module | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
assert isinstance(patch_size, int | Sequence), (
|
||||
@@ -206,6 +208,7 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
stride=patch_size,
|
||||
bias=patch_embed_proj_bias,
|
||||
)
|
||||
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
|
||||
|
||||
if pos_emb_type == "divided_fixed":
|
||||
self.pos_emb = Learnable2DInterpPosEmbDivided_fixed(
|
||||
@@ -225,6 +228,7 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
*,
|
||||
pos_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
x = self.input_norm(x.flatten(1), self.proj.weight.dtype).view_as(x)
|
||||
# forward_native dispatches this non-overlapping patch projection to GEMM.
|
||||
x = self.proj.forward_native(x).view(x.size(0), self.proj.out_channels)
|
||||
if pos_embeds is not None:
|
||||
@@ -703,6 +707,7 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
self,
|
||||
config: KimiK25VisionConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
input_norm: nn.Module | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
@@ -723,6 +728,7 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
pos_emb_interpolation_mode=getattr(
|
||||
config, "pos_emb_interpolation_mode", "bicubic"
|
||||
),
|
||||
input_norm=input_norm,
|
||||
)
|
||||
|
||||
self.encoder = MoonViT3dEncoder(
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors import (
|
||||
compressed_tensors,
|
||||
@@ -65,6 +66,7 @@ class KimiK3ForConditionalGeneration(
|
||||
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
@@ -103,6 +105,7 @@ class KimiK3ForConditionalGeneration(
|
||||
self.vision_tower = MoonViT3dPretrainedModel(
|
||||
config.vision_config,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
input_norm=build_mm_input_norm(vllm_config.model_config),
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
if self._maybe_ignore_quant_config(quant_config) is not None:
|
||||
@@ -163,8 +166,6 @@ class KimiK3ForConditionalGeneration(
|
||||
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
|
||||
)
|
||||
|
||||
target_dtype = next(self.vision_tower.parameters()).dtype
|
||||
pixel_values = pixel_values.to(target_dtype)
|
||||
assert isinstance(grid_thws, torch.Tensor), (
|
||||
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
|
||||
)
|
||||
|
||||
@@ -30,6 +30,9 @@ from vllm.multimodal.processing import (
|
||||
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
|
||||
from vllm.transformers_utils.processor import cached_get_image_processor
|
||||
from vllm.transformers_utils.processors.kimi_k3 import KimiK3Processor
|
||||
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
|
||||
KimiK25FusedVisionProcessor,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -106,6 +109,7 @@ class KimiK3ProcessingInfo(BaseProcessingInfo):
|
||||
self.ctx.model_config.model,
|
||||
revision=self.ctx.model_config.revision,
|
||||
trust_remote_code=self.ctx.model_config.trust_remote_code,
|
||||
processor_cls_overrides=KimiK25FusedVisionProcessor,
|
||||
)
|
||||
|
||||
# Resolve token ID from the tokenizer because transformers v5
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
fused_grouped_topk,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -1736,6 +1737,7 @@ class KimiK3ForConditionalGeneration(
|
||||
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
@@ -1775,6 +1777,7 @@ class KimiK3ForConditionalGeneration(
|
||||
self.vision_tower = MoonViT3dPretrainedModel(
|
||||
config.vision_config,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
input_norm=build_mm_input_norm(vllm_config.model_config),
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
if is_meta_module(self.vision_tower):
|
||||
@@ -1964,13 +1967,13 @@ class KimiK3ForConditionalGeneration(
|
||||
if isinstance(patch_size, int):
|
||||
patch_size = (patch_size, patch_size)
|
||||
total_patches = sum(t * h * w for t, h, w in grid_thws)
|
||||
pixel_values = torch.randn(
|
||||
pixel_values = torch.zeros(
|
||||
total_patches,
|
||||
3,
|
||||
patch_size[0],
|
||||
patch_size[1],
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype,
|
||||
)
|
||||
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
|
||||
grid_thws,
|
||||
@@ -2028,9 +2031,7 @@ class KimiK3ForConditionalGeneration(
|
||||
path: str = "default",
|
||||
) -> torch.Tensor:
|
||||
image_features = self.vision_tower(
|
||||
self._get_pixel_values(mm_kwargs).to(
|
||||
next(self.vision_tower.parameters()).dtype
|
||||
),
|
||||
self._get_pixel_values(mm_kwargs),
|
||||
self._get_grid_thws(mm_kwargs),
|
||||
)
|
||||
return self._project_encoder_features(torch.cat(image_features))
|
||||
@@ -2063,8 +2064,6 @@ class KimiK3ForConditionalGeneration(
|
||||
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
|
||||
)
|
||||
|
||||
target_dtype = next(self.vision_tower.parameters()).dtype
|
||||
pixel_values = pixel_values.to(target_dtype)
|
||||
assert isinstance(grid_thws, torch.Tensor), (
|
||||
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
|
||||
)
|
||||
|
||||
@@ -26,6 +26,8 @@ class KimiK25Processor(ProcessorMixin):
|
||||
text: str | list[str] | None = None,
|
||||
vision_chunks: list[VisionChunk] | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
do_rescale: bool = True,
|
||||
do_normalize: bool = True,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
"""Args:
|
||||
@@ -49,6 +51,8 @@ class KimiK25Processor(ProcessorMixin):
|
||||
mm_inputs = self.image_processor.preprocess(
|
||||
vision_chunks,
|
||||
return_tensors=return_tensors,
|
||||
do_rescale=do_rescale,
|
||||
do_normalize=do_normalize,
|
||||
)
|
||||
else:
|
||||
mm_inputs = {}
|
||||
|
||||
@@ -18,6 +18,8 @@ from vllm.multimodal.image import open_image
|
||||
from vllm.utils.import_utils import is_numba_available
|
||||
from vllm.utils.jit_monitor import numba_workqueue_threading_layer
|
||||
|
||||
_IDENTITY_LUT = np.tile(np.arange(256, dtype=np.uint8)[:, None], (1, 3))
|
||||
|
||||
if is_numba_available():
|
||||
from numba import njit, prange
|
||||
|
||||
@@ -239,7 +241,12 @@ class KimiK25FusedVisionProcessor(BaseImageProcessor):
|
||||
self,
|
||||
medias: list[dict[str, Any]],
|
||||
return_tensors: str | TensorType | None = None,
|
||||
do_rescale: bool = True,
|
||||
do_normalize: bool = True,
|
||||
) -> BatchFeature:
|
||||
assert do_rescale == do_normalize
|
||||
device_normalize = not do_normalize
|
||||
|
||||
if not isinstance(medias, list):
|
||||
medias = [medias]
|
||||
if not medias:
|
||||
@@ -297,8 +304,9 @@ class KimiK25FusedVisionProcessor(BaseImageProcessor):
|
||||
)
|
||||
total_patches += num_patches
|
||||
|
||||
lut = _IDENTITY_LUT if device_normalize else self.normalize_lut
|
||||
pixel_values_np = np.empty(
|
||||
(total_patches, 3, patch_size, patch_size), dtype=np.float32
|
||||
(total_patches, 3, patch_size, patch_size), dtype=lut.dtype
|
||||
)
|
||||
out_offset = 0
|
||||
with numba_workqueue_threading_layer():
|
||||
@@ -319,7 +327,7 @@ class KimiK25FusedVisionProcessor(BaseImageProcessor):
|
||||
padded_height,
|
||||
padded_width,
|
||||
patch_size,
|
||||
self.normalize_lut,
|
||||
lut,
|
||||
)
|
||||
out_offset += num_patches
|
||||
|
||||
|
||||
@@ -35,6 +35,8 @@ class KimiK3Processor(ProcessorMixin):
|
||||
text: str | list[str] | None = None,
|
||||
images: object | list[object] | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
do_rescale: bool = True,
|
||||
do_normalize: bool = True,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
if images is not None:
|
||||
@@ -44,6 +46,8 @@ class KimiK3Processor(ProcessorMixin):
|
||||
mm_inputs = self.image_processor.preprocess(
|
||||
medias,
|
||||
return_tensors=return_tensors,
|
||||
do_rescale=do_rescale,
|
||||
do_normalize=do_normalize,
|
||||
)
|
||||
else:
|
||||
mm_inputs = {}
|
||||
|
||||
Reference in New Issue
Block a user