[Model] Extend device-side mm normalization to Kimi K2.5 / K3 (#59278)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors import (
|
||||
compressed_tensors,
|
||||
@@ -65,6 +66,7 @@ class KimiK3ForConditionalGeneration(
|
||||
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
@@ -103,6 +105,7 @@ class KimiK3ForConditionalGeneration(
|
||||
self.vision_tower = MoonViT3dPretrainedModel(
|
||||
config.vision_config,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
input_norm=build_mm_input_norm(vllm_config.model_config),
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
if self._maybe_ignore_quant_config(quant_config) is not None:
|
||||
@@ -163,8 +166,6 @@ class KimiK3ForConditionalGeneration(
|
||||
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
|
||||
)
|
||||
|
||||
target_dtype = next(self.vision_tower.parameters()).dtype
|
||||
pixel_values = pixel_values.to(target_dtype)
|
||||
assert isinstance(grid_thws, torch.Tensor), (
|
||||
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
|
||||
)
|
||||
|
||||
@@ -30,6 +30,9 @@ from vllm.multimodal.processing import (
|
||||
from vllm.transformers_utils.configs.kimi_k3 import KimiK3Config
|
||||
from vllm.transformers_utils.processor import cached_get_image_processor
|
||||
from vllm.transformers_utils.processors.kimi_k3 import KimiK3Processor
|
||||
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
|
||||
KimiK25FusedVisionProcessor,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -106,6 +109,7 @@ class KimiK3ProcessingInfo(BaseProcessingInfo):
|
||||
self.ctx.model_config.model,
|
||||
revision=self.ctx.model_config.revision,
|
||||
trust_remote_code=self.ctx.model_config.trust_remote_code,
|
||||
processor_cls_overrides=KimiK25FusedVisionProcessor,
|
||||
)
|
||||
|
||||
# Resolve token ID from the tokenizer because transformers v5
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
fused_grouped_topk,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -1736,6 +1737,7 @@ class KimiK3ForConditionalGeneration(
|
||||
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
@@ -1775,6 +1777,7 @@ class KimiK3ForConditionalGeneration(
|
||||
self.vision_tower = MoonViT3dPretrainedModel(
|
||||
config.vision_config,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
input_norm=build_mm_input_norm(vllm_config.model_config),
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
if is_meta_module(self.vision_tower):
|
||||
@@ -1964,13 +1967,13 @@ class KimiK3ForConditionalGeneration(
|
||||
if isinstance(patch_size, int):
|
||||
patch_size = (patch_size, patch_size)
|
||||
total_patches = sum(t * h * w for t, h, w in grid_thws)
|
||||
pixel_values = torch.randn(
|
||||
pixel_values = torch.zeros(
|
||||
total_patches,
|
||||
3,
|
||||
patch_size[0],
|
||||
patch_size[1],
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype,
|
||||
)
|
||||
metadata = self.vision_tower.prepare_encoder_cudagraph_metadata(
|
||||
grid_thws,
|
||||
@@ -2028,9 +2031,7 @@ class KimiK3ForConditionalGeneration(
|
||||
path: str = "default",
|
||||
) -> torch.Tensor:
|
||||
image_features = self.vision_tower(
|
||||
self._get_pixel_values(mm_kwargs).to(
|
||||
next(self.vision_tower.parameters()).dtype
|
||||
),
|
||||
self._get_pixel_values(mm_kwargs),
|
||||
self._get_grid_thws(mm_kwargs),
|
||||
)
|
||||
return self._project_encoder_features(torch.cat(image_features))
|
||||
@@ -2063,8 +2064,6 @@ class KimiK3ForConditionalGeneration(
|
||||
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
|
||||
)
|
||||
|
||||
target_dtype = next(self.vision_tower.parameters()).dtype
|
||||
pixel_values = pixel_values.to(target_dtype)
|
||||
assert isinstance(grid_thws, torch.Tensor), (
|
||||
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user