[Model] Extend device-side mm normalization to Kimi K2.5 / K3 (#59278)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
@@ -17,6 +17,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors import (
|
||||
compressed_tensors,
|
||||
@@ -306,6 +307,7 @@ class KimiK25ForConditionalGeneration(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_encoder_cudagraph: ClassVar[Literal[True]] = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
@@ -350,6 +352,7 @@ class KimiK25ForConditionalGeneration(
|
||||
self.vision_tower = MoonViT3dPretrainedModel(
|
||||
config.vision_config,
|
||||
quant_config=self._maybe_ignore_quant_config(quant_config),
|
||||
input_norm=build_mm_input_norm(vllm_config.model_config),
|
||||
prefix=maybe_prefix(prefix, "vision_tower"),
|
||||
)
|
||||
if self._maybe_ignore_quant_config(quant_config) is not None:
|
||||
@@ -404,9 +407,6 @@ class KimiK25ForConditionalGeneration(
|
||||
pixel_values.shape[0] * pixel_values.shape[1], *pixel_values.shape[2:]
|
||||
)
|
||||
|
||||
# The batch dimension of pixel_values has been flattened into shape[0]
|
||||
target_dtype = next(self.vision_tower.parameters()).dtype
|
||||
pixel_values = pixel_values.to(target_dtype)
|
||||
assert isinstance(grid_thws, torch.Tensor), (
|
||||
f"expect grid_thws to be a tensor, got {type(grid_thws)}"
|
||||
)
|
||||
@@ -623,7 +623,12 @@ class KimiK25ForConditionalGeneration(
|
||||
ps = (ps, ps)
|
||||
total_patches = max_batch_size * ho * kh * wo * kw
|
||||
dummy_pixel_values = torch.zeros(
|
||||
total_patches, 3, ps[0], ps[1], device=device, dtype=dtype
|
||||
total_patches,
|
||||
3,
|
||||
ps[0],
|
||||
ps[1],
|
||||
device=device,
|
||||
dtype=self.vision_tower.patch_embed.input_norm.input_dtype or dtype,
|
||||
)
|
||||
|
||||
# max_seqlen must cover the worst case: one item consuming the full
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
@@ -187,6 +188,7 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
pos_emb_type: str = "divided_fixed",
|
||||
patch_embed_proj_bias: bool = True,
|
||||
pos_emb_interpolation_mode: str = "bicubic",
|
||||
input_norm: nn.Module | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
assert isinstance(patch_size, int | Sequence), (
|
||||
@@ -206,6 +208,7 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
stride=patch_size,
|
||||
bias=patch_embed_proj_bias,
|
||||
)
|
||||
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
|
||||
|
||||
if pos_emb_type == "divided_fixed":
|
||||
self.pos_emb = Learnable2DInterpPosEmbDivided_fixed(
|
||||
@@ -225,6 +228,7 @@ class MoonVision3dPatchEmbed(nn.Module):
|
||||
*,
|
||||
pos_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
x = self.input_norm(x.flatten(1), self.proj.weight.dtype).view_as(x)
|
||||
# forward_native dispatches this non-overlapping patch projection to GEMM.
|
||||
x = self.proj.forward_native(x).view(x.size(0), self.proj.out_channels)
|
||||
if pos_embeds is not None:
|
||||
@@ -703,6 +707,7 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
self,
|
||||
config: KimiK25VisionConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
input_norm: nn.Module | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
@@ -723,6 +728,7 @@ class MoonViT3dPretrainedModel(nn.Module):
|
||||
pos_emb_interpolation_mode=getattr(
|
||||
config, "pos_emb_interpolation_mode", "bicubic"
|
||||
),
|
||||
input_norm=input_norm,
|
||||
)
|
||||
|
||||
self.encoder = MoonViT3dEncoder(
|
||||
|
||||
Reference in New Issue
Block a user