[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -0,0 +1,139 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi-K3 multimodal configuration."""
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class KimiK3VisionConfig(PretrainedConfig):
|
||||
model_type = "kimi_k3_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
init_pos_emb_height: int = 64,
|
||||
init_pos_emb_width: int = 64,
|
||||
init_pos_emb_time: int = 4,
|
||||
pos_emb_type: str = "divided_fixed",
|
||||
vt_num_attention_heads: int = 12,
|
||||
vt_num_hidden_layers: int = 27,
|
||||
vt_hidden_size: int = 1024,
|
||||
vt_intermediate_size: int = 4096,
|
||||
merge_kernel_size: tuple[int, int] = (2, 2),
|
||||
video_attn_type: str = "spatial_temporal",
|
||||
merge_type: str = "sd2_tpool",
|
||||
_attn_implementation: str = "flash_attention_2",
|
||||
mm_projector_type: str = "patchmergerv2",
|
||||
mm_hidden_size: int | None = None,
|
||||
projector_hidden_act: str = "gelu",
|
||||
projector_ln_eps: float = 1e-5,
|
||||
qkv_hidden_size: int = 1536,
|
||||
norm_type: str = "rmsnorm",
|
||||
attn_bias: bool = False,
|
||||
patch_embed_proj_bias: bool = False,
|
||||
mlp_type: str = "mlp2",
|
||||
linear_bias: bool = False,
|
||||
activation_func: str = "gelu_pytorch_tanh",
|
||||
pos_emb_interpolation_mode: str = "bilinear",
|
||||
text_hidden_size: int = 2304,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
self.patch_size = patch_size
|
||||
self.init_pos_emb_height = init_pos_emb_height
|
||||
self.init_pos_emb_width = init_pos_emb_width
|
||||
self.init_pos_emb_time = init_pos_emb_time
|
||||
self.pos_emb_type = pos_emb_type
|
||||
self.vt_num_attention_heads = vt_num_attention_heads
|
||||
self.vt_num_hidden_layers = vt_num_hidden_layers
|
||||
self.vt_hidden_size = vt_hidden_size
|
||||
self.vt_intermediate_size = vt_intermediate_size
|
||||
self.merge_kernel_size = tuple(merge_kernel_size)
|
||||
self.video_attn_type = video_attn_type
|
||||
self.merge_type = merge_type
|
||||
self._attn_implementation = _attn_implementation
|
||||
|
||||
self.mm_projector_type = mm_projector_type
|
||||
self.mm_hidden_size = (
|
||||
mm_hidden_size if mm_hidden_size is not None else vt_hidden_size
|
||||
)
|
||||
self.projector_hidden_act = projector_hidden_act
|
||||
self.projector_ln_eps = projector_ln_eps
|
||||
self.text_hidden_size = text_hidden_size
|
||||
|
||||
self.qkv_hidden_size = qkv_hidden_size
|
||||
self.norm_type = norm_type
|
||||
self.attn_bias = attn_bias
|
||||
self.patch_embed_proj_bias = patch_embed_proj_bias
|
||||
self.mlp_type = mlp_type
|
||||
self.linear_bias = linear_bias
|
||||
self.activation_func = activation_func
|
||||
self.pos_emb_interpolation_mode = pos_emb_interpolation_mode
|
||||
|
||||
# Aliases consumed by the Kimi-K2.5 vision implementation.
|
||||
self.num_attention_heads = vt_num_attention_heads
|
||||
self.num_hidden_layers = vt_num_hidden_layers
|
||||
self.hidden_size = vt_hidden_size
|
||||
self.intermediate_size = vt_intermediate_size
|
||||
|
||||
|
||||
class KimiK3Config(PretrainedConfig):
|
||||
model_type = "kimi_k3"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config: dict | KimiLinearConfig | None = None,
|
||||
vision_config: dict | KimiK3VisionConfig | None = None,
|
||||
ignore_index: int = -100,
|
||||
media_placeholder_token_id: int = 163605,
|
||||
pad_token_id: int = 0,
|
||||
image_placeholder: str = "<|kimi_image_placeholder|>",
|
||||
**kwargs,
|
||||
):
|
||||
if text_config is None:
|
||||
self.text_config = KimiLinearConfig()
|
||||
elif isinstance(text_config, dict):
|
||||
self.text_config = KimiLinearConfig(**text_config)
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
if vision_config is None:
|
||||
self.vision_config = KimiK3VisionConfig()
|
||||
elif isinstance(vision_config, dict):
|
||||
self.vision_config = KimiK3VisionConfig(**vision_config)
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
# K3's vision projector output must match the text model's hidden size.
|
||||
# Override any value provided in vision_config for safety.
|
||||
if self.vision_config.text_hidden_size != self.text_config.hidden_size:
|
||||
logger.info(
|
||||
"Overriding vision_config.text_hidden_size from %s to %s "
|
||||
"to match text_config.hidden_size",
|
||||
self.vision_config.text_hidden_size,
|
||||
self.text_config.hidden_size,
|
||||
)
|
||||
self.vision_config.text_hidden_size = self.text_config.hidden_size
|
||||
|
||||
self.ignore_index = ignore_index
|
||||
self.media_placeholder_token_id = media_placeholder_token_id
|
||||
self.image_placeholder = image_placeholder
|
||||
|
||||
if getattr(self.text_config, "quantization_config", None) is not None:
|
||||
self.quantization_config = self.text_config.quantization_config
|
||||
|
||||
super().__init__(pad_token_id=pad_token_id, **kwargs)
|
||||
|
||||
@property
|
||||
def hidden_size(self) -> int:
|
||||
return self.text_config.hidden_size
|
||||
|
||||
@property
|
||||
def vocab_size(self) -> int:
|
||||
return self.text_config.vocab_size
|
||||
@@ -0,0 +1,61 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from transformers import BaseImageProcessor, BatchFeature, TensorType
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
|
||||
class KimiK3Processor(ProcessorMixin):
|
||||
"""HF-style processor wrapper for the image-only Kimi-K3 model.
|
||||
|
||||
K3 exposes the standard ``image`` modality, so vLLM calls this processor
|
||||
with ``images=[PIL, ...]``. The underlying checkpoint image processor
|
||||
(``KimiK3VisionProcessor``) works on ``{"type": "image", "image": PIL}``
|
||||
media dicts, so this wrapper adapts bare PIL images into that shape before
|
||||
delegating to ``preprocess``.
|
||||
|
||||
Text is only tokenized here; the single ``<|kimi_image_placeholder|>``
|
||||
token per image is expanded into the resolution-aware media block by the
|
||||
model's ``_get_prompt_updates`` on the vLLM side.
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: BaseImageProcessor,
|
||||
tokenizer: HfTokenizer,
|
||||
) -> None:
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str | list[str] | None = None,
|
||||
images: object | list[object] | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
if images is not None:
|
||||
if not isinstance(images, list):
|
||||
images = [images]
|
||||
medias = [{"type": "image", "image": image} for image in images]
|
||||
mm_inputs = self.image_processor.preprocess(
|
||||
medias,
|
||||
return_tensors=return_tensors,
|
||||
)
|
||||
else:
|
||||
mm_inputs = {}
|
||||
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
text_inputs = self.tokenizer(text)
|
||||
else:
|
||||
text_inputs = {}
|
||||
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **mm_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
Reference in New Issue
Block a user