[Models] Kimi-K2.5 (#33131)
Signed-off-by: wanglinian <wanglinian@stu.pku.edu.cn> Signed-off-by: wangln19 <96399074+wangln19@users.noreply.github.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: wanglinian <wanglinian@stu.pku.edu.cn> Co-authored-by: wangln19 <96399074+wangln19@users.noreply.github.com> Co-authored-by: Zaida Zhou <58739961+zhouzaida@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
wanglinian
wangln19
Zaida Zhou
Isotr0py
Nick Hill
youkaichao
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
6c00645712
commit
b539f988e1
@@ -20,6 +20,7 @@ from typing import (
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
from PIL.Image import Image
|
||||
from typing_extensions import NotRequired, TypeVar
|
||||
|
||||
from vllm.utils.collection_utils import full_groupby, is_list_of
|
||||
@@ -29,7 +30,6 @@ from vllm.utils.jsontree import json_map_leaves
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
import torch.types
|
||||
from PIL.Image import Image
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
from .media import MediaWithBytes
|
||||
@@ -105,6 +105,28 @@ The number of data items allowed per modality is restricted by
|
||||
"""
|
||||
|
||||
|
||||
class VisionChunkImage(TypedDict):
|
||||
"""Represents an image wrapped as a vision chunk."""
|
||||
|
||||
type: Literal["image"]
|
||||
image: Image
|
||||
uuid: str | None
|
||||
|
||||
|
||||
class VisionChunkVideo(TypedDict):
|
||||
"""Represents a video chunk with metadata."""
|
||||
|
||||
type: Literal["video_chunk"]
|
||||
video_chunk: list[Image]
|
||||
uuid: str | None
|
||||
prompt: str
|
||||
video_idx: int
|
||||
|
||||
|
||||
VisionChunk = VisionChunkImage | VisionChunkVideo
|
||||
"""A vision chunk is either an image or a video chunk."""
|
||||
|
||||
|
||||
@final
|
||||
class MultiModalDataBuiltins(TypedDict, total=False):
|
||||
"""Type annotations for modality types predefined by vLLM."""
|
||||
@@ -118,6 +140,9 @@ class MultiModalDataBuiltins(TypedDict, total=False):
|
||||
audio: ModalityData[AudioItem]
|
||||
"""The input audio(s)."""
|
||||
|
||||
vision_chunk: ModalityData[VisionChunk]
|
||||
"""The input visual atom(s) - unified modality for images and video chunks."""
|
||||
|
||||
|
||||
MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]
|
||||
"""
|
||||
|
||||
@@ -384,6 +384,13 @@ class VideoEmbeddingItems(EmbeddingItems):
|
||||
super().__init__(data, "video", expected_hidden_size)
|
||||
|
||||
|
||||
class VisionChunkProcessorItems(ProcessorBatchItems[Any]):
|
||||
"""Processor items for vision chunks (unified image and video chunks)."""
|
||||
|
||||
def __init__(self, data: Sequence[Any]) -> None:
|
||||
super().__init__(data, "vision_chunk")
|
||||
|
||||
|
||||
_D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
|
||||
|
||||
|
||||
@@ -652,11 +659,23 @@ class MultiModalDataParser:
|
||||
|
||||
return VideoProcessorItems(new_videos, metadata=metadata_lst)
|
||||
|
||||
def _parse_vision_chunk_data(
|
||||
self,
|
||||
data: ModalityData[Any],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
"""Parse vision chunk data (unified image and video chunks)."""
|
||||
if data is None or self._is_empty(data):
|
||||
return None
|
||||
if self.is_embeddings(data):
|
||||
raise ValueError("Do not support embedding data for vision_chunk right now")
|
||||
return VisionChunkProcessorItems(data)
|
||||
|
||||
def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:
|
||||
return {
|
||||
"audio": self._parse_audio_data,
|
||||
"image": self._parse_image_data,
|
||||
"video": self._parse_video_data,
|
||||
"vision_chunk": self._parse_vision_chunk_data,
|
||||
}
|
||||
|
||||
def parse_mm_data(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
|
||||
|
||||
@@ -235,6 +235,27 @@ class VideoLoader:
|
||||
VIDEO_LOADER_REGISTRY = ExtensionManager()
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register("identity")
|
||||
class IdentityVideoLoader(VideoLoader):
|
||||
"""IdentityVideoLoader returns raw video bytes without decoding.
|
||||
|
||||
This allows the model processor to handle video decoding and
|
||||
is required for models like Kimi-K2.5 that need custom video chunk splitting.
|
||||
|
||||
NOTE: This is temporary for Kimi-K2.5 testing. Remember to change back
|
||||
to opencv before release if needed.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def load_bytes(
|
||||
cls,
|
||||
data: bytes,
|
||||
num_frames: int = -1,
|
||||
**kwargs: Any,
|
||||
) -> tuple[Any, Any]:
|
||||
return data, None
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register("opencv")
|
||||
class OpenCVVideoBackend(VideoLoader):
|
||||
def get_cv2_video_api(self):
|
||||
|
||||
Reference in New Issue
Block a user