[Model] Update Kimi-K25 and Isaac processors to fit HF-style (#37693)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-03-20 18:30:22 +00:00
committed by GitHub
parent d7d2b5e405
commit 37aadf6237
5 changed files with 128 additions and 95 deletions
+54 -32
View File
@@ -6,12 +6,14 @@ import math
from typing import Any
import numpy as np
import PIL.Image
import torch
import torch.nn.functional as F
from PIL import Image
from transformers import BatchFeature, ProcessorMixin, TensorType
from typing_extensions import TypedDict, Unpack
from vllm.tokenizers.hf import HfTokenizer
MAX_PIXELS = 60_000_000 # 60-megapixel ceiling ≈ 8200 × 7300 px
# Vision preprocessing constants
@@ -39,7 +41,7 @@ def _make_writeable(arr: np.ndarray) -> np.ndarray:
return arr.copy()
def extract_image_pil(image: PIL.Image.Image) -> torch.Tensor | None:
def extract_image_pil(image: Image.Image) -> torch.Tensor:
if image.width * image.height > MAX_PIXELS:
raise ValueError(
f"Image (w={image.width}, h={image.height}) > MAX=`{MAX_PIXELS}`"
@@ -314,31 +316,30 @@ class IsaacImageProcessorKwargs(TypedDict, total=False):
class IsaacImageProcessor:
patch_size = 16
max_num_patches = 6144
min_num_patches = 256
pixel_shuffle_scale = 2
valid_kwargs = IsaacImageProcessorKwargs
model_input_names = ["pixel_values", "image_grid_thw"]
def __init__(self, kwargs):
self.patch_size = kwargs.pop("patch_size", self.patch_size)
self.vision_max_num_patches = kwargs.pop(
"vision_max_num_patches", self.max_num_patches
)
self.vision_min_num_patches = kwargs.pop(
"vision_min_num_patches", self.min_num_patches
)
self.pixel_shuffle_scale = kwargs.pop("pixel_shuffle_scale", 2)
def preprocess(
def __init__(
self,
images: list[torch.Tensor],
return_tensors: str | TensorType | None,
patch_size: int = 16,
vision_max_num_patches: int = 6144,
vision_min_num_patches: int = 256,
pixel_shuffle_scale: int = 2,
) -> None:
self.patch_size = patch_size
self.vision_max_num_patches = vision_max_num_patches
self.vision_min_num_patches = vision_min_num_patches
self.pixel_shuffle_scale = pixel_shuffle_scale
def __call__(
self,
images: Image.Image | list[Image.Image],
return_tensors: str | TensorType | None = None,
**kwargs: Unpack[IsaacImageProcessorKwargs],
) -> BatchFeature:
"""Preprocess images into format compatible with vLLM input processing."""
if not isinstance(images, list):
images = [images]
all_pixel_values: list[torch.Tensor] = []
all_image_grids: list[torch.Tensor] = []
@@ -388,23 +389,40 @@ class IsaacImageProcessor:
class IsaacProcessor(ProcessorMixin):
attributes = ["image_processor", "tokenizer"]
def __init__(self, image_processor=None, tokenizer=None, **kwargs):
self.image_token = kwargs.pop("image_token", "<image>")
def __init__(
self,
image_processor: IsaacImageProcessor,
tokenizer: HfTokenizer,
image_token: str = "<image>",
):
self.image_processor = image_processor
self.tokenizer = tokenizer
def __call__(self, text=None, images=None, **kwargs) -> BatchFeature:
result = {}
self.image_token = image_token
def __call__(
self,
text: str | list[str] | None = None,
images: Image.Image | list[Image.Image] | None = None,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
if images is not None:
image_inputs = self.image_processor.preprocess(images, **kwargs)
image_inputs = self.image_processor(
images,
return_tensors=return_tensors,
**kwargs,
)
image_grid_thw = image_inputs["image_grid_thw"]
result.update(image_inputs)
else:
image_inputs = {}
image_grid_thw = []
if text is not None:
if not isinstance(text, list):
text = [text]
if text is not None:
if not isinstance(text, list):
text = [text]
if image_inputs:
text = text.copy() # below lines change text in-place
merge_length = self.image_processor.pixel_shuffle_scale**2
index = 0
@@ -417,10 +435,14 @@ class IsaacProcessor(ProcessorMixin):
index += 1
text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>")
if text is not None:
result.update(self.tokenizer(text, **kwargs))
text_inputs = self.tokenizer(text, return_tensors=return_tensors)
else:
text_inputs = {}
return BatchFeature(result)
return BatchFeature(
data={**text_inputs, **image_inputs},
tensor_type=return_tensors,
)
def apply_chat_template(
self,
+54 -38
View File
@@ -1,38 +1,41 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from transformers import BatchFeature
from transformers import BaseImageProcessor, BatchFeature, TensorType
from transformers.processing_utils import ProcessorMixin
from vllm.multimodal.inputs import VisionChunk
from vllm.tokenizers.hf import HfTokenizer
class KimiK25Processor(ProcessorMixin):
attributes = ["tokenizer"]
tokenizer_class = "AutoTokenizer"
attributes = ["image_processor", "tokenizer"]
def __init__(
self, media_processor=None, tokenizer=None, media_token_id: int | None = None
):
super().__init__(tokenizer)
self.media_processor = media_processor
self,
image_processor: BaseImageProcessor,
tokenizer: HfTokenizer,
media_token_id: int,
) -> None:
self.image_processor = image_processor
self.tokenizer = tokenizer
self.media_token_id = media_token_id
assert self.media_token_id is not None
def __call__(
self,
text: str | list[str] | None = None,
vision_chunks: list[VisionChunk] | None = None,
*,
text: list[int] | str,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
"""
Args:
vision_chunks: List of VisionChunk items to be processed.
For image: VisionChunkImage with type='image', image=PIL.Image
For video_chunk: VisionChunkVideo with type='video_chunk',
video_chunk=list[PIL.Image]
text: The token ids to be fed to a model (required).
text: The text to be field to the model.
vision_chunks: List of `VisionChunk` items to be processed.
For image: `VisionChunkImage` with
`type='image', image=PIL.Image`
For video_chunk: `VisionChunkVideo` with
`type='video_chunk', video_chunk=list[PIL.Image]`
Returns:
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
@@ -42,31 +45,44 @@ class KimiK25Processor(ProcessorMixin):
- **grid_thws** -- list of image 3D grid in LLM.
Returned when `vision_chunks` is not `None`.
"""
mm_inputs = {}
input_ids = self.tokenizer.encode(text) if isinstance(text, str) else text
if vision_chunks is not None:
assert isinstance(vision_chunks, list)
mm_inputs = self.media_processor.preprocess(vision_chunks)
mm_inputs = self.image_processor.preprocess(
vision_chunks,
return_tensors=return_tensors,
)
else:
mm_inputs = {}
num_tokens_per_chunk = [
self.media_processor.media_tokens_calculator(chunk)
for chunk in vision_chunks
]
if text is not None:
if not isinstance(text, list):
text = [text]
new_input_ids = []
for token in input_ids:
if token == self.media_token_id:
new_input_ids.extend(
[self.media_token_id] * num_tokens_per_chunk.pop(0)
)
else:
new_input_ids.append(token)
input_ids = new_input_ids
text_inputs = self.tokenizer(text)
# Note: Modify in-place
input_ids: list[list[int]] = text_inputs["input_ids"] # type: ignore
if vision_chunks is not None:
num_tokens_per_chunk = [
self.image_processor.media_tokens_calculator(chunk)
for chunk in vision_chunks
]
for i in range(len(input_ids)):
new_input_ids = []
for token in input_ids[i]:
if token == self.media_token_id:
new_input_ids.extend(
[self.media_token_id] * num_tokens_per_chunk.pop(0)
)
else:
new_input_ids.append(token)
input_ids[i] = new_input_ids
else:
text_inputs = {}
# XXX: _apply_hf_processor_text_mm will call tolist() on input_ids
return BatchFeature(
data={
"input_ids": torch.tensor([input_ids]),
**mm_inputs,
}
data={**text_inputs, **mm_inputs},
tensor_type=return_tensors,
)
@@ -286,11 +286,9 @@ class Step3VLImageProcessor:
def __call__(
self,
images: Image.Image | list[Image.Image] | None = None,
images: Image.Image | list[Image.Image],
return_tensors: str | TensorType | None = None,
) -> BatchFeature:
if images is None:
images = []
if not isinstance(images, list):
images = [images]