[Mini-VLLM] Remove orphan subsystems: snapshot, CPU workers, dead decoders

- Delete the snapshot package, CLI command, and its launch tests
- Delete CPU workers, cpu_communicator, and the hw_agnostic layer package
- Delete the EC CPU connector, keeping the EC base glue intact
- Delete dead composite/triton_flashinfer backends and third_party/flashmla
- Delete dead video decoders, video-bound tests, and orphan utils
- Remove the deep_gemm_mega_moe option and dead allreduce fusion pair

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 09:27:06 +07:00
co-authored by Hinata
parent 2b544b7ba0
commit bd70a61c20
88 changed files with 4 additions and 20247 deletions
@@ -1,459 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# adapted from https://github.com/deepseek-ai/DeepSeek-OCR/blob/main/DeepSeek-OCR-master/DeepSeek-OCR-vllm/process/image_process.py
# and https://github.com/deepseek-ai/DeepSeek-OCR-2/blob/main/DeepSeek-OCR2-master/DeepSeek-OCR2-vllm/process/image_process.py
import math
from typing import Literal
import torch
import torchvision.transforms as T
from PIL import Image, ImageOps
from transformers import BatchFeature, LlamaTokenizerFast
from transformers.processing_utils import ProcessorMixin
# TODO(Isotr0py): change modes for variants
# see: https://github.com/deepseek-ai/DeepSeek-OCR/blob/8cf003d38821fa1b19c73da3bd1b0dc262ea8136/DeepSeek-OCR-master/DeepSeek-OCR-vllm/config.py#L1-L6
# Tiny: base_size = 512, image_size = 512, crop_mode = False
# Small: base_size = 640, image_size = 640, crop_mode = False
# Base: base_size = 1024, image_size = 1024, crop_mode = False
# Large: base_size = 1280, image_size = 1280, crop_mode = False
# Gundam: base_size = 1024, image_size = 640, crop_mode = True
BASE_SIZE = 1024
IMAGE_SIZE = 640
CROP_MODE = True
# TODO(Isotr0py): Expose as mm_kwargs
MIN_CROPS = 2
MAX_CROPS = 6 # max:9; If your GPU memory is small, it is recommended to set it to 6.
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
best_ratio_diff = float("inf")
best_ratio = (1, 1)
area = width * height
for ratio in target_ratios:
target_aspect_ratio = ratio[0] / ratio[1]
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
if ratio_diff < best_ratio_diff:
best_ratio_diff = ratio_diff
best_ratio = ratio
elif ratio_diff == best_ratio_diff:
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
best_ratio = ratio
return best_ratio
def calculate_aspect_ratios(
min_num: int = MIN_CROPS, max_num: int = MAX_CROPS
) -> list[tuple[int, int]]:
target_ratios: set[tuple[int, int]] = set(
(i, j)
for n in range(min_num, max_num + 1)
for i in range(1, n + 1)
for j in range(1, n + 1)
if i * j <= max_num and i * j >= min_num
)
sorted_target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
return sorted_target_ratios
def count_tiles(
orig_width,
orig_height,
min_num=MIN_CROPS,
max_num=MAX_CROPS,
image_size=640,
use_thumbnail=False,
):
aspect_ratio = orig_width / orig_height
# calculate the existing image aspect ratio
target_ratios = calculate_aspect_ratios(min_num, max_num)
# find the closest aspect ratio to the target
target_aspect_ratio = find_closest_aspect_ratio(
aspect_ratio, target_ratios, orig_width, orig_height, image_size
)
return target_aspect_ratio
def dynamic_preprocess(
image, min_num=MIN_CROPS, max_num=MAX_CROPS, image_size=640, use_thumbnail=False
):
orig_width, orig_height = image.size
aspect_ratio = orig_width / orig_height
# calculate the existing image aspect ratio
target_ratios = calculate_aspect_ratios(min_num, max_num)
# find the closest aspect ratio to the target
target_aspect_ratio = find_closest_aspect_ratio(
aspect_ratio, target_ratios, orig_width, orig_height, image_size
)
# calculate the target width and height
target_width = image_size * target_aspect_ratio[0]
target_height = image_size * target_aspect_ratio[1]
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
# resize the image
resized_img = image.resize((target_width, target_height))
processed_images = []
for i in range(blocks):
box = (
(i % (target_width // image_size)) * image_size,
(i // (target_width // image_size)) * image_size,
((i % (target_width // image_size)) + 1) * image_size,
((i // (target_width // image_size)) + 1) * image_size,
)
# split the image
split_img = resized_img.crop(box)
processed_images.append(split_img)
assert len(processed_images) == blocks
if use_thumbnail and len(processed_images) != 1:
thumbnail_img = image.resize((image_size, image_size))
processed_images.append(thumbnail_img)
return processed_images, target_aspect_ratio
class ImageTransform:
def __init__(
self,
mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
std: tuple[float, float, float] = (0.5, 0.5, 0.5),
normalize: bool = True,
):
self.mean = mean
self.std = std
self.normalize = normalize
transform_pipelines = [T.ToTensor()]
if normalize:
transform_pipelines.append(T.Normalize(mean, std))
self.transform = T.Compose(transform_pipelines)
def __call__(self, pil_img: Image.Image):
x = self.transform(pil_img)
return x
class DeepseekOCRProcessor(ProcessorMixin):
tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast")
attributes = ["tokenizer"]
def __init__(
self,
tokenizer: LlamaTokenizerFast,
patch_size: int = 16,
downsample_ratio: int = 4,
image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
image_std: tuple[float, float, float] = (0.5, 0.5, 0.5),
normalize: bool = True,
image_token: str = "<image>",
pad_token: str = "<|▁pad▁|>",
add_special_token: bool = False,
sft_format: str = "deepseek",
mask_prompt: bool = True,
ignore_id: int = -100,
image_size: int = IMAGE_SIZE,
base_size: int = BASE_SIZE,
strategy: Literal["v1", "v2"] = "v1",
max_crops: int = MAX_CROPS,
**kwargs,
):
self.image_size = image_size
self.base_size = base_size
self.max_crops = max_crops
# image token calculation strategy for
# Deepseek-OCR and Deepseek-OCR-2
self.strategy = strategy
assert strategy in ["v1", "v2"], "Only 'v1' and 'v2' strategies are supported."
self.patch_size = 16
self.image_mean = image_mean
self.image_std = image_std
self.normalize = normalize
self.downsample_ratio = 4
self.image_transform = ImageTransform(
mean=image_mean, std=image_std, normalize=normalize
)
self.tokenizer = tokenizer
self.tokenizer.padding_side = "left" # must set this,padding side with make a difference in batch inference # noqa: E501
# add the pad_token as special token to use 'tokenizer.pad_token'
# and 'tokenizer.pad_token_id'
if self.tokenizer.pad_token is None:
self.tokenizer.add_special_tokens({"pad_token": pad_token})
# add image token
self.image_token_id = self.tokenizer.vocab.get(image_token)
self.image_token = image_token
self.pad_token = pad_token
self.add_special_token = add_special_token
self.sft_format = sft_format
self.mask_prompt = mask_prompt
self.ignore_id = ignore_id
super().__init__(
tokenizer,
**kwargs,
)
@property
def bos_id(self):
return self.tokenizer.bos_token_id
@property
def eos_id(self):
return self.tokenizer.eos_token_id
@property
def pad_id(self):
return self.tokenizer.pad_token_id
def encode(self, text: str, bos: bool = True, eos: bool = False):
t = self.tokenizer.encode(text, add_special_tokens=False)
if bos:
t = [self.bos_id] + t
if eos:
t = t + [self.eos_id]
return t
def decode(self, t: list[int], **kwargs) -> str:
return self.tokenizer.decode(t, **kwargs)
def process_one(
self,
prompt: str,
images: list[Image.Image],
crop_mode: bool = CROP_MODE,
):
"""Args:
prompt (str): the formatted prompt;
images (List[ImageType]): the list of images;
crop_mode (bool): if True, then crop the image;
Returns:
outputs (BaseProcessorOutput): the output of the processor,
- input_ids (torch.LongTensor): [N + image tokens]
- target_ids (torch.LongTensor): [N + image tokens]
- pixel_values (torch.FloatTensor): [n_patches, 3, H, W]
- image_id (int): the id of the image token
- num_image_tokens (List[int]): the number of image tokens
"""
if prompt is None or images is None:
raise ValueError("prompt and images must be used at the same time.")
sft_format = prompt
(
input_ids,
pixel_values,
images_crop,
images_seq_mask,
images_spatial_crop,
num_image_tokens,
_,
) = self.tokenize_with_images(
conversation=sft_format,
images=images,
bos=True,
eos=True,
cropping=crop_mode,
)
prepare = BatchFeature(
data=dict(
input_ids=input_ids,
pixel_values=pixel_values,
images_crop=images_crop,
images_seq_mask=images_seq_mask,
images_spatial_crop=images_spatial_crop,
num_image_tokens=num_image_tokens,
),
tensor_type="pt",
)
return prepare
def __call__(
self,
*,
prompt: str,
images: list[Image.Image],
crop_mode: bool = CROP_MODE,
**kwargs,
):
prepare = self.process_one(
prompt=prompt,
images=images,
crop_mode=crop_mode,
)
return prepare
def tokenize_with_images(
self,
conversation: str,
images: list[Image.Image],
bos: bool = True,
eos: bool = True,
cropping: bool = True,
):
"""Tokenize text with <image> tags."""
num_image_tags = conversation.count(self.image_token)
if num_image_tags != len(images):
raise ValueError(
f"Number of {self.image_token!r} tokens in prompt "
f"({num_image_tags}) does not match number of images "
f"({len(images)})."
)
text_splits = conversation.split(self.image_token)
images_list, images_crop_list, images_seq_mask, images_spatial_crop = (
[],
[],
[],
[],
)
image_shapes = []
num_image_tokens = []
tokenized_str = []
for text_sep, image in zip(text_splits, images):
tokenized_sep = self.encode(text_sep, bos=False, eos=False)
tokenized_str += tokenized_sep
images_seq_mask += [False] * len(tokenized_sep)
image_shapes.append(image.size)
images_crop_raw = []
if image.size[0] <= self.image_size and image.size[1] <= self.image_size:
crop_ratio = [1, 1]
elif cropping:
images_crop_raw, crop_ratio = dynamic_preprocess(
image, image_size=self.image_size, max_num=self.max_crops
)
else:
crop_ratio = [1, 1]
if not cropping:
image = image.resize((self.image_size, self.image_size))
global_view = ImageOps.pad(
image,
(self.base_size, self.base_size),
color=tuple(int(x * 255) for x in self.image_transform.mean),
)
images_list.append(self.image_transform(global_view))
num_width_tiles, num_height_tiles = crop_ratio
images_spatial_crop.append([num_width_tiles, num_height_tiles])
if num_width_tiles > 1 or num_height_tiles > 1:
for cropped_image in images_crop_raw:
images_crop_list.append(self.image_transform(cropped_image))
num_queries = math.ceil(
(self.image_size // self.patch_size) / self.downsample_ratio
)
num_queries_base = math.ceil(
(self.base_size // self.patch_size) / self.downsample_ratio
)
num_tokens_base = (
(num_queries_base * (num_queries_base + 1))
if self.strategy == "v1"
else num_queries_base * num_queries_base
)
tokenized_image = [self.image_token_id] * num_tokens_base
tokenized_image += [self.image_token_id]
if num_width_tiles > 1 or num_height_tiles > 1:
num_tokens_per_row = (
num_queries * num_width_tiles + 1
if self.strategy == "v1"
else num_queries * num_width_tiles
)
local_row = [self.image_token_id] * num_tokens_per_row
tokenized_image += local_row * (num_queries * num_height_tiles)
tokenized_str += tokenized_image
images_seq_mask += [True] * len(tokenized_image)
num_image_tokens.append(len(tokenized_image))
"""process the last text split"""
tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False)
tokenized_str += tokenized_sep
images_seq_mask += [False] * len(tokenized_sep)
"""add the bos and eos tokens"""
if bos:
tokenized_str = [self.bos_id] + tokenized_str
images_seq_mask = [False] + images_seq_mask
if eos:
tokenized_str = tokenized_str + [self.eos_id]
images_seq_mask = images_seq_mask + [False]
assert len(tokenized_str) == len(images_seq_mask), (
f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} "
f"is not equal to images_seq_mask's length {len(images_seq_mask)}."
)
masked_tokenized_str = []
for token_index in tokenized_str:
if token_index != self.image_token_id:
masked_tokenized_str.append(token_index)
else:
masked_tokenized_str.append(self.ignore_id)
assert (
len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str)
), (
f"tokenized_str's length {len(tokenized_str)}, "
f"input_ids' length {len(masked_tokenized_str)}, "
f"images_seq_mask's length {len(images_seq_mask)}, are not equal."
)
input_ids = torch.LongTensor(tokenized_str)
target_ids = torch.LongTensor(masked_tokenized_str)
images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)
# set input_ids < 0 | input_ids == self.image_token_id as ignore_id
target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = (
self.ignore_id
)
input_ids[input_ids < 0] = self.pad_id
# Remove the ending eos token
assert input_ids[-1] == self.eos_id
input_ids = input_ids[:-1]
target_ids = target_ids[:-1]
images_seq_mask = images_seq_mask[:-1]
if len(images_list) == 0:
pixel_values = torch.zeros((0, 3, self.base_size, self.base_size))
images_spatial_crop = torch.zeros((0, 2), dtype=torch.long)
images_crop = torch.zeros((0, 3, self.image_size, self.image_size))
else:
pixel_values = torch.stack(images_list, dim=0)
images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)
if images_crop_list:
images_crop = torch.stack(images_crop_list, dim=0)
else:
images_crop = torch.zeros((0, 3, self.image_size, self.image_size))
input_ids = input_ids.unsqueeze(0)
return (
input_ids,
pixel_values,
images_crop,
images_seq_mask,
images_spatial_crop,
num_image_tokens,
image_shapes,
)
@@ -1,585 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_internvl_chat.py
# --------------------------------------------------------
# InternVL
# Copyright (c) 2023 OpenGVLab
# Licensed under The MIT License [see LICENSE for details]
# --------------------------------------------------------
import numpy.typing as npt
import torch
import torchvision.transforms as T
from PIL import Image
from transformers import (
BaseVideoProcessor,
BatchFeature,
ImageProcessingMixin,
TensorType,
)
from transformers.processing_utils import ProcessorMixin
from vllm.multimodal.image import convert_image_mode
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
from vllm.tokenizers.hf import HfTokenizer
IMAGENET_MEAN = (0.485, 0.456, 0.406)
IMAGENET_STD = (0.229, 0.224, 0.225)
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
def build_transform(input_size: int):
MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
return T.Compose(
[
T.Lambda(lambda img: convert_image_mode(img, "RGB")),
T.Resize(
(input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
),
T.ToTensor(),
T.Normalize(mean=MEAN, std=STD),
]
)
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
def find_closest_aspect_ratio(
aspect_ratio: float,
target_ratios: list[tuple[int, int]],
*,
width: int,
height: int,
image_size: int,
) -> tuple[int, int]:
best_ratio_diff = float("inf")
best_ratio = (1, 1)
area = width * height
for ratio in target_ratios:
target_aspect_ratio = ratio[0] / ratio[1]
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
if ratio_diff < best_ratio_diff:
best_ratio_diff = ratio_diff
best_ratio = ratio
elif ratio_diff == best_ratio_diff:
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
best_ratio = ratio
return best_ratio
def resolve_internvl_min_max_num(
*,
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool,
use_thumbnail: bool,
) -> tuple[int, int]:
min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
if use_thumbnail and max_dynamic_patch != 1:
max_dynamic_patch += 1
return min_dynamic_patch, max_dynamic_patch
def get_internvl_target_ratios(
min_num: int,
max_num: int,
) -> list[tuple[int, int]]:
target_ratios = {
(i, j)
for n in range(min_num, max_num + 1)
for i in range(1, n + 1)
for j in range(1, n + 1)
if min_num <= i * j <= max_num
}
return sorted(target_ratios, key=lambda x: x[0] * x[1])
def calculate_internvl_targets(
*,
orig_width: int,
orig_height: int,
target_ratios: list[tuple[int, int]],
image_size: int,
use_thumbnail: bool,
) -> tuple[int, int, int]:
aspect_ratio = orig_width / orig_height
# find the closest aspect ratio to the target
target_aspect_ratio = find_closest_aspect_ratio(
aspect_ratio,
target_ratios,
width=orig_width,
height=orig_height,
image_size=image_size,
)
# calculate the target width and height
target_width = image_size * target_aspect_ratio[0]
target_height = image_size * target_aspect_ratio[1]
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
# add thumbnail image if num_blocks != 1
if use_thumbnail and blocks != 1:
blocks += 1
return blocks, target_width, target_height
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
def dynamic_preprocess_internvl(
image: Image.Image,
*,
target_ratios: list[tuple[int, int]],
image_size: int,
use_thumbnail: bool,
) -> list[Image.Image]:
orig_width, orig_height = image.size
# calculate the number of blocks without thumbnail
blocks, target_width, target_height = calculate_internvl_targets(
orig_width=orig_width,
orig_height=orig_height,
target_ratios=target_ratios,
image_size=image_size,
use_thumbnail=False,
)
# resize the image
resized_img = image.resize((target_width, target_height))
processed_images = []
for i in range(blocks):
box = (
(i % (target_width // image_size)) * image_size,
(i // (target_width // image_size)) * image_size,
((i % (target_width // image_size)) + 1) * image_size,
((i // (target_width // image_size)) + 1) * image_size,
)
# split the image
split_img = resized_img.crop(box)
processed_images.append(split_img)
assert len(processed_images) == blocks
if use_thumbnail and len(processed_images) != 1:
thumbnail_img = image.resize((image_size, image_size))
processed_images.append(thumbnail_img)
return processed_images
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
def image_to_pixel_values_internvl(
image: Image.Image,
*,
input_size: int,
min_num: int,
max_num: int,
use_thumbnail: bool,
) -> torch.Tensor:
target_ratios = get_internvl_target_ratios(min_num, max_num)
transform = build_transform(input_size=input_size)
images = dynamic_preprocess_internvl(
image,
target_ratios=target_ratios,
image_size=input_size,
use_thumbnail=use_thumbnail,
)
pixel_values = torch.stack([transform(image) for image in images])
return pixel_values
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
def video_to_pixel_values_internvl(
video: npt.NDArray,
*,
input_size: int,
min_num: int,
max_num: int,
use_thumbnail: bool,
) -> torch.Tensor:
target_ratios = get_internvl_target_ratios(min_num, max_num)
transform = build_transform(input_size=input_size)
frames_list = list[Image.Image]()
for frame in video:
pil_frame = dynamic_preprocess_internvl(
Image.fromarray(frame, mode="RGB"),
target_ratios=target_ratios,
image_size=input_size,
use_thumbnail=use_thumbnail,
)
assert len(pil_frame) == 1
frames_list.extend(pil_frame)
pixel_values = torch.stack([transform(image) for image in frames_list])
return pixel_values
class InternVLImageProcessor(ImageProcessingMixin):
def __init__(
self,
image_size: int,
min_dynamic_patch: int,
max_dynamic_patch: int,
dynamic_image_size: bool,
use_thumbnail: bool,
) -> None:
self.image_size = image_size
self.min_dynamic_patch = min_dynamic_patch
self.max_dynamic_patch = max_dynamic_patch
self.dynamic_image_size = dynamic_image_size
self.use_thumbnail = use_thumbnail
def resolve_min_max_num(
self,
*,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
use_thumbnail: bool | None = None,
) -> tuple[int, int]:
if min_dynamic_patch is None:
min_dynamic_patch = self.min_dynamic_patch
if max_dynamic_patch is None:
max_dynamic_patch = self.max_dynamic_patch
if dynamic_image_size is None:
dynamic_image_size = self.dynamic_image_size
if use_thumbnail is None:
use_thumbnail = self.use_thumbnail
return resolve_internvl_min_max_num(
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
use_thumbnail=use_thumbnail,
)
def _images_to_pixel_values_lst(
self,
images: list[Image.Image],
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
) -> list[torch.Tensor]:
if min_dynamic_patch is None:
min_dynamic_patch = self.min_dynamic_patch
if max_dynamic_patch is None:
max_dynamic_patch = self.max_dynamic_patch
if dynamic_image_size is None:
dynamic_image_size = self.dynamic_image_size
min_num, max_num = resolve_internvl_min_max_num(
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
use_thumbnail=False, # Applied in image_to_pixel_values
)
return [
image_to_pixel_values_internvl(
image,
input_size=self.image_size,
min_num=min_num,
max_num=max_num,
use_thumbnail=self.use_thumbnail,
)
for image in images
]
def __call__(
self,
images: Image.Image | list[Image.Image],
*,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
images_lst = [images] if not isinstance(images, list) else images
pixel_values_lst = self._images_to_pixel_values_lst(
images_lst,
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
)
image_inputs = {
"pixel_values_flat": torch.cat(pixel_values_lst),
"image_num_patches": torch.tensor([len(item) for item in pixel_values_lst]),
}
return BatchFeature(image_inputs, tensor_type=return_tensors)
class InternVLVideoProcessor(BaseVideoProcessor):
def __init__(
self,
image_size: int,
) -> None:
self.image_size = image_size
def _videos_to_pixel_values_lst(
self,
videos: list[npt.NDArray],
) -> list[torch.Tensor]:
return [
video_to_pixel_values_internvl(
video,
input_size=self.image_size,
min_num=1,
max_num=1,
use_thumbnail=False,
)
for video in videos
]
def __call__(
self,
videos: npt.NDArray | list[npt.NDArray],
*,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
videos_lst = [videos] if not isinstance(videos, list) else videos
pixel_values_lst = self._videos_to_pixel_values_lst(videos_lst)
image_inputs = {
"pixel_values_flat_video": torch.cat(pixel_values_lst),
"video_num_patches": torch.tensor([len(item) for item in pixel_values_lst]),
}
return BatchFeature(image_inputs, tensor_type=return_tensors)
class InternVLProcessor(ProcessorMixin):
"""This model doesn't define its own HF processor,
so we implement our own one here.
The code to insert image tokens is based on:
https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
Code for video processing is adapted from video example:
https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers
"""
attributes = ["image_processor", "tokenizer", "video_processor"]
def __init__(
self,
image_processor: InternVLImageProcessor,
tokenizer: HfTokenizer,
video_processor: InternVLVideoProcessor | None = None,
*,
image_seq_length: int,
start_image_token: str = "<img>",
end_image_token: str = "</img>",
ctx_image_token: str = "<IMG_CONTEXT>",
ctx_video_token: str | None = None,
) -> None:
self.image_processor = image_processor
self.tokenizer = tokenizer
self.video_processor = video_processor
self.image_seq_length = image_seq_length
self.start_image_token = start_image_token
self.end_image_token = end_image_token
self.ctx_image_token = ctx_image_token
self.ctx_video_token = ctx_video_token
self.start_image_token_id = tokenizer.convert_tokens_to_ids(start_image_token)
self.end_image_token_id = tokenizer.convert_tokens_to_ids(end_image_token)
self.ctx_image_token_id = tokenizer.convert_tokens_to_ids(ctx_image_token)
self.ctx_video_token_id = (
None
if ctx_video_token is None
else tokenizer.convert_tokens_to_ids(ctx_video_token)
)
def resolve_target_ratios(
self,
*,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
use_thumbnail: bool | None = None,
) -> list[tuple[int, int]]:
min_num, max_num = self.image_processor.resolve_min_max_num(
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
use_thumbnail=use_thumbnail,
)
return get_internvl_target_ratios(min_num, max_num)
def get_num_image_tokens(
self,
*,
image_width: int,
image_height: int,
) -> int:
image_processor = self.image_processor
target_ratios = self.resolve_target_ratios(
use_thumbnail=False, # Applied in calculate_targets
)
num_patches, _, _ = calculate_internvl_targets(
orig_width=image_width,
orig_height=image_height,
image_size=image_processor.image_size,
target_ratios=target_ratios,
use_thumbnail=image_processor.use_thumbnail,
)
return num_patches * self.image_seq_length
def get_image_repl(
self,
num_patches: int | None,
num_features: int | None = None,
) -> PromptUpdateDetails:
if num_patches is None:
assert num_features is not None
else:
num_features = num_patches * self.image_seq_length
repl_features = self.ctx_image_token * num_features
repl_full = self.start_image_token + repl_features + self.end_image_token
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
def get_video_repl(self, num_patches: int) -> PromptUpdateDetails:
assert self.ctx_video_token is not None
repl_features = self.ctx_video_token * self.image_seq_length
repl_features_with_sep = (
self.start_image_token + repl_features + self.end_image_token
)
# num_patches is equal to num_frames
repl_full = "".join(
[f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
)
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
assert self.ctx_video_token_id is not None
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_video_token_id)
def __call__(
self,
text: str | list[str] | None = None,
images: Image.Image | list[Image.Image] | None = None,
videos: npt.NDArray | list[npt.NDArray] | None = None,
*,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
return_tensors: str | TensorType | None = None,
**kwargs,
) -> BatchFeature:
if images is not None:
image_inputs = self.image_processor(
images=images,
min_dynamic_patch=min_dynamic_patch,
max_dynamic_patch=max_dynamic_patch,
dynamic_image_size=dynamic_image_size,
return_tensors=return_tensors,
)
image_num_patches = image_inputs["image_num_patches"]
else:
image_inputs = {}
image_num_patches = []
if videos is not None:
if self.video_processor is None:
raise ValueError("This model does not support video inputs")
video_inputs = self.video_processor(
videos=videos,
return_tensors=return_tensors,
)
video_num_patches = video_inputs["video_num_patches"]
else:
video_inputs = {}
video_num_patches = []
if text is not None:
if not isinstance(text, list):
text = [text]
if image_inputs:
image_token = "<image>"
image_index = 0
processed_text = list[str]()
replace_strings = list[str]()
for prompt in text:
new_prompt = prompt
while image_token in new_prompt:
new_prompt = new_prompt.replace(image_token, "<placeholder>", 1)
image_repl = self.get_image_repl(image_num_patches[image_index])
# Convert the token IDs back to text for the
# string-based replacement below
replace_strings.append(
self.tokenizer.decode(
image_repl.full, skip_special_tokens=False
)
)
image_index += 1
while "<placeholder>" in new_prompt:
replace_str = replace_strings.pop(0)
new_prompt = new_prompt.replace("<placeholder>", replace_str, 1)
processed_text.append(new_prompt)
text = processed_text
if video_inputs:
video_token = "<video>"
video_index = 0
processed_text = list[str]()
replace_strings = list[str]()
assert video_token is not None
for prompt in text:
new_prompt = prompt
while video_token in new_prompt:
new_prompt = new_prompt.replace(video_token, "<placeholder>", 1)
video_repl = self.get_video_repl(video_num_patches[video_index])
# Convert the token IDs back to text for the
# string-based replacement below
replace_strings.append(
self.tokenizer.decode(
video_repl.full, skip_special_tokens=False
)
)
video_index += 1
while "<placeholder>" in new_prompt:
replace_str = replace_strings.pop(0)
new_prompt = new_prompt.replace("<placeholder>", replace_str, 1)
processed_text.append(new_prompt)
text = processed_text
text_inputs = self.tokenizer(text, return_tensors=return_tensors)
else:
text_inputs = {}
return BatchFeature(
data={**text_inputs, **image_inputs, **video_inputs},
tensor_type=return_tensors,
)
@@ -1,361 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Optimized CPU image processor for Kimi-K2.5/K2.6 vision chunks."""
import io
import json
import math
from typing import Any
import numpy as np
import pybase64 as base64
import torch
from PIL import Image
from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
from transformers.utils import TensorType
from vllm.multimodal.image import open_image
from vllm.utils.import_utils import is_numba_available
from vllm.utils.jit_monitor import numba_workqueue_threading_layer
_IDENTITY_LUT = np.tile(np.arange(256, dtype=np.uint8)[:, None], (1, 3))
if is_numba_available():
from numba import njit, prange
@njit(parallel=True, cache=True)
def _write_fused_patches(
frames: np.ndarray,
out: np.ndarray,
out_offset: int,
new_h: int,
new_w: int,
padded_h: int,
padded_w: int,
patch_size: int,
normalize_lut: np.ndarray,
) -> None:
# frames: [T, new_h, new_w, 3] uint8, without padding.
# out: [total_patches, 3, patch_size, patch_size] float32.
t_size = frames.shape[0]
patch_h = padded_h // patch_size
patch_w = padded_w // patch_size
total = t_size * padded_h * padded_w * 3
hwc = padded_h * padded_w * 3
wc = padded_w * 3
for linear in prange(total):
t = linear // hwc
rem = linear - t * hwc
y = rem // wc
rem = rem - y * wc
x = rem // 3
c = rem - x * 3
value = frames[t, y, x, c] if y < new_h and x < new_w else 0
patch_idx = (
out_offset
+ t * patch_h * patch_w
+ (y // patch_size) * patch_w
+ (x // patch_size)
)
out[patch_idx, c, y % patch_size, x % patch_size] = normalize_lut[value, c]
else:
def _write_fused_patches(*args: Any, **kwargs: Any) -> None:
raise RuntimeError("numba is required for fused Kimi image preprocessing")
def navit_resize_image(
width: int,
height: int,
patch_size: int,
merge_kernel_size: int,
in_patch_limit: int,
patch_limit_on_one_side: int,
fixed_output_tokens: int | None,
) -> dict[str, int]:
s1 = math.sqrt(
in_patch_limit
/ (max(1.0, width // patch_size) * max(1.0, height // patch_size))
)
s2 = patch_limit_on_one_side * patch_size / width
s3 = patch_limit_on_one_side * patch_size / height
scale = min(1.0, s1, s2, s3)
new_w = min(max(1, int(width * scale)), patch_limit_on_one_side * patch_size)
new_h = min(max(1, int(height * scale)), patch_limit_on_one_side * patch_size)
factor = merge_kernel_size * patch_size
pad_height = (factor - new_h % factor) % factor
pad_width = (factor - new_w % factor) % factor
if fixed_output_tokens is not None:
num_tokens = fixed_output_tokens
else:
token_height = (new_h + pad_height) // factor
token_width = (new_w + pad_width) // factor
num_tokens = token_height * token_width
return {
"num_tokens": num_tokens,
"new_width": new_w,
"new_height": new_h,
"pad_width": pad_width,
"pad_height": pad_height,
"sampled_nframes": 1,
}
def navit_resize_video(
width: int,
height: int,
nframes: int,
avg_fps: float,
sample_fps: float,
patch_size: int,
merge_kernel_size: int,
in_patch_limit_each_frame: int,
patch_limit_on_one_side: int,
in_patch_limit_total: int | None,
max_num_frames_each_video: int | None,
fixed_output_tokens_each_frame: int | None,
) -> dict[str, int]:
sample_fps = min(sample_fps, avg_fps)
sampled_nframes = max(round(nframes * sample_fps / avg_fps), 1)
if max_num_frames_each_video is not None:
sampled_nframes = min(sampled_nframes, max_num_frames_each_video)
if in_patch_limit_total is not None:
in_patch_limit_each_frame = min(
round(in_patch_limit_total / sampled_nframes),
in_patch_limit_each_frame,
)
ret = navit_resize_image(
width,
height,
patch_size,
merge_kernel_size,
in_patch_limit_each_frame,
patch_limit_on_one_side,
fixed_output_tokens_each_frame,
)
ret["sampled_nframes"] = sampled_nframes
return ret
def _to_pil(data: Any) -> Image.Image:
if hasattr(data, "media") and hasattr(data, "original_bytes"):
data = data.media
if isinstance(data, Image.Image):
return data if data.mode == "RGB" else data.convert("RGB")
if isinstance(data, str):
if data.startswith("data:"):
raw_base64 = data.split(",", 1)[1]
return open_image(io.BytesIO(base64.b64decode(raw_base64))).convert("RGB")
return open_image(data).convert("RGB")
if isinstance(data, bytes):
return open_image(io.BytesIO(data)).convert("RGB")
raise ValueError(f"Unsupported data type: {type(data)}")
def _ensure_media_type(media: dict[str, Any]) -> dict[str, Any]:
if media["type"] == "image":
media["image"] = _to_pil(media["image"])
return media
if media["type"] == "video_chunk":
media["video_chunk"] = [_to_pil(frame) for frame in media["video_chunk"]]
return media
raise ValueError(f"Unsupported media type: {media['type']}")
class KimiK25FusedVisionProcessor(BaseImageProcessor):
model_type = "kimi_k25"
def __init__(self, media_proc_cfg: dict[str, Any], **kwargs: Any) -> None:
super().__init__(**kwargs)
media_proc_cfg = dict(media_proc_cfg)
merge_kernel_size = media_proc_cfg["merge_kernel_size"]
if isinstance(merge_kernel_size, (list, tuple)):
media_proc_cfg["merge_kernel_size"] = int(merge_kernel_size[0])
self.media_proc_cfg = media_proc_cfg
self.num_frames_per_chunk = media_proc_cfg["temporal_merge_kernel_size"]
values = np.arange(256, dtype=np.float32)[:, None]
image_mean = np.asarray(media_proc_cfg["image_mean"], dtype=np.float32)
image_std_inv = 1.0 / np.asarray(media_proc_cfg["image_std"], dtype=np.float32)
self.normalize_lut = (values / 255.0 - image_mean[None, :]) * image_std_inv[
None, :
]
def media_tokens_calculator(self, media: dict[str, Any]) -> int:
media = _ensure_media_type(media)
ret = self.get_resize_config(media)
return ret["num_tokens"]
def get_resize_config(self, media_input: dict[str, Any]) -> dict[str, int]:
if media_input["type"] == "image":
width, height = media_input["image"].size
return navit_resize_image(
width,
height,
self.media_proc_cfg["patch_size"],
self.media_proc_cfg["merge_kernel_size"],
self.media_proc_cfg["in_patch_limit"],
self.media_proc_cfg["patch_limit_on_one_side"],
self.media_proc_cfg["fixed_output_tokens"],
)
if media_input["type"] == "video_chunk":
frame = media_input["video_chunk"][0]
width, height = frame.size
num_frames = len(media_input["video_chunk"])
in_patch_limit_each_frame = self.media_proc_cfg["in_patch_limit_each_frame"]
if in_patch_limit_each_frame is None:
in_patch_limit_each_frame = self.media_proc_cfg["in_patch_limit"]
return navit_resize_video(
width,
height,
num_frames,
1.0,
math.inf,
self.media_proc_cfg["patch_size"],
self.media_proc_cfg["merge_kernel_size"],
in_patch_limit_each_frame,
self.media_proc_cfg["patch_limit_on_one_side"],
self.media_proc_cfg["in_patch_limit_video"],
None,
self.media_proc_cfg["fixed_output_tokens"],
)
raise ValueError(f"Unsupported type: {media_input['type']}")
@staticmethod
def resize_image(image: Image.Image, new_width: int, new_height: int) -> np.ndarray:
image = image.resize((new_width, new_height), resample=Image.Resampling.BICUBIC)
return np.asarray(image)
def preprocess(
self,
medias: list[dict[str, Any]],
return_tensors: str | TensorType | None = None,
do_rescale: bool = True,
do_normalize: bool = True,
) -> BatchFeature:
assert do_rescale == do_normalize
device_normalize = not do_normalize
if not isinstance(medias, list):
medias = [medias]
if not medias:
return BatchFeature(data={}, tensor_type=return_tensors)
if njit is None:
raise RuntimeError("numba is required for fused Kimi image preprocessing")
patch_size = int(self.media_proc_cfg["patch_size"])
prepared = []
grid_thws_np = np.empty((len(medias), 3), dtype=np.int64)
total_patches = 0
for idx, item in enumerate(medias):
item = _ensure_media_type(item)
resize_config = self.get_resize_config(item)
new_width = resize_config["new_width"]
new_height = resize_config["new_height"]
pad_width = resize_config["pad_width"]
pad_height = resize_config["pad_height"]
padded_width = new_width + pad_width
padded_height = new_height + pad_height
if item["type"] == "image":
image_np = self.resize_image(item["image"], new_width, new_height)
frames = image_np[np.newaxis, ...]
elif item["type"] == "video_chunk":
frames = np.stack(
[
self.resize_image(frame, new_width, new_height)
for frame in item["video_chunk"]
],
axis=0,
)
else:
raise ValueError(f"Unsupported type: {item['type']}")
t_size = frames.shape[0]
grid_h = padded_height // patch_size
grid_w = padded_width // patch_size
grid_thws_np[idx, 0] = t_size
grid_thws_np[idx, 1] = grid_h
grid_thws_np[idx, 2] = grid_w
num_patches = t_size * grid_h * grid_w
prepared.append(
(
frames,
new_height,
new_width,
padded_height,
padded_width,
num_patches,
)
)
total_patches += num_patches
lut = _IDENTITY_LUT if device_normalize else self.normalize_lut
pixel_values_np = np.empty(
(total_patches, 3, patch_size, patch_size), dtype=lut.dtype
)
out_offset = 0
with numba_workqueue_threading_layer():
for (
frames,
new_height,
new_width,
padded_height,
padded_width,
num_patches,
) in prepared:
_write_fused_patches(
frames,
pixel_values_np,
out_offset,
new_height,
new_width,
padded_height,
padded_width,
patch_size,
lut,
)
out_offset += num_patches
data = {
"pixel_values": torch.from_numpy(pixel_values_np),
"grid_thws": torch.from_numpy(grid_thws_np),
}
return BatchFeature(data=data, tensor_type=return_tensors)
def __repr__(self):
return f"KimiK25FusedVisionProcessor(media_proc_cfg={self.media_proc_cfg})"
def to_dict(self) -> dict[str, Any]:
output = super().to_dict()
output["media_proc_cfg"] = self.media_proc_cfg
if "media_processor" in output:
del output["media_processor"]
return output
@classmethod
def from_dict(cls, config_dict: dict[str, Any], **kwargs):
config = config_dict.copy()
media_proc_cfg = config.pop("media_proc_cfg", {})
return cls(media_proc_cfg=media_proc_cfg, **config, **kwargs)
def to_json_string(self):
dictionary = self.to_dict()
for key, value in dictionary.items():
if hasattr(value, "tolist"):
dictionary[key] = value.tolist()
return json.dumps(dictionary, indent=2, sort_keys=True) + "\n"
@@ -1,582 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""HuggingFace processor for the MuseGlimmer multimodal model (text + image + video).
This packages MuseGlimmer's vision preprocessing and token-span expansion the standard
HF way, so you can do:
processor = AutoProcessor.from_pretrained(hf_dir, trust_remote_code=True)
messages = [{"role": "user", "content": [
{"type": "image"},
{"type": "text", "text": "Describe this image."},
]}]
text = processor.apply_chat_template(messages, add_generation_prompt=True)
inputs = processor(text=text, images=[pil_image], return_tensors="pt")
out = model.generate(**inputs, max_new_tokens=256)
It reproduces the exact token layout the model expects:
image -> <|image_start|> + <|patch|> * N + <|image_end|>
video -> <|vid_start|>
( "Time: X.Xs" + <|video|> * P [+ <|vid_frame_separator|>] )*
+ <|vid_end|>
where N = floor(H/(patch*ds)) * floor(W/(patch*ds)) (<= 4096) for images and
P = the same grid count per temporal frame-group (<= 144) for video, with the
target (H, W) chosen by MuseGlimmerImageProcessor.compute_image_size /
MuseGlimmerVideoProcessor.compute_video_frame_size (the same grid logic as
modeling_muse_glimmer.MuseGlimmerVisionEncoder).
The chat template emits one sentinel per media item -- ``<|image|>`` for images,
``<|video|>`` for videos -- which ``MuseGlimmerProcessor.__call__`` expands into the
spans above using each media item's computed token count. Vision features are
returned in ``pixel_values`` (a list of per-image / per-frame-group tensors) in
the order the sentinels appear, matching how ``MuseGlimmerModel`` merges them at the
``<|patch|>`` / ``<|video|>`` positions.
"""
from __future__ import annotations
import itertools
import math
from pathlib import Path
import torch
from PIL import Image
from torchvision import transforms as T
from transformers.feature_extraction_utils import BatchFeature
from transformers.image_processing_utils import BaseImageProcessor
from transformers.processing_utils import ProcessorMixin
from transformers.video_processing_utils import BaseVideoProcessor
# Single-token sentinels emitted by the chat template, one per media item.
IMAGE_SENTINEL = "<|image|>" # id 200090; absent from the image/video spans
VIDEO_SENTINEL = "<|video|>" # id 200091; also the per-frame token (see __call__)
# Jinja chat template (multimodal superset of convert_muse_glimmer_to_hf.CHAT_TEMPLATE).
# Message content may be a plain string OR a list of parts
# ({"type": "text"|"image"|"video", ...}); image/video parts render as a single
# sentinel that __call__ expands.
MUSE_GLIMMER_MM_CHAT_TEMPLATE = (
"{{- bos_token -}}"
"{%- macro render_parts(content) -%}"
"{%- if content is string -%}{{- content -}}"
"{%- else -%}"
"{%- for part in content -%}"
"{%- if part['type'] == 'image' -%}{{- '<|image|>' -}}"
"{%- elif part['type'] == 'video' -%}{{- '<|video|>' -}}"
"{%- elif part['type'] == 'text' -%}{{- part['text'] -}}"
"{%- endif -%}"
"{%- endfor -%}"
"{%- endif -%}"
"{%- endmacro -%}"
# At inference (add_generation_prompt), inject a default system message when
# the caller supplied none. Full-transcript rendering
# (add_generation_prompt=False) is left unchanged.
"{%- set ns = namespace(has_system=false) -%}"
"{%- for m in messages -%}"
"{%- if m['role'] == 'system' -%}{%- set ns.has_system = true -%}{%- endif -%}"
"{%- endfor -%}"
"{%- if add_generation_prompt and not ns.has_system -%}"
"{{- '<|start|>system<|message|>You are a helpful assistant.<|eot|>' -}}"
"{%- endif -%}"
"{%- for message in messages -%}"
"{%- set role = message['role'] -%}"
"{%- if role == 'assistant' -%}"
"{%- set recipient = message.get('recipient') -%}"
"{%- set end_turn = message.get('end_turn') -%}"
"{%- if end_turn is none -%}"
"{%- set end_turn = not (recipient and recipient != 'user') -%}"
"{%- endif -%}"
"{{- '<|start|>assistant' -}}"
"{%- if recipient -%}{{- ' to=' + recipient -}}{%- endif -%}"
"{{- '<|message|>' -}}{{- render_parts(message['content']) -}}"
"{{- ('<|eot|>' if end_turn else '<|eom|>') -}}"
"{%- elif role == 'tool' -%}"
"{%- set name = message.get('name', '') -%}"
# Tool content is emitted as-is (string body or interleaved image/text parts).
# Any <tool_output ...> wrapper is baked into the SFT data content; the
# tokenizer does not add it, so the template must not either.
"{{- '<|start|>tool ' + name + '<|message|>' -}}"
"{{- render_parts(message['content']) -}}"
"{{- '<|eot|>' -}}"
"{%- else -%}"
"{%- set header = role -%}"
"{%- if message.get('name') -%}"
"{%- set header = role + ' ' + message['name'] -%}"
"{%- endif -%}"
"{{- '<|start|>' + header + '<|message|>' -}}"
"{{- render_parts(message['content']) -}}"
"{{- '<|eot|>' -}}"
"{%- endif -%}"
"{%- endfor -%}"
"{%- if add_generation_prompt -%}{{- '<|start|>assistant' -}}{%- endif -%}"
)
def _grid_size(
img_w: int, img_h: int, patch_hw: int, max_tokens: int
) -> tuple[int, int, int]:
"""Pick the integer (H, W) grid closest to the aspect ratio under the token cap.
Replicates MuseGlimmerVisionEncoder._compute_grid_size
(modeling_muse_glimmer.py) so the processor needs no torch model import.
Returns (target_h, target_w, n_tokens).
"""
i_nph = img_h / patch_hw
i_npw = img_w / patch_hw
ratio = i_npw / i_nph if i_nph > 0 else 1.0
if i_nph * i_npw > max_tokens:
i_nph = (max_tokens / ratio) ** 0.5
i_npw = i_nph * ratio
candidates = list(
set(
itertools.product(
[math.floor(i_nph), math.ceil(i_nph)],
[math.floor(i_npw), math.ceil(i_npw)],
)
)
)
candidates = [
(nph, npw)
for nph, npw in candidates
if nph >= 1 and npw >= 1 and nph * npw <= max_tokens
]
if not candidates:
candidates = [(max(1, round(i_nph)), max(1, round(i_npw)))]
nph, npw = min(candidates, key=lambda c: abs(c[0] / c[1] - img_h / img_w))
return nph * patch_hw, npw * patch_hw, nph * npw
class MuseGlimmerImageProcessor(BaseImageProcessor):
"""Resize + normalize MuseGlimmer images and compute patch-token counts.
Variable-resolution: each image is resized to the grid that best matches its
aspect ratio under the per-image token cap, then normalized with mean/std 0.5.
Returns per-image tensors (not stacked) because MuseGlimmer consumes a list of
variable-size images. Video frames are handled by MuseGlimmerVideoProcessor.
"""
model_input_names = ["pixel_values"]
def __init__(
self,
patch_size: int = 14,
downsample_factor: int = 2,
max_image_tokens: int = 4096,
image_mean: float = 0.5,
image_std: float = 0.5,
**kwargs,
):
super().__init__(**kwargs)
self.patch_size = patch_size
self.downsample_factor = downsample_factor
self.max_image_tokens = max_image_tokens
self.image_mean = image_mean
self.image_std = image_std
def _to_norm_tensor(self, image: Image.Image) -> torch.Tensor:
# Functional normalize -- no stored Normalize object, so the processor
# stays JSON-serializable for to_dict() / save_pretrained.
return T.functional.normalize(
T.functional.to_tensor(image),
[self.image_mean] * 3,
[self.image_std] * 3,
)
# -- size computation (mirrors modeling_muse_glimmer.MuseGlimmerVisionEncoder) --
def compute_image_size(self, img_w: int, img_h: int) -> tuple[int, int, int]:
ph = self.patch_size * self.downsample_factor
return _grid_size(img_w, img_h, ph, self.max_image_tokens)
# -- preprocessing ---------------------------------------------------------
def preprocess_image(self, image: Image.Image) -> tuple[torch.Tensor, int]:
"""Return (pixel tensor [3, H, W], n_patch_tokens) for one image."""
image = image.convert("RGB")
target_h, target_w, n_tokens = self.compute_image_size(
image.width, image.height
)
image = image.resize((target_w, target_h), Image.LANCZOS)
return self._to_norm_tensor(image), n_tokens
# torchcodec is the training-faithful video decoder; other decoders (torchvision /
# PyAV) seek + color-convert differently and diverge from training. Optional at
# import time so text/image paths load without it; decode_video raises if used.
try:
import torchcodec
except Exception:
torchcodec = None
class MuseGlimmerVideoProcessor(BaseVideoProcessor):
"""MuseGlimmer video preprocessing behind the standard HF
``AutoVideoProcessor`` API.
Exposes MuseGlimmer's training-faithful video handling as a first-class video
processor so you can call ``processor(videos="clip.mp4")`` and downstream
users can discover it via the standard HF ``AutoVideoProcessor`` API plus a
``video_preprocessor_config.json``. It wraps:
* torchcodec decode (matches training; other decoders diverge),
* uniform frame sampling to a whole multiple of ``patch_temporal``,
* real per-group PTS timestamps (rendered as ``Time: X.Xs`` by
MuseGlimmerProcessor),
* ``patch_temporal`` frame-grouping (frames cat on the channel axis ->
``[patch_temporal * 3, H, W]``; the encoder detects video by channel count).
It deliberately overrides ``preprocess`` instead of the BaseVideoProcessor fast
pipeline (group_videos_by_shape / smart_resize / single stacked tensor): MuseGlimmer
consumes a LIST of variable-size group tensors and needs the real per-group PTS,
neither of which the stacked fast path models.
"""
model_input_names = ["pixel_values"]
def __init__(
self,
patch_size: int = 14,
downsample_factor: int = 2,
patch_temporal: int = 2,
max_video_frame_tokens: int = 144,
image_mean: float = 0.5,
image_std: float = 0.5,
video_num_frames: int = 96,
video_sampling_fps: float = 2.0,
**kwargs,
):
super().__init__(**kwargs)
self.patch_size = patch_size
self.downsample_factor = downsample_factor
self.patch_temporal = patch_temporal
self.max_video_frame_tokens = max_video_frame_tokens
self.image_mean = image_mean
self.image_std = image_std
self.video_num_frames = video_num_frames
self.video_sampling_fps = video_sampling_fps
def _to_norm_tensor(self, image: Image.Image) -> torch.Tensor:
return T.functional.normalize(
T.functional.to_tensor(image),
[self.image_mean] * 3,
[self.image_std] * 3,
)
def decode_video(self, video_path: str) -> tuple[list[Image.Image], list[float]]:
"""Sample frames + per-group PTS with torchcodec (the training decode path).
``timestamps[g]`` is the ACTUAL decoded PTS of the first frame in temporal
group ``g``; ``len(frames)`` is a whole multiple of ``patch_temporal``.
"""
if torchcodec is None:
raise RuntimeError(
"torchcodec is required for video decoding (it matches the training "
"decode path). See the Environment Setup steps in hf/README.md."
)
pt = self.patch_temporal
reader = torchcodec.decoders.VideoDecoder(video_path)
total = len(reader)
assert reader.metadata.average_fps is not None, (
f"Video has no FPS metadata: {video_path}"
)
fps = reader.metadata.average_fps
assert self.video_sampling_fps and self.video_sampling_fps > 0, (
f"video_sampling_fps must be positive, got {self.video_sampling_fps}"
)
n = min(
int(total * self.video_sampling_fps / fps), self.video_num_frames, total
)
n = max(pt, (n // pt) * pt)
n = min(n, total)
if n < pt:
raise ValueError(
f"Video has only {total} decodable frame(s) but needs at least "
f"{pt} (one temporal patch): {video_path}"
)
indices = torch.linspace(0, total - 1, n).long().tolist()
frames: list[Image.Image] = []
timestamps: list[float] = []
for j, i in enumerate(indices):
fr = reader[i]
frames.append(
Image.fromarray(fr.data.permute(1, 2, 0).numpy()).convert("RGB")
)
if j % pt == 0:
pts = getattr(fr, "pts_seconds", None)
timestamps.append(float(pts) if pts is not None else i / fps)
return frames, timestamps
def compute_video_frame_size(self, img_w: int, img_h: int) -> tuple[int, int, int]:
"""Pick (H, W, tokens_per_frame) for a video frame under the per-frame cap.
Mirrors MuseGlimmerImageProcessor.compute_image_size but uses the (smaller)
per-frame video token budget; shares the same ``_grid_size`` grid logic as
modeling_muse_glimmer.MuseGlimmerVisionEncoder.
"""
ph = self.patch_size * self.downsample_factor
return _grid_size(img_w, img_h, ph, self.max_video_frame_tokens)
def _group_frames(
self, frames: list[Image.Image]
) -> tuple[list[torch.Tensor], int, int]:
"""Resize/normalize frames and cat ``patch_temporal`` frames per group.
Returns (group_tensors, n_groups, tokens_per_group).
"""
pt = self.patch_temporal
if pt <= 0:
raise ValueError(f"patch_temporal must be positive, got {pt}")
if not frames:
raise ValueError("video must contain at least one frame")
if padding := (-len(frames)) % pt:
frames = [*frames, *([frames[-1]] * padding)]
first = frames[0].convert("RGB")
target_h, target_w, n_tokens = self.compute_video_frame_size(
first.width, first.height
)
groups: list[torch.Tensor] = []
for i in range(0, len(frames), pt):
grp = [
self._to_norm_tensor(
frames[i + j]
.convert("RGB")
.resize((target_w, target_h), Image.LANCZOS)
)
for j in range(pt)
]
groups.append(torch.cat(grp, dim=0))
return groups, len(groups), n_tokens
@staticmethod
def _normalize_videos(videos) -> list:
"""Normalize the ``videos`` arg to a list of per-video items.
A video item is a path (str/Path) or a list of PIL frames. Accepts a
single video (one path, or one list of frames) or a list of those.
"""
if isinstance(videos, (str, Path)):
return [videos]
if videos and not isinstance(videos[0], (list, tuple, str, Path)):
return [videos] # a single list of PIL frames
return list(videos or [])
def preprocess_one(
self,
video: str | Path | list[Image.Image],
timestamps: list[float] | None = None,
) -> tuple[list[torch.Tensor], int, int, list[float]]:
"""Decode (if a path) + group ONE video.
Returns (group_tensors, n_groups, tokens_per_group, group_timestamps). This
is the single per-video implementation shared by ``preprocess`` (the HF
AutoVideoProcessor batch API) and ``MuseGlimmerProcessor.__call__`` (prompt
building), so the two paths cannot drift. For a path, frames + real
per-group PTS come from ``decode_video``; for pre-decoded frames the given
``timestamps`` are used verbatim (empty if not supplied).
"""
if isinstance(video, (str, Path)):
frames, ts = self.decode_video(str(video))
else:
frames = [f.convert("RGB") for f in video]
ts = list(timestamps) if timestamps is not None else []
groups, n_groups, tokens_per_group = self._group_frames(frames)
return groups, n_groups, tokens_per_group, ts
def preprocess(
self,
videos,
video_timestamps: list[list[float]] | None = None,
return_tensors: str | None = "pt",
**kwargs,
) -> BatchFeature:
"""Preprocess video file path(s) or pre-decoded frame list(s).
Accepts a single video (path str, or list of PIL frames) or a list of
those. Returns a BatchFeature carrying, in video order:
* ``pixel_values`` -- flat list of [pt*3, H, W] group tensors,
* ``video_num_groups`` -- groups per video,
* ``video_tokens_per_group`` -- patch-token count per group per video,
* ``video_timestamps`` -- per-group PTS per video,
i.e. exactly what MuseGlimmerProcessor needs to build the
``<|vid_start|> ( Time: X.Xs <|video|>*P [sep] )* <|vid_end|>`` block.
"""
videos = self._normalize_videos(videos)
pixel_values: list[torch.Tensor] = []
num_groups: list[int] = []
tokens_per_group: list[int] = []
out_ts: list[list[float]] = []
for idx, v in enumerate(videos):
ts_in = video_timestamps[idx] if video_timestamps else None
groups, ng, tpg, ts = self.preprocess_one(v, ts_in)
pixel_values += groups
num_groups.append(ng)
tokens_per_group.append(tpg)
out_ts.append(ts)
batch = BatchFeature(
data={
"video_num_groups": num_groups,
"video_tokens_per_group": tokens_per_group,
"video_timestamps": out_ts,
},
tensor_type=None,
)
# Variable-size list -- BatchFeature would fail to stack into one tensor.
batch["pixel_values"] = pixel_values
return batch
class MuseGlimmerProcessor(ProcessorMixin):
"""Bundle MuseGlimmerImageProcessor + MuseGlimmerVideoProcessor + tokenizer;
expand media sentinels into spans.
Images go through ``image_processor`` (MuseGlimmerImageProcessor) and videos through
``video_processor`` (MuseGlimmerVideoProcessor) -- one preprocessing implementation
each, no overlap.
"""
attributes = ["image_processor", "video_processor", "tokenizer"]
image_processor_class = "AutoImageProcessor"
video_processor_class = "AutoVideoProcessor"
tokenizer_class = "TokenizersBackend"
def __init__(
self,
image_processor=None,
video_processor=None,
tokenizer=None,
chat_template=None,
**kwargs,
):
if image_processor is None:
image_processor = MuseGlimmerImageProcessor()
if video_processor is None:
video_processor = MuseGlimmerVideoProcessor()
super().__init__(
image_processor,
video_processor,
tokenizer,
chat_template=chat_template or MUSE_GLIMMER_MM_CHAT_TEMPLATE,
**kwargs,
)
def _sid(self, token: str) -> int:
return self.tokenizer.convert_tokens_to_ids(token)
def _image_block(self, n_tokens: int) -> list[int]:
return (
[self._sid("<|image_start|>")]
+ [self._sid("<|patch|>")] * n_tokens
+ [self._sid("<|image_end|>")]
)
def _video_block(
self,
n_groups: int,
tokens_per_group: int,
timestamps: list[float] | None = None,
) -> list[int]:
"""Per-group ``Time: X.Xs`` + <|video|>*P, separated/terminated.
``timestamps`` (one per temporal group, in seconds) should be the ACTUAL
decoded frame times -- training renders the real factored frame PTS, not a
uniform grid. Falls back to ``g*patch_temporal/fps`` only when timestamps
are not supplied (e.g. callers that don't track frame times); that
approximation can differ from training by the per-frame jitter.
"""
vid = self._sid("<|video|>")
sep = self._sid("<|vid_frame_separator|>")
pt = self.video_processor.patch_temporal
fps = self.video_processor.video_sampling_fps
block = [self._sid("<|vid_start|>")]
for g in range(n_groups):
ts = timestamps[g] if timestamps is not None else g * pt / fps
block += self.tokenizer.encode(f"Time: {ts:.1f}s", add_special_tokens=False)
block += [vid] * tokens_per_group
block.append(sep if g < n_groups - 1 else self._sid("<|vid_end|>"))
return block
def __call__(
self,
text: str | list[str] | None = None,
images: list[Image.Image] | None = None,
videos: list[list[Image.Image] | str] | None = None,
video_timestamps: list[list[float]] | None = None,
return_tensors: str | None = "pt",
**kwargs,
) -> BatchFeature:
if text is None:
raise ValueError(
"`text` is required (use apply_chat_template to build it)."
)
if isinstance(text, (list, tuple)):
if len(text) != 1:
raise ValueError(
"MuseGlimmerProcessor supports a single text sample per call."
)
text = text[0]
images = list(images or [])
videos = list(videos or [])
image_sentinel = self._sid(IMAGE_SENTINEL)
video_sentinel = self._sid(VIDEO_SENTINEL)
# Preprocess media up front so we can expand sentinels in document order.
# Images via image_processor; videos via video_processor (each video item
# is a path -- decoded here -- or a pre-decoded list of PIL frames).
prepped_images = [self.image_processor.preprocess_image(im) for im in images]
prepped_videos = [
self.video_processor.preprocess_one(
v, video_timestamps[i] if video_timestamps else None
)
for i, v in enumerate(videos)
]
ids = self.tokenizer.encode(text, add_special_tokens=False)
n_img = sum(1 for t in ids if t == image_sentinel)
n_vid = sum(1 for t in ids if t == video_sentinel)
if n_img != len(prepped_images):
raise ValueError(
f"{n_img} image sentinel(s) in text but "
f"{len(prepped_images)} image(s) given."
)
if n_vid != len(prepped_videos):
raise ValueError(
f"{n_vid} video sentinel(s) in text but "
f"{len(prepped_videos)} video(s) given."
)
out_ids: list[int] = []
pixel_values: list[torch.Tensor] = []
img_i = vid_i = 0
for tid in ids:
if tid == image_sentinel:
tensor, n_tokens = prepped_images[img_i]
img_i += 1
out_ids += self._image_block(n_tokens)
pixel_values.append(tensor)
elif tid == video_sentinel:
groups, n_groups, tokens_per_group, ts = prepped_videos[vid_i]
vid_i += 1
out_ids += self._video_block(n_groups, tokens_per_group, ts or None)
pixel_values += groups
else:
out_ids.append(tid)
data: dict = {
"input_ids": [out_ids],
"attention_mask": [[1] * len(out_ids)],
}
batch = BatchFeature(data=data, tensor_type=return_tensors)
# Keep pixel_values as a list of variable-size tensors (MuseGlimmerModel
# consumes a list); BatchFeature would fail to stack them into one tensor.
if pixel_values:
batch["pixel_values"] = pixel_values
return batch
__all__ = [
"MuseGlimmerImageProcessor",
"MuseGlimmerVideoProcessor",
"MuseGlimmerProcessor",
"MUSE_GLIMMER_MM_CHAT_TEMPLATE",
]
@@ -1,46 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Image processor for Unlimited-OCR (baidu/Unlimited-OCR)."""
from PIL import Image
from vllm.logger import init_logger
from vllm.transformers_utils.processors.deepseek_ocr import DeepseekOCRProcessor
logger = init_logger(__name__)
class UnlimitedOCRProcessor(DeepseekOCRProcessor):
"""DeepseekOCRProcessor variant for Unlimited-OCR.
The only behavioural difference from the base processor is a multi-image
safeguard: when more than one image is present, crop ("gundam") mode is
disabled.
Because the effective crop flag then depends on *how many* images are in the
request, the per-item processing output is no longer invariant of sibling
images. ``UnlimitedOCRMultiModalProcessor`` accounts for this by bypassing
the multimodal processing cache for multi-image requests (see its
``_cached_apply_hf_processor``), so the two paths stay consistent.
DeepSeek-OCR does *not* have this restriction because its ``max_crops=6`` is
small enough to be safe for multi-image use.
"""
def tokenize_with_images(
self,
conversation: str,
images: list[Image.Image],
bos: bool = True,
eos: bool = True,
cropping: bool = True,
):
if len(images) > 1 and cropping:
logger.warning_once(
"Unlimited-OCR: crop mode is not supported for multi-image "
"input. Falling back to cropping=False."
)
cropping = False
return super().tokenize_with_images(
conversation, images, bos=bos, eos=eos, cropping=cropping
)