[Mini-VLLM] Remove orphan subsystems: snapshot, CPU workers, dead decoders
- Delete the snapshot package, CLI command, and its launch tests - Delete CPU workers, cpu_communicator, and the hw_agnostic layer package - Delete the EC CPU connector, keeping the EC base glue intact - Delete dead composite/triton_flashinfer backends and third_party/flashmla - Delete dead video decoders, video-bound tests, and orphan utils - Remove the deep_gemm_mega_moe option and dead allreduce fusion pair Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,459 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# adapted from https://github.com/deepseek-ai/DeepSeek-OCR/blob/main/DeepSeek-OCR-master/DeepSeek-OCR-vllm/process/image_process.py
|
||||
# and https://github.com/deepseek-ai/DeepSeek-OCR-2/blob/main/DeepSeek-OCR2-master/DeepSeek-OCR2-vllm/process/image_process.py
|
||||
import math
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
import torchvision.transforms as T
|
||||
from PIL import Image, ImageOps
|
||||
from transformers import BatchFeature, LlamaTokenizerFast
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
# TODO(Isotr0py): change modes for variants
|
||||
# see: https://github.com/deepseek-ai/DeepSeek-OCR/blob/8cf003d38821fa1b19c73da3bd1b0dc262ea8136/DeepSeek-OCR-master/DeepSeek-OCR-vllm/config.py#L1-L6
|
||||
# Tiny: base_size = 512, image_size = 512, crop_mode = False
|
||||
# Small: base_size = 640, image_size = 640, crop_mode = False
|
||||
# Base: base_size = 1024, image_size = 1024, crop_mode = False
|
||||
# Large: base_size = 1280, image_size = 1280, crop_mode = False
|
||||
# Gundam: base_size = 1024, image_size = 640, crop_mode = True
|
||||
BASE_SIZE = 1024
|
||||
IMAGE_SIZE = 640
|
||||
CROP_MODE = True
|
||||
|
||||
# TODO(Isotr0py): Expose as mm_kwargs
|
||||
MIN_CROPS = 2
|
||||
MAX_CROPS = 6 # max:9; If your GPU memory is small, it is recommended to set it to 6.
|
||||
|
||||
|
||||
def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
|
||||
best_ratio_diff = float("inf")
|
||||
best_ratio = (1, 1)
|
||||
area = width * height
|
||||
for ratio in target_ratios:
|
||||
target_aspect_ratio = ratio[0] / ratio[1]
|
||||
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
|
||||
if ratio_diff < best_ratio_diff:
|
||||
best_ratio_diff = ratio_diff
|
||||
best_ratio = ratio
|
||||
elif ratio_diff == best_ratio_diff:
|
||||
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
|
||||
best_ratio = ratio
|
||||
return best_ratio
|
||||
|
||||
|
||||
def calculate_aspect_ratios(
|
||||
min_num: int = MIN_CROPS, max_num: int = MAX_CROPS
|
||||
) -> list[tuple[int, int]]:
|
||||
target_ratios: set[tuple[int, int]] = set(
|
||||
(i, j)
|
||||
for n in range(min_num, max_num + 1)
|
||||
for i in range(1, n + 1)
|
||||
for j in range(1, n + 1)
|
||||
if i * j <= max_num and i * j >= min_num
|
||||
)
|
||||
sorted_target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
|
||||
return sorted_target_ratios
|
||||
|
||||
|
||||
def count_tiles(
|
||||
orig_width,
|
||||
orig_height,
|
||||
min_num=MIN_CROPS,
|
||||
max_num=MAX_CROPS,
|
||||
image_size=640,
|
||||
use_thumbnail=False,
|
||||
):
|
||||
aspect_ratio = orig_width / orig_height
|
||||
|
||||
# calculate the existing image aspect ratio
|
||||
target_ratios = calculate_aspect_ratios(min_num, max_num)
|
||||
|
||||
# find the closest aspect ratio to the target
|
||||
target_aspect_ratio = find_closest_aspect_ratio(
|
||||
aspect_ratio, target_ratios, orig_width, orig_height, image_size
|
||||
)
|
||||
|
||||
return target_aspect_ratio
|
||||
|
||||
|
||||
def dynamic_preprocess(
|
||||
image, min_num=MIN_CROPS, max_num=MAX_CROPS, image_size=640, use_thumbnail=False
|
||||
):
|
||||
orig_width, orig_height = image.size
|
||||
aspect_ratio = orig_width / orig_height
|
||||
|
||||
# calculate the existing image aspect ratio
|
||||
target_ratios = calculate_aspect_ratios(min_num, max_num)
|
||||
|
||||
# find the closest aspect ratio to the target
|
||||
target_aspect_ratio = find_closest_aspect_ratio(
|
||||
aspect_ratio, target_ratios, orig_width, orig_height, image_size
|
||||
)
|
||||
|
||||
# calculate the target width and height
|
||||
target_width = image_size * target_aspect_ratio[0]
|
||||
target_height = image_size * target_aspect_ratio[1]
|
||||
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
||||
|
||||
# resize the image
|
||||
resized_img = image.resize((target_width, target_height))
|
||||
processed_images = []
|
||||
for i in range(blocks):
|
||||
box = (
|
||||
(i % (target_width // image_size)) * image_size,
|
||||
(i // (target_width // image_size)) * image_size,
|
||||
((i % (target_width // image_size)) + 1) * image_size,
|
||||
((i // (target_width // image_size)) + 1) * image_size,
|
||||
)
|
||||
# split the image
|
||||
split_img = resized_img.crop(box)
|
||||
processed_images.append(split_img)
|
||||
assert len(processed_images) == blocks
|
||||
if use_thumbnail and len(processed_images) != 1:
|
||||
thumbnail_img = image.resize((image_size, image_size))
|
||||
processed_images.append(thumbnail_img)
|
||||
return processed_images, target_aspect_ratio
|
||||
|
||||
|
||||
class ImageTransform:
|
||||
def __init__(
|
||||
self,
|
||||
mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
|
||||
std: tuple[float, float, float] = (0.5, 0.5, 0.5),
|
||||
normalize: bool = True,
|
||||
):
|
||||
self.mean = mean
|
||||
self.std = std
|
||||
self.normalize = normalize
|
||||
|
||||
transform_pipelines = [T.ToTensor()]
|
||||
|
||||
if normalize:
|
||||
transform_pipelines.append(T.Normalize(mean, std))
|
||||
|
||||
self.transform = T.Compose(transform_pipelines)
|
||||
|
||||
def __call__(self, pil_img: Image.Image):
|
||||
x = self.transform(pil_img)
|
||||
return x
|
||||
|
||||
|
||||
class DeepseekOCRProcessor(ProcessorMixin):
|
||||
tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast")
|
||||
attributes = ["tokenizer"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: LlamaTokenizerFast,
|
||||
patch_size: int = 16,
|
||||
downsample_ratio: int = 4,
|
||||
image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
|
||||
image_std: tuple[float, float, float] = (0.5, 0.5, 0.5),
|
||||
normalize: bool = True,
|
||||
image_token: str = "<image>",
|
||||
pad_token: str = "<|▁pad▁|>",
|
||||
add_special_token: bool = False,
|
||||
sft_format: str = "deepseek",
|
||||
mask_prompt: bool = True,
|
||||
ignore_id: int = -100,
|
||||
image_size: int = IMAGE_SIZE,
|
||||
base_size: int = BASE_SIZE,
|
||||
strategy: Literal["v1", "v2"] = "v1",
|
||||
max_crops: int = MAX_CROPS,
|
||||
**kwargs,
|
||||
):
|
||||
self.image_size = image_size
|
||||
self.base_size = base_size
|
||||
self.max_crops = max_crops
|
||||
|
||||
# image token calculation strategy for
|
||||
# Deepseek-OCR and Deepseek-OCR-2
|
||||
self.strategy = strategy
|
||||
assert strategy in ["v1", "v2"], "Only 'v1' and 'v2' strategies are supported."
|
||||
|
||||
self.patch_size = 16
|
||||
self.image_mean = image_mean
|
||||
self.image_std = image_std
|
||||
self.normalize = normalize
|
||||
self.downsample_ratio = 4
|
||||
|
||||
self.image_transform = ImageTransform(
|
||||
mean=image_mean, std=image_std, normalize=normalize
|
||||
)
|
||||
|
||||
self.tokenizer = tokenizer
|
||||
self.tokenizer.padding_side = "left" # must set this,padding side with make a difference in batch inference # noqa: E501
|
||||
|
||||
# add the pad_token as special token to use 'tokenizer.pad_token'
|
||||
# and 'tokenizer.pad_token_id'
|
||||
if self.tokenizer.pad_token is None:
|
||||
self.tokenizer.add_special_tokens({"pad_token": pad_token})
|
||||
|
||||
# add image token
|
||||
self.image_token_id = self.tokenizer.vocab.get(image_token)
|
||||
self.image_token = image_token
|
||||
self.pad_token = pad_token
|
||||
self.add_special_token = add_special_token
|
||||
self.sft_format = sft_format
|
||||
self.mask_prompt = mask_prompt
|
||||
self.ignore_id = ignore_id
|
||||
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def bos_id(self):
|
||||
return self.tokenizer.bos_token_id
|
||||
|
||||
@property
|
||||
def eos_id(self):
|
||||
return self.tokenizer.eos_token_id
|
||||
|
||||
@property
|
||||
def pad_id(self):
|
||||
return self.tokenizer.pad_token_id
|
||||
|
||||
def encode(self, text: str, bos: bool = True, eos: bool = False):
|
||||
t = self.tokenizer.encode(text, add_special_tokens=False)
|
||||
if bos:
|
||||
t = [self.bos_id] + t
|
||||
if eos:
|
||||
t = t + [self.eos_id]
|
||||
return t
|
||||
|
||||
def decode(self, t: list[int], **kwargs) -> str:
|
||||
return self.tokenizer.decode(t, **kwargs)
|
||||
|
||||
def process_one(
|
||||
self,
|
||||
prompt: str,
|
||||
images: list[Image.Image],
|
||||
crop_mode: bool = CROP_MODE,
|
||||
):
|
||||
"""Args:
|
||||
prompt (str): the formatted prompt;
|
||||
images (List[ImageType]): the list of images;
|
||||
crop_mode (bool): if True, then crop the image;
|
||||
|
||||
Returns:
|
||||
outputs (BaseProcessorOutput): the output of the processor,
|
||||
- input_ids (torch.LongTensor): [N + image tokens]
|
||||
- target_ids (torch.LongTensor): [N + image tokens]
|
||||
- pixel_values (torch.FloatTensor): [n_patches, 3, H, W]
|
||||
- image_id (int): the id of the image token
|
||||
- num_image_tokens (List[int]): the number of image tokens
|
||||
|
||||
"""
|
||||
if prompt is None or images is None:
|
||||
raise ValueError("prompt and images must be used at the same time.")
|
||||
|
||||
sft_format = prompt
|
||||
|
||||
(
|
||||
input_ids,
|
||||
pixel_values,
|
||||
images_crop,
|
||||
images_seq_mask,
|
||||
images_spatial_crop,
|
||||
num_image_tokens,
|
||||
_,
|
||||
) = self.tokenize_with_images(
|
||||
conversation=sft_format,
|
||||
images=images,
|
||||
bos=True,
|
||||
eos=True,
|
||||
cropping=crop_mode,
|
||||
)
|
||||
|
||||
prepare = BatchFeature(
|
||||
data=dict(
|
||||
input_ids=input_ids,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_seq_mask=images_seq_mask,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
num_image_tokens=num_image_tokens,
|
||||
),
|
||||
tensor_type="pt",
|
||||
)
|
||||
return prepare
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
prompt: str,
|
||||
images: list[Image.Image],
|
||||
crop_mode: bool = CROP_MODE,
|
||||
**kwargs,
|
||||
):
|
||||
prepare = self.process_one(
|
||||
prompt=prompt,
|
||||
images=images,
|
||||
crop_mode=crop_mode,
|
||||
)
|
||||
|
||||
return prepare
|
||||
|
||||
def tokenize_with_images(
|
||||
self,
|
||||
conversation: str,
|
||||
images: list[Image.Image],
|
||||
bos: bool = True,
|
||||
eos: bool = True,
|
||||
cropping: bool = True,
|
||||
):
|
||||
"""Tokenize text with <image> tags."""
|
||||
num_image_tags = conversation.count(self.image_token)
|
||||
if num_image_tags != len(images):
|
||||
raise ValueError(
|
||||
f"Number of {self.image_token!r} tokens in prompt "
|
||||
f"({num_image_tags}) does not match number of images "
|
||||
f"({len(images)})."
|
||||
)
|
||||
text_splits = conversation.split(self.image_token)
|
||||
images_list, images_crop_list, images_seq_mask, images_spatial_crop = (
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
)
|
||||
image_shapes = []
|
||||
num_image_tokens = []
|
||||
tokenized_str = []
|
||||
for text_sep, image in zip(text_splits, images):
|
||||
tokenized_sep = self.encode(text_sep, bos=False, eos=False)
|
||||
tokenized_str += tokenized_sep
|
||||
images_seq_mask += [False] * len(tokenized_sep)
|
||||
|
||||
image_shapes.append(image.size)
|
||||
|
||||
images_crop_raw = []
|
||||
if image.size[0] <= self.image_size and image.size[1] <= self.image_size:
|
||||
crop_ratio = [1, 1]
|
||||
elif cropping:
|
||||
images_crop_raw, crop_ratio = dynamic_preprocess(
|
||||
image, image_size=self.image_size, max_num=self.max_crops
|
||||
)
|
||||
else:
|
||||
crop_ratio = [1, 1]
|
||||
|
||||
if not cropping:
|
||||
image = image.resize((self.image_size, self.image_size))
|
||||
|
||||
global_view = ImageOps.pad(
|
||||
image,
|
||||
(self.base_size, self.base_size),
|
||||
color=tuple(int(x * 255) for x in self.image_transform.mean),
|
||||
)
|
||||
images_list.append(self.image_transform(global_view))
|
||||
|
||||
num_width_tiles, num_height_tiles = crop_ratio
|
||||
images_spatial_crop.append([num_width_tiles, num_height_tiles])
|
||||
|
||||
if num_width_tiles > 1 or num_height_tiles > 1:
|
||||
for cropped_image in images_crop_raw:
|
||||
images_crop_list.append(self.image_transform(cropped_image))
|
||||
|
||||
num_queries = math.ceil(
|
||||
(self.image_size // self.patch_size) / self.downsample_ratio
|
||||
)
|
||||
num_queries_base = math.ceil(
|
||||
(self.base_size // self.patch_size) / self.downsample_ratio
|
||||
)
|
||||
|
||||
num_tokens_base = (
|
||||
(num_queries_base * (num_queries_base + 1))
|
||||
if self.strategy == "v1"
|
||||
else num_queries_base * num_queries_base
|
||||
)
|
||||
tokenized_image = [self.image_token_id] * num_tokens_base
|
||||
|
||||
tokenized_image += [self.image_token_id]
|
||||
if num_width_tiles > 1 or num_height_tiles > 1:
|
||||
num_tokens_per_row = (
|
||||
num_queries * num_width_tiles + 1
|
||||
if self.strategy == "v1"
|
||||
else num_queries * num_width_tiles
|
||||
)
|
||||
local_row = [self.image_token_id] * num_tokens_per_row
|
||||
tokenized_image += local_row * (num_queries * num_height_tiles)
|
||||
tokenized_str += tokenized_image
|
||||
images_seq_mask += [True] * len(tokenized_image)
|
||||
num_image_tokens.append(len(tokenized_image))
|
||||
|
||||
"""process the last text split"""
|
||||
tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False)
|
||||
tokenized_str += tokenized_sep
|
||||
images_seq_mask += [False] * len(tokenized_sep)
|
||||
|
||||
"""add the bos and eos tokens"""
|
||||
if bos:
|
||||
tokenized_str = [self.bos_id] + tokenized_str
|
||||
images_seq_mask = [False] + images_seq_mask
|
||||
if eos:
|
||||
tokenized_str = tokenized_str + [self.eos_id]
|
||||
images_seq_mask = images_seq_mask + [False]
|
||||
|
||||
assert len(tokenized_str) == len(images_seq_mask), (
|
||||
f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} "
|
||||
f"is not equal to images_seq_mask's length {len(images_seq_mask)}."
|
||||
)
|
||||
|
||||
masked_tokenized_str = []
|
||||
for token_index in tokenized_str:
|
||||
if token_index != self.image_token_id:
|
||||
masked_tokenized_str.append(token_index)
|
||||
else:
|
||||
masked_tokenized_str.append(self.ignore_id)
|
||||
|
||||
assert (
|
||||
len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str)
|
||||
), (
|
||||
f"tokenized_str's length {len(tokenized_str)}, "
|
||||
f"input_ids' length {len(masked_tokenized_str)}, "
|
||||
f"images_seq_mask's length {len(images_seq_mask)}, are not equal."
|
||||
)
|
||||
|
||||
input_ids = torch.LongTensor(tokenized_str)
|
||||
target_ids = torch.LongTensor(masked_tokenized_str)
|
||||
images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)
|
||||
|
||||
# set input_ids < 0 | input_ids == self.image_token_id as ignore_id
|
||||
target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = (
|
||||
self.ignore_id
|
||||
)
|
||||
input_ids[input_ids < 0] = self.pad_id
|
||||
|
||||
# Remove the ending eos token
|
||||
assert input_ids[-1] == self.eos_id
|
||||
input_ids = input_ids[:-1]
|
||||
target_ids = target_ids[:-1]
|
||||
images_seq_mask = images_seq_mask[:-1]
|
||||
|
||||
if len(images_list) == 0:
|
||||
pixel_values = torch.zeros((0, 3, self.base_size, self.base_size))
|
||||
images_spatial_crop = torch.zeros((0, 2), dtype=torch.long)
|
||||
images_crop = torch.zeros((0, 3, self.image_size, self.image_size))
|
||||
else:
|
||||
pixel_values = torch.stack(images_list, dim=0)
|
||||
images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)
|
||||
if images_crop_list:
|
||||
images_crop = torch.stack(images_crop_list, dim=0)
|
||||
else:
|
||||
images_crop = torch.zeros((0, 3, self.image_size, self.image_size))
|
||||
|
||||
input_ids = input_ids.unsqueeze(0)
|
||||
|
||||
return (
|
||||
input_ids,
|
||||
pixel_values,
|
||||
images_crop,
|
||||
images_seq_mask,
|
||||
images_spatial_crop,
|
||||
num_image_tokens,
|
||||
image_shapes,
|
||||
)
|
||||
@@ -1,585 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_internvl_chat.py
|
||||
# --------------------------------------------------------
|
||||
# InternVL
|
||||
# Copyright (c) 2023 OpenGVLab
|
||||
# Licensed under The MIT License [see LICENSE for details]
|
||||
# --------------------------------------------------------
|
||||
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
import torchvision.transforms as T
|
||||
from PIL import Image
|
||||
from transformers import (
|
||||
BaseVideoProcessor,
|
||||
BatchFeature,
|
||||
ImageProcessingMixin,
|
||||
TensorType,
|
||||
)
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.processing import PromptUpdateDetails, cached_encode
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
IMAGENET_MEAN = (0.485, 0.456, 0.406)
|
||||
IMAGENET_STD = (0.229, 0.224, 0.225)
|
||||
|
||||
|
||||
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
|
||||
def build_transform(input_size: int):
|
||||
MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
|
||||
return T.Compose(
|
||||
[
|
||||
T.Lambda(lambda img: convert_image_mode(img, "RGB")),
|
||||
T.Resize(
|
||||
(input_size, input_size), interpolation=T.InterpolationMode.BICUBIC
|
||||
),
|
||||
T.ToTensor(),
|
||||
T.Normalize(mean=MEAN, std=STD),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
|
||||
def find_closest_aspect_ratio(
|
||||
aspect_ratio: float,
|
||||
target_ratios: list[tuple[int, int]],
|
||||
*,
|
||||
width: int,
|
||||
height: int,
|
||||
image_size: int,
|
||||
) -> tuple[int, int]:
|
||||
best_ratio_diff = float("inf")
|
||||
best_ratio = (1, 1)
|
||||
area = width * height
|
||||
for ratio in target_ratios:
|
||||
target_aspect_ratio = ratio[0] / ratio[1]
|
||||
ratio_diff = abs(aspect_ratio - target_aspect_ratio)
|
||||
if ratio_diff < best_ratio_diff:
|
||||
best_ratio_diff = ratio_diff
|
||||
best_ratio = ratio
|
||||
elif ratio_diff == best_ratio_diff:
|
||||
if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
|
||||
best_ratio = ratio
|
||||
return best_ratio
|
||||
|
||||
|
||||
def resolve_internvl_min_max_num(
|
||||
*,
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool,
|
||||
use_thumbnail: bool,
|
||||
) -> tuple[int, int]:
|
||||
min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
|
||||
max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
|
||||
|
||||
if use_thumbnail and max_dynamic_patch != 1:
|
||||
max_dynamic_patch += 1
|
||||
|
||||
return min_dynamic_patch, max_dynamic_patch
|
||||
|
||||
|
||||
def get_internvl_target_ratios(
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
) -> list[tuple[int, int]]:
|
||||
target_ratios = {
|
||||
(i, j)
|
||||
for n in range(min_num, max_num + 1)
|
||||
for i in range(1, n + 1)
|
||||
for j in range(1, n + 1)
|
||||
if min_num <= i * j <= max_num
|
||||
}
|
||||
return sorted(target_ratios, key=lambda x: x[0] * x[1])
|
||||
|
||||
|
||||
def calculate_internvl_targets(
|
||||
*,
|
||||
orig_width: int,
|
||||
orig_height: int,
|
||||
target_ratios: list[tuple[int, int]],
|
||||
image_size: int,
|
||||
use_thumbnail: bool,
|
||||
) -> tuple[int, int, int]:
|
||||
aspect_ratio = orig_width / orig_height
|
||||
|
||||
# find the closest aspect ratio to the target
|
||||
target_aspect_ratio = find_closest_aspect_ratio(
|
||||
aspect_ratio,
|
||||
target_ratios,
|
||||
width=orig_width,
|
||||
height=orig_height,
|
||||
image_size=image_size,
|
||||
)
|
||||
|
||||
# calculate the target width and height
|
||||
target_width = image_size * target_aspect_ratio[0]
|
||||
target_height = image_size * target_aspect_ratio[1]
|
||||
blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
|
||||
|
||||
# add thumbnail image if num_blocks != 1
|
||||
if use_thumbnail and blocks != 1:
|
||||
blocks += 1
|
||||
|
||||
return blocks, target_width, target_height
|
||||
|
||||
|
||||
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
|
||||
def dynamic_preprocess_internvl(
|
||||
image: Image.Image,
|
||||
*,
|
||||
target_ratios: list[tuple[int, int]],
|
||||
image_size: int,
|
||||
use_thumbnail: bool,
|
||||
) -> list[Image.Image]:
|
||||
orig_width, orig_height = image.size
|
||||
|
||||
# calculate the number of blocks without thumbnail
|
||||
blocks, target_width, target_height = calculate_internvl_targets(
|
||||
orig_width=orig_width,
|
||||
orig_height=orig_height,
|
||||
target_ratios=target_ratios,
|
||||
image_size=image_size,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
|
||||
# resize the image
|
||||
resized_img = image.resize((target_width, target_height))
|
||||
processed_images = []
|
||||
for i in range(blocks):
|
||||
box = (
|
||||
(i % (target_width // image_size)) * image_size,
|
||||
(i // (target_width // image_size)) * image_size,
|
||||
((i % (target_width // image_size)) + 1) * image_size,
|
||||
((i // (target_width // image_size)) + 1) * image_size,
|
||||
)
|
||||
# split the image
|
||||
split_img = resized_img.crop(box)
|
||||
processed_images.append(split_img)
|
||||
|
||||
assert len(processed_images) == blocks
|
||||
|
||||
if use_thumbnail and len(processed_images) != 1:
|
||||
thumbnail_img = image.resize((image_size, image_size))
|
||||
processed_images.append(thumbnail_img)
|
||||
|
||||
return processed_images
|
||||
|
||||
|
||||
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
|
||||
def image_to_pixel_values_internvl(
|
||||
image: Image.Image,
|
||||
*,
|
||||
input_size: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
use_thumbnail: bool,
|
||||
) -> torch.Tensor:
|
||||
target_ratios = get_internvl_target_ratios(min_num, max_num)
|
||||
|
||||
transform = build_transform(input_size=input_size)
|
||||
images = dynamic_preprocess_internvl(
|
||||
image,
|
||||
target_ratios=target_ratios,
|
||||
image_size=input_size,
|
||||
use_thumbnail=use_thumbnail,
|
||||
)
|
||||
|
||||
pixel_values = torch.stack([transform(image) for image in images])
|
||||
return pixel_values
|
||||
|
||||
|
||||
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
|
||||
def video_to_pixel_values_internvl(
|
||||
video: npt.NDArray,
|
||||
*,
|
||||
input_size: int,
|
||||
min_num: int,
|
||||
max_num: int,
|
||||
use_thumbnail: bool,
|
||||
) -> torch.Tensor:
|
||||
target_ratios = get_internvl_target_ratios(min_num, max_num)
|
||||
|
||||
transform = build_transform(input_size=input_size)
|
||||
frames_list = list[Image.Image]()
|
||||
for frame in video:
|
||||
pil_frame = dynamic_preprocess_internvl(
|
||||
Image.fromarray(frame, mode="RGB"),
|
||||
target_ratios=target_ratios,
|
||||
image_size=input_size,
|
||||
use_thumbnail=use_thumbnail,
|
||||
)
|
||||
assert len(pil_frame) == 1
|
||||
frames_list.extend(pil_frame)
|
||||
|
||||
pixel_values = torch.stack([transform(image) for image in frames_list])
|
||||
return pixel_values
|
||||
|
||||
|
||||
class InternVLImageProcessor(ImageProcessingMixin):
|
||||
def __init__(
|
||||
self,
|
||||
image_size: int,
|
||||
min_dynamic_patch: int,
|
||||
max_dynamic_patch: int,
|
||||
dynamic_image_size: bool,
|
||||
use_thumbnail: bool,
|
||||
) -> None:
|
||||
self.image_size = image_size
|
||||
self.min_dynamic_patch = min_dynamic_patch
|
||||
self.max_dynamic_patch = max_dynamic_patch
|
||||
self.dynamic_image_size = dynamic_image_size
|
||||
self.use_thumbnail = use_thumbnail
|
||||
|
||||
def resolve_min_max_num(
|
||||
self,
|
||||
*,
|
||||
min_dynamic_patch: int | None = None,
|
||||
max_dynamic_patch: int | None = None,
|
||||
dynamic_image_size: bool | None = None,
|
||||
use_thumbnail: bool | None = None,
|
||||
) -> tuple[int, int]:
|
||||
if min_dynamic_patch is None:
|
||||
min_dynamic_patch = self.min_dynamic_patch
|
||||
if max_dynamic_patch is None:
|
||||
max_dynamic_patch = self.max_dynamic_patch
|
||||
if dynamic_image_size is None:
|
||||
dynamic_image_size = self.dynamic_image_size
|
||||
if use_thumbnail is None:
|
||||
use_thumbnail = self.use_thumbnail
|
||||
|
||||
return resolve_internvl_min_max_num(
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
use_thumbnail=use_thumbnail,
|
||||
)
|
||||
|
||||
def _images_to_pixel_values_lst(
|
||||
self,
|
||||
images: list[Image.Image],
|
||||
min_dynamic_patch: int | None = None,
|
||||
max_dynamic_patch: int | None = None,
|
||||
dynamic_image_size: bool | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
if min_dynamic_patch is None:
|
||||
min_dynamic_patch = self.min_dynamic_patch
|
||||
if max_dynamic_patch is None:
|
||||
max_dynamic_patch = self.max_dynamic_patch
|
||||
if dynamic_image_size is None:
|
||||
dynamic_image_size = self.dynamic_image_size
|
||||
|
||||
min_num, max_num = resolve_internvl_min_max_num(
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
use_thumbnail=False, # Applied in image_to_pixel_values
|
||||
)
|
||||
|
||||
return [
|
||||
image_to_pixel_values_internvl(
|
||||
image,
|
||||
input_size=self.image_size,
|
||||
min_num=min_num,
|
||||
max_num=max_num,
|
||||
use_thumbnail=self.use_thumbnail,
|
||||
)
|
||||
for image in images
|
||||
]
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
images: Image.Image | list[Image.Image],
|
||||
*,
|
||||
min_dynamic_patch: int | None = None,
|
||||
max_dynamic_patch: int | None = None,
|
||||
dynamic_image_size: bool | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
images_lst = [images] if not isinstance(images, list) else images
|
||||
|
||||
pixel_values_lst = self._images_to_pixel_values_lst(
|
||||
images_lst,
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
)
|
||||
|
||||
image_inputs = {
|
||||
"pixel_values_flat": torch.cat(pixel_values_lst),
|
||||
"image_num_patches": torch.tensor([len(item) for item in pixel_values_lst]),
|
||||
}
|
||||
return BatchFeature(image_inputs, tensor_type=return_tensors)
|
||||
|
||||
|
||||
class InternVLVideoProcessor(BaseVideoProcessor):
|
||||
def __init__(
|
||||
self,
|
||||
image_size: int,
|
||||
) -> None:
|
||||
self.image_size = image_size
|
||||
|
||||
def _videos_to_pixel_values_lst(
|
||||
self,
|
||||
videos: list[npt.NDArray],
|
||||
) -> list[torch.Tensor]:
|
||||
return [
|
||||
video_to_pixel_values_internvl(
|
||||
video,
|
||||
input_size=self.image_size,
|
||||
min_num=1,
|
||||
max_num=1,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
for video in videos
|
||||
]
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
videos: npt.NDArray | list[npt.NDArray],
|
||||
*,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
videos_lst = [videos] if not isinstance(videos, list) else videos
|
||||
|
||||
pixel_values_lst = self._videos_to_pixel_values_lst(videos_lst)
|
||||
|
||||
image_inputs = {
|
||||
"pixel_values_flat_video": torch.cat(pixel_values_lst),
|
||||
"video_num_patches": torch.tensor([len(item) for item in pixel_values_lst]),
|
||||
}
|
||||
return BatchFeature(image_inputs, tensor_type=return_tensors)
|
||||
|
||||
|
||||
class InternVLProcessor(ProcessorMixin):
|
||||
"""This model doesn't define its own HF processor,
|
||||
so we implement our own one here.
|
||||
|
||||
The code to insert image tokens is based on:
|
||||
https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252
|
||||
|
||||
Code for video processing is adapted from video example:
|
||||
https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "tokenizer", "video_processor"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor: InternVLImageProcessor,
|
||||
tokenizer: HfTokenizer,
|
||||
video_processor: InternVLVideoProcessor | None = None,
|
||||
*,
|
||||
image_seq_length: int,
|
||||
start_image_token: str = "<img>",
|
||||
end_image_token: str = "</img>",
|
||||
ctx_image_token: str = "<IMG_CONTEXT>",
|
||||
ctx_video_token: str | None = None,
|
||||
) -> None:
|
||||
self.image_processor = image_processor
|
||||
self.tokenizer = tokenizer
|
||||
self.video_processor = video_processor
|
||||
|
||||
self.image_seq_length = image_seq_length
|
||||
self.start_image_token = start_image_token
|
||||
self.end_image_token = end_image_token
|
||||
self.ctx_image_token = ctx_image_token
|
||||
self.ctx_video_token = ctx_video_token
|
||||
|
||||
self.start_image_token_id = tokenizer.convert_tokens_to_ids(start_image_token)
|
||||
self.end_image_token_id = tokenizer.convert_tokens_to_ids(end_image_token)
|
||||
self.ctx_image_token_id = tokenizer.convert_tokens_to_ids(ctx_image_token)
|
||||
self.ctx_video_token_id = (
|
||||
None
|
||||
if ctx_video_token is None
|
||||
else tokenizer.convert_tokens_to_ids(ctx_video_token)
|
||||
)
|
||||
|
||||
def resolve_target_ratios(
|
||||
self,
|
||||
*,
|
||||
min_dynamic_patch: int | None = None,
|
||||
max_dynamic_patch: int | None = None,
|
||||
dynamic_image_size: bool | None = None,
|
||||
use_thumbnail: bool | None = None,
|
||||
) -> list[tuple[int, int]]:
|
||||
min_num, max_num = self.image_processor.resolve_min_max_num(
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
use_thumbnail=use_thumbnail,
|
||||
)
|
||||
|
||||
return get_internvl_target_ratios(min_num, max_num)
|
||||
|
||||
def get_num_image_tokens(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
) -> int:
|
||||
image_processor = self.image_processor
|
||||
target_ratios = self.resolve_target_ratios(
|
||||
use_thumbnail=False, # Applied in calculate_targets
|
||||
)
|
||||
|
||||
num_patches, _, _ = calculate_internvl_targets(
|
||||
orig_width=image_width,
|
||||
orig_height=image_height,
|
||||
image_size=image_processor.image_size,
|
||||
target_ratios=target_ratios,
|
||||
use_thumbnail=image_processor.use_thumbnail,
|
||||
)
|
||||
|
||||
return num_patches * self.image_seq_length
|
||||
|
||||
def get_image_repl(
|
||||
self,
|
||||
num_patches: int | None,
|
||||
num_features: int | None = None,
|
||||
) -> PromptUpdateDetails:
|
||||
if num_patches is None:
|
||||
assert num_features is not None
|
||||
else:
|
||||
num_features = num_patches * self.image_seq_length
|
||||
|
||||
repl_features = self.ctx_image_token * num_features
|
||||
repl_full = self.start_image_token + repl_features + self.end_image_token
|
||||
|
||||
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
|
||||
|
||||
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id)
|
||||
|
||||
def get_video_repl(self, num_patches: int) -> PromptUpdateDetails:
|
||||
assert self.ctx_video_token is not None
|
||||
|
||||
repl_features = self.ctx_video_token * self.image_seq_length
|
||||
repl_features_with_sep = (
|
||||
self.start_image_token + repl_features + self.end_image_token
|
||||
)
|
||||
# num_patches is equal to num_frames
|
||||
repl_full = "".join(
|
||||
[f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)]
|
||||
)
|
||||
|
||||
full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False)
|
||||
assert self.ctx_video_token_id is not None
|
||||
|
||||
return PromptUpdateDetails.select_token_id(full_ids, self.ctx_video_token_id)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str | list[str] | None = None,
|
||||
images: Image.Image | list[Image.Image] | None = None,
|
||||
videos: npt.NDArray | list[npt.NDArray] | None = None,
|
||||
*,
|
||||
min_dynamic_patch: int | None = None,
|
||||
max_dynamic_patch: int | None = None,
|
||||
dynamic_image_size: bool | None = None,
|
||||
return_tensors: str | TensorType | None = None,
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
if images is not None:
|
||||
image_inputs = self.image_processor(
|
||||
images=images,
|
||||
min_dynamic_patch=min_dynamic_patch,
|
||||
max_dynamic_patch=max_dynamic_patch,
|
||||
dynamic_image_size=dynamic_image_size,
|
||||
return_tensors=return_tensors,
|
||||
)
|
||||
image_num_patches = image_inputs["image_num_patches"]
|
||||
else:
|
||||
image_inputs = {}
|
||||
image_num_patches = []
|
||||
|
||||
if videos is not None:
|
||||
if self.video_processor is None:
|
||||
raise ValueError("This model does not support video inputs")
|
||||
|
||||
video_inputs = self.video_processor(
|
||||
videos=videos,
|
||||
return_tensors=return_tensors,
|
||||
)
|
||||
video_num_patches = video_inputs["video_num_patches"]
|
||||
else:
|
||||
video_inputs = {}
|
||||
video_num_patches = []
|
||||
|
||||
if text is not None:
|
||||
if not isinstance(text, list):
|
||||
text = [text]
|
||||
|
||||
if image_inputs:
|
||||
image_token = "<image>"
|
||||
image_index = 0
|
||||
processed_text = list[str]()
|
||||
replace_strings = list[str]()
|
||||
|
||||
for prompt in text:
|
||||
new_prompt = prompt
|
||||
|
||||
while image_token in new_prompt:
|
||||
new_prompt = new_prompt.replace(image_token, "<placeholder>", 1)
|
||||
image_repl = self.get_image_repl(image_num_patches[image_index])
|
||||
# Convert the token IDs back to text for the
|
||||
# string-based replacement below
|
||||
replace_strings.append(
|
||||
self.tokenizer.decode(
|
||||
image_repl.full, skip_special_tokens=False
|
||||
)
|
||||
)
|
||||
image_index += 1
|
||||
|
||||
while "<placeholder>" in new_prompt:
|
||||
replace_str = replace_strings.pop(0)
|
||||
new_prompt = new_prompt.replace("<placeholder>", replace_str, 1)
|
||||
|
||||
processed_text.append(new_prompt)
|
||||
|
||||
text = processed_text
|
||||
|
||||
if video_inputs:
|
||||
video_token = "<video>"
|
||||
video_index = 0
|
||||
processed_text = list[str]()
|
||||
replace_strings = list[str]()
|
||||
|
||||
assert video_token is not None
|
||||
|
||||
for prompt in text:
|
||||
new_prompt = prompt
|
||||
|
||||
while video_token in new_prompt:
|
||||
new_prompt = new_prompt.replace(video_token, "<placeholder>", 1)
|
||||
video_repl = self.get_video_repl(video_num_patches[video_index])
|
||||
# Convert the token IDs back to text for the
|
||||
# string-based replacement below
|
||||
replace_strings.append(
|
||||
self.tokenizer.decode(
|
||||
video_repl.full, skip_special_tokens=False
|
||||
)
|
||||
)
|
||||
video_index += 1
|
||||
|
||||
while "<placeholder>" in new_prompt:
|
||||
replace_str = replace_strings.pop(0)
|
||||
new_prompt = new_prompt.replace("<placeholder>", replace_str, 1)
|
||||
|
||||
processed_text.append(new_prompt)
|
||||
|
||||
text = processed_text
|
||||
|
||||
text_inputs = self.tokenizer(text, return_tensors=return_tensors)
|
||||
else:
|
||||
text_inputs = {}
|
||||
|
||||
return BatchFeature(
|
||||
data={**text_inputs, **image_inputs, **video_inputs},
|
||||
tensor_type=return_tensors,
|
||||
)
|
||||
@@ -1,361 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Optimized CPU image processor for Kimi-K2.5/K2.6 vision chunks."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
|
||||
from transformers.utils import TensorType
|
||||
|
||||
from vllm.multimodal.image import open_image
|
||||
from vllm.utils.import_utils import is_numba_available
|
||||
from vllm.utils.jit_monitor import numba_workqueue_threading_layer
|
||||
|
||||
_IDENTITY_LUT = np.tile(np.arange(256, dtype=np.uint8)[:, None], (1, 3))
|
||||
|
||||
if is_numba_available():
|
||||
from numba import njit, prange
|
||||
|
||||
@njit(parallel=True, cache=True)
|
||||
def _write_fused_patches(
|
||||
frames: np.ndarray,
|
||||
out: np.ndarray,
|
||||
out_offset: int,
|
||||
new_h: int,
|
||||
new_w: int,
|
||||
padded_h: int,
|
||||
padded_w: int,
|
||||
patch_size: int,
|
||||
normalize_lut: np.ndarray,
|
||||
) -> None:
|
||||
# frames: [T, new_h, new_w, 3] uint8, without padding.
|
||||
# out: [total_patches, 3, patch_size, patch_size] float32.
|
||||
t_size = frames.shape[0]
|
||||
patch_h = padded_h // patch_size
|
||||
patch_w = padded_w // patch_size
|
||||
total = t_size * padded_h * padded_w * 3
|
||||
hwc = padded_h * padded_w * 3
|
||||
wc = padded_w * 3
|
||||
|
||||
for linear in prange(total):
|
||||
t = linear // hwc
|
||||
rem = linear - t * hwc
|
||||
y = rem // wc
|
||||
rem = rem - y * wc
|
||||
x = rem // 3
|
||||
c = rem - x * 3
|
||||
|
||||
value = frames[t, y, x, c] if y < new_h and x < new_w else 0
|
||||
|
||||
patch_idx = (
|
||||
out_offset
|
||||
+ t * patch_h * patch_w
|
||||
+ (y // patch_size) * patch_w
|
||||
+ (x // patch_size)
|
||||
)
|
||||
out[patch_idx, c, y % patch_size, x % patch_size] = normalize_lut[value, c]
|
||||
|
||||
else:
|
||||
|
||||
def _write_fused_patches(*args: Any, **kwargs: Any) -> None:
|
||||
raise RuntimeError("numba is required for fused Kimi image preprocessing")
|
||||
|
||||
|
||||
def navit_resize_image(
|
||||
width: int,
|
||||
height: int,
|
||||
patch_size: int,
|
||||
merge_kernel_size: int,
|
||||
in_patch_limit: int,
|
||||
patch_limit_on_one_side: int,
|
||||
fixed_output_tokens: int | None,
|
||||
) -> dict[str, int]:
|
||||
s1 = math.sqrt(
|
||||
in_patch_limit
|
||||
/ (max(1.0, width // patch_size) * max(1.0, height // patch_size))
|
||||
)
|
||||
s2 = patch_limit_on_one_side * patch_size / width
|
||||
s3 = patch_limit_on_one_side * patch_size / height
|
||||
scale = min(1.0, s1, s2, s3)
|
||||
new_w = min(max(1, int(width * scale)), patch_limit_on_one_side * patch_size)
|
||||
new_h = min(max(1, int(height * scale)), patch_limit_on_one_side * patch_size)
|
||||
|
||||
factor = merge_kernel_size * patch_size
|
||||
pad_height = (factor - new_h % factor) % factor
|
||||
pad_width = (factor - new_w % factor) % factor
|
||||
|
||||
if fixed_output_tokens is not None:
|
||||
num_tokens = fixed_output_tokens
|
||||
else:
|
||||
token_height = (new_h + pad_height) // factor
|
||||
token_width = (new_w + pad_width) // factor
|
||||
num_tokens = token_height * token_width
|
||||
|
||||
return {
|
||||
"num_tokens": num_tokens,
|
||||
"new_width": new_w,
|
||||
"new_height": new_h,
|
||||
"pad_width": pad_width,
|
||||
"pad_height": pad_height,
|
||||
"sampled_nframes": 1,
|
||||
}
|
||||
|
||||
|
||||
def navit_resize_video(
|
||||
width: int,
|
||||
height: int,
|
||||
nframes: int,
|
||||
avg_fps: float,
|
||||
sample_fps: float,
|
||||
patch_size: int,
|
||||
merge_kernel_size: int,
|
||||
in_patch_limit_each_frame: int,
|
||||
patch_limit_on_one_side: int,
|
||||
in_patch_limit_total: int | None,
|
||||
max_num_frames_each_video: int | None,
|
||||
fixed_output_tokens_each_frame: int | None,
|
||||
) -> dict[str, int]:
|
||||
sample_fps = min(sample_fps, avg_fps)
|
||||
sampled_nframes = max(round(nframes * sample_fps / avg_fps), 1)
|
||||
if max_num_frames_each_video is not None:
|
||||
sampled_nframes = min(sampled_nframes, max_num_frames_each_video)
|
||||
|
||||
if in_patch_limit_total is not None:
|
||||
in_patch_limit_each_frame = min(
|
||||
round(in_patch_limit_total / sampled_nframes),
|
||||
in_patch_limit_each_frame,
|
||||
)
|
||||
|
||||
ret = navit_resize_image(
|
||||
width,
|
||||
height,
|
||||
patch_size,
|
||||
merge_kernel_size,
|
||||
in_patch_limit_each_frame,
|
||||
patch_limit_on_one_side,
|
||||
fixed_output_tokens_each_frame,
|
||||
)
|
||||
ret["sampled_nframes"] = sampled_nframes
|
||||
return ret
|
||||
|
||||
|
||||
def _to_pil(data: Any) -> Image.Image:
|
||||
if hasattr(data, "media") and hasattr(data, "original_bytes"):
|
||||
data = data.media
|
||||
if isinstance(data, Image.Image):
|
||||
return data if data.mode == "RGB" else data.convert("RGB")
|
||||
if isinstance(data, str):
|
||||
if data.startswith("data:"):
|
||||
raw_base64 = data.split(",", 1)[1]
|
||||
return open_image(io.BytesIO(base64.b64decode(raw_base64))).convert("RGB")
|
||||
return open_image(data).convert("RGB")
|
||||
if isinstance(data, bytes):
|
||||
return open_image(io.BytesIO(data)).convert("RGB")
|
||||
raise ValueError(f"Unsupported data type: {type(data)}")
|
||||
|
||||
|
||||
def _ensure_media_type(media: dict[str, Any]) -> dict[str, Any]:
|
||||
if media["type"] == "image":
|
||||
media["image"] = _to_pil(media["image"])
|
||||
return media
|
||||
if media["type"] == "video_chunk":
|
||||
media["video_chunk"] = [_to_pil(frame) for frame in media["video_chunk"]]
|
||||
return media
|
||||
raise ValueError(f"Unsupported media type: {media['type']}")
|
||||
|
||||
|
||||
class KimiK25FusedVisionProcessor(BaseImageProcessor):
|
||||
model_type = "kimi_k25"
|
||||
|
||||
def __init__(self, media_proc_cfg: dict[str, Any], **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
media_proc_cfg = dict(media_proc_cfg)
|
||||
merge_kernel_size = media_proc_cfg["merge_kernel_size"]
|
||||
if isinstance(merge_kernel_size, (list, tuple)):
|
||||
media_proc_cfg["merge_kernel_size"] = int(merge_kernel_size[0])
|
||||
self.media_proc_cfg = media_proc_cfg
|
||||
self.num_frames_per_chunk = media_proc_cfg["temporal_merge_kernel_size"]
|
||||
values = np.arange(256, dtype=np.float32)[:, None]
|
||||
image_mean = np.asarray(media_proc_cfg["image_mean"], dtype=np.float32)
|
||||
image_std_inv = 1.0 / np.asarray(media_proc_cfg["image_std"], dtype=np.float32)
|
||||
self.normalize_lut = (values / 255.0 - image_mean[None, :]) * image_std_inv[
|
||||
None, :
|
||||
]
|
||||
|
||||
def media_tokens_calculator(self, media: dict[str, Any]) -> int:
|
||||
media = _ensure_media_type(media)
|
||||
ret = self.get_resize_config(media)
|
||||
return ret["num_tokens"]
|
||||
|
||||
def get_resize_config(self, media_input: dict[str, Any]) -> dict[str, int]:
|
||||
if media_input["type"] == "image":
|
||||
width, height = media_input["image"].size
|
||||
return navit_resize_image(
|
||||
width,
|
||||
height,
|
||||
self.media_proc_cfg["patch_size"],
|
||||
self.media_proc_cfg["merge_kernel_size"],
|
||||
self.media_proc_cfg["in_patch_limit"],
|
||||
self.media_proc_cfg["patch_limit_on_one_side"],
|
||||
self.media_proc_cfg["fixed_output_tokens"],
|
||||
)
|
||||
|
||||
if media_input["type"] == "video_chunk":
|
||||
frame = media_input["video_chunk"][0]
|
||||
width, height = frame.size
|
||||
num_frames = len(media_input["video_chunk"])
|
||||
in_patch_limit_each_frame = self.media_proc_cfg["in_patch_limit_each_frame"]
|
||||
if in_patch_limit_each_frame is None:
|
||||
in_patch_limit_each_frame = self.media_proc_cfg["in_patch_limit"]
|
||||
|
||||
return navit_resize_video(
|
||||
width,
|
||||
height,
|
||||
num_frames,
|
||||
1.0,
|
||||
math.inf,
|
||||
self.media_proc_cfg["patch_size"],
|
||||
self.media_proc_cfg["merge_kernel_size"],
|
||||
in_patch_limit_each_frame,
|
||||
self.media_proc_cfg["patch_limit_on_one_side"],
|
||||
self.media_proc_cfg["in_patch_limit_video"],
|
||||
None,
|
||||
self.media_proc_cfg["fixed_output_tokens"],
|
||||
)
|
||||
|
||||
raise ValueError(f"Unsupported type: {media_input['type']}")
|
||||
|
||||
@staticmethod
|
||||
def resize_image(image: Image.Image, new_width: int, new_height: int) -> np.ndarray:
|
||||
image = image.resize((new_width, new_height), resample=Image.Resampling.BICUBIC)
|
||||
return np.asarray(image)
|
||||
|
||||
def preprocess(
|
||||
self,
|
||||
medias: list[dict[str, Any]],
|
||||
return_tensors: str | TensorType | None = None,
|
||||
do_rescale: bool = True,
|
||||
do_normalize: bool = True,
|
||||
) -> BatchFeature:
|
||||
assert do_rescale == do_normalize
|
||||
device_normalize = not do_normalize
|
||||
|
||||
if not isinstance(medias, list):
|
||||
medias = [medias]
|
||||
if not medias:
|
||||
return BatchFeature(data={}, tensor_type=return_tensors)
|
||||
|
||||
if njit is None:
|
||||
raise RuntimeError("numba is required for fused Kimi image preprocessing")
|
||||
|
||||
patch_size = int(self.media_proc_cfg["patch_size"])
|
||||
prepared = []
|
||||
grid_thws_np = np.empty((len(medias), 3), dtype=np.int64)
|
||||
total_patches = 0
|
||||
|
||||
for idx, item in enumerate(medias):
|
||||
item = _ensure_media_type(item)
|
||||
resize_config = self.get_resize_config(item)
|
||||
new_width = resize_config["new_width"]
|
||||
new_height = resize_config["new_height"]
|
||||
pad_width = resize_config["pad_width"]
|
||||
pad_height = resize_config["pad_height"]
|
||||
padded_width = new_width + pad_width
|
||||
padded_height = new_height + pad_height
|
||||
|
||||
if item["type"] == "image":
|
||||
image_np = self.resize_image(item["image"], new_width, new_height)
|
||||
frames = image_np[np.newaxis, ...]
|
||||
elif item["type"] == "video_chunk":
|
||||
frames = np.stack(
|
||||
[
|
||||
self.resize_image(frame, new_width, new_height)
|
||||
for frame in item["video_chunk"]
|
||||
],
|
||||
axis=0,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported type: {item['type']}")
|
||||
|
||||
t_size = frames.shape[0]
|
||||
grid_h = padded_height // patch_size
|
||||
grid_w = padded_width // patch_size
|
||||
grid_thws_np[idx, 0] = t_size
|
||||
grid_thws_np[idx, 1] = grid_h
|
||||
grid_thws_np[idx, 2] = grid_w
|
||||
|
||||
num_patches = t_size * grid_h * grid_w
|
||||
prepared.append(
|
||||
(
|
||||
frames,
|
||||
new_height,
|
||||
new_width,
|
||||
padded_height,
|
||||
padded_width,
|
||||
num_patches,
|
||||
)
|
||||
)
|
||||
total_patches += num_patches
|
||||
|
||||
lut = _IDENTITY_LUT if device_normalize else self.normalize_lut
|
||||
pixel_values_np = np.empty(
|
||||
(total_patches, 3, patch_size, patch_size), dtype=lut.dtype
|
||||
)
|
||||
out_offset = 0
|
||||
with numba_workqueue_threading_layer():
|
||||
for (
|
||||
frames,
|
||||
new_height,
|
||||
new_width,
|
||||
padded_height,
|
||||
padded_width,
|
||||
num_patches,
|
||||
) in prepared:
|
||||
_write_fused_patches(
|
||||
frames,
|
||||
pixel_values_np,
|
||||
out_offset,
|
||||
new_height,
|
||||
new_width,
|
||||
padded_height,
|
||||
padded_width,
|
||||
patch_size,
|
||||
lut,
|
||||
)
|
||||
out_offset += num_patches
|
||||
|
||||
data = {
|
||||
"pixel_values": torch.from_numpy(pixel_values_np),
|
||||
"grid_thws": torch.from_numpy(grid_thws_np),
|
||||
}
|
||||
return BatchFeature(data=data, tensor_type=return_tensors)
|
||||
|
||||
def __repr__(self):
|
||||
return f"KimiK25FusedVisionProcessor(media_proc_cfg={self.media_proc_cfg})"
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
output = super().to_dict()
|
||||
output["media_proc_cfg"] = self.media_proc_cfg
|
||||
if "media_processor" in output:
|
||||
del output["media_processor"]
|
||||
return output
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, config_dict: dict[str, Any], **kwargs):
|
||||
config = config_dict.copy()
|
||||
media_proc_cfg = config.pop("media_proc_cfg", {})
|
||||
return cls(media_proc_cfg=media_proc_cfg, **config, **kwargs)
|
||||
|
||||
def to_json_string(self):
|
||||
dictionary = self.to_dict()
|
||||
for key, value in dictionary.items():
|
||||
if hasattr(value, "tolist"):
|
||||
dictionary[key] = value.tolist()
|
||||
return json.dumps(dictionary, indent=2, sort_keys=True) + "\n"
|
||||
@@ -1,582 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HuggingFace processor for the MuseGlimmer multimodal model (text + image + video).
|
||||
|
||||
This packages MuseGlimmer's vision preprocessing and token-span expansion the standard
|
||||
HF way, so you can do:
|
||||
|
||||
processor = AutoProcessor.from_pretrained(hf_dir, trust_remote_code=True)
|
||||
messages = [{"role": "user", "content": [
|
||||
{"type": "image"},
|
||||
{"type": "text", "text": "Describe this image."},
|
||||
]}]
|
||||
text = processor.apply_chat_template(messages, add_generation_prompt=True)
|
||||
inputs = processor(text=text, images=[pil_image], return_tensors="pt")
|
||||
out = model.generate(**inputs, max_new_tokens=256)
|
||||
|
||||
It reproduces the exact token layout the model expects:
|
||||
|
||||
image -> <|image_start|> + <|patch|> * N + <|image_end|>
|
||||
video -> <|vid_start|>
|
||||
( "Time: X.Xs" + <|video|> * P [+ <|vid_frame_separator|>] )*
|
||||
+ <|vid_end|>
|
||||
|
||||
where N = floor(H/(patch*ds)) * floor(W/(patch*ds)) (<= 4096) for images and
|
||||
P = the same grid count per temporal frame-group (<= 144) for video, with the
|
||||
target (H, W) chosen by MuseGlimmerImageProcessor.compute_image_size /
|
||||
MuseGlimmerVideoProcessor.compute_video_frame_size (the same grid logic as
|
||||
modeling_muse_glimmer.MuseGlimmerVisionEncoder).
|
||||
|
||||
The chat template emits one sentinel per media item -- ``<|image|>`` for images,
|
||||
``<|video|>`` for videos -- which ``MuseGlimmerProcessor.__call__`` expands into the
|
||||
spans above using each media item's computed token count. Vision features are
|
||||
returned in ``pixel_values`` (a list of per-image / per-frame-group tensors) in
|
||||
the order the sentinels appear, matching how ``MuseGlimmerModel`` merges them at the
|
||||
``<|patch|>`` / ``<|video|>`` positions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torchvision import transforms as T
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.image_processing_utils import BaseImageProcessor
|
||||
from transformers.processing_utils import ProcessorMixin
|
||||
from transformers.video_processing_utils import BaseVideoProcessor
|
||||
|
||||
# Single-token sentinels emitted by the chat template, one per media item.
|
||||
IMAGE_SENTINEL = "<|image|>" # id 200090; absent from the image/video spans
|
||||
VIDEO_SENTINEL = "<|video|>" # id 200091; also the per-frame token (see __call__)
|
||||
|
||||
# Jinja chat template (multimodal superset of convert_muse_glimmer_to_hf.CHAT_TEMPLATE).
|
||||
# Message content may be a plain string OR a list of parts
|
||||
# ({"type": "text"|"image"|"video", ...}); image/video parts render as a single
|
||||
# sentinel that __call__ expands.
|
||||
MUSE_GLIMMER_MM_CHAT_TEMPLATE = (
|
||||
"{{- bos_token -}}"
|
||||
"{%- macro render_parts(content) -%}"
|
||||
"{%- if content is string -%}{{- content -}}"
|
||||
"{%- else -%}"
|
||||
"{%- for part in content -%}"
|
||||
"{%- if part['type'] == 'image' -%}{{- '<|image|>' -}}"
|
||||
"{%- elif part['type'] == 'video' -%}{{- '<|video|>' -}}"
|
||||
"{%- elif part['type'] == 'text' -%}{{- part['text'] -}}"
|
||||
"{%- endif -%}"
|
||||
"{%- endfor -%}"
|
||||
"{%- endif -%}"
|
||||
"{%- endmacro -%}"
|
||||
# At inference (add_generation_prompt), inject a default system message when
|
||||
# the caller supplied none. Full-transcript rendering
|
||||
# (add_generation_prompt=False) is left unchanged.
|
||||
"{%- set ns = namespace(has_system=false) -%}"
|
||||
"{%- for m in messages -%}"
|
||||
"{%- if m['role'] == 'system' -%}{%- set ns.has_system = true -%}{%- endif -%}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt and not ns.has_system -%}"
|
||||
"{{- '<|start|>system<|message|>You are a helpful assistant.<|eot|>' -}}"
|
||||
"{%- endif -%}"
|
||||
"{%- for message in messages -%}"
|
||||
"{%- set role = message['role'] -%}"
|
||||
"{%- if role == 'assistant' -%}"
|
||||
"{%- set recipient = message.get('recipient') -%}"
|
||||
"{%- set end_turn = message.get('end_turn') -%}"
|
||||
"{%- if end_turn is none -%}"
|
||||
"{%- set end_turn = not (recipient and recipient != 'user') -%}"
|
||||
"{%- endif -%}"
|
||||
"{{- '<|start|>assistant' -}}"
|
||||
"{%- if recipient -%}{{- ' to=' + recipient -}}{%- endif -%}"
|
||||
"{{- '<|message|>' -}}{{- render_parts(message['content']) -}}"
|
||||
"{{- ('<|eot|>' if end_turn else '<|eom|>') -}}"
|
||||
"{%- elif role == 'tool' -%}"
|
||||
"{%- set name = message.get('name', '') -%}"
|
||||
# Tool content is emitted as-is (string body or interleaved image/text parts).
|
||||
# Any <tool_output ...> wrapper is baked into the SFT data content; the
|
||||
# tokenizer does not add it, so the template must not either.
|
||||
"{{- '<|start|>tool ' + name + '<|message|>' -}}"
|
||||
"{{- render_parts(message['content']) -}}"
|
||||
"{{- '<|eot|>' -}}"
|
||||
"{%- else -%}"
|
||||
"{%- set header = role -%}"
|
||||
"{%- if message.get('name') -%}"
|
||||
"{%- set header = role + ' ' + message['name'] -%}"
|
||||
"{%- endif -%}"
|
||||
"{{- '<|start|>' + header + '<|message|>' -}}"
|
||||
"{{- render_parts(message['content']) -}}"
|
||||
"{{- '<|eot|>' -}}"
|
||||
"{%- endif -%}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt -%}{{- '<|start|>assistant' -}}{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
def _grid_size(
|
||||
img_w: int, img_h: int, patch_hw: int, max_tokens: int
|
||||
) -> tuple[int, int, int]:
|
||||
"""Pick the integer (H, W) grid closest to the aspect ratio under the token cap.
|
||||
|
||||
Replicates MuseGlimmerVisionEncoder._compute_grid_size
|
||||
(modeling_muse_glimmer.py) so the processor needs no torch model import.
|
||||
Returns (target_h, target_w, n_tokens).
|
||||
"""
|
||||
i_nph = img_h / patch_hw
|
||||
i_npw = img_w / patch_hw
|
||||
ratio = i_npw / i_nph if i_nph > 0 else 1.0
|
||||
if i_nph * i_npw > max_tokens:
|
||||
i_nph = (max_tokens / ratio) ** 0.5
|
||||
i_npw = i_nph * ratio
|
||||
candidates = list(
|
||||
set(
|
||||
itertools.product(
|
||||
[math.floor(i_nph), math.ceil(i_nph)],
|
||||
[math.floor(i_npw), math.ceil(i_npw)],
|
||||
)
|
||||
)
|
||||
)
|
||||
candidates = [
|
||||
(nph, npw)
|
||||
for nph, npw in candidates
|
||||
if nph >= 1 and npw >= 1 and nph * npw <= max_tokens
|
||||
]
|
||||
if not candidates:
|
||||
candidates = [(max(1, round(i_nph)), max(1, round(i_npw)))]
|
||||
nph, npw = min(candidates, key=lambda c: abs(c[0] / c[1] - img_h / img_w))
|
||||
return nph * patch_hw, npw * patch_hw, nph * npw
|
||||
|
||||
|
||||
class MuseGlimmerImageProcessor(BaseImageProcessor):
|
||||
"""Resize + normalize MuseGlimmer images and compute patch-token counts.
|
||||
|
||||
Variable-resolution: each image is resized to the grid that best matches its
|
||||
aspect ratio under the per-image token cap, then normalized with mean/std 0.5.
|
||||
Returns per-image tensors (not stacked) because MuseGlimmer consumes a list of
|
||||
variable-size images. Video frames are handled by MuseGlimmerVideoProcessor.
|
||||
"""
|
||||
|
||||
model_input_names = ["pixel_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
downsample_factor: int = 2,
|
||||
max_image_tokens: int = 4096,
|
||||
image_mean: float = 0.5,
|
||||
image_std: float = 0.5,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.patch_size = patch_size
|
||||
self.downsample_factor = downsample_factor
|
||||
self.max_image_tokens = max_image_tokens
|
||||
self.image_mean = image_mean
|
||||
self.image_std = image_std
|
||||
|
||||
def _to_norm_tensor(self, image: Image.Image) -> torch.Tensor:
|
||||
# Functional normalize -- no stored Normalize object, so the processor
|
||||
# stays JSON-serializable for to_dict() / save_pretrained.
|
||||
return T.functional.normalize(
|
||||
T.functional.to_tensor(image),
|
||||
[self.image_mean] * 3,
|
||||
[self.image_std] * 3,
|
||||
)
|
||||
|
||||
# -- size computation (mirrors modeling_muse_glimmer.MuseGlimmerVisionEncoder) --
|
||||
def compute_image_size(self, img_w: int, img_h: int) -> tuple[int, int, int]:
|
||||
ph = self.patch_size * self.downsample_factor
|
||||
return _grid_size(img_w, img_h, ph, self.max_image_tokens)
|
||||
|
||||
# -- preprocessing ---------------------------------------------------------
|
||||
def preprocess_image(self, image: Image.Image) -> tuple[torch.Tensor, int]:
|
||||
"""Return (pixel tensor [3, H, W], n_patch_tokens) for one image."""
|
||||
image = image.convert("RGB")
|
||||
target_h, target_w, n_tokens = self.compute_image_size(
|
||||
image.width, image.height
|
||||
)
|
||||
image = image.resize((target_w, target_h), Image.LANCZOS)
|
||||
return self._to_norm_tensor(image), n_tokens
|
||||
|
||||
|
||||
# torchcodec is the training-faithful video decoder; other decoders (torchvision /
|
||||
# PyAV) seek + color-convert differently and diverge from training. Optional at
|
||||
# import time so text/image paths load without it; decode_video raises if used.
|
||||
try:
|
||||
import torchcodec
|
||||
except Exception:
|
||||
torchcodec = None
|
||||
|
||||
|
||||
class MuseGlimmerVideoProcessor(BaseVideoProcessor):
|
||||
"""MuseGlimmer video preprocessing behind the standard HF
|
||||
``AutoVideoProcessor`` API.
|
||||
|
||||
Exposes MuseGlimmer's training-faithful video handling as a first-class video
|
||||
processor so you can call ``processor(videos="clip.mp4")`` and downstream
|
||||
users can discover it via the standard HF ``AutoVideoProcessor`` API plus a
|
||||
``video_preprocessor_config.json``. It wraps:
|
||||
|
||||
* torchcodec decode (matches training; other decoders diverge),
|
||||
* uniform frame sampling to a whole multiple of ``patch_temporal``,
|
||||
* real per-group PTS timestamps (rendered as ``Time: X.Xs`` by
|
||||
MuseGlimmerProcessor),
|
||||
* ``patch_temporal`` frame-grouping (frames cat on the channel axis ->
|
||||
``[patch_temporal * 3, H, W]``; the encoder detects video by channel count).
|
||||
|
||||
It deliberately overrides ``preprocess`` instead of the BaseVideoProcessor fast
|
||||
pipeline (group_videos_by_shape / smart_resize / single stacked tensor): MuseGlimmer
|
||||
consumes a LIST of variable-size group tensors and needs the real per-group PTS,
|
||||
neither of which the stacked fast path models.
|
||||
"""
|
||||
|
||||
model_input_names = ["pixel_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
patch_size: int = 14,
|
||||
downsample_factor: int = 2,
|
||||
patch_temporal: int = 2,
|
||||
max_video_frame_tokens: int = 144,
|
||||
image_mean: float = 0.5,
|
||||
image_std: float = 0.5,
|
||||
video_num_frames: int = 96,
|
||||
video_sampling_fps: float = 2.0,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.patch_size = patch_size
|
||||
self.downsample_factor = downsample_factor
|
||||
self.patch_temporal = patch_temporal
|
||||
self.max_video_frame_tokens = max_video_frame_tokens
|
||||
self.image_mean = image_mean
|
||||
self.image_std = image_std
|
||||
self.video_num_frames = video_num_frames
|
||||
self.video_sampling_fps = video_sampling_fps
|
||||
|
||||
def _to_norm_tensor(self, image: Image.Image) -> torch.Tensor:
|
||||
return T.functional.normalize(
|
||||
T.functional.to_tensor(image),
|
||||
[self.image_mean] * 3,
|
||||
[self.image_std] * 3,
|
||||
)
|
||||
|
||||
def decode_video(self, video_path: str) -> tuple[list[Image.Image], list[float]]:
|
||||
"""Sample frames + per-group PTS with torchcodec (the training decode path).
|
||||
|
||||
``timestamps[g]`` is the ACTUAL decoded PTS of the first frame in temporal
|
||||
group ``g``; ``len(frames)`` is a whole multiple of ``patch_temporal``.
|
||||
"""
|
||||
if torchcodec is None:
|
||||
raise RuntimeError(
|
||||
"torchcodec is required for video decoding (it matches the training "
|
||||
"decode path). See the Environment Setup steps in hf/README.md."
|
||||
)
|
||||
pt = self.patch_temporal
|
||||
reader = torchcodec.decoders.VideoDecoder(video_path)
|
||||
total = len(reader)
|
||||
assert reader.metadata.average_fps is not None, (
|
||||
f"Video has no FPS metadata: {video_path}"
|
||||
)
|
||||
fps = reader.metadata.average_fps
|
||||
assert self.video_sampling_fps and self.video_sampling_fps > 0, (
|
||||
f"video_sampling_fps must be positive, got {self.video_sampling_fps}"
|
||||
)
|
||||
n = min(
|
||||
int(total * self.video_sampling_fps / fps), self.video_num_frames, total
|
||||
)
|
||||
n = max(pt, (n // pt) * pt)
|
||||
n = min(n, total)
|
||||
if n < pt:
|
||||
raise ValueError(
|
||||
f"Video has only {total} decodable frame(s) but needs at least "
|
||||
f"{pt} (one temporal patch): {video_path}"
|
||||
)
|
||||
indices = torch.linspace(0, total - 1, n).long().tolist()
|
||||
frames: list[Image.Image] = []
|
||||
timestamps: list[float] = []
|
||||
for j, i in enumerate(indices):
|
||||
fr = reader[i]
|
||||
frames.append(
|
||||
Image.fromarray(fr.data.permute(1, 2, 0).numpy()).convert("RGB")
|
||||
)
|
||||
if j % pt == 0:
|
||||
pts = getattr(fr, "pts_seconds", None)
|
||||
timestamps.append(float(pts) if pts is not None else i / fps)
|
||||
return frames, timestamps
|
||||
|
||||
def compute_video_frame_size(self, img_w: int, img_h: int) -> tuple[int, int, int]:
|
||||
"""Pick (H, W, tokens_per_frame) for a video frame under the per-frame cap.
|
||||
|
||||
Mirrors MuseGlimmerImageProcessor.compute_image_size but uses the (smaller)
|
||||
per-frame video token budget; shares the same ``_grid_size`` grid logic as
|
||||
modeling_muse_glimmer.MuseGlimmerVisionEncoder.
|
||||
"""
|
||||
ph = self.patch_size * self.downsample_factor
|
||||
return _grid_size(img_w, img_h, ph, self.max_video_frame_tokens)
|
||||
|
||||
def _group_frames(
|
||||
self, frames: list[Image.Image]
|
||||
) -> tuple[list[torch.Tensor], int, int]:
|
||||
"""Resize/normalize frames and cat ``patch_temporal`` frames per group.
|
||||
|
||||
Returns (group_tensors, n_groups, tokens_per_group).
|
||||
"""
|
||||
pt = self.patch_temporal
|
||||
if pt <= 0:
|
||||
raise ValueError(f"patch_temporal must be positive, got {pt}")
|
||||
if not frames:
|
||||
raise ValueError("video must contain at least one frame")
|
||||
if padding := (-len(frames)) % pt:
|
||||
frames = [*frames, *([frames[-1]] * padding)]
|
||||
first = frames[0].convert("RGB")
|
||||
target_h, target_w, n_tokens = self.compute_video_frame_size(
|
||||
first.width, first.height
|
||||
)
|
||||
groups: list[torch.Tensor] = []
|
||||
for i in range(0, len(frames), pt):
|
||||
grp = [
|
||||
self._to_norm_tensor(
|
||||
frames[i + j]
|
||||
.convert("RGB")
|
||||
.resize((target_w, target_h), Image.LANCZOS)
|
||||
)
|
||||
for j in range(pt)
|
||||
]
|
||||
groups.append(torch.cat(grp, dim=0))
|
||||
return groups, len(groups), n_tokens
|
||||
|
||||
@staticmethod
|
||||
def _normalize_videos(videos) -> list:
|
||||
"""Normalize the ``videos`` arg to a list of per-video items.
|
||||
|
||||
A video item is a path (str/Path) or a list of PIL frames. Accepts a
|
||||
single video (one path, or one list of frames) or a list of those.
|
||||
"""
|
||||
if isinstance(videos, (str, Path)):
|
||||
return [videos]
|
||||
if videos and not isinstance(videos[0], (list, tuple, str, Path)):
|
||||
return [videos] # a single list of PIL frames
|
||||
return list(videos or [])
|
||||
|
||||
def preprocess_one(
|
||||
self,
|
||||
video: str | Path | list[Image.Image],
|
||||
timestamps: list[float] | None = None,
|
||||
) -> tuple[list[torch.Tensor], int, int, list[float]]:
|
||||
"""Decode (if a path) + group ONE video.
|
||||
|
||||
Returns (group_tensors, n_groups, tokens_per_group, group_timestamps). This
|
||||
is the single per-video implementation shared by ``preprocess`` (the HF
|
||||
AutoVideoProcessor batch API) and ``MuseGlimmerProcessor.__call__`` (prompt
|
||||
building), so the two paths cannot drift. For a path, frames + real
|
||||
per-group PTS come from ``decode_video``; for pre-decoded frames the given
|
||||
``timestamps`` are used verbatim (empty if not supplied).
|
||||
"""
|
||||
if isinstance(video, (str, Path)):
|
||||
frames, ts = self.decode_video(str(video))
|
||||
else:
|
||||
frames = [f.convert("RGB") for f in video]
|
||||
ts = list(timestamps) if timestamps is not None else []
|
||||
groups, n_groups, tokens_per_group = self._group_frames(frames)
|
||||
return groups, n_groups, tokens_per_group, ts
|
||||
|
||||
def preprocess(
|
||||
self,
|
||||
videos,
|
||||
video_timestamps: list[list[float]] | None = None,
|
||||
return_tensors: str | None = "pt",
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
"""Preprocess video file path(s) or pre-decoded frame list(s).
|
||||
|
||||
Accepts a single video (path str, or list of PIL frames) or a list of
|
||||
those. Returns a BatchFeature carrying, in video order:
|
||||
* ``pixel_values`` -- flat list of [pt*3, H, W] group tensors,
|
||||
* ``video_num_groups`` -- groups per video,
|
||||
* ``video_tokens_per_group`` -- patch-token count per group per video,
|
||||
* ``video_timestamps`` -- per-group PTS per video,
|
||||
i.e. exactly what MuseGlimmerProcessor needs to build the
|
||||
``<|vid_start|> ( Time: X.Xs <|video|>*P [sep] )* <|vid_end|>`` block.
|
||||
"""
|
||||
videos = self._normalize_videos(videos)
|
||||
pixel_values: list[torch.Tensor] = []
|
||||
num_groups: list[int] = []
|
||||
tokens_per_group: list[int] = []
|
||||
out_ts: list[list[float]] = []
|
||||
for idx, v in enumerate(videos):
|
||||
ts_in = video_timestamps[idx] if video_timestamps else None
|
||||
groups, ng, tpg, ts = self.preprocess_one(v, ts_in)
|
||||
pixel_values += groups
|
||||
num_groups.append(ng)
|
||||
tokens_per_group.append(tpg)
|
||||
out_ts.append(ts)
|
||||
batch = BatchFeature(
|
||||
data={
|
||||
"video_num_groups": num_groups,
|
||||
"video_tokens_per_group": tokens_per_group,
|
||||
"video_timestamps": out_ts,
|
||||
},
|
||||
tensor_type=None,
|
||||
)
|
||||
# Variable-size list -- BatchFeature would fail to stack into one tensor.
|
||||
batch["pixel_values"] = pixel_values
|
||||
return batch
|
||||
|
||||
|
||||
class MuseGlimmerProcessor(ProcessorMixin):
|
||||
"""Bundle MuseGlimmerImageProcessor + MuseGlimmerVideoProcessor + tokenizer;
|
||||
expand media sentinels into spans.
|
||||
|
||||
Images go through ``image_processor`` (MuseGlimmerImageProcessor) and videos through
|
||||
``video_processor`` (MuseGlimmerVideoProcessor) -- one preprocessing implementation
|
||||
each, no overlap.
|
||||
"""
|
||||
|
||||
attributes = ["image_processor", "video_processor", "tokenizer"]
|
||||
image_processor_class = "AutoImageProcessor"
|
||||
video_processor_class = "AutoVideoProcessor"
|
||||
tokenizer_class = "TokenizersBackend"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_processor=None,
|
||||
video_processor=None,
|
||||
tokenizer=None,
|
||||
chat_template=None,
|
||||
**kwargs,
|
||||
):
|
||||
if image_processor is None:
|
||||
image_processor = MuseGlimmerImageProcessor()
|
||||
if video_processor is None:
|
||||
video_processor = MuseGlimmerVideoProcessor()
|
||||
super().__init__(
|
||||
image_processor,
|
||||
video_processor,
|
||||
tokenizer,
|
||||
chat_template=chat_template or MUSE_GLIMMER_MM_CHAT_TEMPLATE,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _sid(self, token: str) -> int:
|
||||
return self.tokenizer.convert_tokens_to_ids(token)
|
||||
|
||||
def _image_block(self, n_tokens: int) -> list[int]:
|
||||
return (
|
||||
[self._sid("<|image_start|>")]
|
||||
+ [self._sid("<|patch|>")] * n_tokens
|
||||
+ [self._sid("<|image_end|>")]
|
||||
)
|
||||
|
||||
def _video_block(
|
||||
self,
|
||||
n_groups: int,
|
||||
tokens_per_group: int,
|
||||
timestamps: list[float] | None = None,
|
||||
) -> list[int]:
|
||||
"""Per-group ``Time: X.Xs`` + <|video|>*P, separated/terminated.
|
||||
|
||||
``timestamps`` (one per temporal group, in seconds) should be the ACTUAL
|
||||
decoded frame times -- training renders the real factored frame PTS, not a
|
||||
uniform grid. Falls back to ``g*patch_temporal/fps`` only when timestamps
|
||||
are not supplied (e.g. callers that don't track frame times); that
|
||||
approximation can differ from training by the per-frame jitter.
|
||||
"""
|
||||
vid = self._sid("<|video|>")
|
||||
sep = self._sid("<|vid_frame_separator|>")
|
||||
pt = self.video_processor.patch_temporal
|
||||
fps = self.video_processor.video_sampling_fps
|
||||
block = [self._sid("<|vid_start|>")]
|
||||
for g in range(n_groups):
|
||||
ts = timestamps[g] if timestamps is not None else g * pt / fps
|
||||
block += self.tokenizer.encode(f"Time: {ts:.1f}s", add_special_tokens=False)
|
||||
block += [vid] * tokens_per_group
|
||||
block.append(sep if g < n_groups - 1 else self._sid("<|vid_end|>"))
|
||||
return block
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str | list[str] | None = None,
|
||||
images: list[Image.Image] | None = None,
|
||||
videos: list[list[Image.Image] | str] | None = None,
|
||||
video_timestamps: list[list[float]] | None = None,
|
||||
return_tensors: str | None = "pt",
|
||||
**kwargs,
|
||||
) -> BatchFeature:
|
||||
if text is None:
|
||||
raise ValueError(
|
||||
"`text` is required (use apply_chat_template to build it)."
|
||||
)
|
||||
if isinstance(text, (list, tuple)):
|
||||
if len(text) != 1:
|
||||
raise ValueError(
|
||||
"MuseGlimmerProcessor supports a single text sample per call."
|
||||
)
|
||||
text = text[0]
|
||||
|
||||
images = list(images or [])
|
||||
videos = list(videos or [])
|
||||
image_sentinel = self._sid(IMAGE_SENTINEL)
|
||||
video_sentinel = self._sid(VIDEO_SENTINEL)
|
||||
|
||||
# Preprocess media up front so we can expand sentinels in document order.
|
||||
# Images via image_processor; videos via video_processor (each video item
|
||||
# is a path -- decoded here -- or a pre-decoded list of PIL frames).
|
||||
prepped_images = [self.image_processor.preprocess_image(im) for im in images]
|
||||
prepped_videos = [
|
||||
self.video_processor.preprocess_one(
|
||||
v, video_timestamps[i] if video_timestamps else None
|
||||
)
|
||||
for i, v in enumerate(videos)
|
||||
]
|
||||
|
||||
ids = self.tokenizer.encode(text, add_special_tokens=False)
|
||||
n_img = sum(1 for t in ids if t == image_sentinel)
|
||||
n_vid = sum(1 for t in ids if t == video_sentinel)
|
||||
if n_img != len(prepped_images):
|
||||
raise ValueError(
|
||||
f"{n_img} image sentinel(s) in text but "
|
||||
f"{len(prepped_images)} image(s) given."
|
||||
)
|
||||
if n_vid != len(prepped_videos):
|
||||
raise ValueError(
|
||||
f"{n_vid} video sentinel(s) in text but "
|
||||
f"{len(prepped_videos)} video(s) given."
|
||||
)
|
||||
|
||||
out_ids: list[int] = []
|
||||
pixel_values: list[torch.Tensor] = []
|
||||
img_i = vid_i = 0
|
||||
for tid in ids:
|
||||
if tid == image_sentinel:
|
||||
tensor, n_tokens = prepped_images[img_i]
|
||||
img_i += 1
|
||||
out_ids += self._image_block(n_tokens)
|
||||
pixel_values.append(tensor)
|
||||
elif tid == video_sentinel:
|
||||
groups, n_groups, tokens_per_group, ts = prepped_videos[vid_i]
|
||||
vid_i += 1
|
||||
out_ids += self._video_block(n_groups, tokens_per_group, ts or None)
|
||||
pixel_values += groups
|
||||
else:
|
||||
out_ids.append(tid)
|
||||
|
||||
data: dict = {
|
||||
"input_ids": [out_ids],
|
||||
"attention_mask": [[1] * len(out_ids)],
|
||||
}
|
||||
batch = BatchFeature(data=data, tensor_type=return_tensors)
|
||||
# Keep pixel_values as a list of variable-size tensors (MuseGlimmerModel
|
||||
# consumes a list); BatchFeature would fail to stack them into one tensor.
|
||||
if pixel_values:
|
||||
batch["pixel_values"] = pixel_values
|
||||
return batch
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MuseGlimmerImageProcessor",
|
||||
"MuseGlimmerVideoProcessor",
|
||||
"MuseGlimmerProcessor",
|
||||
"MUSE_GLIMMER_MM_CHAT_TEMPLATE",
|
||||
]
|
||||
@@ -1,46 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Image processor for Unlimited-OCR (baidu/Unlimited-OCR)."""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import DeepseekOCRProcessor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class UnlimitedOCRProcessor(DeepseekOCRProcessor):
|
||||
"""DeepseekOCRProcessor variant for Unlimited-OCR.
|
||||
|
||||
The only behavioural difference from the base processor is a multi-image
|
||||
safeguard: when more than one image is present, crop ("gundam") mode is
|
||||
disabled.
|
||||
|
||||
Because the effective crop flag then depends on *how many* images are in the
|
||||
request, the per-item processing output is no longer invariant of sibling
|
||||
images. ``UnlimitedOCRMultiModalProcessor`` accounts for this by bypassing
|
||||
the multimodal processing cache for multi-image requests (see its
|
||||
``_cached_apply_hf_processor``), so the two paths stay consistent.
|
||||
|
||||
DeepSeek-OCR does *not* have this restriction because its ``max_crops=6`` is
|
||||
small enough to be safe for multi-image use.
|
||||
"""
|
||||
|
||||
def tokenize_with_images(
|
||||
self,
|
||||
conversation: str,
|
||||
images: list[Image.Image],
|
||||
bos: bool = True,
|
||||
eos: bool = True,
|
||||
cropping: bool = True,
|
||||
):
|
||||
if len(images) > 1 and cropping:
|
||||
logger.warning_once(
|
||||
"Unlimited-OCR: crop mode is not supported for multi-image "
|
||||
"input. Falling back to cropping=False."
|
||||
)
|
||||
cropping = False
|
||||
return super().tokenize_with_images(
|
||||
conversation, images, bos=bos, eos=eos, cropping=cropping
|
||||
)
|
||||
Reference in New Issue
Block a user