Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
352 lines
14 KiB
Python
352 lines
14 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Inkling multimodal preprocessing."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from collections.abc import Iterable, Mapping, Sequence
|
|
from typing import Any, cast
|
|
|
|
import numpy as np
|
|
import regex as re
|
|
import torch
|
|
from PIL import Image
|
|
from transformers import InklingProcessor
|
|
from transformers.feature_extraction_utils import BatchFeature
|
|
|
|
from vllm.config.multimodal import (
|
|
MultiModalDummyOptions,
|
|
)
|
|
from vllm.inputs import MultiModalDataDict
|
|
from vllm.multimodal.inputs import (
|
|
MultiModalFieldConfig,
|
|
MultiModalKwargsItems,
|
|
)
|
|
from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
|
|
from vllm.multimodal.processing import (
|
|
BaseDummyInputsBuilder,
|
|
BaseMultiModalProcessor,
|
|
BaseProcessingInfo,
|
|
PromptReplacement,
|
|
PromptUpdate,
|
|
PromptUpdateDetails,
|
|
)
|
|
|
|
from ..configs import InklingMMConfig
|
|
|
|
# Block-start markers (<|content_image|>, <|content_audio_input|>), kept verbatim
|
|
IMAGE_MARKER_ID = 200005
|
|
AUDIO_MARKER_ID = 200020
|
|
# Per-patch / per-frame placeholders (<|unused_200054|>, <|unused_200053|>)
|
|
IMAGE_TOKEN_ID = 200054
|
|
AUDIO_TOKEN_ID = 200053
|
|
|
|
# Long-edge upscale applied before patchifying (upstream defaults to no rescale)
|
|
DEFAULT_RESCALE_IMAGE_FRAC = 2.0
|
|
|
|
# Maximum audio tokens accepted per clip. At the dMel rate of 20 tokens/s
|
|
# (50 ms hop) this is ~10 minutes of audio. It bounds the persistent per-request
|
|
# buffers and the encoder/memory budget; longer clips are rejected up front.
|
|
MAX_AUDIO_TOKENS = 12_000
|
|
|
|
|
|
class InklingMultiModalDataParser(MultiModalDataParser):
|
|
def _parse_audio_data(self, data: Any) -> Any:
|
|
if isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 2:
|
|
raise ValueError(
|
|
"Inkling raw 2-D audio has an ambiguous channel layout. "
|
|
"Provide encoded audio or a list of mono waveforms."
|
|
)
|
|
return super()._parse_audio_data(data)
|
|
|
|
|
|
def _rescale_image(
|
|
image: Image.Image, frac: float | None, max_upscaled_long_edge: int | None
|
|
) -> Image.Image:
|
|
"""Scale the long edge by `frac` with PIL LANCZOS, as the reference does."""
|
|
image = image.convert("RGB")
|
|
if frac is None:
|
|
return image
|
|
long_edge = max(image.size)
|
|
target_long_edge = long_edge * frac
|
|
if max_upscaled_long_edge is not None:
|
|
target_long_edge = min(target_long_edge, max(max_upscaled_long_edge, long_edge))
|
|
ratio = target_long_edge / long_edge
|
|
if ratio == 1.0:
|
|
return image
|
|
size = tuple(max(1, math.floor(d * ratio + 0.5)) for d in image.size)
|
|
return image.resize(size, resample=Image.Resampling.LANCZOS)
|
|
|
|
|
|
def inkling_vision_enabled(config: InklingMMConfig) -> bool:
|
|
return getattr(config.vision_config, "decoder_dmodel", None) is not None
|
|
|
|
|
|
def inkling_audio_enabled(config: InklingMMConfig) -> bool:
|
|
return getattr(config.audio_config, "decoder_dmodel", None) is not None
|
|
|
|
|
|
class InklingProcessingInfo(BaseProcessingInfo):
|
|
def get_hf_config(self) -> InklingMMConfig:
|
|
return self.ctx.get_hf_config(InklingMMConfig)
|
|
|
|
def get_hf_processor(self, **kwargs: object) -> InklingProcessor:
|
|
return self.ctx.get_hf_processor(InklingProcessor, **kwargs)
|
|
|
|
def get_data_parser(self) -> MultiModalDataParser:
|
|
# The feature extractor requires audio at its sampling rate
|
|
feature_extractor = self.get_hf_processor().feature_extractor
|
|
return InklingMultiModalDataParser(
|
|
target_sr=feature_extractor.sampling_rate,
|
|
target_channels=1,
|
|
expected_hidden_size=self._get_expected_hidden_size(),
|
|
)
|
|
|
|
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
|
config = self.get_hf_config()
|
|
limits: dict[str, int | None] = {}
|
|
if inkling_vision_enabled(config):
|
|
limits["image"] = None
|
|
if inkling_audio_enabled(config):
|
|
limits["audio"] = None
|
|
return limits
|
|
|
|
def get_mm_max_tokens_per_item(
|
|
self,
|
|
seq_len: int,
|
|
mm_counts: Mapping[str, int],
|
|
) -> Mapping[str, int] | None:
|
|
# Let vLLM profile dummy inputs to determine the max token counts; the
|
|
# image patch count is data-dependent, and the dummy audio is sized to
|
|
# MAX_AUDIO_TOKENS so audio is profiled/budgeted at its allowed maximum.
|
|
return None
|
|
|
|
|
|
class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
|
|
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
|
# One placeholder per media item; the processor expands each into N
|
|
# copies once feature row counts are known.
|
|
num_images = mm_counts.get("image", 0)
|
|
num_audios = mm_counts.get("audio", 0)
|
|
# Use spellings the renderer would emit; tokenization is bypassed in
|
|
# _apply_hf_processor_main (we build input_ids directly), so the exact
|
|
# text only needs to be a stable per-item marker.
|
|
return ("<|content_image|>" * num_images) + (
|
|
"<|content_audio_input|>" * num_audios
|
|
)
|
|
|
|
def get_dummy_mm_data(
|
|
self,
|
|
seq_len: int,
|
|
mm_counts: Mapping[str, int],
|
|
mm_options: MultiModalDummyOptions,
|
|
) -> MultiModalDataDict:
|
|
config = self.info.get_hf_config()
|
|
num_images = mm_counts.get("image", 0)
|
|
num_audios = mm_counts.get("audio", 0)
|
|
|
|
mm_data: dict[str, Any] = {}
|
|
if num_images:
|
|
patch_size = getattr(config.vision_config, "patch_size", 40)
|
|
# A square image ~4 patches wide so the dummy emits several patches.
|
|
side = patch_size * 4
|
|
mm_data["image"] = self._get_dummy_images(
|
|
width=side,
|
|
height=side,
|
|
num_images=num_images,
|
|
overrides=mm_options.get("image"),
|
|
)
|
|
if num_audios:
|
|
# Size the dummy at the maximum allowed audio so memory/encoder
|
|
# budgeting reflects the largest clip we accept (MAX_AUDIO_TOKENS).
|
|
feature_extractor = self.info.get_hf_processor().feature_extractor
|
|
audio_len = MAX_AUDIO_TOKENS * feature_extractor.hop_length
|
|
mm_data["audio"] = self._get_dummy_audios(
|
|
length=audio_len,
|
|
num_audios=num_audios,
|
|
overrides=mm_options.get("audio"),
|
|
)
|
|
return mm_data
|
|
|
|
|
|
class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
|
|
def _apply_hf_processor_main(
|
|
self,
|
|
mm_items: MultiModalDataItems,
|
|
hf_kwargs: Mapping[str, object],
|
|
) -> BatchFeature:
|
|
mm_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
|
mm_items, hf_kwargs
|
|
)
|
|
|
|
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
|
|
|
processor = self.info.get_hf_processor(**hf_kwargs)
|
|
tokenizer = self.info.get_tokenizer()
|
|
|
|
images = mm_data.get("images") or []
|
|
audios = mm_data.get("audio") or []
|
|
if not isinstance(images, list):
|
|
images = list(cast(Iterable[Any], images))
|
|
if not isinstance(audios, list):
|
|
audios = list(cast(Iterable[Any], audios))
|
|
|
|
prompt_ids = self._tokenize_with_placeholders(
|
|
prompt_text, tokenizer, len(images), len(audios)
|
|
)
|
|
|
|
data: dict[str, Any] = {"input_ids": [prompt_ids]}
|
|
|
|
if images:
|
|
image_processor = processor.image_processor
|
|
frac = image_processor.rescale_image_frac or DEFAULT_RESCALE_IMAGE_FRAC
|
|
max_long_edge = image_processor.rescale_image_max_upscaled_long_edge
|
|
# Resize with PIL: torchvision LANCZOS does not match the reference
|
|
images = [_rescale_image(img, frac, max_long_edge) for img in images]
|
|
img_feat = image_processor(
|
|
images, rescale_image_frac=None, return_tensors="pt"
|
|
)
|
|
data["pixel_values"] = img_feat["pixel_values"].to(torch.bfloat16)
|
|
data["num_patches"] = img_feat["num_patches"].to(torch.int64)
|
|
|
|
if audios:
|
|
feature_extractor = processor.feature_extractor
|
|
aud_feat = feature_extractor(
|
|
audios,
|
|
sampling_rate=feature_extractor.sampling_rate,
|
|
return_tensors="pt",
|
|
)
|
|
audio_input_ids = processor._extract_dmel_bins(aud_feat["input_features"])
|
|
num_audio_tokens = aud_feat["input_features_mask"].sum(-1)
|
|
for i, n in enumerate(num_audio_tokens.tolist()):
|
|
if n > MAX_AUDIO_TOKENS:
|
|
raise ValueError(
|
|
f"Audio clip {i} produces {n} tokens, exceeding the "
|
|
f"maximum of {MAX_AUDIO_TOKENS} (~10 min at 20 tokens/s). "
|
|
"Provide a shorter clip."
|
|
)
|
|
data["input_audio_features"] = torch.cat(
|
|
[ids[:n] for ids, n in zip(audio_input_ids, num_audio_tokens)]
|
|
)
|
|
data["num_audio_tokens"] = num_audio_tokens.to(torch.int64)
|
|
|
|
processed_data = BatchFeature(data=data, tensor_type=None)
|
|
return self._finalize_hf_mm_data(
|
|
mm_data, hf_kwargs, passthrough_data, processed_data
|
|
)
|
|
|
|
def _tokenize_with_placeholders(
|
|
self,
|
|
prompt: str,
|
|
tokenizer: Any,
|
|
num_images: int,
|
|
num_audios: int,
|
|
) -> list[int]:
|
|
"""Tokenize `prompt`, emitting the block-start marker id per media item.
|
|
|
|
Each marker (kept verbatim) is later expanded by ``_get_prompt_updates``
|
|
into ``<marker> + <placeholder> * N``.
|
|
"""
|
|
image_marker = "<|content_image|>"
|
|
audio_marker = "<|content_audio_input|>"
|
|
|
|
pattern = f"({re.escape(image_marker)}|{re.escape(audio_marker)})"
|
|
chunks = re.split(pattern, prompt)
|
|
|
|
ids: list[int] = []
|
|
seen_img = seen_aud = 0
|
|
for chunk in chunks:
|
|
if chunk == image_marker:
|
|
ids.append(IMAGE_MARKER_ID)
|
|
seen_img += 1
|
|
elif chunk == audio_marker:
|
|
ids.append(AUDIO_MARKER_ID)
|
|
seen_aud += 1
|
|
elif chunk:
|
|
ids.extend(tokenizer.encode(chunk, add_special_tokens=False))
|
|
|
|
# Reconcile against the declared media counts only when media is
|
|
# present. With no media items, emit the markers verbatim; the
|
|
# marker<->item correspondence is enforced later by
|
|
# ``_get_prompt_updates`` once the media features are available.
|
|
if num_images or num_audios:
|
|
# Fail clearly on a placeholder/media-count mismatch instead of
|
|
# crashing with an IndexError deep in the per-item replacement logic.
|
|
if num_images and seen_img != num_images:
|
|
raise ValueError(
|
|
f"Prompt contains {seen_img} image placeholder(s), but only "
|
|
f"{num_images} image(s) were provided."
|
|
)
|
|
if num_audios and seen_aud != num_audios:
|
|
raise ValueError(
|
|
f"Prompt contains {seen_aud} audio placeholder(s), but only "
|
|
f"{num_audios} audio input(s) were provided."
|
|
)
|
|
return ids
|
|
|
|
def _get_mm_fields_config(
|
|
self,
|
|
hf_inputs: BatchFeature,
|
|
hf_processor_mm_kwargs: Mapping[str, object],
|
|
) -> Mapping[str, MultiModalFieldConfig]:
|
|
num_patches = hf_inputs.get("num_patches", torch.empty(0, dtype=torch.int64))
|
|
num_audio_tokens = hf_inputs.get(
|
|
"num_audio_tokens", torch.empty(0, dtype=torch.int64)
|
|
)
|
|
return dict(
|
|
# Ragged per-image patches, grouped by num_patches.
|
|
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches),
|
|
num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
|
# Ragged per-audio frames, grouped by num_audio_tokens.
|
|
input_audio_features=MultiModalFieldConfig.flat_from_sizes(
|
|
"audio", num_audio_tokens
|
|
),
|
|
num_audio_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
|
)
|
|
|
|
def _get_prompt_updates(
|
|
self,
|
|
mm_items: Any,
|
|
hf_processor_mm_kwargs: Mapping[str, Any],
|
|
out_mm_kwargs: MultiModalKwargsItems,
|
|
) -> Sequence[PromptUpdate]:
|
|
out_mm_data = out_mm_kwargs.get_data()
|
|
num_patches: Any = out_mm_data.get("num_patches")
|
|
num_audio_tokens: Any = out_mm_data.get("num_audio_tokens")
|
|
|
|
# Keep the block-start marker and append N placeholder tokens after it;
|
|
# only the placeholder positions are flagged as embeddings (is_embed), so
|
|
# the marker stays a normal text token while the tower features scatter
|
|
# into the placeholders.
|
|
def image_replacement(item_idx: int) -> PromptUpdateDetails:
|
|
n = int(num_patches[item_idx])
|
|
return PromptUpdateDetails.select_token_id(
|
|
[IMAGE_MARKER_ID] + [IMAGE_TOKEN_ID] * n, IMAGE_TOKEN_ID
|
|
)
|
|
|
|
def audio_replacement(item_idx: int) -> PromptUpdateDetails:
|
|
n = int(num_audio_tokens[item_idx])
|
|
return PromptUpdateDetails.select_token_id(
|
|
[AUDIO_MARKER_ID] + [AUDIO_TOKEN_ID] * n, AUDIO_TOKEN_ID
|
|
)
|
|
|
|
updates: list[PromptUpdate] = []
|
|
if num_patches is not None and len(num_patches) > 0:
|
|
updates.append(
|
|
PromptReplacement(
|
|
modality="image",
|
|
target=[IMAGE_MARKER_ID],
|
|
replacement=image_replacement,
|
|
)
|
|
)
|
|
if num_audio_tokens is not None and len(num_audio_tokens) > 0:
|
|
updates.append(
|
|
PromptReplacement(
|
|
modality="audio",
|
|
target=[AUDIO_MARKER_ID],
|
|
replacement=audio_replacement,
|
|
)
|
|
)
|
|
return updates
|