[Model] Support MiniCPM-V 4.7 (#58674)

Signed-off-by: tc-mb <tianchi_cai@icloud.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
tc-mb
2026-10-11 10:21:10 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent f425a21208
commit c85616efc6
13 changed files with 2366 additions and 62 deletions
+270 -61
View File
@@ -45,6 +45,7 @@ from .interfaces import (
HasInnerState,
IsHybrid,
MultiModalEmbeddings,
SupportsEagle3,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
@@ -87,35 +88,186 @@ def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]):
fields["use_vit_merger"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
if "video_use_vit_merger" in hf_inputs:
fields["video_use_vit_merger"] = MultiModalFieldConfig.batched(
"video", keep_on_cpu=True
)
return fields
def _expand_flags(value: object, num_items: int) -> list[bool | None]:
"""Broadcast a batched per-item flag to exactly one entry per item."""
if value is None:
return [None] * num_items
raw: list[object]
if isinstance(value, torch.Tensor):
raw = value.reshape(-1).tolist()
elif isinstance(value, list | tuple):
raw = []
for item in value:
if isinstance(item, torch.Tensor):
raw.extend(item.reshape(-1).tolist())
else:
raw.append(item)
else:
raw = [value]
if len(raw) == num_items:
return [bool(v) for v in raw]
if len(raw) == 1:
return [bool(raw[0])] * num_items
# Unexpected shape: keep one decision for the whole batch instead of
# failing the request.
return [any(bool(v) for v in raw)] * num_items
def _vision_downsample_mode(flag: bool | None) -> str | None:
if flag is None:
return None
return "16x" if flag else "4x"
def _select_vision_items(
image_input: MiniCPMVImagePixelInputs,
item_indices: list[int],
num_slices: list[int],
) -> MiniCPMVImagePixelInputs:
"""Slice a vision input down to the given items, keeping their order."""
offsets = [0]
for n in num_slices:
offsets.append(offsets[-1] + n)
pixel_values: list[torch.Tensor] = []
tgt_sizes: list[torch.Tensor] = []
sizes: list[int] = []
for i in item_indices:
start, end = offsets[i], offsets[i + 1]
pixel_values.extend(image_input["pixel_values"][start:end])
tgt_sizes.append(image_input["tgt_sizes"][start:end])
sizes.append(end - start)
return MiniCPMVImagePixelInputs(
type="pixel_values",
pixel_values=pixel_values,
tgt_sizes=torch.cat(tgt_sizes, dim=0),
num_slices=torch.tensor(sizes),
)
# HF-style modality-scoped keys inside `mm_processor_kwargs`. They scope a value
# to a single modality and must not be forwarded to the processors, which take
# flat arguments only.
_MODALITY_SCOPED_MM_KWARGS = ("images_kwargs", "videos_kwargs", "audio_kwargs")
def _flat_processor_kwargs(
mm_kwargs: Mapping[str, object],
downsample_mode: str,
) -> dict[str, object]:
"""Drop the modality-scoped nesting and pin the resolved downsample mode.
The placeholder count and the vision tower must read the same mode, so the
value resolved for this modality is passed down explicitly rather than
leaving the processor to fall back to its own default.
"""
kwargs = {
key: value
for key, value in mm_kwargs.items()
if key not in _MODALITY_SCOPED_MM_KWARGS
}
kwargs["downsample_mode"] = downsample_mode
return kwargs
_MODALITY_TO_MM_KWARGS_SCOPE = {
"image": "images_kwargs",
"video": "videos_kwargs",
}
def _scoped_mm_kwarg(
merged: Mapping[str, object],
modality: str,
key: str,
) -> Any:
"""Read a processor kwarg for one modality.
``merged`` is what ``_merge_and_resolve_mm_processor_kwargs`` returns: it
routes a flat key into every scope whose processor declares it, and a value
scoped to the other modality stays there, so it is not visible here.
"""
scoped = merged.get(_MODALITY_TO_MM_KWARGS_SCOPE[modality])
if isinstance(scoped, Mapping):
return scoped.get(key)
return None
class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
def _resolve_downsample_mode(
self,
mm_kwargs: Mapping[str, object],
modality: str = "image",
) -> str:
ds = mm_kwargs.get("downsample_mode")
if ds is not None:
return str(ds)
# A flat `downsample_mode` applies to both modalities; the nested
# `images_kwargs` / `videos_kwargs` forms scope it to one, matching
# MiniCPMV4_6Processor.
info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
downsample_mode = _scoped_mm_kwarg(merged, modality, "downsample_mode")
if downsample_mode is not None:
return str(downsample_mode)
return info._get_downsample_mode()
def _resolve_max_slice_nums(
self,
mm_kwargs: Mapping[str, object],
modality: str = "image",
) -> int | None:
# Per-request max_slice_nums. None keeps the processor default.
# Resolved like `downsample_mode` so the nested per-modality form
# (`images_kwargs` / `videos_kwargs`) is honoured too. This must match
# what `process_images` passes to the image processor: the placeholder
# count and the visual token count are derived from it separately, so
# they diverge (and the engine aborts) if the two sides read different
# values.
info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
max_slice = _scoped_mm_kwarg(merged, modality, "max_slice_nums")
if max_slice is None:
return None
return int(max_slice)
def get_image_prompt_texts(
self,
image_size,
image_idx: int = 0,
downsample_mode: str | None = None,
max_slice_nums: int | None = None,
) -> str:
info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
return info.get_slice_image_placeholder(
image_size,
image_idx=image_idx,
max_slice_nums=max_slice_nums,
downsample_mode=downsample_mode,
)
def _video_local_id_prefix(self, video_idx: int) -> str:
"""`<image_id>{idx}</image_id>` prefix emitted in front of a video placeholder.
MiniCPM-V 4.6 numbers each video with a local id, matching
MiniCPMV4_6Processor. 4.7 treats a video as one temporal sequence and is
trained without these ids, so it overrides this to return an empty string.
"""
tokenizer = self.info.get_tokenizer()
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
return f"{id_start}{video_idx}{id_end}"
def get_video_prompt_texts(
self,
image_size,
@@ -144,8 +296,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
image_end = getattr(tokenizer, "image_end_token", "</image>")
slice_start = getattr(tokenizer, "slice_start_token", "<slice>")
slice_end = getattr(tokenizer, "slice_end_token", "</slice>")
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
per_frame = image_start + video_token * source_tokens + image_end
if grids[0] > 0 and grids[1] > 0 and patch_tokens > 0:
@@ -154,7 +304,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
per_frame += "\n".join(rows)
body = per_frame * num_frames
return f"{id_start}{video_idx}{id_end}" + body
return self._video_local_id_prefix(video_idx) + body
def process_images(
self,
@@ -179,10 +329,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
n_images = len(parsed_images)
image_processor = self.info.get_image_processor()
patch_size = image_processor.patch_size
ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="image")
image_proc_kwargs = _flat_processor_kwargs(mm_kwargs, ds_mode)
# `_flat_processor_kwargs` drops the nested form, so the value resolved
# from it has to be pinned back for the processor and the prompt text
# to agree on the slice count.
image_max_slice = self._resolve_max_slice_nums(mm_kwargs, modality="image")
if image_max_slice is not None:
image_proc_kwargs["max_slice_nums"] = image_max_slice
per_image_pixel_values: list[torch.Tensor] = []
per_image_tgt_sizes: list[torch.Tensor] = []
for image in parsed_images:
ip_out = image_processor([image], **mm_kwargs)
ip_out = image_processor([image], **image_proc_kwargs)
pv = ip_out["pixel_values"] # (1, C, P, sum_W)
ts = ip_out["target_sizes"] # (n_slices, 2)
if pv.ndim == 4 and pv.shape[0] == 1:
@@ -210,7 +368,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
"tgt_sizes": per_image_tgt_sizes,
}
ds_mode = self._resolve_downsample_mode(mm_kwargs)
insert_layer_id = getattr(
self.info.get_hf_config(),
"insert_layer_id",
@@ -245,7 +402,11 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
image_processor = self.info.get_image_processor()
patch_size = image_processor.patch_size
video_max_slice = self.info.get_video_max_slice_num()
video_mm_kwargs = {**mm_kwargs, "max_slice_nums": video_max_slice}
video_ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="video")
video_mm_kwargs = {
**_flat_processor_kwargs(mm_kwargs, video_ds_mode),
"max_slice_nums": video_max_slice,
}
per_video_pixel_values: list[torch.Tensor] = []
per_video_tgt_sizes: list[torch.Tensor] = []
@@ -312,10 +473,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
if not per_video_pixel_values:
return {}
insert_layer_id = getattr(self.info.get_hf_config(), "insert_layer_id", -1)
merger_flag = video_ds_mode != "4x" and insert_layer_id >= 0
return {
"video_pixel_values": per_video_pixel_values,
"video_tgt_sizes": per_video_tgt_sizes,
"video_image_sizes": per_video_image_sizes,
"video_use_vit_merger": [
torch.tensor([merger_flag], dtype=torch.bool)
for _ in per_video_pixel_values
],
}
def _get_prompt_updates(
@@ -324,7 +492,13 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs,
):
ds_mode = self._resolve_downsample_mode(hf_processor_mm_kwargs)
image_ds_mode = self._resolve_downsample_mode(
hf_processor_mm_kwargs, modality="image"
)
video_ds_mode = self._resolve_downsample_mode(
hf_processor_mm_kwargs, modality="video"
)
image_max_slice = self._resolve_max_slice_nums(hf_processor_mm_kwargs)
placeholders = [
("image", self.info.image_pattern),
@@ -360,7 +534,8 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self.get_image_prompt_texts(
image_size,
item_idx,
downsample_mode=ds_mode,
downsample_mode=image_ds_mode,
max_slice_nums=image_max_slice,
),
add_special_tokens=False,
),
@@ -389,7 +564,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
downsample_mode=video_ds_mode,
video_idx=item_idx,
),
add_special_tokens=False,
@@ -409,7 +584,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self.get_video_prompt_texts(
frame_size,
num_frames,
downsample_mode=ds_mode,
downsample_mode=video_ds_mode,
video_idx=item_idx,
),
add_special_tokens=False,
@@ -437,31 +612,28 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
new_update = super()._recompute_cached_prompt_update(
cached_update, new_item_idx
)
# MiniCPM-V 4.6 prefixes video placeholders with `<image_id>{idx}</image_id>`
# (the base class only rewrites the image modality).
# Video placeholders carry an `<image_id>{idx}</image_id>` prefix that
# the base class does not rewrite. 4.7 has no prefix, so this is a
# no-op there.
if cached_update.modality == "video":
tokenizer = self.info.get_tokenizer()
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
old_prefix = self._video_local_id_prefix(cached_update.item_idx)
new_prefix = self._video_local_id_prefix(new_item_idx)
if old_prefix != new_prefix:
tokenizer = self.info.get_tokenizer()
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
text = tokenizer.decode(cached_update.content.full)
prev_item_idx = cached_update.item_idx
text = tokenizer.decode(cached_update.content.full)
new_update = new_update.with_content(
PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
text.replace(
f"{id_start}{prev_item_idx}{id_end}",
f"{id_start}{new_item_idx}{id_end}",
1,
new_update = new_update.with_content(
PromptUpdateDetails.select_token_ids(
cached_encode(
tokenizer,
text.replace(old_prefix, new_prefix, 1),
add_special_tokens=False,
),
add_special_tokens=False,
),
cached_encode(tokenizer, video_token, add_special_tokens=False),
cached_encode(tokenizer, video_token, add_special_tokens=False),
)
)
)
return new_update
def _get_mm_fields_config(
@@ -676,6 +848,31 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
return source_tokens + grids[0] * grids[1] * patch_tokens
def _stack_vit_merger_qkv(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[tuple[str, torch.Tensor]]:
"""Fold separate vit_merger q/k/v projections into the fused qkv_proj module.
HF checkpoints store the window-attention projections separately, while
vLLM keeps them fused. The shard_id mechanism on the submodule's
WeightsMapper cannot do this: AutoWeightsLoader still recurses over every
incoming key after calling a submodule's load_weights, so a "k_proj" key
with no matching child raises before the mapping is honoured.
"""
pairs = (
("vit_merger.self_attn.q_proj.", "q"),
("vit_merger.self_attn.k_proj.", "k"),
("vit_merger.self_attn.v_proj.", "v"),
)
for name, tensor in weights:
for src, shard in pairs:
if src in name:
name = name.replace(src, "vit_merger.self_attn.qkv_proj.", 1)
tensor.shard_id = shard
break
yield name, tensor
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
def __init__(
self,
@@ -957,6 +1154,7 @@ class MiniCPMV4_6ForConditionalGeneration(
SupportsPP,
HasInnerState,
IsHybrid,
SupportsEagle3,
SupportsMRoPE,
):
supports_encoder_tp_data = True
@@ -1031,7 +1229,7 @@ class MiniCPMV4_6ForConditionalGeneration(
apply_encoder_attention_mask=True,
prefix=maybe_prefix(prefix, "vpm"),
)
if config.drop_vision_last_layer:
if self._drop_vision_last_layer():
self.vpm.encoder.layers = self.vpm.encoder.layers[:-1]
self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger(
@@ -1062,6 +1260,14 @@ class MiniCPMV4_6ForConditionalGeneration(
self.language_model.make_empty_intermediate_tensors
)
def _drop_vision_last_layer(self) -> bool:
"""Whether the last vision encoder layer is dropped.
MiniCPM-V 4.6 checkpoints always define this. 4.7's config no longer
carries the field, so it overrides this.
"""
return self.config.drop_vision_last_layer
# ----- Multimodal parsing -----
def _parse_and_validate_vision_input(
@@ -1174,35 +1380,35 @@ class MiniCPMV4_6ForConditionalGeneration(
if image_input["type"] == "image_embeds":
return image_input["image_embeds"]
downsample_mode = None
if use_vit_merger is not None:
downsample_mode = "16x" if use_vit_merger else "4x"
image_features = self.get_vision_hidden_states(
image_input,
downsample_mode=downsample_mode,
)
num_slices = image_input["num_slices"]
results = []
idx = 0
for n in num_slices.tolist():
group = image_features[idx : idx + n]
results.append(torch.cat(group, dim=0))
idx += n
return results
num_slices = image_input["num_slices"].tolist()
flags = _expand_flags(use_vit_merger, len(num_slices))
# The merger flag is per request, but the model runner may batch items
# from several concurrent requests. Encoding such a mixed batch would
# apply one request's downsample mode to another request's items, so
# group by flag and encode each group separately.
encoded: dict[int, torch.Tensor] = {}
for flag in dict.fromkeys(flags):
item_indices = [i for i, f in enumerate(flags) if f == flag]
group_input = _select_vision_items(image_input, item_indices, num_slices)
group_features = self.get_vision_hidden_states(
group_input,
downsample_mode=_vision_downsample_mode(flag),
)
idx = 0
for i in item_indices:
n = num_slices[i]
encoded[i] = torch.cat(group_features[idx : idx + n], dim=0)
idx += n
return [encoded[i] for i in range(len(num_slices))]
# ----- Multimodal embedding interface -----
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
use_vit_merger_tensors = kwargs.pop("use_vit_merger", None)
use_vit_merger = None
if use_vit_merger_tensors is not None:
if isinstance(use_vit_merger_tensors, torch.Tensor):
use_vit_merger = bool(use_vit_merger_tensors.any().item())
elif isinstance(use_vit_merger_tensors, list | tuple):
use_vit_merger = any(
bool(t.any().item()) if isinstance(t, torch.Tensor) else bool(t)
for t in use_vit_merger_tensors
)
use_vit_merger = kwargs.pop("use_vit_merger", None)
video_use_vit_merger = kwargs.pop("video_use_vit_merger", None)
image_kwargs = {
k: v
@@ -1235,7 +1441,7 @@ class MiniCPMV4_6ForConditionalGeneration(
multimodal_embeddings += tuple(
self._process_vision_input(
video_input,
use_vit_merger=use_vit_merger,
use_vit_merger=video_use_vit_merger,
)
)
@@ -1298,7 +1504,10 @@ class MiniCPMV4_6ForConditionalGeneration(
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
return loader.load_weights(
_stack_vit_merger_qkv(weights),
mapper=self.hf_to_vllm_mapper,
)
def get_mm_mapping(self) -> MultiModelKeys:
return MultiModelKeys.from_string_field(
+764
View File
@@ -0,0 +1,764 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only MiniCPM-V 4.7 model.
Same vision/LLM stack as MiniCPM-V 4.6, plus canvas 3D M-RoPE.
"""
import math
from collections.abc import Mapping
from typing import Any, NamedTuple
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig
from .minicpmv import MiniCPMVDummyInputsBuilder
from .minicpmv4_6 import (
MiniCPMV4_6ForConditionalGeneration,
MiniCPMV4_6MultiModalProcessor,
MiniCPMV4_6ProcessingInfo,
_scoped_mm_kwarg,
)
logger = init_logger(__name__)
class MiniCPMV4_7ProcessingInfo(MiniCPMV4_6ProcessingInfo):
def _mm_max_slice_nums(self, **kwargs: object) -> int | None:
# No schema resolution here: it would call back into `get_hf_processor`
# below, which resolves this same value.
merged = self.ctx.get_merged_mm_kwargs(kwargs)
max_slice = _scoped_mm_kwarg(merged, "image", "max_slice_nums")
if max_slice is None:
max_slice = merged.get("max_slice_nums")
if max_slice is None:
return None
return int(max_slice)
def _configured_max_slice_nums(self) -> int | None:
# Configured (server-level) value only: per-request values are passed
# explicitly to the image processor call, so they must not be written
# back to the shared processor instance.
return self._mm_max_slice_nums()
def get_hf_processor(self, **kwargs: object):
hf_processor = super().get_hf_processor(**kwargs)
image_processor = getattr(hf_processor, "image_processor", None)
configured = self._configured_max_slice_nums()
if image_processor is not None and configured is not None:
image_processor.max_slice_nums = configured
return hf_processor
def get_image_max_slice_num(self) -> int:
max_slice = self._configured_max_slice_nums()
if max_slice is not None:
return max_slice
return super().get_image_max_slice_num()
class MiniCPMV4_7MultiModalProcessor(MiniCPMV4_6MultiModalProcessor):
def _get_mm_fields_config(
self,
hf_inputs,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
fields = dict(super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs))
# Canvas M-RoPE places slices from `tgt_sizes`. Stripping the payload of
# a prefix-cache-covered item only keeps CPU-side metadata, so without
# this a cached image loses its target sizes, `image_bounds` and
# `target_sizes` disagree, and the whole request silently falls back to
# sequential positions.
for key, modality in (("tgt_sizes", "image"), ("video_tgt_sizes", "video")):
if key in fields:
fields[key] = MultiModalFieldConfig.batched(modality, keep_on_cpu=True)
return fields
def _video_local_id_prefix(self, video_idx: int) -> str:
# MiniCPMV4_7Processor emits no local `<image_id>` in front of video
# placeholders: a video is one temporal sequence and 4.7 was trained
# without them.
return ""
def get_image_prompt_texts(
self,
image_size,
image_idx: int = 0,
downsample_mode: str | None = None,
max_slice_nums: int | None = None,
) -> str:
info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
if max_slice_nums is None:
max_slice_nums = info.get_image_max_slice_num()
return info.get_slice_image_placeholder(
image_size,
image_idx=image_idx,
max_slice_nums=max_slice_nums,
downsample_mode=downsample_mode,
)
def _as_hw_pairs(tensor: torch.Tensor) -> torch.Tensor:
data = tensor.to(torch.long)
if data.numel() == 0:
return data.reshape(0, 2)
if data.shape[-1] != 2:
raise ValueError(f"expected last dim 2, got {tuple(data.shape)}")
return data.reshape(-1, 2)
def _sequential_positions(seq_len: int) -> torch.Tensor:
"""Positions for the fallback where canvas M-RoPE cannot be computed."""
return torch.arange(seq_len).unsqueeze(0).expand(3, -1)
@MULTIMODAL_REGISTRY.register_processor(
MiniCPMV4_7MultiModalProcessor,
info=MiniCPMV4_7ProcessingInfo,
dummy_inputs=MiniCPMVDummyInputsBuilder,
)
class MiniCPMV4_7ForConditionalGeneration(MiniCPMV4_6ForConditionalGeneration):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
self._init_canvas_mrope(vllm_config)
def _drop_vision_last_layer(self) -> bool:
# MiniCPMV4_7Config no longer carries this field (transformers marks it
# removed), and 4.7 keeps every vision encoder layer.
return False
def _init_canvas_mrope(self, vllm_config: VllmConfig) -> None:
config = self.config
uses_canvas = bool(getattr(config, "uses_mrope_canvas", False))
if not uses_canvas:
mrope_mode = getattr(config, "mrope_mode", None)
uses_canvas = mrope_mode is not None and "canvas" in str(mrope_mode).lower()
self._uses_canvas_mrope = uses_canvas
self._canvas_special_ids: dict[str, int] | None = None
if not uses_canvas:
return
# MiniCPMV4_7Config pins the structural ids in config.json; the
# conversion script resolves them from the tokenizer. Prefer them so the
# canvas matches training, and fall back to the tokenizer for checkpoints
# whose config predates these fields.
configured_ids: dict[str, int | None] = {
"im_start_id": getattr(config, "image_start_id", None),
"im_end_id": getattr(config, "image_end_id", None),
"slice_start_id": getattr(config, "slice_start_id", None),
"slice_end_id": getattr(config, "slice_end_id", None),
"newline_id": getattr(config, "newline_id", None),
}
if all(value is not None for value in configured_ids.values()):
self._canvas_special_ids = {
name: int(value)
for name, value in configured_ids.items()
if value is not None
}
return
from vllm.tokenizers.registry import get_tokenizer
model_config = vllm_config.model_config
tokenizer = get_tokenizer(
model_config.tokenizer,
revision=model_config.tokenizer_revision,
trust_remote_code=model_config.trust_remote_code,
)
self._canvas_special_ids = {
"im_start_id": tokenizer.convert_tokens_to_ids(
getattr(tokenizer, "image_start_token", "<image>")
),
"im_end_id": tokenizer.convert_tokens_to_ids(
getattr(tokenizer, "image_end_token", "</image>")
),
"slice_start_id": tokenizer.convert_tokens_to_ids(
getattr(tokenizer, "slice_start_token", "<slice>")
),
"slice_end_id": tokenizer.convert_tokens_to_ids(
getattr(tokenizer, "slice_end_token", "</slice>")
),
"newline_id": tokenizer.encode("\n", add_special_tokens=False)[0],
}
def get_mrope_input_positions(
self,
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec],
) -> tuple[torch.Tensor, int]:
seq_len = len(input_tokens)
if (
not getattr(self, "_uses_canvas_mrope", False)
or self._canvas_special_ids is None
or not mm_features
):
return _sequential_positions(seq_len), 0
input_ids = torch.tensor(input_tokens, dtype=torch.long)
image_bounds = build_image_bounds(input_ids, self._canvas_special_ids)
target_sizes_list: list[torch.Tensor] = []
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
if mm_feature.data is None:
continue
for key in ("tgt_sizes", "video_tgt_sizes"):
tgt = mm_feature.data.get(key)
if tgt is None or not isinstance(tgt.data, torch.Tensor):
continue
try:
target_sizes_list.append(_as_hw_pairs(tgt.data))
except ValueError:
# Not an (N, 2) size table. The bounds/sizes match below
# turns this into a sequential fallback.
continue
if not target_sizes_list or image_bounds.numel() == 0:
logger.warning(
"MiniCPM-V 4.7 canvas M-RoPE missing inputs: "
"tgt_chunks=%s image_bounds=%s; using sequential positions.",
len(target_sizes_list),
tuple(image_bounds.shape) if image_bounds is not None else None,
)
return _sequential_positions(seq_len), 0
target_sizes = torch.cat(target_sizes_list, dim=0)
if target_sizes.ndim != 2 or target_sizes.shape[-1] != 2:
logger.warning(
"MiniCPM-V 4.7 canvas M-RoPE unexpected tgt_sizes shape %s; "
"using sequential positions.",
tuple(target_sizes.shape),
)
return _sequential_positions(seq_len), 0
if image_bounds.shape[0] != target_sizes.shape[0]:
logger.warning(
"MiniCPM-V 4.7 canvas M-RoPE size mismatch: "
"image_bounds=%s target_sizes=%s; using sequential positions.",
tuple(image_bounds.shape),
tuple(target_sizes.shape),
)
return _sequential_positions(seq_len), 0
try:
pos3d = _compute_canvas_single(
input_ids,
torch.arange(seq_len, dtype=torch.long),
image_bounds,
target_sizes,
self._canvas_special_ids,
)
except Exception:
logger.warning(
"MiniCPM-V 4.7 canvas M-RoPE failed; "
"falling back to sequential positions.",
exc_info=True,
)
return _sequential_positions(seq_len), 0
return pos3d, canvas_rope_delta(pos3d, seq_len)
# Canvas 3D M-RoPE: position ids for the slice canvas.
def compute_llm_grid(
n_tokens: int, vision_grid_height: int, vision_grid_width: int
) -> tuple[int, int]:
if n_tokens <= 0 or vision_grid_height <= 0 or vision_grid_width <= 0:
return 0, 0
vit_total = vision_grid_height * vision_grid_width
if vit_total == n_tokens:
return vision_grid_height, vision_grid_width
if vit_total < n_tokens:
return 0, 0
factor = int(round(math.sqrt(vit_total / n_tokens)))
if (
factor > 0
and vision_grid_height % factor == 0
and vision_grid_width % factor == 0
):
height, width = vision_grid_height // factor, vision_grid_width // factor
if height * width == n_tokens:
return height, width
ratio = vision_grid_height / vision_grid_width
width = max(1, int(round(math.sqrt(n_tokens / ratio))))
height = n_tokens // width
if height * width == n_tokens and height > 0:
return height, width
return 0, 0
def build_image_bounds(
input_ids: torch.LongTensor, special_token_ids: dict
) -> torch.LongTensor:
start_ids = [
x
for x in (
special_token_ids.get("im_start_id"),
special_token_ids.get("slice_start_id"),
)
if x is not None
]
end_ids = [
x
for x in (
special_token_ids.get("im_end_id"),
special_token_ids.get("slice_end_id"),
)
if x is not None
]
if not start_ids or not end_ids:
return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
start_cond = torch.zeros_like(input_ids, dtype=torch.bool)
end_cond = torch.zeros_like(input_ids, dtype=torch.bool)
for sid in start_ids:
start_cond |= input_ids == sid
for eid in end_ids:
end_cond |= input_ids == eid
starts = torch.where(start_cond)[0] + 1
ends = torch.where(end_cond)[0]
n = min(len(starts), len(ends))
if n == 0:
return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
return torch.stack([starts[:n], ends[:n]], dim=-1)
def _compute_canvas_single(
input_ids, position_ids_2d, image_bound, target_sizes, special_token_ids
):
# Inference always handles one unpadded sequence. The packed (cu_seqlens)
# form below is kept only to mirror the transformers reference
# implementation of its canvas position computation.
seq_len = input_ids.shape[0]
cu_seqlens = torch.tensor([0, seq_len], device=input_ids.device, dtype=torch.long)
return _compute_canvas_packed(
position_ids_2d.unsqueeze(0),
cu_seqlens,
image_bound,
target_sizes,
input_ids.unsqueeze(0),
special_token_ids,
)[:, 0, :]
def _target_hw(target_sizes: torch.Tensor, index: int) -> tuple[int, int]:
"""Patch-grid height/width recorded for one image or slice."""
return int(target_sizes[index, 0]), int(target_sizes[index, 1])
class _CanvasLayout(NamedTuple):
"""Grid of one canvas measured in LLM tokens.
A canvas is a thumbnail plus the slices cut from it, tiled row-major.
"""
thumb_h: int
thumb_w: int
"""Thumbnail grid."""
slice_h: int
slice_w: int
"""Per-slice grid. Zero when the canvas has no slices."""
cols: int
"""Slices sharing a canvas row. Zero when the canvas has no slices."""
height: int
width: int
"""Assembled canvas."""
def _canvas_layout(
thumbnail: tuple[int, int, int],
slices: list[tuple[int, int, int]],
target_sizes: torch.Tensor,
) -> _CanvasLayout:
"""Measure the canvas for one ``(thumbnail, slices)`` group.
``thumbnail`` and each slice are ``(start, end, target_index)`` token spans.
"""
t_bs, t_be, t_gi = thumbnail
thumb_h, thumb_w = compute_llm_grid(t_be - t_bs, *_target_hw(target_sizes, t_gi))
if not slices:
return _CanvasLayout(
thumb_h, thumb_w, 0, 0, 0, max(thumb_h, 1), max(thumb_w, 1)
)
cols = len(slices)
for k in range(len(slices) - 1):
if slices[k + 1][0] - slices[k][1] > 2:
cols = k + 1
break
rows = len(slices) // cols if cols > 0 else 1
if rows * cols != len(slices):
rows, cols = 1, len(slices)
s0_bs, s0_be, s0_gi = slices[0]
slice_h, slice_w = compute_llm_grid(s0_be - s0_bs, *_target_hw(target_sizes, s0_gi))
return _CanvasLayout(
thumb_h, thumb_w, slice_h, slice_w, cols, rows * slice_h, cols * slice_w
)
def _assign_canvas_span(
pos3d: torch.Tensor,
*,
thumbnail: tuple[int, int, int],
span_end: int,
slices: list[tuple[int, int, int]],
base: int,
target_sizes: torch.Tensor,
) -> int:
"""Place one canvas at ``base``, writing its 3-D positions into ``pos3d``.
Shared by the video-frame and single-image paths; they differ only in how
the per-canvas base advances, not in the geometry placed here.
Mutates ``pos3d`` in place, as the packed canvas computation has always
done. Returns the distance to the next canvas base, so each caller keeps
only its own base advancement.
"""
t_bs, t_be, _ = thumbnail
device = pos3d.device
span_start = t_bs - 1 # the <im_start> token
n_thumb = t_be - t_bs
layout = _canvas_layout(thumbnail, slices, target_sizes)
cols, canvas_H, canvas_W = layout.cols, layout.height, layout.width
# base coat
pos3d[0, 0, span_start:span_end] = base
pos3d[1, 0, span_start:span_end] = base
pos3d[2, 0, span_start:span_end] = base
# <im_start> -> halo
halo_lo = max(base - 1, 0)
pos3d[1, 0, span_start] = halo_lo
pos3d[2, 0, span_start] = halo_lo
# <im_end> -> halo (right after thumbnail)
if t_be < span_end:
pos3d[1, 0, t_be] = base + canvas_H
pos3d[2, 0, t_be] = base + canvas_W
# thumbnail visual tokens
llm_th, llm_tw = layout.thumb_h, layout.thumb_w
if llm_th > 0 and llm_tw > 0 and llm_th * llm_tw == n_thumb:
if slices and canvas_H > 0 and canvas_W > 0:
h_c = torch.linspace(0, canvas_H - 1, llm_th, device=device).round().long()
w_c = torch.linspace(0, canvas_W - 1, llm_tw, device=device).round().long()
else:
h_c = torch.arange(llm_th, device=device)
w_c = torch.arange(llm_tw, device=device)
h_idx = h_c.view(-1, 1).expand(-1, llm_tw).reshape(-1)
w_idx = w_c.view(1, -1).expand(llm_th, -1).reshape(-1)
pos3d[0, 0, t_bs:t_be] = base
pos3d[1, 0, t_bs:t_be] = h_idx + base
pos3d[2, 0, t_bs:t_be] = w_idx + base
else:
fallback = torch.arange(n_thumb, device=device, dtype=torch.long) + base
for d in range(3):
pos3d[d, 0, t_bs:t_be] = fallback
# slice visual tokens + slice specials
for k, (s_bs, s_be, s_gi) in enumerate(slices):
n_s = s_be - s_bs
sh_k, sw_k = compute_llm_grid(n_s, *_target_hw(target_sizes, s_gi))
h_off = (k // cols) * layout.slice_h
w_off = (k % cols) * layout.slice_w
slice_start_pos = s_bs - 1
if slice_start_pos >= span_start:
pos3d[1, 0, slice_start_pos] = base + h_off
pos3d[2, 0, slice_start_pos] = base + w_off
slice_end_pos = s_be
if sh_k > 0 and sw_k > 0:
end_h = h_off + sh_k - 1
end_w = w_off + sw_k - 1
else:
end_h = h_off
end_w = w_off
if slice_end_pos < span_end:
pos3d[1, 0, slice_end_pos] = base + end_h
pos3d[2, 0, slice_end_pos] = base + end_w
if sh_k > 0 and sw_k > 0 and sh_k * sw_k == n_s:
h_idx = (
torch.arange(sh_k, device=device)
.view(-1, 1)
.expand(-1, sw_k)
.reshape(-1)
)
w_idx = (
torch.arange(sw_k, device=device)
.view(1, -1)
.expand(sh_k, -1)
.reshape(-1)
)
pos3d[0, 0, s_bs:s_be] = base
pos3d[1, 0, s_bs:s_be] = h_idx + h_off + base
pos3d[2, 0, s_bs:s_be] = w_idx + w_off + base
else:
fallback = torch.arange(n_s, device=device, dtype=torch.long) + base
for d in range(3):
pos3d[d, 0, s_bs:s_be] = fallback
# \n between slice rows -> W = right edge + 1.
# H stays within the ended row (does NOT cross to next row): the \n sits at
# the right-outside of the row that just ended, at H = that row's last row
# and W = one column past the rightmost slice.
for k in range(len(slices) - 1):
gap_s = slices[k][1]
gap_e = slices[k + 1][0]
if gap_e - gap_s <= 2:
continue
boundary_h = ((k // cols) + 1) * layout.slice_h - 1
right_edge_w = cols * layout.slice_w
for nl_pos in range(gap_s + 1, gap_e - 1):
pos3d[1, 0, nl_pos] = base + boundary_h
pos3d[2, 0, nl_pos] = base + right_edge_w
return max(canvas_H, canvas_W) + 1
def _compute_canvas_packed(
position_ids_2d,
cu_seqlens,
image_bound,
target_sizes,
input_ids,
special_token_ids,
):
if position_ids_2d.ndim == 1:
position_ids_2d = position_ids_2d.unsqueeze(0)
if input_ids.ndim == 1:
input_ids = input_ids.unsqueeze(0)
has_images = (
isinstance(image_bound, torch.Tensor)
and image_bound.numel() > 0
and isinstance(target_sizes, torch.Tensor)
and target_sizes.numel() > 0
)
if not has_images:
return position_ids_2d.unsqueeze(0).expand(3, -1, -1).contiguous()
device = position_ids_2d.device
pos3d = position_ids_2d.unsqueeze(0).expand(3, -1, -1).clone()
ids_flat = input_ids.view(-1)
im_start_id = special_token_ids.get("im_start_id")
im_end_id = special_token_ids.get("im_end_id")
slice_start_id = special_token_ids.get("slice_start_id")
slice_end_id = special_token_ids.get("slice_end_id")
structural_ids = {
v
for v in (im_start_id, im_end_id, slice_start_id, slice_end_id)
if v is not None
}
num_seqs = len(cu_seqlens) - 1
img_ptr = 0
num_imgs = len(image_bound)
newline_id = special_token_ids.get("newline_id")
gap_ok_ids = structural_ids | ({newline_id} if newline_id is not None else set())
def _group_end_pos(g, s_end):
"""First position after a group's trailing structural tokens.
Matches the transformers reference, which closes a span over every
marker in ``markers`` -- newlines included. The chat template joins
content parts with ``\\n``, so an image followed by text always ends with
one; leaving it out moves that newline out of the canvas and pushes
every later position one step forward.
"""
le = g["slices"][-1][1] if g["slices"] else g["thumbnail"][1]
ge = le
while ge < s_end and ids_flat[ge].item() in gap_ok_ids:
ge += 1
return min(ge, s_end)
def _is_tight_gap(g_prev, g_next, s_end):
"""Check if only structural/nl tokens sit between two groups."""
ge = _group_end_pos(g_prev, s_end)
ns = g_next["thumbnail"][0] - 1 # im_start of next group
return all(ids_flat[p].item() in gap_ok_ids for p in range(ge, ns))
for si in range(num_seqs):
s_start = cu_seqlens[si].item()
s_end = cu_seqlens[si + 1].item()
if s_start >= s_end:
continue
seq_imgs = []
while img_ptr < num_imgs:
bs_ = image_bound[img_ptr, 0].item()
be_ = image_bound[img_ptr, 1].item()
if bs_ >= s_start and be_ <= s_end:
seq_imgs.append((bs_, be_, img_ptr))
img_ptr += 1
else:
break
if not seq_imgs:
continue
# ---- group images: thumbnail + following slices ----
# merge consecutive no-slice frames into one video group
raw_groups = []
cur_group: dict[str, Any] | None = None
for bs_, be_, gi in seq_imgs:
marker_pos = bs_ - 1
is_slice = (
slice_start_id is not None
and marker_pos >= s_start
and ids_flat[marker_pos].item() == slice_start_id
)
if is_slice and cur_group is not None:
cur_group["slices"].append((bs_, be_, gi))
else:
if cur_group is not None:
raw_groups.append(cur_group)
cur_group = {"thumbnail": (bs_, be_, gi), "slices": []}
if cur_group is not None:
raw_groups.append(cur_group)
# merge consecutive tightly-adjacent groups into video groups.
# Two groups are "tightly adjacent" if the gap between them
# contains only structural tokens and \n (no real text).
# This handles both no-slice frames and frames-with-slices.
groups = []
i_g = 0
while i_g < len(raw_groups):
video_frames = [raw_groups[i_g]]
j_g = i_g + 1
while j_g < len(raw_groups) and _is_tight_gap(
raw_groups[j_g - 1], raw_groups[j_g], s_end
):
video_frames.append(raw_groups[j_g])
j_g += 1
if len(video_frames) == 1:
groups.append(raw_groups[i_g])
else:
groups.append({"video_frames": video_frames})
i_g = j_g
# ---- assign 3-D positions ----
pos = 0
cursor = s_start
for group in groups:
is_video_group = "video_frames" in group
if is_video_group:
# === Video group: multiple frames merged ===
# Each frame gets its own base_f so different frames have
# distinct spatial positions. Inter-frame gap tokens (\n
# etc.) are pure 1D text. After each gap, pos advances
# by 1 extra so halo_lo of the next frame does not
# collide with the last gap token.
frames = group["video_frames"]
first_frame = frames[0]
last_frame = frames[-1]
# group_start = <im_start> of first frame
group_start = first_frame["thumbnail"][0] - 1
# group_end = after last structural token of last frame
group_end = _group_end_pos(last_frame, s_end)
# text before video group
text_len = group_start - cursor
if text_len > 0:
t_pos = (
torch.arange(text_len, device=device, dtype=torch.long) + pos
)
for d in range(3):
pos3d[d, 0, cursor:group_start] = t_pos
pos += text_len
# process each frame with its own per-frame base
frame_cursor = group_start
for vf in frames:
t_bs = vf["thumbnail"][0]
vf_slices = vf["slices"]
im_start_pos = t_bs - 1
# --- inter-frame gap: pure 1D text T=H=W=pos ---
gap_len = im_start_pos - frame_cursor
if gap_len > 0:
t_pos = (
torch.arange(gap_len, device=device, dtype=torch.long) + pos
)
for d in range(3):
pos3d[d, 0, frame_cursor:im_start_pos] = t_pos
pos += gap_len
pos += 1 # buffer: prevents halo_lo collision
# --- per-frame base ---
base_f = pos
frame_end = _group_end_pos(vf, group_end)
pos = base_f + _assign_canvas_span(
pos3d,
thumbnail=vf["thumbnail"],
span_end=frame_end,
slices=vf_slices,
base=base_f,
target_sizes=target_sizes,
)
frame_cursor = frame_end
# trailing tokens after last frame (if any)
if frame_cursor < group_end:
trail_len = group_end - frame_cursor
t_pos = (
torch.arange(trail_len, device=device, dtype=torch.long) + pos
)
for d in range(3):
pos3d[d, 0, frame_cursor:group_end] = t_pos
pos += trail_len
cursor = group_end
else:
# === Single image group (thumbnail + optional slices) ===
slices = group["slices"]
group_start = group["thumbnail"][0] - 1
group_end = _group_end_pos(group, s_end)
# text before group
text_len = group_start - cursor
if text_len > 0:
t_pos = (
torch.arange(text_len, device=device, dtype=torch.long) + pos
)
for d in range(3):
pos3d[d, 0, cursor:group_start] = t_pos
pos += text_len
base = pos
pos = base + _assign_canvas_span(
pos3d,
thumbnail=group["thumbnail"],
span_end=group_end,
slices=slices,
base=base,
target_sizes=target_sizes,
)
cursor = group_end
# remaining text
rem = s_end - cursor
if rem > 0:
t_pos = torch.arange(rem, device=device, dtype=torch.long) + pos
for d in range(3):
pos3d[d, 0, cursor:s_end] = t_pos
return pos3d
def canvas_rope_delta(position_ids: torch.Tensor, seq_len: int) -> int:
"""Offset applied to the 1-D positions of tokens decoded after the prompt.
The first decoded token sits one past the last prefill position.
"""
return int(position_ids.amax().item()) + 1 - seq_len
+4
View File
@@ -514,6 +514,10 @@ _MULTIMODAL_MODELS = {
"minicpmv4_6",
"MiniCPMV4_6ForConditionalGeneration",
),
"MiniCPMV4_7ForConditionalGeneration": (
"minicpmv4_7",
"MiniCPMV4_7ForConditionalGeneration",
),
"Mistral3ForConditionalGeneration": (
"mistral3",
"Mistral3ForConditionalGeneration",