[Model] Support MiniCPM-V 4.7 (#58674)
Signed-off-by: tc-mb <tianchi_cai@icloud.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -45,6 +45,7 @@ from .interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
@@ -87,35 +88,186 @@ def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
fields["use_vit_merger"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
if "video_use_vit_merger" in hf_inputs:
|
||||
fields["video_use_vit_merger"] = MultiModalFieldConfig.batched(
|
||||
"video", keep_on_cpu=True
|
||||
)
|
||||
return fields
|
||||
|
||||
|
||||
def _expand_flags(value: object, num_items: int) -> list[bool | None]:
|
||||
"""Broadcast a batched per-item flag to exactly one entry per item."""
|
||||
if value is None:
|
||||
return [None] * num_items
|
||||
|
||||
raw: list[object]
|
||||
if isinstance(value, torch.Tensor):
|
||||
raw = value.reshape(-1).tolist()
|
||||
elif isinstance(value, list | tuple):
|
||||
raw = []
|
||||
for item in value:
|
||||
if isinstance(item, torch.Tensor):
|
||||
raw.extend(item.reshape(-1).tolist())
|
||||
else:
|
||||
raw.append(item)
|
||||
else:
|
||||
raw = [value]
|
||||
|
||||
if len(raw) == num_items:
|
||||
return [bool(v) for v in raw]
|
||||
if len(raw) == 1:
|
||||
return [bool(raw[0])] * num_items
|
||||
# Unexpected shape: keep one decision for the whole batch instead of
|
||||
# failing the request.
|
||||
return [any(bool(v) for v in raw)] * num_items
|
||||
|
||||
|
||||
def _vision_downsample_mode(flag: bool | None) -> str | None:
|
||||
if flag is None:
|
||||
return None
|
||||
return "16x" if flag else "4x"
|
||||
|
||||
|
||||
def _select_vision_items(
|
||||
image_input: MiniCPMVImagePixelInputs,
|
||||
item_indices: list[int],
|
||||
num_slices: list[int],
|
||||
) -> MiniCPMVImagePixelInputs:
|
||||
"""Slice a vision input down to the given items, keeping their order."""
|
||||
offsets = [0]
|
||||
for n in num_slices:
|
||||
offsets.append(offsets[-1] + n)
|
||||
|
||||
pixel_values: list[torch.Tensor] = []
|
||||
tgt_sizes: list[torch.Tensor] = []
|
||||
sizes: list[int] = []
|
||||
for i in item_indices:
|
||||
start, end = offsets[i], offsets[i + 1]
|
||||
pixel_values.extend(image_input["pixel_values"][start:end])
|
||||
tgt_sizes.append(image_input["tgt_sizes"][start:end])
|
||||
sizes.append(end - start)
|
||||
|
||||
return MiniCPMVImagePixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values,
|
||||
tgt_sizes=torch.cat(tgt_sizes, dim=0),
|
||||
num_slices=torch.tensor(sizes),
|
||||
)
|
||||
|
||||
|
||||
# HF-style modality-scoped keys inside `mm_processor_kwargs`. They scope a value
|
||||
# to a single modality and must not be forwarded to the processors, which take
|
||||
# flat arguments only.
|
||||
_MODALITY_SCOPED_MM_KWARGS = ("images_kwargs", "videos_kwargs", "audio_kwargs")
|
||||
|
||||
|
||||
def _flat_processor_kwargs(
|
||||
mm_kwargs: Mapping[str, object],
|
||||
downsample_mode: str,
|
||||
) -> dict[str, object]:
|
||||
"""Drop the modality-scoped nesting and pin the resolved downsample mode.
|
||||
|
||||
The placeholder count and the vision tower must read the same mode, so the
|
||||
value resolved for this modality is passed down explicitly rather than
|
||||
leaving the processor to fall back to its own default.
|
||||
"""
|
||||
kwargs = {
|
||||
key: value
|
||||
for key, value in mm_kwargs.items()
|
||||
if key not in _MODALITY_SCOPED_MM_KWARGS
|
||||
}
|
||||
kwargs["downsample_mode"] = downsample_mode
|
||||
return kwargs
|
||||
|
||||
|
||||
_MODALITY_TO_MM_KWARGS_SCOPE = {
|
||||
"image": "images_kwargs",
|
||||
"video": "videos_kwargs",
|
||||
}
|
||||
|
||||
|
||||
def _scoped_mm_kwarg(
|
||||
merged: Mapping[str, object],
|
||||
modality: str,
|
||||
key: str,
|
||||
) -> Any:
|
||||
"""Read a processor kwarg for one modality.
|
||||
|
||||
``merged`` is what ``_merge_and_resolve_mm_processor_kwargs`` returns: it
|
||||
routes a flat key into every scope whose processor declares it, and a value
|
||||
scoped to the other modality stays there, so it is not visible here.
|
||||
"""
|
||||
scoped = merged.get(_MODALITY_TO_MM_KWARGS_SCOPE[modality])
|
||||
if isinstance(scoped, Mapping):
|
||||
return scoped.get(key)
|
||||
return None
|
||||
|
||||
|
||||
class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
def _resolve_downsample_mode(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str = "image",
|
||||
) -> str:
|
||||
ds = mm_kwargs.get("downsample_mode")
|
||||
if ds is not None:
|
||||
return str(ds)
|
||||
# A flat `downsample_mode` applies to both modalities; the nested
|
||||
# `images_kwargs` / `videos_kwargs` forms scope it to one, matching
|
||||
# MiniCPMV4_6Processor.
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
downsample_mode = _scoped_mm_kwarg(merged, modality, "downsample_mode")
|
||||
if downsample_mode is not None:
|
||||
return str(downsample_mode)
|
||||
return info._get_downsample_mode()
|
||||
|
||||
def _resolve_max_slice_nums(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str = "image",
|
||||
) -> int | None:
|
||||
# Per-request max_slice_nums. None keeps the processor default.
|
||||
# Resolved like `downsample_mode` so the nested per-modality form
|
||||
# (`images_kwargs` / `videos_kwargs`) is honoured too. This must match
|
||||
# what `process_images` passes to the image processor: the placeholder
|
||||
# count and the visual token count are derived from it separately, so
|
||||
# they diverge (and the engine aborts) if the two sides read different
|
||||
# values.
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
max_slice = _scoped_mm_kwarg(merged, modality, "max_slice_nums")
|
||||
if max_slice is None:
|
||||
return None
|
||||
return int(max_slice)
|
||||
|
||||
def get_image_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
image_idx: int = 0,
|
||||
downsample_mode: str | None = None,
|
||||
max_slice_nums: int | None = None,
|
||||
) -> str:
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
return info.get_slice_image_placeholder(
|
||||
image_size,
|
||||
image_idx=image_idx,
|
||||
max_slice_nums=max_slice_nums,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
|
||||
def _video_local_id_prefix(self, video_idx: int) -> str:
|
||||
"""`<image_id>{idx}</image_id>` prefix emitted in front of a video placeholder.
|
||||
|
||||
MiniCPM-V 4.6 numbers each video with a local id, matching
|
||||
MiniCPMV4_6Processor. 4.7 treats a video as one temporal sequence and is
|
||||
trained without these ids, so it overrides this to return an empty string.
|
||||
"""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
return f"{id_start}{video_idx}{id_end}"
|
||||
|
||||
def get_video_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
@@ -144,8 +296,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
image_end = getattr(tokenizer, "image_end_token", "</image>")
|
||||
slice_start = getattr(tokenizer, "slice_start_token", "<slice>")
|
||||
slice_end = getattr(tokenizer, "slice_end_token", "</slice>")
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
|
||||
per_frame = image_start + video_token * source_tokens + image_end
|
||||
if grids[0] > 0 and grids[1] > 0 and patch_tokens > 0:
|
||||
@@ -154,7 +304,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
per_frame += "\n".join(rows)
|
||||
|
||||
body = per_frame * num_frames
|
||||
return f"{id_start}{video_idx}{id_end}" + body
|
||||
return self._video_local_id_prefix(video_idx) + body
|
||||
|
||||
def process_images(
|
||||
self,
|
||||
@@ -179,10 +329,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
n_images = len(parsed_images)
|
||||
image_processor = self.info.get_image_processor()
|
||||
patch_size = image_processor.patch_size
|
||||
ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="image")
|
||||
image_proc_kwargs = _flat_processor_kwargs(mm_kwargs, ds_mode)
|
||||
# `_flat_processor_kwargs` drops the nested form, so the value resolved
|
||||
# from it has to be pinned back for the processor and the prompt text
|
||||
# to agree on the slice count.
|
||||
image_max_slice = self._resolve_max_slice_nums(mm_kwargs, modality="image")
|
||||
if image_max_slice is not None:
|
||||
image_proc_kwargs["max_slice_nums"] = image_max_slice
|
||||
per_image_pixel_values: list[torch.Tensor] = []
|
||||
per_image_tgt_sizes: list[torch.Tensor] = []
|
||||
for image in parsed_images:
|
||||
ip_out = image_processor([image], **mm_kwargs)
|
||||
ip_out = image_processor([image], **image_proc_kwargs)
|
||||
pv = ip_out["pixel_values"] # (1, C, P, sum_W)
|
||||
ts = ip_out["target_sizes"] # (n_slices, 2)
|
||||
if pv.ndim == 4 and pv.shape[0] == 1:
|
||||
@@ -210,7 +368,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
"tgt_sizes": per_image_tgt_sizes,
|
||||
}
|
||||
|
||||
ds_mode = self._resolve_downsample_mode(mm_kwargs)
|
||||
insert_layer_id = getattr(
|
||||
self.info.get_hf_config(),
|
||||
"insert_layer_id",
|
||||
@@ -245,7 +402,11 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
image_processor = self.info.get_image_processor()
|
||||
patch_size = image_processor.patch_size
|
||||
video_max_slice = self.info.get_video_max_slice_num()
|
||||
video_mm_kwargs = {**mm_kwargs, "max_slice_nums": video_max_slice}
|
||||
video_ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="video")
|
||||
video_mm_kwargs = {
|
||||
**_flat_processor_kwargs(mm_kwargs, video_ds_mode),
|
||||
"max_slice_nums": video_max_slice,
|
||||
}
|
||||
|
||||
per_video_pixel_values: list[torch.Tensor] = []
|
||||
per_video_tgt_sizes: list[torch.Tensor] = []
|
||||
@@ -312,10 +473,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
if not per_video_pixel_values:
|
||||
return {}
|
||||
|
||||
insert_layer_id = getattr(self.info.get_hf_config(), "insert_layer_id", -1)
|
||||
merger_flag = video_ds_mode != "4x" and insert_layer_id >= 0
|
||||
|
||||
return {
|
||||
"video_pixel_values": per_video_pixel_values,
|
||||
"video_tgt_sizes": per_video_tgt_sizes,
|
||||
"video_image_sizes": per_video_image_sizes,
|
||||
"video_use_vit_merger": [
|
||||
torch.tensor([merger_flag], dtype=torch.bool)
|
||||
for _ in per_video_pixel_values
|
||||
],
|
||||
}
|
||||
|
||||
def _get_prompt_updates(
|
||||
@@ -324,7 +492,13 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs,
|
||||
):
|
||||
ds_mode = self._resolve_downsample_mode(hf_processor_mm_kwargs)
|
||||
image_ds_mode = self._resolve_downsample_mode(
|
||||
hf_processor_mm_kwargs, modality="image"
|
||||
)
|
||||
video_ds_mode = self._resolve_downsample_mode(
|
||||
hf_processor_mm_kwargs, modality="video"
|
||||
)
|
||||
image_max_slice = self._resolve_max_slice_nums(hf_processor_mm_kwargs)
|
||||
|
||||
placeholders = [
|
||||
("image", self.info.image_pattern),
|
||||
@@ -360,7 +534,8 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_image_prompt_texts(
|
||||
image_size,
|
||||
item_idx,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=image_ds_mode,
|
||||
max_slice_nums=image_max_slice,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
@@ -389,7 +564,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=video_ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
@@ -409,7 +584,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=video_ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
@@ -437,31 +612,28 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
new_update = super()._recompute_cached_prompt_update(
|
||||
cached_update, new_item_idx
|
||||
)
|
||||
# MiniCPM-V 4.6 prefixes video placeholders with `<image_id>{idx}</image_id>`
|
||||
# (the base class only rewrites the image modality).
|
||||
# Video placeholders carry an `<image_id>{idx}</image_id>` prefix that
|
||||
# the base class does not rewrite. 4.7 has no prefix, so this is a
|
||||
# no-op there.
|
||||
if cached_update.modality == "video":
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
old_prefix = self._video_local_id_prefix(cached_update.item_idx)
|
||||
new_prefix = self._video_local_id_prefix(new_item_idx)
|
||||
if old_prefix != new_prefix:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(
|
||||
f"{id_start}{prev_item_idx}{id_end}",
|
||||
f"{id_start}{new_item_idx}{id_end}",
|
||||
1,
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(old_prefix, new_prefix, 1),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
cached_encode(tokenizer, video_token, add_special_tokens=False),
|
||||
cached_encode(tokenizer, video_token, add_special_tokens=False),
|
||||
)
|
||||
)
|
||||
)
|
||||
return new_update
|
||||
|
||||
def _get_mm_fields_config(
|
||||
@@ -676,6 +848,31 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
|
||||
return source_tokens + grids[0] * grids[1] * patch_tokens
|
||||
|
||||
|
||||
def _stack_vit_merger_qkv(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Fold separate vit_merger q/k/v projections into the fused qkv_proj module.
|
||||
|
||||
HF checkpoints store the window-attention projections separately, while
|
||||
vLLM keeps them fused. The shard_id mechanism on the submodule's
|
||||
WeightsMapper cannot do this: AutoWeightsLoader still recurses over every
|
||||
incoming key after calling a submodule's load_weights, so a "k_proj" key
|
||||
with no matching child raises before the mapping is honoured.
|
||||
"""
|
||||
pairs = (
|
||||
("vit_merger.self_attn.q_proj.", "q"),
|
||||
("vit_merger.self_attn.k_proj.", "k"),
|
||||
("vit_merger.self_attn.v_proj.", "v"),
|
||||
)
|
||||
for name, tensor in weights:
|
||||
for src, shard in pairs:
|
||||
if src in name:
|
||||
name = name.replace(src, "vit_merger.self_attn.qkv_proj.", 1)
|
||||
tensor.shard_id = shard
|
||||
break
|
||||
yield name, tensor
|
||||
|
||||
|
||||
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -957,6 +1154,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
SupportsPP,
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsEagle3,
|
||||
SupportsMRoPE,
|
||||
):
|
||||
supports_encoder_tp_data = True
|
||||
@@ -1031,7 +1229,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
apply_encoder_attention_mask=True,
|
||||
prefix=maybe_prefix(prefix, "vpm"),
|
||||
)
|
||||
if config.drop_vision_last_layer:
|
||||
if self._drop_vision_last_layer():
|
||||
self.vpm.encoder.layers = self.vpm.encoder.layers[:-1]
|
||||
|
||||
self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger(
|
||||
@@ -1062,6 +1260,14 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _drop_vision_last_layer(self) -> bool:
|
||||
"""Whether the last vision encoder layer is dropped.
|
||||
|
||||
MiniCPM-V 4.6 checkpoints always define this. 4.7's config no longer
|
||||
carries the field, so it overrides this.
|
||||
"""
|
||||
return self.config.drop_vision_last_layer
|
||||
|
||||
# ----- Multimodal parsing -----
|
||||
|
||||
def _parse_and_validate_vision_input(
|
||||
@@ -1174,35 +1380,35 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
if image_input["type"] == "image_embeds":
|
||||
return image_input["image_embeds"]
|
||||
|
||||
downsample_mode = None
|
||||
if use_vit_merger is not None:
|
||||
downsample_mode = "16x" if use_vit_merger else "4x"
|
||||
image_features = self.get_vision_hidden_states(
|
||||
image_input,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
num_slices = image_input["num_slices"]
|
||||
results = []
|
||||
idx = 0
|
||||
for n in num_slices.tolist():
|
||||
group = image_features[idx : idx + n]
|
||||
results.append(torch.cat(group, dim=0))
|
||||
idx += n
|
||||
return results
|
||||
num_slices = image_input["num_slices"].tolist()
|
||||
flags = _expand_flags(use_vit_merger, len(num_slices))
|
||||
|
||||
# The merger flag is per request, but the model runner may batch items
|
||||
# from several concurrent requests. Encoding such a mixed batch would
|
||||
# apply one request's downsample mode to another request's items, so
|
||||
# group by flag and encode each group separately.
|
||||
encoded: dict[int, torch.Tensor] = {}
|
||||
for flag in dict.fromkeys(flags):
|
||||
item_indices = [i for i, f in enumerate(flags) if f == flag]
|
||||
group_input = _select_vision_items(image_input, item_indices, num_slices)
|
||||
group_features = self.get_vision_hidden_states(
|
||||
group_input,
|
||||
downsample_mode=_vision_downsample_mode(flag),
|
||||
)
|
||||
|
||||
idx = 0
|
||||
for i in item_indices:
|
||||
n = num_slices[i]
|
||||
encoded[i] = torch.cat(group_features[idx : idx + n], dim=0)
|
||||
idx += n
|
||||
|
||||
return [encoded[i] for i in range(len(num_slices))]
|
||||
|
||||
# ----- Multimodal embedding interface -----
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
use_vit_merger_tensors = kwargs.pop("use_vit_merger", None)
|
||||
use_vit_merger = None
|
||||
if use_vit_merger_tensors is not None:
|
||||
if isinstance(use_vit_merger_tensors, torch.Tensor):
|
||||
use_vit_merger = bool(use_vit_merger_tensors.any().item())
|
||||
elif isinstance(use_vit_merger_tensors, list | tuple):
|
||||
use_vit_merger = any(
|
||||
bool(t.any().item()) if isinstance(t, torch.Tensor) else bool(t)
|
||||
for t in use_vit_merger_tensors
|
||||
)
|
||||
use_vit_merger = kwargs.pop("use_vit_merger", None)
|
||||
video_use_vit_merger = kwargs.pop("video_use_vit_merger", None)
|
||||
|
||||
image_kwargs = {
|
||||
k: v
|
||||
@@ -1235,7 +1441,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
multimodal_embeddings += tuple(
|
||||
self._process_vision_input(
|
||||
video_input,
|
||||
use_vit_merger=use_vit_merger,
|
||||
use_vit_merger=video_use_vit_merger,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1298,7 +1504,10 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
return loader.load_weights(
|
||||
_stack_vit_merger_qkv(weights),
|
||||
mapper=self.hf_to_vllm_mapper,
|
||||
)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
return MultiModelKeys.from_string_field(
|
||||
|
||||
@@ -0,0 +1,764 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only MiniCPM-V 4.7 model.
|
||||
|
||||
Same vision/LLM stack as MiniCPM-V 4.6, plus canvas 3D M-RoPE.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig
|
||||
|
||||
from .minicpmv import MiniCPMVDummyInputsBuilder
|
||||
from .minicpmv4_6 import (
|
||||
MiniCPMV4_6ForConditionalGeneration,
|
||||
MiniCPMV4_6MultiModalProcessor,
|
||||
MiniCPMV4_6ProcessingInfo,
|
||||
_scoped_mm_kwarg,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MiniCPMV4_7ProcessingInfo(MiniCPMV4_6ProcessingInfo):
|
||||
def _mm_max_slice_nums(self, **kwargs: object) -> int | None:
|
||||
# No schema resolution here: it would call back into `get_hf_processor`
|
||||
# below, which resolves this same value.
|
||||
merged = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
max_slice = _scoped_mm_kwarg(merged, "image", "max_slice_nums")
|
||||
if max_slice is None:
|
||||
max_slice = merged.get("max_slice_nums")
|
||||
if max_slice is None:
|
||||
return None
|
||||
return int(max_slice)
|
||||
|
||||
def _configured_max_slice_nums(self) -> int | None:
|
||||
# Configured (server-level) value only: per-request values are passed
|
||||
# explicitly to the image processor call, so they must not be written
|
||||
# back to the shared processor instance.
|
||||
return self._mm_max_slice_nums()
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
hf_processor = super().get_hf_processor(**kwargs)
|
||||
image_processor = getattr(hf_processor, "image_processor", None)
|
||||
configured = self._configured_max_slice_nums()
|
||||
if image_processor is not None and configured is not None:
|
||||
image_processor.max_slice_nums = configured
|
||||
return hf_processor
|
||||
|
||||
def get_image_max_slice_num(self) -> int:
|
||||
max_slice = self._configured_max_slice_nums()
|
||||
if max_slice is not None:
|
||||
return max_slice
|
||||
return super().get_image_max_slice_num()
|
||||
|
||||
|
||||
class MiniCPMV4_7MultiModalProcessor(MiniCPMV4_6MultiModalProcessor):
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
fields = dict(super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs))
|
||||
# Canvas M-RoPE places slices from `tgt_sizes`. Stripping the payload of
|
||||
# a prefix-cache-covered item only keeps CPU-side metadata, so without
|
||||
# this a cached image loses its target sizes, `image_bounds` and
|
||||
# `target_sizes` disagree, and the whole request silently falls back to
|
||||
# sequential positions.
|
||||
for key, modality in (("tgt_sizes", "image"), ("video_tgt_sizes", "video")):
|
||||
if key in fields:
|
||||
fields[key] = MultiModalFieldConfig.batched(modality, keep_on_cpu=True)
|
||||
return fields
|
||||
|
||||
def _video_local_id_prefix(self, video_idx: int) -> str:
|
||||
# MiniCPMV4_7Processor emits no local `<image_id>` in front of video
|
||||
# placeholders: a video is one temporal sequence and 4.7 was trained
|
||||
# without them.
|
||||
return ""
|
||||
|
||||
def get_image_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
image_idx: int = 0,
|
||||
downsample_mode: str | None = None,
|
||||
max_slice_nums: int | None = None,
|
||||
) -> str:
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
if max_slice_nums is None:
|
||||
max_slice_nums = info.get_image_max_slice_num()
|
||||
return info.get_slice_image_placeholder(
|
||||
image_size,
|
||||
image_idx=image_idx,
|
||||
max_slice_nums=max_slice_nums,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
|
||||
|
||||
def _as_hw_pairs(tensor: torch.Tensor) -> torch.Tensor:
|
||||
data = tensor.to(torch.long)
|
||||
if data.numel() == 0:
|
||||
return data.reshape(0, 2)
|
||||
if data.shape[-1] != 2:
|
||||
raise ValueError(f"expected last dim 2, got {tuple(data.shape)}")
|
||||
return data.reshape(-1, 2)
|
||||
|
||||
|
||||
def _sequential_positions(seq_len: int) -> torch.Tensor:
|
||||
"""Positions for the fallback where canvas M-RoPE cannot be computed."""
|
||||
return torch.arange(seq_len).unsqueeze(0).expand(3, -1)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MiniCPMV4_7MultiModalProcessor,
|
||||
info=MiniCPMV4_7ProcessingInfo,
|
||||
dummy_inputs=MiniCPMVDummyInputsBuilder,
|
||||
)
|
||||
class MiniCPMV4_7ForConditionalGeneration(MiniCPMV4_6ForConditionalGeneration):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
self._init_canvas_mrope(vllm_config)
|
||||
|
||||
def _drop_vision_last_layer(self) -> bool:
|
||||
# MiniCPMV4_7Config no longer carries this field (transformers marks it
|
||||
# removed), and 4.7 keeps every vision encoder layer.
|
||||
return False
|
||||
|
||||
def _init_canvas_mrope(self, vllm_config: VllmConfig) -> None:
|
||||
config = self.config
|
||||
uses_canvas = bool(getattr(config, "uses_mrope_canvas", False))
|
||||
if not uses_canvas:
|
||||
mrope_mode = getattr(config, "mrope_mode", None)
|
||||
uses_canvas = mrope_mode is not None and "canvas" in str(mrope_mode).lower()
|
||||
|
||||
self._uses_canvas_mrope = uses_canvas
|
||||
self._canvas_special_ids: dict[str, int] | None = None
|
||||
if not uses_canvas:
|
||||
return
|
||||
|
||||
# MiniCPMV4_7Config pins the structural ids in config.json; the
|
||||
# conversion script resolves them from the tokenizer. Prefer them so the
|
||||
# canvas matches training, and fall back to the tokenizer for checkpoints
|
||||
# whose config predates these fields.
|
||||
configured_ids: dict[str, int | None] = {
|
||||
"im_start_id": getattr(config, "image_start_id", None),
|
||||
"im_end_id": getattr(config, "image_end_id", None),
|
||||
"slice_start_id": getattr(config, "slice_start_id", None),
|
||||
"slice_end_id": getattr(config, "slice_end_id", None),
|
||||
"newline_id": getattr(config, "newline_id", None),
|
||||
}
|
||||
if all(value is not None for value in configured_ids.values()):
|
||||
self._canvas_special_ids = {
|
||||
name: int(value)
|
||||
for name, value in configured_ids.items()
|
||||
if value is not None
|
||||
}
|
||||
return
|
||||
|
||||
from vllm.tokenizers.registry import get_tokenizer
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
tokenizer = get_tokenizer(
|
||||
model_config.tokenizer,
|
||||
revision=model_config.tokenizer_revision,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
)
|
||||
self._canvas_special_ids = {
|
||||
"im_start_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "image_start_token", "<image>")
|
||||
),
|
||||
"im_end_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "image_end_token", "</image>")
|
||||
),
|
||||
"slice_start_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "slice_start_token", "<slice>")
|
||||
),
|
||||
"slice_end_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "slice_end_token", "</slice>")
|
||||
),
|
||||
"newline_id": tokenizer.encode("\n", add_special_tokens=False)[0],
|
||||
}
|
||||
|
||||
def get_mrope_input_positions(
|
||||
self,
|
||||
input_tokens: list[int],
|
||||
mm_features: list[MultiModalFeatureSpec],
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
seq_len = len(input_tokens)
|
||||
if (
|
||||
not getattr(self, "_uses_canvas_mrope", False)
|
||||
or self._canvas_special_ids is None
|
||||
or not mm_features
|
||||
):
|
||||
return _sequential_positions(seq_len), 0
|
||||
|
||||
input_ids = torch.tensor(input_tokens, dtype=torch.long)
|
||||
image_bounds = build_image_bounds(input_ids, self._canvas_special_ids)
|
||||
|
||||
target_sizes_list: list[torch.Tensor] = []
|
||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||
if mm_feature.data is None:
|
||||
continue
|
||||
for key in ("tgt_sizes", "video_tgt_sizes"):
|
||||
tgt = mm_feature.data.get(key)
|
||||
if tgt is None or not isinstance(tgt.data, torch.Tensor):
|
||||
continue
|
||||
try:
|
||||
target_sizes_list.append(_as_hw_pairs(tgt.data))
|
||||
except ValueError:
|
||||
# Not an (N, 2) size table. The bounds/sizes match below
|
||||
# turns this into a sequential fallback.
|
||||
continue
|
||||
|
||||
if not target_sizes_list or image_bounds.numel() == 0:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE missing inputs: "
|
||||
"tgt_chunks=%s image_bounds=%s; using sequential positions.",
|
||||
len(target_sizes_list),
|
||||
tuple(image_bounds.shape) if image_bounds is not None else None,
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
|
||||
target_sizes = torch.cat(target_sizes_list, dim=0)
|
||||
if target_sizes.ndim != 2 or target_sizes.shape[-1] != 2:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE unexpected tgt_sizes shape %s; "
|
||||
"using sequential positions.",
|
||||
tuple(target_sizes.shape),
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
if image_bounds.shape[0] != target_sizes.shape[0]:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE size mismatch: "
|
||||
"image_bounds=%s target_sizes=%s; using sequential positions.",
|
||||
tuple(image_bounds.shape),
|
||||
tuple(target_sizes.shape),
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
|
||||
try:
|
||||
pos3d = _compute_canvas_single(
|
||||
input_ids,
|
||||
torch.arange(seq_len, dtype=torch.long),
|
||||
image_bounds,
|
||||
target_sizes,
|
||||
self._canvas_special_ids,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE failed; "
|
||||
"falling back to sequential positions.",
|
||||
exc_info=True,
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
return pos3d, canvas_rope_delta(pos3d, seq_len)
|
||||
|
||||
|
||||
# Canvas 3D M-RoPE: position ids for the slice canvas.
|
||||
def compute_llm_grid(
|
||||
n_tokens: int, vision_grid_height: int, vision_grid_width: int
|
||||
) -> tuple[int, int]:
|
||||
if n_tokens <= 0 or vision_grid_height <= 0 or vision_grid_width <= 0:
|
||||
return 0, 0
|
||||
vit_total = vision_grid_height * vision_grid_width
|
||||
if vit_total == n_tokens:
|
||||
return vision_grid_height, vision_grid_width
|
||||
if vit_total < n_tokens:
|
||||
return 0, 0
|
||||
factor = int(round(math.sqrt(vit_total / n_tokens)))
|
||||
if (
|
||||
factor > 0
|
||||
and vision_grid_height % factor == 0
|
||||
and vision_grid_width % factor == 0
|
||||
):
|
||||
height, width = vision_grid_height // factor, vision_grid_width // factor
|
||||
if height * width == n_tokens:
|
||||
return height, width
|
||||
ratio = vision_grid_height / vision_grid_width
|
||||
width = max(1, int(round(math.sqrt(n_tokens / ratio))))
|
||||
height = n_tokens // width
|
||||
if height * width == n_tokens and height > 0:
|
||||
return height, width
|
||||
return 0, 0
|
||||
|
||||
|
||||
def build_image_bounds(
|
||||
input_ids: torch.LongTensor, special_token_ids: dict
|
||||
) -> torch.LongTensor:
|
||||
start_ids = [
|
||||
x
|
||||
for x in (
|
||||
special_token_ids.get("im_start_id"),
|
||||
special_token_ids.get("slice_start_id"),
|
||||
)
|
||||
if x is not None
|
||||
]
|
||||
end_ids = [
|
||||
x
|
||||
for x in (
|
||||
special_token_ids.get("im_end_id"),
|
||||
special_token_ids.get("slice_end_id"),
|
||||
)
|
||||
if x is not None
|
||||
]
|
||||
if not start_ids or not end_ids:
|
||||
return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
|
||||
start_cond = torch.zeros_like(input_ids, dtype=torch.bool)
|
||||
end_cond = torch.zeros_like(input_ids, dtype=torch.bool)
|
||||
for sid in start_ids:
|
||||
start_cond |= input_ids == sid
|
||||
for eid in end_ids:
|
||||
end_cond |= input_ids == eid
|
||||
starts = torch.where(start_cond)[0] + 1
|
||||
ends = torch.where(end_cond)[0]
|
||||
n = min(len(starts), len(ends))
|
||||
if n == 0:
|
||||
return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
|
||||
return torch.stack([starts[:n], ends[:n]], dim=-1)
|
||||
|
||||
|
||||
def _compute_canvas_single(
|
||||
input_ids, position_ids_2d, image_bound, target_sizes, special_token_ids
|
||||
):
|
||||
# Inference always handles one unpadded sequence. The packed (cu_seqlens)
|
||||
# form below is kept only to mirror the transformers reference
|
||||
# implementation of its canvas position computation.
|
||||
seq_len = input_ids.shape[0]
|
||||
cu_seqlens = torch.tensor([0, seq_len], device=input_ids.device, dtype=torch.long)
|
||||
return _compute_canvas_packed(
|
||||
position_ids_2d.unsqueeze(0),
|
||||
cu_seqlens,
|
||||
image_bound,
|
||||
target_sizes,
|
||||
input_ids.unsqueeze(0),
|
||||
special_token_ids,
|
||||
)[:, 0, :]
|
||||
|
||||
|
||||
def _target_hw(target_sizes: torch.Tensor, index: int) -> tuple[int, int]:
|
||||
"""Patch-grid height/width recorded for one image or slice."""
|
||||
return int(target_sizes[index, 0]), int(target_sizes[index, 1])
|
||||
|
||||
|
||||
class _CanvasLayout(NamedTuple):
|
||||
"""Grid of one canvas measured in LLM tokens.
|
||||
|
||||
A canvas is a thumbnail plus the slices cut from it, tiled row-major.
|
||||
"""
|
||||
|
||||
thumb_h: int
|
||||
thumb_w: int
|
||||
"""Thumbnail grid."""
|
||||
slice_h: int
|
||||
slice_w: int
|
||||
"""Per-slice grid. Zero when the canvas has no slices."""
|
||||
cols: int
|
||||
"""Slices sharing a canvas row. Zero when the canvas has no slices."""
|
||||
height: int
|
||||
width: int
|
||||
"""Assembled canvas."""
|
||||
|
||||
|
||||
def _canvas_layout(
|
||||
thumbnail: tuple[int, int, int],
|
||||
slices: list[tuple[int, int, int]],
|
||||
target_sizes: torch.Tensor,
|
||||
) -> _CanvasLayout:
|
||||
"""Measure the canvas for one ``(thumbnail, slices)`` group.
|
||||
|
||||
``thumbnail`` and each slice are ``(start, end, target_index)`` token spans.
|
||||
"""
|
||||
t_bs, t_be, t_gi = thumbnail
|
||||
thumb_h, thumb_w = compute_llm_grid(t_be - t_bs, *_target_hw(target_sizes, t_gi))
|
||||
if not slices:
|
||||
return _CanvasLayout(
|
||||
thumb_h, thumb_w, 0, 0, 0, max(thumb_h, 1), max(thumb_w, 1)
|
||||
)
|
||||
|
||||
cols = len(slices)
|
||||
for k in range(len(slices) - 1):
|
||||
if slices[k + 1][0] - slices[k][1] > 2:
|
||||
cols = k + 1
|
||||
break
|
||||
rows = len(slices) // cols if cols > 0 else 1
|
||||
if rows * cols != len(slices):
|
||||
rows, cols = 1, len(slices)
|
||||
|
||||
s0_bs, s0_be, s0_gi = slices[0]
|
||||
slice_h, slice_w = compute_llm_grid(s0_be - s0_bs, *_target_hw(target_sizes, s0_gi))
|
||||
return _CanvasLayout(
|
||||
thumb_h, thumb_w, slice_h, slice_w, cols, rows * slice_h, cols * slice_w
|
||||
)
|
||||
|
||||
|
||||
def _assign_canvas_span(
|
||||
pos3d: torch.Tensor,
|
||||
*,
|
||||
thumbnail: tuple[int, int, int],
|
||||
span_end: int,
|
||||
slices: list[tuple[int, int, int]],
|
||||
base: int,
|
||||
target_sizes: torch.Tensor,
|
||||
) -> int:
|
||||
"""Place one canvas at ``base``, writing its 3-D positions into ``pos3d``.
|
||||
|
||||
Shared by the video-frame and single-image paths; they differ only in how
|
||||
the per-canvas base advances, not in the geometry placed here.
|
||||
|
||||
Mutates ``pos3d`` in place, as the packed canvas computation has always
|
||||
done. Returns the distance to the next canvas base, so each caller keeps
|
||||
only its own base advancement.
|
||||
"""
|
||||
t_bs, t_be, _ = thumbnail
|
||||
device = pos3d.device
|
||||
span_start = t_bs - 1 # the <im_start> token
|
||||
n_thumb = t_be - t_bs
|
||||
layout = _canvas_layout(thumbnail, slices, target_sizes)
|
||||
cols, canvas_H, canvas_W = layout.cols, layout.height, layout.width
|
||||
|
||||
# base coat
|
||||
pos3d[0, 0, span_start:span_end] = base
|
||||
pos3d[1, 0, span_start:span_end] = base
|
||||
pos3d[2, 0, span_start:span_end] = base
|
||||
|
||||
# <im_start> -> halo
|
||||
halo_lo = max(base - 1, 0)
|
||||
pos3d[1, 0, span_start] = halo_lo
|
||||
pos3d[2, 0, span_start] = halo_lo
|
||||
|
||||
# <im_end> -> halo (right after thumbnail)
|
||||
if t_be < span_end:
|
||||
pos3d[1, 0, t_be] = base + canvas_H
|
||||
pos3d[2, 0, t_be] = base + canvas_W
|
||||
|
||||
# thumbnail visual tokens
|
||||
llm_th, llm_tw = layout.thumb_h, layout.thumb_w
|
||||
if llm_th > 0 and llm_tw > 0 and llm_th * llm_tw == n_thumb:
|
||||
if slices and canvas_H > 0 and canvas_W > 0:
|
||||
h_c = torch.linspace(0, canvas_H - 1, llm_th, device=device).round().long()
|
||||
w_c = torch.linspace(0, canvas_W - 1, llm_tw, device=device).round().long()
|
||||
else:
|
||||
h_c = torch.arange(llm_th, device=device)
|
||||
w_c = torch.arange(llm_tw, device=device)
|
||||
h_idx = h_c.view(-1, 1).expand(-1, llm_tw).reshape(-1)
|
||||
w_idx = w_c.view(1, -1).expand(llm_th, -1).reshape(-1)
|
||||
pos3d[0, 0, t_bs:t_be] = base
|
||||
pos3d[1, 0, t_bs:t_be] = h_idx + base
|
||||
pos3d[2, 0, t_bs:t_be] = w_idx + base
|
||||
else:
|
||||
fallback = torch.arange(n_thumb, device=device, dtype=torch.long) + base
|
||||
for d in range(3):
|
||||
pos3d[d, 0, t_bs:t_be] = fallback
|
||||
|
||||
# slice visual tokens + slice specials
|
||||
for k, (s_bs, s_be, s_gi) in enumerate(slices):
|
||||
n_s = s_be - s_bs
|
||||
sh_k, sw_k = compute_llm_grid(n_s, *_target_hw(target_sizes, s_gi))
|
||||
h_off = (k // cols) * layout.slice_h
|
||||
w_off = (k % cols) * layout.slice_w
|
||||
|
||||
slice_start_pos = s_bs - 1
|
||||
if slice_start_pos >= span_start:
|
||||
pos3d[1, 0, slice_start_pos] = base + h_off
|
||||
pos3d[2, 0, slice_start_pos] = base + w_off
|
||||
|
||||
slice_end_pos = s_be
|
||||
if sh_k > 0 and sw_k > 0:
|
||||
end_h = h_off + sh_k - 1
|
||||
end_w = w_off + sw_k - 1
|
||||
else:
|
||||
end_h = h_off
|
||||
end_w = w_off
|
||||
if slice_end_pos < span_end:
|
||||
pos3d[1, 0, slice_end_pos] = base + end_h
|
||||
pos3d[2, 0, slice_end_pos] = base + end_w
|
||||
|
||||
if sh_k > 0 and sw_k > 0 and sh_k * sw_k == n_s:
|
||||
h_idx = (
|
||||
torch.arange(sh_k, device=device)
|
||||
.view(-1, 1)
|
||||
.expand(-1, sw_k)
|
||||
.reshape(-1)
|
||||
)
|
||||
w_idx = (
|
||||
torch.arange(sw_k, device=device)
|
||||
.view(1, -1)
|
||||
.expand(sh_k, -1)
|
||||
.reshape(-1)
|
||||
)
|
||||
pos3d[0, 0, s_bs:s_be] = base
|
||||
pos3d[1, 0, s_bs:s_be] = h_idx + h_off + base
|
||||
pos3d[2, 0, s_bs:s_be] = w_idx + w_off + base
|
||||
else:
|
||||
fallback = torch.arange(n_s, device=device, dtype=torch.long) + base
|
||||
for d in range(3):
|
||||
pos3d[d, 0, s_bs:s_be] = fallback
|
||||
|
||||
# \n between slice rows -> W = right edge + 1.
|
||||
# H stays within the ended row (does NOT cross to next row): the \n sits at
|
||||
# the right-outside of the row that just ended, at H = that row's last row
|
||||
# and W = one column past the rightmost slice.
|
||||
for k in range(len(slices) - 1):
|
||||
gap_s = slices[k][1]
|
||||
gap_e = slices[k + 1][0]
|
||||
if gap_e - gap_s <= 2:
|
||||
continue
|
||||
boundary_h = ((k // cols) + 1) * layout.slice_h - 1
|
||||
right_edge_w = cols * layout.slice_w
|
||||
for nl_pos in range(gap_s + 1, gap_e - 1):
|
||||
pos3d[1, 0, nl_pos] = base + boundary_h
|
||||
pos3d[2, 0, nl_pos] = base + right_edge_w
|
||||
|
||||
return max(canvas_H, canvas_W) + 1
|
||||
|
||||
|
||||
def _compute_canvas_packed(
|
||||
position_ids_2d,
|
||||
cu_seqlens,
|
||||
image_bound,
|
||||
target_sizes,
|
||||
input_ids,
|
||||
special_token_ids,
|
||||
):
|
||||
if position_ids_2d.ndim == 1:
|
||||
position_ids_2d = position_ids_2d.unsqueeze(0)
|
||||
if input_ids.ndim == 1:
|
||||
input_ids = input_ids.unsqueeze(0)
|
||||
|
||||
has_images = (
|
||||
isinstance(image_bound, torch.Tensor)
|
||||
and image_bound.numel() > 0
|
||||
and isinstance(target_sizes, torch.Tensor)
|
||||
and target_sizes.numel() > 0
|
||||
)
|
||||
if not has_images:
|
||||
return position_ids_2d.unsqueeze(0).expand(3, -1, -1).contiguous()
|
||||
|
||||
device = position_ids_2d.device
|
||||
pos3d = position_ids_2d.unsqueeze(0).expand(3, -1, -1).clone()
|
||||
ids_flat = input_ids.view(-1)
|
||||
|
||||
im_start_id = special_token_ids.get("im_start_id")
|
||||
im_end_id = special_token_ids.get("im_end_id")
|
||||
slice_start_id = special_token_ids.get("slice_start_id")
|
||||
slice_end_id = special_token_ids.get("slice_end_id")
|
||||
|
||||
structural_ids = {
|
||||
v
|
||||
for v in (im_start_id, im_end_id, slice_start_id, slice_end_id)
|
||||
if v is not None
|
||||
}
|
||||
|
||||
num_seqs = len(cu_seqlens) - 1
|
||||
img_ptr = 0
|
||||
num_imgs = len(image_bound)
|
||||
|
||||
newline_id = special_token_ids.get("newline_id")
|
||||
gap_ok_ids = structural_ids | ({newline_id} if newline_id is not None else set())
|
||||
|
||||
def _group_end_pos(g, s_end):
|
||||
"""First position after a group's trailing structural tokens.
|
||||
|
||||
Matches the transformers reference, which closes a span over every
|
||||
marker in ``markers`` -- newlines included. The chat template joins
|
||||
content parts with ``\\n``, so an image followed by text always ends with
|
||||
one; leaving it out moves that newline out of the canvas and pushes
|
||||
every later position one step forward.
|
||||
"""
|
||||
le = g["slices"][-1][1] if g["slices"] else g["thumbnail"][1]
|
||||
ge = le
|
||||
while ge < s_end and ids_flat[ge].item() in gap_ok_ids:
|
||||
ge += 1
|
||||
return min(ge, s_end)
|
||||
|
||||
def _is_tight_gap(g_prev, g_next, s_end):
|
||||
"""Check if only structural/nl tokens sit between two groups."""
|
||||
ge = _group_end_pos(g_prev, s_end)
|
||||
ns = g_next["thumbnail"][0] - 1 # im_start of next group
|
||||
return all(ids_flat[p].item() in gap_ok_ids for p in range(ge, ns))
|
||||
|
||||
for si in range(num_seqs):
|
||||
s_start = cu_seqlens[si].item()
|
||||
s_end = cu_seqlens[si + 1].item()
|
||||
if s_start >= s_end:
|
||||
continue
|
||||
|
||||
seq_imgs = []
|
||||
while img_ptr < num_imgs:
|
||||
bs_ = image_bound[img_ptr, 0].item()
|
||||
be_ = image_bound[img_ptr, 1].item()
|
||||
if bs_ >= s_start and be_ <= s_end:
|
||||
seq_imgs.append((bs_, be_, img_ptr))
|
||||
img_ptr += 1
|
||||
else:
|
||||
break
|
||||
if not seq_imgs:
|
||||
continue
|
||||
|
||||
# ---- group images: thumbnail + following slices ----
|
||||
# merge consecutive no-slice frames into one video group
|
||||
raw_groups = []
|
||||
cur_group: dict[str, Any] | None = None
|
||||
for bs_, be_, gi in seq_imgs:
|
||||
marker_pos = bs_ - 1
|
||||
is_slice = (
|
||||
slice_start_id is not None
|
||||
and marker_pos >= s_start
|
||||
and ids_flat[marker_pos].item() == slice_start_id
|
||||
)
|
||||
if is_slice and cur_group is not None:
|
||||
cur_group["slices"].append((bs_, be_, gi))
|
||||
else:
|
||||
if cur_group is not None:
|
||||
raw_groups.append(cur_group)
|
||||
cur_group = {"thumbnail": (bs_, be_, gi), "slices": []}
|
||||
if cur_group is not None:
|
||||
raw_groups.append(cur_group)
|
||||
|
||||
# merge consecutive tightly-adjacent groups into video groups.
|
||||
# Two groups are "tightly adjacent" if the gap between them
|
||||
# contains only structural tokens and \n (no real text).
|
||||
# This handles both no-slice frames and frames-with-slices.
|
||||
groups = []
|
||||
i_g = 0
|
||||
while i_g < len(raw_groups):
|
||||
video_frames = [raw_groups[i_g]]
|
||||
j_g = i_g + 1
|
||||
while j_g < len(raw_groups) and _is_tight_gap(
|
||||
raw_groups[j_g - 1], raw_groups[j_g], s_end
|
||||
):
|
||||
video_frames.append(raw_groups[j_g])
|
||||
j_g += 1
|
||||
if len(video_frames) == 1:
|
||||
groups.append(raw_groups[i_g])
|
||||
else:
|
||||
groups.append({"video_frames": video_frames})
|
||||
i_g = j_g
|
||||
|
||||
# ---- assign 3-D positions ----
|
||||
pos = 0
|
||||
cursor = s_start
|
||||
|
||||
for group in groups:
|
||||
is_video_group = "video_frames" in group
|
||||
|
||||
if is_video_group:
|
||||
# === Video group: multiple frames merged ===
|
||||
# Each frame gets its own base_f so different frames have
|
||||
# distinct spatial positions. Inter-frame gap tokens (\n
|
||||
# etc.) are pure 1D text. After each gap, pos advances
|
||||
# by 1 extra so halo_lo of the next frame does not
|
||||
# collide with the last gap token.
|
||||
frames = group["video_frames"]
|
||||
first_frame = frames[0]
|
||||
last_frame = frames[-1]
|
||||
|
||||
# group_start = <im_start> of first frame
|
||||
group_start = first_frame["thumbnail"][0] - 1
|
||||
# group_end = after last structural token of last frame
|
||||
group_end = _group_end_pos(last_frame, s_end)
|
||||
|
||||
# text before video group
|
||||
text_len = group_start - cursor
|
||||
if text_len > 0:
|
||||
t_pos = (
|
||||
torch.arange(text_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, cursor:group_start] = t_pos
|
||||
pos += text_len
|
||||
|
||||
# process each frame with its own per-frame base
|
||||
frame_cursor = group_start
|
||||
for vf in frames:
|
||||
t_bs = vf["thumbnail"][0]
|
||||
vf_slices = vf["slices"]
|
||||
im_start_pos = t_bs - 1
|
||||
|
||||
# --- inter-frame gap: pure 1D text T=H=W=pos ---
|
||||
gap_len = im_start_pos - frame_cursor
|
||||
if gap_len > 0:
|
||||
t_pos = (
|
||||
torch.arange(gap_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, frame_cursor:im_start_pos] = t_pos
|
||||
pos += gap_len
|
||||
pos += 1 # buffer: prevents halo_lo collision
|
||||
|
||||
# --- per-frame base ---
|
||||
base_f = pos
|
||||
frame_end = _group_end_pos(vf, group_end)
|
||||
pos = base_f + _assign_canvas_span(
|
||||
pos3d,
|
||||
thumbnail=vf["thumbnail"],
|
||||
span_end=frame_end,
|
||||
slices=vf_slices,
|
||||
base=base_f,
|
||||
target_sizes=target_sizes,
|
||||
)
|
||||
frame_cursor = frame_end
|
||||
|
||||
# trailing tokens after last frame (if any)
|
||||
if frame_cursor < group_end:
|
||||
trail_len = group_end - frame_cursor
|
||||
t_pos = (
|
||||
torch.arange(trail_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, frame_cursor:group_end] = t_pos
|
||||
pos += trail_len
|
||||
|
||||
cursor = group_end
|
||||
|
||||
else:
|
||||
# === Single image group (thumbnail + optional slices) ===
|
||||
slices = group["slices"]
|
||||
|
||||
group_start = group["thumbnail"][0] - 1
|
||||
group_end = _group_end_pos(group, s_end)
|
||||
|
||||
# text before group
|
||||
text_len = group_start - cursor
|
||||
if text_len > 0:
|
||||
t_pos = (
|
||||
torch.arange(text_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, cursor:group_start] = t_pos
|
||||
pos += text_len
|
||||
|
||||
base = pos
|
||||
pos = base + _assign_canvas_span(
|
||||
pos3d,
|
||||
thumbnail=group["thumbnail"],
|
||||
span_end=group_end,
|
||||
slices=slices,
|
||||
base=base,
|
||||
target_sizes=target_sizes,
|
||||
)
|
||||
cursor = group_end
|
||||
|
||||
# remaining text
|
||||
rem = s_end - cursor
|
||||
if rem > 0:
|
||||
t_pos = torch.arange(rem, device=device, dtype=torch.long) + pos
|
||||
for d in range(3):
|
||||
pos3d[d, 0, cursor:s_end] = t_pos
|
||||
|
||||
return pos3d
|
||||
|
||||
|
||||
def canvas_rope_delta(position_ids: torch.Tensor, seq_len: int) -> int:
|
||||
"""Offset applied to the 1-D positions of tokens decoded after the prompt.
|
||||
|
||||
The first decoded token sits one past the last prefill position.
|
||||
"""
|
||||
return int(position_ids.amax().item()) + 1 - seq_len
|
||||
@@ -514,6 +514,10 @@ _MULTIMODAL_MODELS = {
|
||||
"minicpmv4_6",
|
||||
"MiniCPMV4_6ForConditionalGeneration",
|
||||
),
|
||||
"MiniCPMV4_7ForConditionalGeneration": (
|
||||
"minicpmv4_7",
|
||||
"MiniCPMV4_7ForConditionalGeneration",
|
||||
),
|
||||
"Mistral3ForConditionalGeneration": (
|
||||
"mistral3",
|
||||
"Mistral3ForConditionalGeneration",
|
||||
|
||||
Reference in New Issue
Block a user