[Model] Support MiniCPM-V 4.7 (#58674)
Signed-off-by: tc-mb <tianchi_cai@icloud.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
tc-mb
@@ -592,7 +592,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| `MiDashengLMModel` | MiDashengLM | T + A<sup>+</sup> | `mispeech/midashenglm-7b` | | ✅︎ |
|
||||
| `MiMoV2OmniForCausalLM` | MiMo-V2.5-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `XiaomiMiMo/MiMo-V2.5-Omni` | | ✅︎ |
|
||||
| `MiniCPMO` | MiniCPM-O | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>E+</sup> | `openbmb/MiniCPM-o-2_6`, etc. | ✅︎ | ✅︎ |
|
||||
| `MiniCPMV` | MiniCPM-V | T + I<sup>E+</sup> + V<sup>E+</sup> | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, `openbmb/MiniCPM-V-4_6`, etc. | ✅︎ | |
|
||||
| `MiniCPMV` | MiniCPM-V | T + I<sup>E+</sup> + V<sup>E+</sup> | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, `openbmb/MiniCPM-V-4_6`, `openbmb/MiniCPM-V-4_7`, etc. | ✅︎ | |
|
||||
| `MiniMaxM3SparseForConditionalGeneration` | MiniMax-M3 | T + I<sup>+</sup> + V<sup>+</sup> | `MiniMaxAI/MiniMax-M3`, `MiniMaxAI/MiniMax-M3-MXFP8`, etc. | | ✅︎ |
|
||||
| `MiniMaxVL01ForConditionalGeneration` | MiniMax-VL | T + I<sup>E+</sup> | `MiniMaxAI/MiniMax-VL-01`, etc. | | ✅︎ |
|
||||
| `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I<sup>+</sup> | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ |
|
||||
|
||||
@@ -6,6 +6,9 @@ import torch
|
||||
from vllm.model_executor.models.minicpmv import MiniCPMV2_6
|
||||
from vllm.model_executor.models.minicpmv4_6 import (
|
||||
MiniCPMV4_6ForConditionalGeneration,
|
||||
_expand_flags,
|
||||
_select_vision_items,
|
||||
_vision_downsample_mode,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
@@ -54,3 +57,114 @@ def test_image_and_video_embeds_stay_in_their_own_modality():
|
||||
|
||||
assert torch.equal(modalities["images"]["image_embeds"], image_embeds)
|
||||
assert torch.equal(modalities["videos"]["image_embeds"], video_embeds)
|
||||
|
||||
|
||||
class TestPerItemMergerFlags:
|
||||
"""The merger flag is per request but items from several requests can be
|
||||
batched together, so it has to be applied per item."""
|
||||
|
||||
def test_expand_flags_broadcasts_a_single_value(self):
|
||||
assert _expand_flags(None, 3) == [None, None, None]
|
||||
assert _expand_flags(torch.tensor([True]), 3) == [True, True, True]
|
||||
assert _expand_flags(True, 2) == [True, True]
|
||||
|
||||
def test_expand_flags_keeps_per_item_values(self):
|
||||
flags = [torch.tensor([True]), torch.tensor([False])]
|
||||
|
||||
assert _expand_flags(flags, 2) == [True, False]
|
||||
|
||||
def test_expand_flags_falls_back_on_a_shape_mismatch(self):
|
||||
# Too many values to map onto the items: keep one decision for the
|
||||
# batch rather than failing the request.
|
||||
assert _expand_flags([True, False, True], 2) == [True, True]
|
||||
|
||||
def test_vision_downsample_mode(self):
|
||||
assert _vision_downsample_mode(None) is None
|
||||
assert _vision_downsample_mode(True) == "16x"
|
||||
assert _vision_downsample_mode(False) == "4x"
|
||||
|
||||
def test_select_vision_items_slices_by_item(self):
|
||||
# 3 items holding 2, 1 and 2 slices; one channel, one patch, width 4.
|
||||
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(5)]
|
||||
tgt_sizes = torch.arange(10, dtype=torch.long).reshape(5, 2)
|
||||
image_input = {
|
||||
"pixel_values": pixel_values,
|
||||
"tgt_sizes": tgt_sizes,
|
||||
"num_slices": torch.tensor([2, 1, 2]),
|
||||
}
|
||||
|
||||
selected = _select_vision_items(image_input, [2], [2, 1, 2])
|
||||
|
||||
assert len(selected["pixel_values"]) == 2
|
||||
assert bool(selected["pixel_values"][0][0, 0, 0] == 3.0)
|
||||
assert bool(selected["pixel_values"][1][0, 0, 0] == 4.0)
|
||||
assert selected["num_slices"].tolist() == [2]
|
||||
assert torch.equal(selected["tgt_sizes"], tgt_sizes[3:5])
|
||||
|
||||
def test_select_vision_items_preserves_order(self):
|
||||
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(3)]
|
||||
image_input = {
|
||||
"pixel_values": pixel_values,
|
||||
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1, 1]),
|
||||
}
|
||||
|
||||
selected = _select_vision_items(image_input, [2, 0], [1, 1, 1])
|
||||
|
||||
assert selected["num_slices"].tolist() == [1, 1]
|
||||
assert bool(selected["pixel_values"][0][0, 0, 0] == 2.0)
|
||||
assert bool(selected["pixel_values"][1][0, 0, 0] == 0.0)
|
||||
|
||||
def test_process_vision_input_applies_flags_per_item(self):
|
||||
calls: list[str | None] = []
|
||||
|
||||
def fake_get_vision_hidden_states(data, downsample_mode=None):
|
||||
calls.append(downsample_mode)
|
||||
tag = 1.0 if downsample_mode == "16x" else 0.0
|
||||
return [torch.full((1,), tag) for _ in data["pixel_values"]]
|
||||
|
||||
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
|
||||
model.get_vision_hidden_states = fake_get_vision_hidden_states
|
||||
|
||||
image_input = {
|
||||
"type": "pixel_values",
|
||||
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(3)],
|
||||
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1, 1]),
|
||||
}
|
||||
|
||||
result = model._process_vision_input(
|
||||
image_input,
|
||||
use_vit_merger=[
|
||||
torch.tensor([True]),
|
||||
torch.tensor([False]),
|
||||
torch.tensor([True]),
|
||||
],
|
||||
)
|
||||
|
||||
# Each item keeps its own mode even though they share one call.
|
||||
assert [float(r[0]) for r in result] == [1.0, 0.0, 1.0]
|
||||
# Mixed flags are encoded as separate groups, not one collapsed batch.
|
||||
assert calls == ["16x", "4x"]
|
||||
|
||||
def test_process_vision_input_uses_one_group_for_uniform_flags(self):
|
||||
calls: list[str | None] = []
|
||||
|
||||
def fake_get_vision_hidden_states(data, downsample_mode=None):
|
||||
calls.append(downsample_mode)
|
||||
return [torch.zeros(1) for _ in data["pixel_values"]]
|
||||
|
||||
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
|
||||
model.get_vision_hidden_states = fake_get_vision_hidden_states
|
||||
|
||||
image_input = {
|
||||
"type": "pixel_values",
|
||||
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(2)],
|
||||
"tgt_sizes": torch.zeros(2, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1]),
|
||||
}
|
||||
|
||||
model._process_vision_input(image_input, use_vit_merger=torch.tensor([True]))
|
||||
|
||||
# The common case still makes a single encoder call.
|
||||
assert calls == ["16x"]
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.minicpmv4_6 import _stack_vit_merger_qkv
|
||||
from vllm.model_executor.models.minicpmv4_7 import (
|
||||
MiniCPMV4_7MultiModalProcessor,
|
||||
_compute_canvas_single,
|
||||
build_image_bounds,
|
||||
canvas_rope_delta,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def test_canvas_mrope_sizes_survive_prefix_cache_stripping():
|
||||
"""A cached image must keep `tgt_sizes`, or canvas M-RoPE falls back.
|
||||
|
||||
`strip_covered_mm_data` drops the payload of a prefix-cache-covered item and
|
||||
keeps only CPU-side metadata. Canvas M-RoPE places slices from `tgt_sizes`,
|
||||
so 4.7 marks it `keep_on_cpu` while the shared 4.6 config stays as it was.
|
||||
"""
|
||||
from vllm.model_executor.models.minicpmv4_6 import _minicpmv4_6_field_config
|
||||
|
||||
shared = _minicpmv4_6_field_config({})
|
||||
assert shared["tgt_sizes"].field.keep_on_cpu is False
|
||||
assert shared["video_tgt_sizes"].field.keep_on_cpu is False
|
||||
|
||||
processor = MiniCPMV4_7MultiModalProcessor.__new__(MiniCPMV4_7MultiModalProcessor)
|
||||
fields = processor._get_mm_fields_config({}, {})
|
||||
|
||||
assert fields["tgt_sizes"].field.keep_on_cpu is True
|
||||
assert fields["video_tgt_sizes"].field.keep_on_cpu is True
|
||||
# Other fields keep their previous setting.
|
||||
assert fields["pixel_values"].field.keep_on_cpu is False
|
||||
assert fields["video_image_sizes"].field.keep_on_cpu is True
|
||||
|
||||
|
||||
def test_modality_mm_kwargs_resolution():
|
||||
"""Flat keys reach both modalities; scopes stay scoped, and win on conflict.
|
||||
|
||||
`_merge_and_resolve_mm_processor_kwargs` routes a flat key into every scope
|
||||
whose HF processor declares it, so reading is a plain scope lookup. The
|
||||
routing itself is upstream's, tested here against the MiniCPM-V schema to
|
||||
pin the precedence rule (`scoped > flat`).
|
||||
"""
|
||||
from vllm.model_executor.models.minicpmv4_6 import (
|
||||
_flat_processor_kwargs,
|
||||
_scoped_mm_kwarg,
|
||||
)
|
||||
from vllm.multimodal.processing.context import _resolve_mm_processor_kwargs
|
||||
|
||||
# What MiniCPMV4_6{Image,Video}ProcessorKwargs declare.
|
||||
processor_keys = {"downsample_mode", "max_slice_nums"}
|
||||
schema = {
|
||||
"text_kwargs": set(),
|
||||
"images_kwargs": set(processor_keys),
|
||||
"videos_kwargs": set(processor_keys),
|
||||
"audio_kwargs": set(),
|
||||
}
|
||||
|
||||
def read(mm_kwargs, modality, key):
|
||||
merged = _resolve_mm_processor_kwargs(dict(mm_kwargs), schema)
|
||||
return _scoped_mm_kwarg(merged, modality, key)
|
||||
|
||||
assert read({}, "image", "downsample_mode") is None
|
||||
# A flat key is routed into both scopes.
|
||||
assert read({"downsample_mode": "4x"}, "image", "downsample_mode") == "4x"
|
||||
assert read({"downsample_mode": "4x"}, "video", "downsample_mode") == "4x"
|
||||
# A scoped key stays in its scope.
|
||||
assert (
|
||||
read({"images_kwargs": {"downsample_mode": "16x"}}, "image", "downsample_mode")
|
||||
== "16x"
|
||||
)
|
||||
assert (
|
||||
read({"images_kwargs": {"downsample_mode": "16x"}}, "video", "downsample_mode")
|
||||
is None
|
||||
)
|
||||
videos_only = {"videos_kwargs": {"max_slice_nums": 1}}
|
||||
assert read(videos_only, "video", "max_slice_nums") == 1
|
||||
# Scoped to videos, so image must not see it.
|
||||
assert read(videos_only, "image", "max_slice_nums") is None
|
||||
# An explicit scope wins over the flat key it is merged with.
|
||||
assert (
|
||||
read(
|
||||
{"downsample_mode": "16x", "images_kwargs": {"downsample_mode": "4x"}},
|
||||
"image",
|
||||
"downsample_mode",
|
||||
)
|
||||
== "4x"
|
||||
)
|
||||
# A non-mapping scope is ignored rather than raising.
|
||||
assert read({"images_kwargs": None}, "image", "downsample_mode") is None
|
||||
|
||||
# `_flat_processor_kwargs` drops the scoped keys and pins the mode.
|
||||
flat = _flat_processor_kwargs(
|
||||
{
|
||||
"downsample_mode": "4x",
|
||||
"images_kwargs": {"a": 1},
|
||||
"videos_kwargs": {"b": 2},
|
||||
"audio_kwargs": {"c": 3},
|
||||
},
|
||||
"16x",
|
||||
)
|
||||
assert flat == {"downsample_mode": "16x"}
|
||||
|
||||
|
||||
def test_vit_merger_qkv_is_stacked_before_load():
|
||||
mapped = list(
|
||||
_stack_vit_merger_qkv(
|
||||
[
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.q_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.k_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.v_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
("vit_merger.self_attn.qkv_proj.weight", torch.zeros(1)),
|
||||
]
|
||||
)
|
||||
)
|
||||
names = [name for name, _ in mapped]
|
||||
shards = [getattr(tensor, "shard_id", None) for _, tensor in mapped]
|
||||
assert names == [
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"vit_merger.self_attn.qkv_proj.weight",
|
||||
]
|
||||
assert shards == ["q", "k", "v", None]
|
||||
|
||||
|
||||
# Canvas M-RoPE goldens, cross-checked against the reference implementation in
|
||||
# transformers PR 48979 (`modular_minicpmv4_7.py`). Synthetic ids keep the token
|
||||
# sequences readable: the canvas math only compares ids for equality.
|
||||
_IM_START = 10
|
||||
_IM_END = 11
|
||||
_SLICE_START = 12
|
||||
_SLICE_END = 13
|
||||
_NEWLINE = 14
|
||||
_IMAGE_PAD = 20
|
||||
_VIDEO_PAD = 21
|
||||
_TEXT = 100
|
||||
|
||||
_CANVAS_IDS = {
|
||||
"im_start_id": _IM_START,
|
||||
"im_end_id": _IM_END,
|
||||
"slice_start_id": _SLICE_START,
|
||||
"slice_end_id": _SLICE_END,
|
||||
"newline_id": _NEWLINE,
|
||||
}
|
||||
|
||||
|
||||
def _canvas_positions(tokens, target_sizes):
|
||||
input_ids = torch.tensor([tokens], dtype=torch.long)
|
||||
targets = torch.tensor(target_sizes, dtype=torch.long)
|
||||
bounds = build_image_bounds(input_ids[0], _CANVAS_IDS)
|
||||
positions = _compute_canvas_single(
|
||||
input_ids[0],
|
||||
torch.arange(len(tokens), dtype=torch.long),
|
||||
bounds,
|
||||
targets,
|
||||
_CANVAS_IDS,
|
||||
)
|
||||
return positions.tolist(), canvas_rope_delta(positions, len(tokens))
|
||||
|
||||
|
||||
def test_canvas_mrope_trailing_newline_stays_inside_the_canvas():
|
||||
"""A newline right after `</image>` belongs to the canvas, not to the text.
|
||||
|
||||
The chat template joins content parts with ``\\n``, so an image followed by
|
||||
text always leaves one there. The transformers reference closes a frame over
|
||||
every marker including newlines; leaving it out pushes every later position
|
||||
one step forward and makes `rope_delta` differ by one.
|
||||
"""
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _NEWLINE, _TEXT, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 3, 4],
|
||||
[0, 0, 1, 2, 1, 3, 4],
|
||||
[0, 0, 1, 2, 1, 3, 4],
|
||||
]
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_canvas_mrope_thumbnail_matches_reference():
|
||||
# <text> <image> PAD </image> <text>; 16x turns a 4x4 patch grid into 1 token.
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 3],
|
||||
[0, 0, 1, 2, 3],
|
||||
[0, 0, 1, 2, 3],
|
||||
]
|
||||
assert delta == -1
|
||||
|
||||
|
||||
def test_canvas_mrope_slice_row_matches_reference():
|
||||
# Thumbnail then a 1x2 slice row; the slices tile the thumbnail's canvas.
|
||||
tokens = [
|
||||
_TEXT,
|
||||
_IM_START,
|
||||
_IMAGE_PAD,
|
||||
_IM_END,
|
||||
_SLICE_START,
|
||||
_IMAGE_PAD,
|
||||
_SLICE_END,
|
||||
_SLICE_START,
|
||||
_IMAGE_PAD,
|
||||
_SLICE_END,
|
||||
_TEXT,
|
||||
]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4], [4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 2, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 3, 1, 1, 1, 2, 2, 2, 4],
|
||||
]
|
||||
assert delta == -6
|
||||
|
||||
|
||||
def test_canvas_mrope_resamples_thumbnail_over_canvas():
|
||||
# A 8x8 patch grid is 2x2 LLM tokens, so the canvas is resampled (not arange).
|
||||
tokens = [_TEXT, _IM_START] + [_IMAGE_PAD] * 4 + [_IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[8, 8]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 1, 2, 2, 3, 4],
|
||||
[0, 0, 1, 2, 1, 2, 3, 4],
|
||||
]
|
||||
assert delta == -3
|
||||
|
||||
|
||||
def test_canvas_mrope_video_frames_share_one_canvas():
|
||||
tokens = [
|
||||
_TEXT,
|
||||
_IM_START,
|
||||
_VIDEO_PAD,
|
||||
_IM_END,
|
||||
_IM_START,
|
||||
_VIDEO_PAD,
|
||||
_IM_END,
|
||||
_TEXT,
|
||||
]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 3, 3, 3, 5],
|
||||
[0, 0, 1, 2, 2, 3, 4, 5],
|
||||
[0, 0, 1, 2, 2, 3, 4, 5],
|
||||
]
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_canvas_mrope_interleaved_image_and_video():
|
||||
tokens = (
|
||||
[_TEXT, _IM_START]
|
||||
+ [_IMAGE_PAD] * 4
|
||||
+ [_IM_END, _IM_START]
|
||||
+ [_VIDEO_PAD] * 4
|
||||
+ [_IM_END, _TEXT]
|
||||
)
|
||||
positions, delta = _canvas_positions(tokens, [[8, 8], [8, 8]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 4, 4, 4, 4, 4, 4, 7],
|
||||
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7],
|
||||
[0, 0, 1, 2, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7],
|
||||
]
|
||||
assert delta == -6
|
||||
|
||||
|
||||
def test_canvas_span_is_shared_by_image_and_video():
|
||||
"""A video frame and a same-geometry image must place identical offsets.
|
||||
|
||||
Both paths go through `_assign_canvas_span`; this pins that they cannot
|
||||
drift apart now that the duplicated geometry is gone.
|
||||
"""
|
||||
thumb = [_IM_START] + [_IMAGE_PAD] * 4 + [_IM_END]
|
||||
|
||||
img_pos, _ = _canvas_positions([_TEXT, *thumb, _TEXT], [[8, 8]])
|
||||
vid_pos, _ = _canvas_positions([_TEXT, *thumb, *thumb, _TEXT], [[8, 8], [8, 8]])
|
||||
|
||||
def offsets(pos, start):
|
||||
# The <im_start> halo is one below the canvas base.
|
||||
base = pos[1][start] + 1
|
||||
return [[c[start + k] - base for k in range(3)] for c in pos]
|
||||
|
||||
assert offsets(img_pos, 0) == offsets(vid_pos, 0)
|
||||
|
||||
|
||||
def test_canvas_mrope_delta_leaves_room_for_the_first_decoded_token():
|
||||
# First decoded token must land on max_prefill_position + 1, matching
|
||||
# transformers and the other native M-RoPE models.
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
max_position = max(max(channel) for channel in positions)
|
||||
assert delta == max_position + 1 - len(tokens)
|
||||
@@ -1118,6 +1118,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM-V-4_6",
|
||||
),
|
||||
"MiniCPMV4_7ForConditionalGeneration": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM-V-4_7",
|
||||
min_transformers_version="5.18.0",
|
||||
is_available_online=False,
|
||||
),
|
||||
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
|
||||
"MiniMaxAI/MiniMax-M3",
|
||||
trust_remote_code=True,
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MiniCPM-V family parser."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.minicpmv import (
|
||||
EscapedNewlineNormalizer,
|
||||
MiniCPMVOutputNormalizer,
|
||||
recover_newlines,
|
||||
)
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
_VOCAB = {
|
||||
"<think>": 10,
|
||||
"</think>": 11,
|
||||
"<|im_end|>": 12,
|
||||
"<tool_call>": 13,
|
||||
"</tool_call>": 14,
|
||||
}
|
||||
|
||||
_TOOLS = [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
return make_mock_tokenizer(_VOCAB)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def request_obj():
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
|
||||
def _make_parser(tokenizer, *, enable_thinking=None, tools=None):
|
||||
"""Build the parser the serving stack resolves for MiniCPM-V.
|
||||
|
||||
Everything is built through ``ParserManager`` so the tests exercise the
|
||||
``DelegatingParser`` the server actually drives, not the parser engine on
|
||||
its own. Tool calling goes through ``qwen3_coder``: MiniCPM-V emits that
|
||||
XML syntax verbatim and has no tool parser of its own.
|
||||
"""
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="qwen3_coder" if tools else None,
|
||||
reasoning_parser_name="minicpmv",
|
||||
enable_auto_tools=bool(tools),
|
||||
)
|
||||
assert parser_cls is not None
|
||||
kwargs = {}
|
||||
if enable_thinking is not None:
|
||||
kwargs["chat_template_kwargs"] = {"enable_thinking": enable_thinking}
|
||||
return parser_cls(tokenizer, tools, **kwargs)
|
||||
|
||||
|
||||
def test_reasoning_parser_is_registered():
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser("minicpmv")
|
||||
|
||||
assert parser_cls.__name__ == "MiniCPMVParserReasoningAdapter"
|
||||
|
||||
|
||||
def test_thinking_defaults_to_the_chat_template(mock_tokenizer):
|
||||
"""Without `enable_thinking`, the grammar must wait for reasoning.
|
||||
|
||||
The chat template ends the assistant turn with `<think>` unless the flag is
|
||||
explicitly false, so the model reasons by default. This is the flag
|
||||
`serving.py` latches into the structured-output gate, so a non-thinking
|
||||
default both disables the grammar and leaks the thinking into `content`.
|
||||
"""
|
||||
|
||||
def reasoning_pending(**kwargs):
|
||||
return not _make_parser(mock_tokenizer, **kwargs).is_reasoning_end([1, 2])
|
||||
|
||||
assert reasoning_pending() is True # default matches the template
|
||||
assert reasoning_pending(enable_thinking=True) is True
|
||||
assert reasoning_pending(enable_thinking=False) is False
|
||||
|
||||
|
||||
def test_no_minicpmv_tool_parser_is_registered():
|
||||
"""MiniCPM-V deliberately registers no tool parser.
|
||||
|
||||
The family emits the Qwen3-Coder XML syntax verbatim, so
|
||||
``--tool-call-parser qwen3_coder`` already covers it. Registering a
|
||||
``minicpmv`` tool parser would duplicate that name for no parsing gain.
|
||||
"""
|
||||
with pytest.raises(KeyError):
|
||||
ToolParserManager.get_tool_parser("minicpmv")
|
||||
|
||||
|
||||
def test_composes_with_qwen3_coder_tool_parser(mock_tokenizer):
|
||||
"""`--tool-call-parser qwen3_coder` must work through the serving Parser."""
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=_TOOLS,
|
||||
)
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="qwen3_coder",
|
||||
reasoning_parser_name="minicpmv",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert parser_cls is not None
|
||||
parser = parser_cls(mock_tokenizer, request.tools)
|
||||
|
||||
_, _, tool_calls = parser.parse(
|
||||
"<tool_call><function=get_weather>"
|
||||
"<parameter=city>SF</parameter></function></tool_call>",
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert "SF" in tool_calls[0].arguments
|
||||
|
||||
|
||||
class TestNonThinking:
|
||||
@pytest.fixture
|
||||
def parser(self, mock_tokenizer):
|
||||
return _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
|
||||
def test_plain_content_is_unchanged(self, parser):
|
||||
reasoning, content = parser.extract_reasoning("The answer is 42.", None)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "The answer is 42."
|
||||
|
||||
def test_reserved_markers_are_removed(self, parser):
|
||||
output = (
|
||||
"<reserved_12>reasoning<reserved_13><|reserved_14|>tool call<|reserved_15|>"
|
||||
)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(output, None)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "reasoningtool call"
|
||||
|
||||
def test_unrelated_reserved_text_is_unchanged(self, parser):
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<reserved_11>content<|reserved_16|>",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "<reserved_11>content<|reserved_16|>"
|
||||
|
||||
def test_dangling_standard_think_end_is_removed(self, parser):
|
||||
output = "first part </think>\n\nfinal part"
|
||||
|
||||
reasoning, content = parser.extract_reasoning(output, None)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "first part \n\nfinal part"
|
||||
|
||||
def test_truncated_standard_reasoning_does_not_leak(self, parser):
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"prefix<think>private reasoning",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "prefix"
|
||||
|
||||
def test_streaming_split_standard_markers(self, parser):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
[
|
||||
"<",
|
||||
"think>",
|
||||
"private reasoning",
|
||||
"</",
|
||||
"think>",
|
||||
"final answer",
|
||||
],
|
||||
)
|
||||
|
||||
assert reasoning == ""
|
||||
assert content == "final answer"
|
||||
|
||||
def test_streaming_split_reserved_markers(self, parser):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
[
|
||||
"<reser",
|
||||
"ved_12>",
|
||||
"reasoning",
|
||||
"<|reserved",
|
||||
"_13|>",
|
||||
"final answer",
|
||||
],
|
||||
)
|
||||
|
||||
assert reasoning == ""
|
||||
assert content == "reasoningfinal answer"
|
||||
|
||||
def test_tool_calls_are_parsed(self, mock_tokenizer):
|
||||
"""Thinking is off by default, and tool calls must still be parsed."""
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=_TOOLS,
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
"<tool_call><function=get_weather>"
|
||||
"<parameter=city>SF</parameter></function></tool_call>",
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert not content
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert "SF" in tool_calls[0].arguments
|
||||
|
||||
def test_streaming_tool_calls_are_parsed(self, mock_tokenizer):
|
||||
"""Regression: tool calls must stream when thinking is explicitly off.
|
||||
|
||||
The reasoning parser used to report ``reasoning_ended=False`` in
|
||||
non-thinking mode, which kept the delegating parser in its reasoning
|
||||
phase and streamed the raw ``<tool_call>`` XML to the client as content.
|
||||
"""
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=_TOOLS,
|
||||
)
|
||||
|
||||
delta = parser.parse_delta(
|
||||
"<tool_call><function=get_weather>"
|
||||
"<parameter=city>SF</parameter></function></tool_call>",
|
||||
[],
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
assert delta is not None
|
||||
assert delta.tool_calls
|
||||
assert delta.tool_calls[0].function is not None
|
||||
assert delta.tool_calls[0].function.name == "get_weather"
|
||||
assert "SF" in (delta.tool_calls[0].function.arguments or "")
|
||||
assert not delta.content
|
||||
|
||||
|
||||
class TestThinking:
|
||||
def test_standard_think_tags_are_parsed(self, mock_tokenizer):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=True)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<think>private reasoning</think>final answer",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning == "private reasoning"
|
||||
assert content == "final answer"
|
||||
|
||||
def test_reserved_markers_are_removed(self, mock_tokenizer):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=True)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<think><reserved_12>private reasoning</think><|reserved_13|>final answer",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning == "private reasoning"
|
||||
assert content == "final answer"
|
||||
|
||||
def test_streaming_im_end_is_removed(self):
|
||||
tokenizer = make_mock_tokenizer(_VOCAB, special_tokens=[])
|
||||
parser = _make_parser(tokenizer, enable_thinking=True)
|
||||
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["Answer: 111", "<|im_end|>"],
|
||||
)
|
||||
|
||||
assert reasoning == "Answer: 111"
|
||||
assert content == ""
|
||||
|
||||
|
||||
class TestNewlineRecovery:
|
||||
"""The tokenizer writes a newline as the two characters ``\\`` and ``n``."""
|
||||
|
||||
def test_recover_newlines(self):
|
||||
text = (
|
||||
"first\\nsecond\\rthird\\r\\nfourth "
|
||||
"`code\\nvalue` "
|
||||
"```python\\nvalue = '\\\\n'\\n``` "
|
||||
"$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
|
||||
"escaped \\\\n"
|
||||
)
|
||||
|
||||
assert recover_newlines(text) == (
|
||||
"first\nsecond\nthird\nfourth "
|
||||
"`code\\nvalue` "
|
||||
"```python\\nvalue = '\\\\n'\\n``` "
|
||||
"$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
|
||||
"escaped \\\\n"
|
||||
)
|
||||
|
||||
def test_streaming_matches_complete_recovery(self):
|
||||
text = (
|
||||
"first\\nsecond `code\\nvalue` $x\\ny$ ```text\\nvalue\\n``` last\\r\\nline"
|
||||
)
|
||||
expected = recover_newlines(text)
|
||||
|
||||
for split in range(len(text) + 1):
|
||||
normalizer = EscapedNewlineNormalizer()
|
||||
actual = normalizer.feed(text[:split])
|
||||
actual += normalizer.feed(text[split:], final=True)
|
||||
assert actual == expected
|
||||
|
||||
normalizer = EscapedNewlineNormalizer()
|
||||
actual = "".join(normalizer.feed(char) for char in text)
|
||||
actual += normalizer.feed("", final=True)
|
||||
assert actual == expected
|
||||
|
||||
def test_backslash_runs_stay_consistent_across_splits(self):
|
||||
# A run of backslashes can be split over chunks. Its parity decides
|
||||
# whether a following `n` is a newline, so streaming and non-streaming
|
||||
# must agree for every split point.
|
||||
for count in range(1, 7):
|
||||
text = "\\" * count + "n"
|
||||
expected = recover_newlines(text)
|
||||
|
||||
for split in range(len(text) + 1):
|
||||
normalizer = EscapedNewlineNormalizer()
|
||||
actual = normalizer.feed(text[:split])
|
||||
actual += normalizer.feed(text[split:], final=True)
|
||||
assert actual == expected, (count, split)
|
||||
|
||||
def test_streaming_recovers_reasoning_and_content(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
|
||||
first = normalizer.normalize_delta(
|
||||
DeltaMessage(reasoning="reason\\"),
|
||||
finished=False,
|
||||
)
|
||||
second = normalizer.normalize_delta(
|
||||
DeltaMessage(reasoning="nnext\\", content="answer\\"),
|
||||
finished=False,
|
||||
)
|
||||
final = normalizer.normalize_delta(
|
||||
DeltaMessage(content="nend"),
|
||||
finished=True,
|
||||
)
|
||||
|
||||
assert first == DeltaMessage(reasoning="reason")
|
||||
assert second == DeltaMessage(reasoning="\nnext\\", content="answer")
|
||||
assert final == DeltaMessage(content="\nend")
|
||||
|
||||
def test_streaming_does_not_modify_tool_arguments(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
arguments = '{"value":"first\\\\nsecond"}'
|
||||
delta = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
index=0,
|
||||
function=DeltaFunctionCall(arguments=arguments),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
result = normalizer.normalize_delta(delta, finished=True)
|
||||
|
||||
assert result is not None
|
||||
assert result.tool_calls[0].function is not None
|
||||
assert result.tool_calls[0].function.arguments == arguments
|
||||
|
||||
def test_parse_recovers_content(self, mock_tokenizer, request_obj):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
"first\\nsecond",
|
||||
request_obj,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "first\nsecond"
|
||||
assert not tool_calls
|
||||
|
||||
def test_parse_delta_recovers_content(self, mock_tokenizer, request_obj):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
|
||||
chunk = parser.parse_delta(
|
||||
"first\\",
|
||||
[],
|
||||
request_obj,
|
||||
finished=False,
|
||||
)
|
||||
rest = parser.parse_delta(
|
||||
"nsecond",
|
||||
[],
|
||||
request_obj,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
# The trailing half of the escape is held back until it is complete.
|
||||
assert chunk is not None
|
||||
assert chunk.content == "first"
|
||||
assert rest is not None
|
||||
assert rest.content == "\nsecond"
|
||||
|
||||
def test_buffered_delta_is_dropped_not_sent_empty(
|
||||
self, mock_tokenizer, request_obj
|
||||
):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
parser.parse_delta("hello", [], request_obj, finished=False)
|
||||
|
||||
# The lone backtick is held back for the next chunk. Reporting it as an
|
||||
# all-None DeltaMessage would reach the client as an empty SSE chunk.
|
||||
assert parser.parse_delta("`", [], request_obj, finished=False) is None
|
||||
|
||||
def test_normalizer_drops_empty_delta(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
|
||||
assert (
|
||||
normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_normalizer_flushes_buffered_content_when_finished(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
|
||||
|
||||
final = normalizer.normalize_delta(DeltaMessage(), finished=True)
|
||||
|
||||
assert final is not None
|
||||
assert final.content == "`"
|
||||
@@ -45,6 +45,7 @@ from .interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
@@ -87,35 +88,186 @@ def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
fields["use_vit_merger"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
if "video_use_vit_merger" in hf_inputs:
|
||||
fields["video_use_vit_merger"] = MultiModalFieldConfig.batched(
|
||||
"video", keep_on_cpu=True
|
||||
)
|
||||
return fields
|
||||
|
||||
|
||||
def _expand_flags(value: object, num_items: int) -> list[bool | None]:
|
||||
"""Broadcast a batched per-item flag to exactly one entry per item."""
|
||||
if value is None:
|
||||
return [None] * num_items
|
||||
|
||||
raw: list[object]
|
||||
if isinstance(value, torch.Tensor):
|
||||
raw = value.reshape(-1).tolist()
|
||||
elif isinstance(value, list | tuple):
|
||||
raw = []
|
||||
for item in value:
|
||||
if isinstance(item, torch.Tensor):
|
||||
raw.extend(item.reshape(-1).tolist())
|
||||
else:
|
||||
raw.append(item)
|
||||
else:
|
||||
raw = [value]
|
||||
|
||||
if len(raw) == num_items:
|
||||
return [bool(v) for v in raw]
|
||||
if len(raw) == 1:
|
||||
return [bool(raw[0])] * num_items
|
||||
# Unexpected shape: keep one decision for the whole batch instead of
|
||||
# failing the request.
|
||||
return [any(bool(v) for v in raw)] * num_items
|
||||
|
||||
|
||||
def _vision_downsample_mode(flag: bool | None) -> str | None:
|
||||
if flag is None:
|
||||
return None
|
||||
return "16x" if flag else "4x"
|
||||
|
||||
|
||||
def _select_vision_items(
|
||||
image_input: MiniCPMVImagePixelInputs,
|
||||
item_indices: list[int],
|
||||
num_slices: list[int],
|
||||
) -> MiniCPMVImagePixelInputs:
|
||||
"""Slice a vision input down to the given items, keeping their order."""
|
||||
offsets = [0]
|
||||
for n in num_slices:
|
||||
offsets.append(offsets[-1] + n)
|
||||
|
||||
pixel_values: list[torch.Tensor] = []
|
||||
tgt_sizes: list[torch.Tensor] = []
|
||||
sizes: list[int] = []
|
||||
for i in item_indices:
|
||||
start, end = offsets[i], offsets[i + 1]
|
||||
pixel_values.extend(image_input["pixel_values"][start:end])
|
||||
tgt_sizes.append(image_input["tgt_sizes"][start:end])
|
||||
sizes.append(end - start)
|
||||
|
||||
return MiniCPMVImagePixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values,
|
||||
tgt_sizes=torch.cat(tgt_sizes, dim=0),
|
||||
num_slices=torch.tensor(sizes),
|
||||
)
|
||||
|
||||
|
||||
# HF-style modality-scoped keys inside `mm_processor_kwargs`. They scope a value
|
||||
# to a single modality and must not be forwarded to the processors, which take
|
||||
# flat arguments only.
|
||||
_MODALITY_SCOPED_MM_KWARGS = ("images_kwargs", "videos_kwargs", "audio_kwargs")
|
||||
|
||||
|
||||
def _flat_processor_kwargs(
|
||||
mm_kwargs: Mapping[str, object],
|
||||
downsample_mode: str,
|
||||
) -> dict[str, object]:
|
||||
"""Drop the modality-scoped nesting and pin the resolved downsample mode.
|
||||
|
||||
The placeholder count and the vision tower must read the same mode, so the
|
||||
value resolved for this modality is passed down explicitly rather than
|
||||
leaving the processor to fall back to its own default.
|
||||
"""
|
||||
kwargs = {
|
||||
key: value
|
||||
for key, value in mm_kwargs.items()
|
||||
if key not in _MODALITY_SCOPED_MM_KWARGS
|
||||
}
|
||||
kwargs["downsample_mode"] = downsample_mode
|
||||
return kwargs
|
||||
|
||||
|
||||
_MODALITY_TO_MM_KWARGS_SCOPE = {
|
||||
"image": "images_kwargs",
|
||||
"video": "videos_kwargs",
|
||||
}
|
||||
|
||||
|
||||
def _scoped_mm_kwarg(
|
||||
merged: Mapping[str, object],
|
||||
modality: str,
|
||||
key: str,
|
||||
) -> Any:
|
||||
"""Read a processor kwarg for one modality.
|
||||
|
||||
``merged`` is what ``_merge_and_resolve_mm_processor_kwargs`` returns: it
|
||||
routes a flat key into every scope whose processor declares it, and a value
|
||||
scoped to the other modality stays there, so it is not visible here.
|
||||
"""
|
||||
scoped = merged.get(_MODALITY_TO_MM_KWARGS_SCOPE[modality])
|
||||
if isinstance(scoped, Mapping):
|
||||
return scoped.get(key)
|
||||
return None
|
||||
|
||||
|
||||
class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
def _resolve_downsample_mode(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str = "image",
|
||||
) -> str:
|
||||
ds = mm_kwargs.get("downsample_mode")
|
||||
if ds is not None:
|
||||
return str(ds)
|
||||
# A flat `downsample_mode` applies to both modalities; the nested
|
||||
# `images_kwargs` / `videos_kwargs` forms scope it to one, matching
|
||||
# MiniCPMV4_6Processor.
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
downsample_mode = _scoped_mm_kwarg(merged, modality, "downsample_mode")
|
||||
if downsample_mode is not None:
|
||||
return str(downsample_mode)
|
||||
return info._get_downsample_mode()
|
||||
|
||||
def _resolve_max_slice_nums(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str = "image",
|
||||
) -> int | None:
|
||||
# Per-request max_slice_nums. None keeps the processor default.
|
||||
# Resolved like `downsample_mode` so the nested per-modality form
|
||||
# (`images_kwargs` / `videos_kwargs`) is honoured too. This must match
|
||||
# what `process_images` passes to the image processor: the placeholder
|
||||
# count and the visual token count are derived from it separately, so
|
||||
# they diverge (and the engine aborts) if the two sides read different
|
||||
# values.
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
max_slice = _scoped_mm_kwarg(merged, modality, "max_slice_nums")
|
||||
if max_slice is None:
|
||||
return None
|
||||
return int(max_slice)
|
||||
|
||||
def get_image_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
image_idx: int = 0,
|
||||
downsample_mode: str | None = None,
|
||||
max_slice_nums: int | None = None,
|
||||
) -> str:
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
return info.get_slice_image_placeholder(
|
||||
image_size,
|
||||
image_idx=image_idx,
|
||||
max_slice_nums=max_slice_nums,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
|
||||
def _video_local_id_prefix(self, video_idx: int) -> str:
|
||||
"""`<image_id>{idx}</image_id>` prefix emitted in front of a video placeholder.
|
||||
|
||||
MiniCPM-V 4.6 numbers each video with a local id, matching
|
||||
MiniCPMV4_6Processor. 4.7 treats a video as one temporal sequence and is
|
||||
trained without these ids, so it overrides this to return an empty string.
|
||||
"""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
return f"{id_start}{video_idx}{id_end}"
|
||||
|
||||
def get_video_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
@@ -144,8 +296,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
image_end = getattr(tokenizer, "image_end_token", "</image>")
|
||||
slice_start = getattr(tokenizer, "slice_start_token", "<slice>")
|
||||
slice_end = getattr(tokenizer, "slice_end_token", "</slice>")
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
|
||||
per_frame = image_start + video_token * source_tokens + image_end
|
||||
if grids[0] > 0 and grids[1] > 0 and patch_tokens > 0:
|
||||
@@ -154,7 +304,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
per_frame += "\n".join(rows)
|
||||
|
||||
body = per_frame * num_frames
|
||||
return f"{id_start}{video_idx}{id_end}" + body
|
||||
return self._video_local_id_prefix(video_idx) + body
|
||||
|
||||
def process_images(
|
||||
self,
|
||||
@@ -179,10 +329,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
n_images = len(parsed_images)
|
||||
image_processor = self.info.get_image_processor()
|
||||
patch_size = image_processor.patch_size
|
||||
ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="image")
|
||||
image_proc_kwargs = _flat_processor_kwargs(mm_kwargs, ds_mode)
|
||||
# `_flat_processor_kwargs` drops the nested form, so the value resolved
|
||||
# from it has to be pinned back for the processor and the prompt text
|
||||
# to agree on the slice count.
|
||||
image_max_slice = self._resolve_max_slice_nums(mm_kwargs, modality="image")
|
||||
if image_max_slice is not None:
|
||||
image_proc_kwargs["max_slice_nums"] = image_max_slice
|
||||
per_image_pixel_values: list[torch.Tensor] = []
|
||||
per_image_tgt_sizes: list[torch.Tensor] = []
|
||||
for image in parsed_images:
|
||||
ip_out = image_processor([image], **mm_kwargs)
|
||||
ip_out = image_processor([image], **image_proc_kwargs)
|
||||
pv = ip_out["pixel_values"] # (1, C, P, sum_W)
|
||||
ts = ip_out["target_sizes"] # (n_slices, 2)
|
||||
if pv.ndim == 4 and pv.shape[0] == 1:
|
||||
@@ -210,7 +368,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
"tgt_sizes": per_image_tgt_sizes,
|
||||
}
|
||||
|
||||
ds_mode = self._resolve_downsample_mode(mm_kwargs)
|
||||
insert_layer_id = getattr(
|
||||
self.info.get_hf_config(),
|
||||
"insert_layer_id",
|
||||
@@ -245,7 +402,11 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
image_processor = self.info.get_image_processor()
|
||||
patch_size = image_processor.patch_size
|
||||
video_max_slice = self.info.get_video_max_slice_num()
|
||||
video_mm_kwargs = {**mm_kwargs, "max_slice_nums": video_max_slice}
|
||||
video_ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="video")
|
||||
video_mm_kwargs = {
|
||||
**_flat_processor_kwargs(mm_kwargs, video_ds_mode),
|
||||
"max_slice_nums": video_max_slice,
|
||||
}
|
||||
|
||||
per_video_pixel_values: list[torch.Tensor] = []
|
||||
per_video_tgt_sizes: list[torch.Tensor] = []
|
||||
@@ -312,10 +473,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
if not per_video_pixel_values:
|
||||
return {}
|
||||
|
||||
insert_layer_id = getattr(self.info.get_hf_config(), "insert_layer_id", -1)
|
||||
merger_flag = video_ds_mode != "4x" and insert_layer_id >= 0
|
||||
|
||||
return {
|
||||
"video_pixel_values": per_video_pixel_values,
|
||||
"video_tgt_sizes": per_video_tgt_sizes,
|
||||
"video_image_sizes": per_video_image_sizes,
|
||||
"video_use_vit_merger": [
|
||||
torch.tensor([merger_flag], dtype=torch.bool)
|
||||
for _ in per_video_pixel_values
|
||||
],
|
||||
}
|
||||
|
||||
def _get_prompt_updates(
|
||||
@@ -324,7 +492,13 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs,
|
||||
):
|
||||
ds_mode = self._resolve_downsample_mode(hf_processor_mm_kwargs)
|
||||
image_ds_mode = self._resolve_downsample_mode(
|
||||
hf_processor_mm_kwargs, modality="image"
|
||||
)
|
||||
video_ds_mode = self._resolve_downsample_mode(
|
||||
hf_processor_mm_kwargs, modality="video"
|
||||
)
|
||||
image_max_slice = self._resolve_max_slice_nums(hf_processor_mm_kwargs)
|
||||
|
||||
placeholders = [
|
||||
("image", self.info.image_pattern),
|
||||
@@ -360,7 +534,8 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_image_prompt_texts(
|
||||
image_size,
|
||||
item_idx,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=image_ds_mode,
|
||||
max_slice_nums=image_max_slice,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
@@ -389,7 +564,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=video_ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
@@ -409,7 +584,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=video_ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
@@ -437,31 +612,28 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
new_update = super()._recompute_cached_prompt_update(
|
||||
cached_update, new_item_idx
|
||||
)
|
||||
# MiniCPM-V 4.6 prefixes video placeholders with `<image_id>{idx}</image_id>`
|
||||
# (the base class only rewrites the image modality).
|
||||
# Video placeholders carry an `<image_id>{idx}</image_id>` prefix that
|
||||
# the base class does not rewrite. 4.7 has no prefix, so this is a
|
||||
# no-op there.
|
||||
if cached_update.modality == "video":
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
old_prefix = self._video_local_id_prefix(cached_update.item_idx)
|
||||
new_prefix = self._video_local_id_prefix(new_item_idx)
|
||||
if old_prefix != new_prefix:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(
|
||||
f"{id_start}{prev_item_idx}{id_end}",
|
||||
f"{id_start}{new_item_idx}{id_end}",
|
||||
1,
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(old_prefix, new_prefix, 1),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
cached_encode(tokenizer, video_token, add_special_tokens=False),
|
||||
cached_encode(tokenizer, video_token, add_special_tokens=False),
|
||||
)
|
||||
)
|
||||
)
|
||||
return new_update
|
||||
|
||||
def _get_mm_fields_config(
|
||||
@@ -676,6 +848,31 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
|
||||
return source_tokens + grids[0] * grids[1] * patch_tokens
|
||||
|
||||
|
||||
def _stack_vit_merger_qkv(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Fold separate vit_merger q/k/v projections into the fused qkv_proj module.
|
||||
|
||||
HF checkpoints store the window-attention projections separately, while
|
||||
vLLM keeps them fused. The shard_id mechanism on the submodule's
|
||||
WeightsMapper cannot do this: AutoWeightsLoader still recurses over every
|
||||
incoming key after calling a submodule's load_weights, so a "k_proj" key
|
||||
with no matching child raises before the mapping is honoured.
|
||||
"""
|
||||
pairs = (
|
||||
("vit_merger.self_attn.q_proj.", "q"),
|
||||
("vit_merger.self_attn.k_proj.", "k"),
|
||||
("vit_merger.self_attn.v_proj.", "v"),
|
||||
)
|
||||
for name, tensor in weights:
|
||||
for src, shard in pairs:
|
||||
if src in name:
|
||||
name = name.replace(src, "vit_merger.self_attn.qkv_proj.", 1)
|
||||
tensor.shard_id = shard
|
||||
break
|
||||
yield name, tensor
|
||||
|
||||
|
||||
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -957,6 +1154,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
SupportsPP,
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsEagle3,
|
||||
SupportsMRoPE,
|
||||
):
|
||||
supports_encoder_tp_data = True
|
||||
@@ -1031,7 +1229,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
apply_encoder_attention_mask=True,
|
||||
prefix=maybe_prefix(prefix, "vpm"),
|
||||
)
|
||||
if config.drop_vision_last_layer:
|
||||
if self._drop_vision_last_layer():
|
||||
self.vpm.encoder.layers = self.vpm.encoder.layers[:-1]
|
||||
|
||||
self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger(
|
||||
@@ -1062,6 +1260,14 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _drop_vision_last_layer(self) -> bool:
|
||||
"""Whether the last vision encoder layer is dropped.
|
||||
|
||||
MiniCPM-V 4.6 checkpoints always define this. 4.7's config no longer
|
||||
carries the field, so it overrides this.
|
||||
"""
|
||||
return self.config.drop_vision_last_layer
|
||||
|
||||
# ----- Multimodal parsing -----
|
||||
|
||||
def _parse_and_validate_vision_input(
|
||||
@@ -1174,35 +1380,35 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
if image_input["type"] == "image_embeds":
|
||||
return image_input["image_embeds"]
|
||||
|
||||
downsample_mode = None
|
||||
if use_vit_merger is not None:
|
||||
downsample_mode = "16x" if use_vit_merger else "4x"
|
||||
image_features = self.get_vision_hidden_states(
|
||||
image_input,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
num_slices = image_input["num_slices"]
|
||||
results = []
|
||||
idx = 0
|
||||
for n in num_slices.tolist():
|
||||
group = image_features[idx : idx + n]
|
||||
results.append(torch.cat(group, dim=0))
|
||||
idx += n
|
||||
return results
|
||||
num_slices = image_input["num_slices"].tolist()
|
||||
flags = _expand_flags(use_vit_merger, len(num_slices))
|
||||
|
||||
# The merger flag is per request, but the model runner may batch items
|
||||
# from several concurrent requests. Encoding such a mixed batch would
|
||||
# apply one request's downsample mode to another request's items, so
|
||||
# group by flag and encode each group separately.
|
||||
encoded: dict[int, torch.Tensor] = {}
|
||||
for flag in dict.fromkeys(flags):
|
||||
item_indices = [i for i, f in enumerate(flags) if f == flag]
|
||||
group_input = _select_vision_items(image_input, item_indices, num_slices)
|
||||
group_features = self.get_vision_hidden_states(
|
||||
group_input,
|
||||
downsample_mode=_vision_downsample_mode(flag),
|
||||
)
|
||||
|
||||
idx = 0
|
||||
for i in item_indices:
|
||||
n = num_slices[i]
|
||||
encoded[i] = torch.cat(group_features[idx : idx + n], dim=0)
|
||||
idx += n
|
||||
|
||||
return [encoded[i] for i in range(len(num_slices))]
|
||||
|
||||
# ----- Multimodal embedding interface -----
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
use_vit_merger_tensors = kwargs.pop("use_vit_merger", None)
|
||||
use_vit_merger = None
|
||||
if use_vit_merger_tensors is not None:
|
||||
if isinstance(use_vit_merger_tensors, torch.Tensor):
|
||||
use_vit_merger = bool(use_vit_merger_tensors.any().item())
|
||||
elif isinstance(use_vit_merger_tensors, list | tuple):
|
||||
use_vit_merger = any(
|
||||
bool(t.any().item()) if isinstance(t, torch.Tensor) else bool(t)
|
||||
for t in use_vit_merger_tensors
|
||||
)
|
||||
use_vit_merger = kwargs.pop("use_vit_merger", None)
|
||||
video_use_vit_merger = kwargs.pop("video_use_vit_merger", None)
|
||||
|
||||
image_kwargs = {
|
||||
k: v
|
||||
@@ -1235,7 +1441,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
multimodal_embeddings += tuple(
|
||||
self._process_vision_input(
|
||||
video_input,
|
||||
use_vit_merger=use_vit_merger,
|
||||
use_vit_merger=video_use_vit_merger,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1298,7 +1504,10 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
return loader.load_weights(
|
||||
_stack_vit_merger_qkv(weights),
|
||||
mapper=self.hf_to_vllm_mapper,
|
||||
)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
return MultiModelKeys.from_string_field(
|
||||
|
||||
@@ -0,0 +1,764 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only MiniCPM-V 4.7 model.
|
||||
|
||||
Same vision/LLM stack as MiniCPM-V 4.6, plus canvas 3D M-RoPE.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig
|
||||
|
||||
from .minicpmv import MiniCPMVDummyInputsBuilder
|
||||
from .minicpmv4_6 import (
|
||||
MiniCPMV4_6ForConditionalGeneration,
|
||||
MiniCPMV4_6MultiModalProcessor,
|
||||
MiniCPMV4_6ProcessingInfo,
|
||||
_scoped_mm_kwarg,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MiniCPMV4_7ProcessingInfo(MiniCPMV4_6ProcessingInfo):
|
||||
def _mm_max_slice_nums(self, **kwargs: object) -> int | None:
|
||||
# No schema resolution here: it would call back into `get_hf_processor`
|
||||
# below, which resolves this same value.
|
||||
merged = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
max_slice = _scoped_mm_kwarg(merged, "image", "max_slice_nums")
|
||||
if max_slice is None:
|
||||
max_slice = merged.get("max_slice_nums")
|
||||
if max_slice is None:
|
||||
return None
|
||||
return int(max_slice)
|
||||
|
||||
def _configured_max_slice_nums(self) -> int | None:
|
||||
# Configured (server-level) value only: per-request values are passed
|
||||
# explicitly to the image processor call, so they must not be written
|
||||
# back to the shared processor instance.
|
||||
return self._mm_max_slice_nums()
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
hf_processor = super().get_hf_processor(**kwargs)
|
||||
image_processor = getattr(hf_processor, "image_processor", None)
|
||||
configured = self._configured_max_slice_nums()
|
||||
if image_processor is not None and configured is not None:
|
||||
image_processor.max_slice_nums = configured
|
||||
return hf_processor
|
||||
|
||||
def get_image_max_slice_num(self) -> int:
|
||||
max_slice = self._configured_max_slice_nums()
|
||||
if max_slice is not None:
|
||||
return max_slice
|
||||
return super().get_image_max_slice_num()
|
||||
|
||||
|
||||
class MiniCPMV4_7MultiModalProcessor(MiniCPMV4_6MultiModalProcessor):
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
fields = dict(super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs))
|
||||
# Canvas M-RoPE places slices from `tgt_sizes`. Stripping the payload of
|
||||
# a prefix-cache-covered item only keeps CPU-side metadata, so without
|
||||
# this a cached image loses its target sizes, `image_bounds` and
|
||||
# `target_sizes` disagree, and the whole request silently falls back to
|
||||
# sequential positions.
|
||||
for key, modality in (("tgt_sizes", "image"), ("video_tgt_sizes", "video")):
|
||||
if key in fields:
|
||||
fields[key] = MultiModalFieldConfig.batched(modality, keep_on_cpu=True)
|
||||
return fields
|
||||
|
||||
def _video_local_id_prefix(self, video_idx: int) -> str:
|
||||
# MiniCPMV4_7Processor emits no local `<image_id>` in front of video
|
||||
# placeholders: a video is one temporal sequence and 4.7 was trained
|
||||
# without them.
|
||||
return ""
|
||||
|
||||
def get_image_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
image_idx: int = 0,
|
||||
downsample_mode: str | None = None,
|
||||
max_slice_nums: int | None = None,
|
||||
) -> str:
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
if max_slice_nums is None:
|
||||
max_slice_nums = info.get_image_max_slice_num()
|
||||
return info.get_slice_image_placeholder(
|
||||
image_size,
|
||||
image_idx=image_idx,
|
||||
max_slice_nums=max_slice_nums,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
|
||||
|
||||
def _as_hw_pairs(tensor: torch.Tensor) -> torch.Tensor:
|
||||
data = tensor.to(torch.long)
|
||||
if data.numel() == 0:
|
||||
return data.reshape(0, 2)
|
||||
if data.shape[-1] != 2:
|
||||
raise ValueError(f"expected last dim 2, got {tuple(data.shape)}")
|
||||
return data.reshape(-1, 2)
|
||||
|
||||
|
||||
def _sequential_positions(seq_len: int) -> torch.Tensor:
|
||||
"""Positions for the fallback where canvas M-RoPE cannot be computed."""
|
||||
return torch.arange(seq_len).unsqueeze(0).expand(3, -1)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MiniCPMV4_7MultiModalProcessor,
|
||||
info=MiniCPMV4_7ProcessingInfo,
|
||||
dummy_inputs=MiniCPMVDummyInputsBuilder,
|
||||
)
|
||||
class MiniCPMV4_7ForConditionalGeneration(MiniCPMV4_6ForConditionalGeneration):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
self._init_canvas_mrope(vllm_config)
|
||||
|
||||
def _drop_vision_last_layer(self) -> bool:
|
||||
# MiniCPMV4_7Config no longer carries this field (transformers marks it
|
||||
# removed), and 4.7 keeps every vision encoder layer.
|
||||
return False
|
||||
|
||||
def _init_canvas_mrope(self, vllm_config: VllmConfig) -> None:
|
||||
config = self.config
|
||||
uses_canvas = bool(getattr(config, "uses_mrope_canvas", False))
|
||||
if not uses_canvas:
|
||||
mrope_mode = getattr(config, "mrope_mode", None)
|
||||
uses_canvas = mrope_mode is not None and "canvas" in str(mrope_mode).lower()
|
||||
|
||||
self._uses_canvas_mrope = uses_canvas
|
||||
self._canvas_special_ids: dict[str, int] | None = None
|
||||
if not uses_canvas:
|
||||
return
|
||||
|
||||
# MiniCPMV4_7Config pins the structural ids in config.json; the
|
||||
# conversion script resolves them from the tokenizer. Prefer them so the
|
||||
# canvas matches training, and fall back to the tokenizer for checkpoints
|
||||
# whose config predates these fields.
|
||||
configured_ids: dict[str, int | None] = {
|
||||
"im_start_id": getattr(config, "image_start_id", None),
|
||||
"im_end_id": getattr(config, "image_end_id", None),
|
||||
"slice_start_id": getattr(config, "slice_start_id", None),
|
||||
"slice_end_id": getattr(config, "slice_end_id", None),
|
||||
"newline_id": getattr(config, "newline_id", None),
|
||||
}
|
||||
if all(value is not None for value in configured_ids.values()):
|
||||
self._canvas_special_ids = {
|
||||
name: int(value)
|
||||
for name, value in configured_ids.items()
|
||||
if value is not None
|
||||
}
|
||||
return
|
||||
|
||||
from vllm.tokenizers.registry import get_tokenizer
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
tokenizer = get_tokenizer(
|
||||
model_config.tokenizer,
|
||||
revision=model_config.tokenizer_revision,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
)
|
||||
self._canvas_special_ids = {
|
||||
"im_start_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "image_start_token", "<image>")
|
||||
),
|
||||
"im_end_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "image_end_token", "</image>")
|
||||
),
|
||||
"slice_start_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "slice_start_token", "<slice>")
|
||||
),
|
||||
"slice_end_id": tokenizer.convert_tokens_to_ids(
|
||||
getattr(tokenizer, "slice_end_token", "</slice>")
|
||||
),
|
||||
"newline_id": tokenizer.encode("\n", add_special_tokens=False)[0],
|
||||
}
|
||||
|
||||
def get_mrope_input_positions(
|
||||
self,
|
||||
input_tokens: list[int],
|
||||
mm_features: list[MultiModalFeatureSpec],
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
seq_len = len(input_tokens)
|
||||
if (
|
||||
not getattr(self, "_uses_canvas_mrope", False)
|
||||
or self._canvas_special_ids is None
|
||||
or not mm_features
|
||||
):
|
||||
return _sequential_positions(seq_len), 0
|
||||
|
||||
input_ids = torch.tensor(input_tokens, dtype=torch.long)
|
||||
image_bounds = build_image_bounds(input_ids, self._canvas_special_ids)
|
||||
|
||||
target_sizes_list: list[torch.Tensor] = []
|
||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||
if mm_feature.data is None:
|
||||
continue
|
||||
for key in ("tgt_sizes", "video_tgt_sizes"):
|
||||
tgt = mm_feature.data.get(key)
|
||||
if tgt is None or not isinstance(tgt.data, torch.Tensor):
|
||||
continue
|
||||
try:
|
||||
target_sizes_list.append(_as_hw_pairs(tgt.data))
|
||||
except ValueError:
|
||||
# Not an (N, 2) size table. The bounds/sizes match below
|
||||
# turns this into a sequential fallback.
|
||||
continue
|
||||
|
||||
if not target_sizes_list or image_bounds.numel() == 0:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE missing inputs: "
|
||||
"tgt_chunks=%s image_bounds=%s; using sequential positions.",
|
||||
len(target_sizes_list),
|
||||
tuple(image_bounds.shape) if image_bounds is not None else None,
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
|
||||
target_sizes = torch.cat(target_sizes_list, dim=0)
|
||||
if target_sizes.ndim != 2 or target_sizes.shape[-1] != 2:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE unexpected tgt_sizes shape %s; "
|
||||
"using sequential positions.",
|
||||
tuple(target_sizes.shape),
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
if image_bounds.shape[0] != target_sizes.shape[0]:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE size mismatch: "
|
||||
"image_bounds=%s target_sizes=%s; using sequential positions.",
|
||||
tuple(image_bounds.shape),
|
||||
tuple(target_sizes.shape),
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
|
||||
try:
|
||||
pos3d = _compute_canvas_single(
|
||||
input_ids,
|
||||
torch.arange(seq_len, dtype=torch.long),
|
||||
image_bounds,
|
||||
target_sizes,
|
||||
self._canvas_special_ids,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"MiniCPM-V 4.7 canvas M-RoPE failed; "
|
||||
"falling back to sequential positions.",
|
||||
exc_info=True,
|
||||
)
|
||||
return _sequential_positions(seq_len), 0
|
||||
return pos3d, canvas_rope_delta(pos3d, seq_len)
|
||||
|
||||
|
||||
# Canvas 3D M-RoPE: position ids for the slice canvas.
|
||||
def compute_llm_grid(
|
||||
n_tokens: int, vision_grid_height: int, vision_grid_width: int
|
||||
) -> tuple[int, int]:
|
||||
if n_tokens <= 0 or vision_grid_height <= 0 or vision_grid_width <= 0:
|
||||
return 0, 0
|
||||
vit_total = vision_grid_height * vision_grid_width
|
||||
if vit_total == n_tokens:
|
||||
return vision_grid_height, vision_grid_width
|
||||
if vit_total < n_tokens:
|
||||
return 0, 0
|
||||
factor = int(round(math.sqrt(vit_total / n_tokens)))
|
||||
if (
|
||||
factor > 0
|
||||
and vision_grid_height % factor == 0
|
||||
and vision_grid_width % factor == 0
|
||||
):
|
||||
height, width = vision_grid_height // factor, vision_grid_width // factor
|
||||
if height * width == n_tokens:
|
||||
return height, width
|
||||
ratio = vision_grid_height / vision_grid_width
|
||||
width = max(1, int(round(math.sqrt(n_tokens / ratio))))
|
||||
height = n_tokens // width
|
||||
if height * width == n_tokens and height > 0:
|
||||
return height, width
|
||||
return 0, 0
|
||||
|
||||
|
||||
def build_image_bounds(
|
||||
input_ids: torch.LongTensor, special_token_ids: dict
|
||||
) -> torch.LongTensor:
|
||||
start_ids = [
|
||||
x
|
||||
for x in (
|
||||
special_token_ids.get("im_start_id"),
|
||||
special_token_ids.get("slice_start_id"),
|
||||
)
|
||||
if x is not None
|
||||
]
|
||||
end_ids = [
|
||||
x
|
||||
for x in (
|
||||
special_token_ids.get("im_end_id"),
|
||||
special_token_ids.get("slice_end_id"),
|
||||
)
|
||||
if x is not None
|
||||
]
|
||||
if not start_ids or not end_ids:
|
||||
return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
|
||||
start_cond = torch.zeros_like(input_ids, dtype=torch.bool)
|
||||
end_cond = torch.zeros_like(input_ids, dtype=torch.bool)
|
||||
for sid in start_ids:
|
||||
start_cond |= input_ids == sid
|
||||
for eid in end_ids:
|
||||
end_cond |= input_ids == eid
|
||||
starts = torch.where(start_cond)[0] + 1
|
||||
ends = torch.where(end_cond)[0]
|
||||
n = min(len(starts), len(ends))
|
||||
if n == 0:
|
||||
return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
|
||||
return torch.stack([starts[:n], ends[:n]], dim=-1)
|
||||
|
||||
|
||||
def _compute_canvas_single(
|
||||
input_ids, position_ids_2d, image_bound, target_sizes, special_token_ids
|
||||
):
|
||||
# Inference always handles one unpadded sequence. The packed (cu_seqlens)
|
||||
# form below is kept only to mirror the transformers reference
|
||||
# implementation of its canvas position computation.
|
||||
seq_len = input_ids.shape[0]
|
||||
cu_seqlens = torch.tensor([0, seq_len], device=input_ids.device, dtype=torch.long)
|
||||
return _compute_canvas_packed(
|
||||
position_ids_2d.unsqueeze(0),
|
||||
cu_seqlens,
|
||||
image_bound,
|
||||
target_sizes,
|
||||
input_ids.unsqueeze(0),
|
||||
special_token_ids,
|
||||
)[:, 0, :]
|
||||
|
||||
|
||||
def _target_hw(target_sizes: torch.Tensor, index: int) -> tuple[int, int]:
|
||||
"""Patch-grid height/width recorded for one image or slice."""
|
||||
return int(target_sizes[index, 0]), int(target_sizes[index, 1])
|
||||
|
||||
|
||||
class _CanvasLayout(NamedTuple):
|
||||
"""Grid of one canvas measured in LLM tokens.
|
||||
|
||||
A canvas is a thumbnail plus the slices cut from it, tiled row-major.
|
||||
"""
|
||||
|
||||
thumb_h: int
|
||||
thumb_w: int
|
||||
"""Thumbnail grid."""
|
||||
slice_h: int
|
||||
slice_w: int
|
||||
"""Per-slice grid. Zero when the canvas has no slices."""
|
||||
cols: int
|
||||
"""Slices sharing a canvas row. Zero when the canvas has no slices."""
|
||||
height: int
|
||||
width: int
|
||||
"""Assembled canvas."""
|
||||
|
||||
|
||||
def _canvas_layout(
|
||||
thumbnail: tuple[int, int, int],
|
||||
slices: list[tuple[int, int, int]],
|
||||
target_sizes: torch.Tensor,
|
||||
) -> _CanvasLayout:
|
||||
"""Measure the canvas for one ``(thumbnail, slices)`` group.
|
||||
|
||||
``thumbnail`` and each slice are ``(start, end, target_index)`` token spans.
|
||||
"""
|
||||
t_bs, t_be, t_gi = thumbnail
|
||||
thumb_h, thumb_w = compute_llm_grid(t_be - t_bs, *_target_hw(target_sizes, t_gi))
|
||||
if not slices:
|
||||
return _CanvasLayout(
|
||||
thumb_h, thumb_w, 0, 0, 0, max(thumb_h, 1), max(thumb_w, 1)
|
||||
)
|
||||
|
||||
cols = len(slices)
|
||||
for k in range(len(slices) - 1):
|
||||
if slices[k + 1][0] - slices[k][1] > 2:
|
||||
cols = k + 1
|
||||
break
|
||||
rows = len(slices) // cols if cols > 0 else 1
|
||||
if rows * cols != len(slices):
|
||||
rows, cols = 1, len(slices)
|
||||
|
||||
s0_bs, s0_be, s0_gi = slices[0]
|
||||
slice_h, slice_w = compute_llm_grid(s0_be - s0_bs, *_target_hw(target_sizes, s0_gi))
|
||||
return _CanvasLayout(
|
||||
thumb_h, thumb_w, slice_h, slice_w, cols, rows * slice_h, cols * slice_w
|
||||
)
|
||||
|
||||
|
||||
def _assign_canvas_span(
|
||||
pos3d: torch.Tensor,
|
||||
*,
|
||||
thumbnail: tuple[int, int, int],
|
||||
span_end: int,
|
||||
slices: list[tuple[int, int, int]],
|
||||
base: int,
|
||||
target_sizes: torch.Tensor,
|
||||
) -> int:
|
||||
"""Place one canvas at ``base``, writing its 3-D positions into ``pos3d``.
|
||||
|
||||
Shared by the video-frame and single-image paths; they differ only in how
|
||||
the per-canvas base advances, not in the geometry placed here.
|
||||
|
||||
Mutates ``pos3d`` in place, as the packed canvas computation has always
|
||||
done. Returns the distance to the next canvas base, so each caller keeps
|
||||
only its own base advancement.
|
||||
"""
|
||||
t_bs, t_be, _ = thumbnail
|
||||
device = pos3d.device
|
||||
span_start = t_bs - 1 # the <im_start> token
|
||||
n_thumb = t_be - t_bs
|
||||
layout = _canvas_layout(thumbnail, slices, target_sizes)
|
||||
cols, canvas_H, canvas_W = layout.cols, layout.height, layout.width
|
||||
|
||||
# base coat
|
||||
pos3d[0, 0, span_start:span_end] = base
|
||||
pos3d[1, 0, span_start:span_end] = base
|
||||
pos3d[2, 0, span_start:span_end] = base
|
||||
|
||||
# <im_start> -> halo
|
||||
halo_lo = max(base - 1, 0)
|
||||
pos3d[1, 0, span_start] = halo_lo
|
||||
pos3d[2, 0, span_start] = halo_lo
|
||||
|
||||
# <im_end> -> halo (right after thumbnail)
|
||||
if t_be < span_end:
|
||||
pos3d[1, 0, t_be] = base + canvas_H
|
||||
pos3d[2, 0, t_be] = base + canvas_W
|
||||
|
||||
# thumbnail visual tokens
|
||||
llm_th, llm_tw = layout.thumb_h, layout.thumb_w
|
||||
if llm_th > 0 and llm_tw > 0 and llm_th * llm_tw == n_thumb:
|
||||
if slices and canvas_H > 0 and canvas_W > 0:
|
||||
h_c = torch.linspace(0, canvas_H - 1, llm_th, device=device).round().long()
|
||||
w_c = torch.linspace(0, canvas_W - 1, llm_tw, device=device).round().long()
|
||||
else:
|
||||
h_c = torch.arange(llm_th, device=device)
|
||||
w_c = torch.arange(llm_tw, device=device)
|
||||
h_idx = h_c.view(-1, 1).expand(-1, llm_tw).reshape(-1)
|
||||
w_idx = w_c.view(1, -1).expand(llm_th, -1).reshape(-1)
|
||||
pos3d[0, 0, t_bs:t_be] = base
|
||||
pos3d[1, 0, t_bs:t_be] = h_idx + base
|
||||
pos3d[2, 0, t_bs:t_be] = w_idx + base
|
||||
else:
|
||||
fallback = torch.arange(n_thumb, device=device, dtype=torch.long) + base
|
||||
for d in range(3):
|
||||
pos3d[d, 0, t_bs:t_be] = fallback
|
||||
|
||||
# slice visual tokens + slice specials
|
||||
for k, (s_bs, s_be, s_gi) in enumerate(slices):
|
||||
n_s = s_be - s_bs
|
||||
sh_k, sw_k = compute_llm_grid(n_s, *_target_hw(target_sizes, s_gi))
|
||||
h_off = (k // cols) * layout.slice_h
|
||||
w_off = (k % cols) * layout.slice_w
|
||||
|
||||
slice_start_pos = s_bs - 1
|
||||
if slice_start_pos >= span_start:
|
||||
pos3d[1, 0, slice_start_pos] = base + h_off
|
||||
pos3d[2, 0, slice_start_pos] = base + w_off
|
||||
|
||||
slice_end_pos = s_be
|
||||
if sh_k > 0 and sw_k > 0:
|
||||
end_h = h_off + sh_k - 1
|
||||
end_w = w_off + sw_k - 1
|
||||
else:
|
||||
end_h = h_off
|
||||
end_w = w_off
|
||||
if slice_end_pos < span_end:
|
||||
pos3d[1, 0, slice_end_pos] = base + end_h
|
||||
pos3d[2, 0, slice_end_pos] = base + end_w
|
||||
|
||||
if sh_k > 0 and sw_k > 0 and sh_k * sw_k == n_s:
|
||||
h_idx = (
|
||||
torch.arange(sh_k, device=device)
|
||||
.view(-1, 1)
|
||||
.expand(-1, sw_k)
|
||||
.reshape(-1)
|
||||
)
|
||||
w_idx = (
|
||||
torch.arange(sw_k, device=device)
|
||||
.view(1, -1)
|
||||
.expand(sh_k, -1)
|
||||
.reshape(-1)
|
||||
)
|
||||
pos3d[0, 0, s_bs:s_be] = base
|
||||
pos3d[1, 0, s_bs:s_be] = h_idx + h_off + base
|
||||
pos3d[2, 0, s_bs:s_be] = w_idx + w_off + base
|
||||
else:
|
||||
fallback = torch.arange(n_s, device=device, dtype=torch.long) + base
|
||||
for d in range(3):
|
||||
pos3d[d, 0, s_bs:s_be] = fallback
|
||||
|
||||
# \n between slice rows -> W = right edge + 1.
|
||||
# H stays within the ended row (does NOT cross to next row): the \n sits at
|
||||
# the right-outside of the row that just ended, at H = that row's last row
|
||||
# and W = one column past the rightmost slice.
|
||||
for k in range(len(slices) - 1):
|
||||
gap_s = slices[k][1]
|
||||
gap_e = slices[k + 1][0]
|
||||
if gap_e - gap_s <= 2:
|
||||
continue
|
||||
boundary_h = ((k // cols) + 1) * layout.slice_h - 1
|
||||
right_edge_w = cols * layout.slice_w
|
||||
for nl_pos in range(gap_s + 1, gap_e - 1):
|
||||
pos3d[1, 0, nl_pos] = base + boundary_h
|
||||
pos3d[2, 0, nl_pos] = base + right_edge_w
|
||||
|
||||
return max(canvas_H, canvas_W) + 1
|
||||
|
||||
|
||||
def _compute_canvas_packed(
|
||||
position_ids_2d,
|
||||
cu_seqlens,
|
||||
image_bound,
|
||||
target_sizes,
|
||||
input_ids,
|
||||
special_token_ids,
|
||||
):
|
||||
if position_ids_2d.ndim == 1:
|
||||
position_ids_2d = position_ids_2d.unsqueeze(0)
|
||||
if input_ids.ndim == 1:
|
||||
input_ids = input_ids.unsqueeze(0)
|
||||
|
||||
has_images = (
|
||||
isinstance(image_bound, torch.Tensor)
|
||||
and image_bound.numel() > 0
|
||||
and isinstance(target_sizes, torch.Tensor)
|
||||
and target_sizes.numel() > 0
|
||||
)
|
||||
if not has_images:
|
||||
return position_ids_2d.unsqueeze(0).expand(3, -1, -1).contiguous()
|
||||
|
||||
device = position_ids_2d.device
|
||||
pos3d = position_ids_2d.unsqueeze(0).expand(3, -1, -1).clone()
|
||||
ids_flat = input_ids.view(-1)
|
||||
|
||||
im_start_id = special_token_ids.get("im_start_id")
|
||||
im_end_id = special_token_ids.get("im_end_id")
|
||||
slice_start_id = special_token_ids.get("slice_start_id")
|
||||
slice_end_id = special_token_ids.get("slice_end_id")
|
||||
|
||||
structural_ids = {
|
||||
v
|
||||
for v in (im_start_id, im_end_id, slice_start_id, slice_end_id)
|
||||
if v is not None
|
||||
}
|
||||
|
||||
num_seqs = len(cu_seqlens) - 1
|
||||
img_ptr = 0
|
||||
num_imgs = len(image_bound)
|
||||
|
||||
newline_id = special_token_ids.get("newline_id")
|
||||
gap_ok_ids = structural_ids | ({newline_id} if newline_id is not None else set())
|
||||
|
||||
def _group_end_pos(g, s_end):
|
||||
"""First position after a group's trailing structural tokens.
|
||||
|
||||
Matches the transformers reference, which closes a span over every
|
||||
marker in ``markers`` -- newlines included. The chat template joins
|
||||
content parts with ``\\n``, so an image followed by text always ends with
|
||||
one; leaving it out moves that newline out of the canvas and pushes
|
||||
every later position one step forward.
|
||||
"""
|
||||
le = g["slices"][-1][1] if g["slices"] else g["thumbnail"][1]
|
||||
ge = le
|
||||
while ge < s_end and ids_flat[ge].item() in gap_ok_ids:
|
||||
ge += 1
|
||||
return min(ge, s_end)
|
||||
|
||||
def _is_tight_gap(g_prev, g_next, s_end):
|
||||
"""Check if only structural/nl tokens sit between two groups."""
|
||||
ge = _group_end_pos(g_prev, s_end)
|
||||
ns = g_next["thumbnail"][0] - 1 # im_start of next group
|
||||
return all(ids_flat[p].item() in gap_ok_ids for p in range(ge, ns))
|
||||
|
||||
for si in range(num_seqs):
|
||||
s_start = cu_seqlens[si].item()
|
||||
s_end = cu_seqlens[si + 1].item()
|
||||
if s_start >= s_end:
|
||||
continue
|
||||
|
||||
seq_imgs = []
|
||||
while img_ptr < num_imgs:
|
||||
bs_ = image_bound[img_ptr, 0].item()
|
||||
be_ = image_bound[img_ptr, 1].item()
|
||||
if bs_ >= s_start and be_ <= s_end:
|
||||
seq_imgs.append((bs_, be_, img_ptr))
|
||||
img_ptr += 1
|
||||
else:
|
||||
break
|
||||
if not seq_imgs:
|
||||
continue
|
||||
|
||||
# ---- group images: thumbnail + following slices ----
|
||||
# merge consecutive no-slice frames into one video group
|
||||
raw_groups = []
|
||||
cur_group: dict[str, Any] | None = None
|
||||
for bs_, be_, gi in seq_imgs:
|
||||
marker_pos = bs_ - 1
|
||||
is_slice = (
|
||||
slice_start_id is not None
|
||||
and marker_pos >= s_start
|
||||
and ids_flat[marker_pos].item() == slice_start_id
|
||||
)
|
||||
if is_slice and cur_group is not None:
|
||||
cur_group["slices"].append((bs_, be_, gi))
|
||||
else:
|
||||
if cur_group is not None:
|
||||
raw_groups.append(cur_group)
|
||||
cur_group = {"thumbnail": (bs_, be_, gi), "slices": []}
|
||||
if cur_group is not None:
|
||||
raw_groups.append(cur_group)
|
||||
|
||||
# merge consecutive tightly-adjacent groups into video groups.
|
||||
# Two groups are "tightly adjacent" if the gap between them
|
||||
# contains only structural tokens and \n (no real text).
|
||||
# This handles both no-slice frames and frames-with-slices.
|
||||
groups = []
|
||||
i_g = 0
|
||||
while i_g < len(raw_groups):
|
||||
video_frames = [raw_groups[i_g]]
|
||||
j_g = i_g + 1
|
||||
while j_g < len(raw_groups) and _is_tight_gap(
|
||||
raw_groups[j_g - 1], raw_groups[j_g], s_end
|
||||
):
|
||||
video_frames.append(raw_groups[j_g])
|
||||
j_g += 1
|
||||
if len(video_frames) == 1:
|
||||
groups.append(raw_groups[i_g])
|
||||
else:
|
||||
groups.append({"video_frames": video_frames})
|
||||
i_g = j_g
|
||||
|
||||
# ---- assign 3-D positions ----
|
||||
pos = 0
|
||||
cursor = s_start
|
||||
|
||||
for group in groups:
|
||||
is_video_group = "video_frames" in group
|
||||
|
||||
if is_video_group:
|
||||
# === Video group: multiple frames merged ===
|
||||
# Each frame gets its own base_f so different frames have
|
||||
# distinct spatial positions. Inter-frame gap tokens (\n
|
||||
# etc.) are pure 1D text. After each gap, pos advances
|
||||
# by 1 extra so halo_lo of the next frame does not
|
||||
# collide with the last gap token.
|
||||
frames = group["video_frames"]
|
||||
first_frame = frames[0]
|
||||
last_frame = frames[-1]
|
||||
|
||||
# group_start = <im_start> of first frame
|
||||
group_start = first_frame["thumbnail"][0] - 1
|
||||
# group_end = after last structural token of last frame
|
||||
group_end = _group_end_pos(last_frame, s_end)
|
||||
|
||||
# text before video group
|
||||
text_len = group_start - cursor
|
||||
if text_len > 0:
|
||||
t_pos = (
|
||||
torch.arange(text_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, cursor:group_start] = t_pos
|
||||
pos += text_len
|
||||
|
||||
# process each frame with its own per-frame base
|
||||
frame_cursor = group_start
|
||||
for vf in frames:
|
||||
t_bs = vf["thumbnail"][0]
|
||||
vf_slices = vf["slices"]
|
||||
im_start_pos = t_bs - 1
|
||||
|
||||
# --- inter-frame gap: pure 1D text T=H=W=pos ---
|
||||
gap_len = im_start_pos - frame_cursor
|
||||
if gap_len > 0:
|
||||
t_pos = (
|
||||
torch.arange(gap_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, frame_cursor:im_start_pos] = t_pos
|
||||
pos += gap_len
|
||||
pos += 1 # buffer: prevents halo_lo collision
|
||||
|
||||
# --- per-frame base ---
|
||||
base_f = pos
|
||||
frame_end = _group_end_pos(vf, group_end)
|
||||
pos = base_f + _assign_canvas_span(
|
||||
pos3d,
|
||||
thumbnail=vf["thumbnail"],
|
||||
span_end=frame_end,
|
||||
slices=vf_slices,
|
||||
base=base_f,
|
||||
target_sizes=target_sizes,
|
||||
)
|
||||
frame_cursor = frame_end
|
||||
|
||||
# trailing tokens after last frame (if any)
|
||||
if frame_cursor < group_end:
|
||||
trail_len = group_end - frame_cursor
|
||||
t_pos = (
|
||||
torch.arange(trail_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, frame_cursor:group_end] = t_pos
|
||||
pos += trail_len
|
||||
|
||||
cursor = group_end
|
||||
|
||||
else:
|
||||
# === Single image group (thumbnail + optional slices) ===
|
||||
slices = group["slices"]
|
||||
|
||||
group_start = group["thumbnail"][0] - 1
|
||||
group_end = _group_end_pos(group, s_end)
|
||||
|
||||
# text before group
|
||||
text_len = group_start - cursor
|
||||
if text_len > 0:
|
||||
t_pos = (
|
||||
torch.arange(text_len, device=device, dtype=torch.long) + pos
|
||||
)
|
||||
for d in range(3):
|
||||
pos3d[d, 0, cursor:group_start] = t_pos
|
||||
pos += text_len
|
||||
|
||||
base = pos
|
||||
pos = base + _assign_canvas_span(
|
||||
pos3d,
|
||||
thumbnail=group["thumbnail"],
|
||||
span_end=group_end,
|
||||
slices=slices,
|
||||
base=base,
|
||||
target_sizes=target_sizes,
|
||||
)
|
||||
cursor = group_end
|
||||
|
||||
# remaining text
|
||||
rem = s_end - cursor
|
||||
if rem > 0:
|
||||
t_pos = torch.arange(rem, device=device, dtype=torch.long) + pos
|
||||
for d in range(3):
|
||||
pos3d[d, 0, cursor:s_end] = t_pos
|
||||
|
||||
return pos3d
|
||||
|
||||
|
||||
def canvas_rope_delta(position_ids: torch.Tensor, seq_len: int) -> int:
|
||||
"""Offset applied to the 1-D positions of tokens decoded after the prompt.
|
||||
|
||||
The first decoded token sits one past the last prefill position.
|
||||
"""
|
||||
return int(position_ids.amax().item()) + 1 - seq_len
|
||||
@@ -514,6 +514,10 @@ _MULTIMODAL_MODELS = {
|
||||
"minicpmv4_6",
|
||||
"MiniCPMV4_6ForConditionalGeneration",
|
||||
),
|
||||
"MiniCPMV4_7ForConditionalGeneration": (
|
||||
"minicpmv4_7",
|
||||
"MiniCPMV4_7ForConditionalGeneration",
|
||||
),
|
||||
"Mistral3ForConditionalGeneration": (
|
||||
"mistral3",
|
||||
"Mistral3ForConditionalGeneration",
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiniCPM-V family parser.
|
||||
|
||||
Wraps the Qwen3 grammar (``<think>`` reasoning + XML tool calls) with two
|
||||
MiniCPM-V specifics:
|
||||
|
||||
- ``<|im_end|>`` and the reserved marker tokens are passthrough terminals,
|
||||
so they never leak into ``content``.
|
||||
- The tokenizer decodes a newline as the two characters ``\\`` and ``n``
|
||||
(e.g. token id 1639) instead of ``\\n``, so the decoded text is recovered
|
||||
before it reaches the client. Code blocks, inline code and LaTeX are left
|
||||
alone, as are tool call arguments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
|
||||
_RESERVED_MARKER_IDS = range(12, 16)
|
||||
_IM_END = "<|im_end|>"
|
||||
|
||||
|
||||
@functools.cache
|
||||
def minicpmv_config(thinking: bool) -> ParserEngineConfig:
|
||||
"""MiniCPM-V grammar: the qwen3 grammar plus MiniCPM-V markers.
|
||||
|
||||
Both modes keep the qwen3 tool call terminals and transitions, so
|
||||
``<tool_call>`` output is parsed whether or not thinking is enabled.
|
||||
"""
|
||||
name = "minicpmv" if thinking else "minicpmv_no_thinking"
|
||||
base = qwen3_config(thinking=thinking, name=name)
|
||||
|
||||
transitions = dict(base.transitions)
|
||||
if not thinking:
|
||||
# Default mode: the model answers directly, so `<think>` must not open
|
||||
# a reasoning span that would swallow the answer. A stray
|
||||
# `<think>...</think>` in the output is still dropped, not printed.
|
||||
transitions[(ParserState.CONTENT, "THINK_START")] = Transition(
|
||||
ParserState.REASONING
|
||||
)
|
||||
base = replace(base, strip_trailing_reasoning_whitespace=False)
|
||||
|
||||
terminals = dict(base.terminals)
|
||||
token_id_terminals = dict(base.token_id_terminals)
|
||||
terminals["IM_END"] = _IM_END
|
||||
token_id_terminals["IM_END"] = _IM_END
|
||||
for state in (ParserState.CONTENT, ParserState.REASONING):
|
||||
transitions[(state, "IM_END")] = Transition(state)
|
||||
|
||||
for index in _RESERVED_MARKER_IDS:
|
||||
text_name = f"RESERVED_TEXT_{index}"
|
||||
token_name = f"RESERVED_TOKEN_{index}"
|
||||
terminals[text_name] = f"<reserved_{index}>"
|
||||
terminals[token_name] = f"<|reserved_{index}|>"
|
||||
token_id_terminals[token_name] = f"<|reserved_{index}|>"
|
||||
for state in (ParserState.CONTENT, ParserState.REASONING):
|
||||
transitions[(state, text_name)] = Transition(state)
|
||||
transitions[(state, token_name)] = Transition(state)
|
||||
|
||||
return replace(
|
||||
base,
|
||||
terminals=terminals,
|
||||
token_id_terminals=token_id_terminals,
|
||||
transitions=transitions,
|
||||
)
|
||||
|
||||
|
||||
class EscapedNewlineNormalizer:
|
||||
"""Recover literal escape sequences written by the model as real text.
|
||||
|
||||
Scans character by character so that a backslash inside a code block,
|
||||
inline code, LaTeX span or a doubled escape is left untouched. A trailing
|
||||
half of an escape is held in ``_pending`` until the next chunk arrives,
|
||||
which keeps streaming output identical to the non-streaming result.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._closing_marker: str | None = None
|
||||
self._pending = ""
|
||||
|
||||
def feed(self, text: str, *, final: bool = False) -> str:
|
||||
text = self._pending + text
|
||||
self._pending = ""
|
||||
output: list[str] = []
|
||||
index = 0
|
||||
|
||||
while index < len(text):
|
||||
if self._closing_marker is not None:
|
||||
marker = self._closing_marker
|
||||
remaining = text[index:]
|
||||
if remaining.startswith(marker):
|
||||
output.append(marker)
|
||||
index += len(marker)
|
||||
self._closing_marker = None
|
||||
continue
|
||||
if not final and marker.startswith(remaining):
|
||||
self._pending = remaining
|
||||
break
|
||||
output.append(text[index])
|
||||
index += 1
|
||||
continue
|
||||
|
||||
remaining = text[index:]
|
||||
if text[index] == "`":
|
||||
if not final and remaining in ("`", "``"):
|
||||
self._pending = remaining
|
||||
break
|
||||
if remaining.startswith("```"):
|
||||
output.append("```")
|
||||
index += 3
|
||||
self._closing_marker = "```"
|
||||
else:
|
||||
output.append("`")
|
||||
index += 1
|
||||
self._closing_marker = "`"
|
||||
continue
|
||||
|
||||
if text[index] == "$":
|
||||
if not final and remaining == "$":
|
||||
self._pending = remaining
|
||||
break
|
||||
if remaining.startswith("$$"):
|
||||
output.append("$$")
|
||||
index += 2
|
||||
self._closing_marker = "$$"
|
||||
else:
|
||||
output.append("$")
|
||||
index += 1
|
||||
self._closing_marker = "$"
|
||||
continue
|
||||
|
||||
if text[index] != "\\":
|
||||
output.append(text[index])
|
||||
index += 1
|
||||
continue
|
||||
|
||||
slash_end = index + 1
|
||||
while slash_end < len(text) and text[slash_end] == "\\":
|
||||
slash_end += 1
|
||||
if slash_end - index > 1:
|
||||
if not final and slash_end == len(text):
|
||||
# The run may still grow, and its parity decides whether a
|
||||
# following `n` is a newline or stays literal. Hold it until
|
||||
# the run is complete.
|
||||
self._pending = text[index:slash_end]
|
||||
break
|
||||
output.append(text[index:slash_end])
|
||||
index = slash_end
|
||||
continue
|
||||
|
||||
if not final and remaining in ("\\", "\\r", "\\r\\"):
|
||||
self._pending = remaining
|
||||
break
|
||||
if remaining.startswith("\\r\\n"):
|
||||
output.append("\n")
|
||||
index += 4
|
||||
elif remaining.startswith(("\\n", "\\r")):
|
||||
output.append("\n")
|
||||
index += 2
|
||||
elif remaining.startswith("\\("):
|
||||
output.append("\\(")
|
||||
index += 2
|
||||
self._closing_marker = "\\)"
|
||||
elif remaining.startswith("\\["):
|
||||
output.append("\\[")
|
||||
index += 2
|
||||
self._closing_marker = "\\]"
|
||||
else:
|
||||
output.append("\\")
|
||||
index += 1
|
||||
|
||||
if final and self._pending:
|
||||
output.append(self._pending)
|
||||
self._pending = ""
|
||||
return "".join(output)
|
||||
|
||||
|
||||
def recover_newlines(text: str | None) -> str | None:
|
||||
if text is None or "\\" not in text:
|
||||
return text
|
||||
return EscapedNewlineNormalizer().feed(text, final=True)
|
||||
|
||||
|
||||
class MiniCPMVOutputNormalizer:
|
||||
"""Apply :class:`EscapedNewlineNormalizer` to a whole stream.
|
||||
|
||||
Reasoning and content are tracked separately so that a half-written escape
|
||||
at the end of a reasoning chunk does not swallow the first characters of
|
||||
the content that follows.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._reasoning = EscapedNewlineNormalizer()
|
||||
self._content = EscapedNewlineNormalizer()
|
||||
|
||||
def normalize_delta(
|
||||
self,
|
||||
delta: DeltaMessage | None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
reasoning_finished = finished or (
|
||||
delta is not None and (delta.content is not None or bool(delta.tool_calls))
|
||||
)
|
||||
reasoning = self._reasoning.feed(
|
||||
delta.reasoning if delta and delta.reasoning else "",
|
||||
final=reasoning_finished,
|
||||
)
|
||||
content = self._content.feed(
|
||||
delta.content if delta and delta.content else "",
|
||||
final=finished,
|
||||
)
|
||||
|
||||
if delta is None:
|
||||
if not reasoning and not content:
|
||||
return None
|
||||
return DeltaMessage(
|
||||
reasoning=reasoning or None,
|
||||
content=content or None,
|
||||
)
|
||||
|
||||
if reasoning:
|
||||
delta.reasoning = reasoning
|
||||
elif delta.reasoning is not None:
|
||||
delta.reasoning = None
|
||||
if content:
|
||||
delta.content = content
|
||||
elif delta.content is not None:
|
||||
delta.content = None
|
||||
|
||||
# A chunk that normalizes to nothing must be dropped, not returned as an
|
||||
# all-None DeltaMessage: the engine only skips on `None`, so anything
|
||||
# else reaches the client as an empty delta.
|
||||
if not delta.content and not delta.reasoning and not delta.tool_calls:
|
||||
return None
|
||||
return delta
|
||||
|
||||
|
||||
class MiniCPMVParser(Qwen3Parser):
|
||||
"""Parse MiniCPM-V family output in thinking and non-thinking modes."""
|
||||
|
||||
CONFIG_NAME = "minicpmv"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
# The chat template emits `<think>` unless `enable_thinking` is
|
||||
# explicitly false, so the grammar must default to thinking too, or the
|
||||
# reasoning span is misread as content.
|
||||
thinking_enabled = chat_kwargs.get("enable_thinking", True)
|
||||
kwargs.setdefault("parser_engine_config", minicpmv_config(thinking_enabled))
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
# ``Qwen3Parser.__init__`` reads the same key with a default of True,
|
||||
# which does not apply here: MiniCPM-V defaults to thinking disabled.
|
||||
# Keep the attribute in sync with the grammar built above.
|
||||
self.thinking_enabled = thinking_enabled
|
||||
|
||||
@property
|
||||
def reasoning_ended(self) -> bool:
|
||||
# The engine derives this from the presence of THINK terminals, which
|
||||
# both MiniCPM-V configs keep, so the base property stays False in
|
||||
# non-thinking mode. There is no reasoning span to end there.
|
||||
if not self.thinking_enabled:
|
||||
return True
|
||||
return super().reasoning_ended
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if self.thinking_enabled:
|
||||
return super().extract_reasoning(model_output, request)
|
||||
_, content = ParserEngine.extract_reasoning(self, model_output, request)
|
||||
return None, content
|
||||
|
||||
def extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> DeltaMessage | None:
|
||||
if self.thinking_enabled:
|
||||
return super().extract_reasoning_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
)
|
||||
|
||||
delta = ParserEngine.extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
)
|
||||
if delta is None:
|
||||
return None
|
||||
delta.reasoning = None
|
||||
if delta.content is None and not delta.tool_calls:
|
||||
return None
|
||||
return delta
|
||||
|
||||
def get_streaming_fallback_content(
|
||||
self,
|
||||
text: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> str | None:
|
||||
if self.thinking_enabled:
|
||||
return super().get_streaming_fallback_content(text, request)
|
||||
delta = ParserEngine.finish_streaming(self)
|
||||
return delta.content if delta is not None else None
|
||||
|
||||
|
||||
class MiniCPMVUnifiedParser(DelegatingParser):
|
||||
"""MiniCPM-V serving parser: the adapters plus newline recovery.
|
||||
|
||||
Recovery cannot live in :class:`MiniCPMVParser`: the serving stack drives
|
||||
the reasoning/tool adapters, never ``ParserEngine.parse`` /
|
||||
``ParserEngine.parse_delta``, so a rewrite applied there would only be
|
||||
visible to callers that construct the engine directly.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(tokenizer, tools, *args, **kwargs)
|
||||
self._output_normalizer = MiniCPMVOutputNormalizer()
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
model_output_token_ids: Sequence[int] = (),
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
reasoning, content, tool_calls = super().parse(
|
||||
model_output,
|
||||
request,
|
||||
enable_auto_tools,
|
||||
model_output_token_ids,
|
||||
)
|
||||
# Tool call arguments are JSON, where a literal `\n` is meaningful and
|
||||
# must survive as written.
|
||||
return (
|
||||
recover_newlines(reasoning),
|
||||
recover_newlines(content),
|
||||
tool_calls,
|
||||
)
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
delta = super().parse_delta(
|
||||
delta_text,
|
||||
delta_token_ids,
|
||||
request,
|
||||
prompt_token_ids,
|
||||
finished=finished,
|
||||
)
|
||||
return self._output_normalizer.normalize_delta(delta, finished=finished)
|
||||
@@ -242,6 +242,19 @@ class ParserManager:
|
||||
|
||||
return _CohereCommandParser
|
||||
|
||||
if "minicpmv" in {reasoning_parser_name, tool_parser_name}:
|
||||
from vllm.parser.minicpmv import MiniCPMVUnifiedParser
|
||||
|
||||
r_cls = reasoning_parser_cls
|
||||
t_cls = tool_parser_cls
|
||||
|
||||
class _MiniCPMVParser(MiniCPMVUnifiedParser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
|
||||
return _MiniCPMVParser
|
||||
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
|
||||
r_cls = reasoning_parser_cls
|
||||
|
||||
@@ -116,6 +116,10 @@ _REASONING_PARSERS_TO_REGISTER = {
|
||||
"mimo_engine_reasoning_parser",
|
||||
"MiMoParserReasoningAdapter",
|
||||
),
|
||||
"minicpmv": (
|
||||
"minicpmv_reasoning_parser",
|
||||
"MiniCPMVParserReasoningAdapter",
|
||||
),
|
||||
"minimax_m2": (
|
||||
"minimax_m2_reasoning_parser",
|
||||
"MiniMaxM2ReasoningParser",
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
|
||||
from vllm.parser.minicpmv import MiniCPMVParser
|
||||
|
||||
|
||||
class MiniCPMVParserReasoningAdapter(ParserEngineReasoningAdapter):
|
||||
# Reasoning-only on purpose: this family registers no tool parser, because
|
||||
# MiniCPM-V emits the Qwen3-Coder XML syntax verbatim and `qwen3_coder`
|
||||
# already covers it. ``make_adapters`` only builds reasoning/tool pairs, so
|
||||
# the two halves are declared explicitly here.
|
||||
_parser_engine_cls = MiniCPMVParser
|
||||
|
||||
|
||||
__all__ = ["MiniCPMVParserReasoningAdapter"]
|
||||
@@ -32,6 +32,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
|
||||
"deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
|
||||
"minicpmv": _get_minicpmv_chat_template_fallback,
|
||||
"minicpmv4_6": _get_minicpmv_chat_template_fallback,
|
||||
"minicpmv4_7": _get_minicpmv_chat_template_fallback,
|
||||
"paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
||||
"siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
||||
"siglip2": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
||||
|
||||
Reference in New Issue
Block a user