diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index 221ab877ea..bbcb94f32c 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -592,7 +592,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `MiDashengLMModel` | MiDashengLM | T + A+ | `mispeech/midashenglm-7b` | | ✅︎ |
| `MiMoV2OmniForCausalLM` | MiMo-V2.5-Omni | T + IE+ + VE+ + A+ | `XiaomiMiMo/MiMo-V2.5-Omni` | | ✅︎ |
| `MiniCPMO` | MiniCPM-O | T + IE+ + VE+ + AE+ | `openbmb/MiniCPM-o-2_6`, etc. | ✅︎ | ✅︎ |
-| `MiniCPMV` | MiniCPM-V | T + IE+ + VE+ | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, `openbmb/MiniCPM-V-4_6`, etc. | ✅︎ | |
+| `MiniCPMV` | MiniCPM-V | T + IE+ + VE+ | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, `openbmb/MiniCPM-V-4_6`, `openbmb/MiniCPM-V-4_7`, etc. | ✅︎ | |
| `MiniMaxM3SparseForConditionalGeneration` | MiniMax-M3 | T + I+ + V+ | `MiniMaxAI/MiniMax-M3`, `MiniMaxAI/MiniMax-M3-MXFP8`, etc. | | ✅︎ |
| `MiniMaxVL01ForConditionalGeneration` | MiniMax-VL | T + IE+ | `MiniMaxAI/MiniMax-VL-01`, etc. | | ✅︎ |
| `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I+ | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ |
diff --git a/tests/model_executor/test_minicpmv.py b/tests/model_executor/test_minicpmv.py
index 96b80101b4..cffdea2600 100644
--- a/tests/model_executor/test_minicpmv.py
+++ b/tests/model_executor/test_minicpmv.py
@@ -6,6 +6,9 @@ import torch
from vllm.model_executor.models.minicpmv import MiniCPMV2_6
from vllm.model_executor.models.minicpmv4_6 import (
MiniCPMV4_6ForConditionalGeneration,
+ _expand_flags,
+ _select_vision_items,
+ _vision_downsample_mode,
)
pytestmark = pytest.mark.skip_global_cleanup
@@ -54,3 +57,114 @@ def test_image_and_video_embeds_stay_in_their_own_modality():
assert torch.equal(modalities["images"]["image_embeds"], image_embeds)
assert torch.equal(modalities["videos"]["image_embeds"], video_embeds)
+
+
+class TestPerItemMergerFlags:
+ """The merger flag is per request but items from several requests can be
+ batched together, so it has to be applied per item."""
+
+ def test_expand_flags_broadcasts_a_single_value(self):
+ assert _expand_flags(None, 3) == [None, None, None]
+ assert _expand_flags(torch.tensor([True]), 3) == [True, True, True]
+ assert _expand_flags(True, 2) == [True, True]
+
+ def test_expand_flags_keeps_per_item_values(self):
+ flags = [torch.tensor([True]), torch.tensor([False])]
+
+ assert _expand_flags(flags, 2) == [True, False]
+
+ def test_expand_flags_falls_back_on_a_shape_mismatch(self):
+ # Too many values to map onto the items: keep one decision for the
+ # batch rather than failing the request.
+ assert _expand_flags([True, False, True], 2) == [True, True]
+
+ def test_vision_downsample_mode(self):
+ assert _vision_downsample_mode(None) is None
+ assert _vision_downsample_mode(True) == "16x"
+ assert _vision_downsample_mode(False) == "4x"
+
+ def test_select_vision_items_slices_by_item(self):
+ # 3 items holding 2, 1 and 2 slices; one channel, one patch, width 4.
+ pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(5)]
+ tgt_sizes = torch.arange(10, dtype=torch.long).reshape(5, 2)
+ image_input = {
+ "pixel_values": pixel_values,
+ "tgt_sizes": tgt_sizes,
+ "num_slices": torch.tensor([2, 1, 2]),
+ }
+
+ selected = _select_vision_items(image_input, [2], [2, 1, 2])
+
+ assert len(selected["pixel_values"]) == 2
+ assert bool(selected["pixel_values"][0][0, 0, 0] == 3.0)
+ assert bool(selected["pixel_values"][1][0, 0, 0] == 4.0)
+ assert selected["num_slices"].tolist() == [2]
+ assert torch.equal(selected["tgt_sizes"], tgt_sizes[3:5])
+
+ def test_select_vision_items_preserves_order(self):
+ pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(3)]
+ image_input = {
+ "pixel_values": pixel_values,
+ "tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
+ "num_slices": torch.tensor([1, 1, 1]),
+ }
+
+ selected = _select_vision_items(image_input, [2, 0], [1, 1, 1])
+
+ assert selected["num_slices"].tolist() == [1, 1]
+ assert bool(selected["pixel_values"][0][0, 0, 0] == 2.0)
+ assert bool(selected["pixel_values"][1][0, 0, 0] == 0.0)
+
+ def test_process_vision_input_applies_flags_per_item(self):
+ calls: list[str | None] = []
+
+ def fake_get_vision_hidden_states(data, downsample_mode=None):
+ calls.append(downsample_mode)
+ tag = 1.0 if downsample_mode == "16x" else 0.0
+ return [torch.full((1,), tag) for _ in data["pixel_values"]]
+
+ model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
+ model.get_vision_hidden_states = fake_get_vision_hidden_states
+
+ image_input = {
+ "type": "pixel_values",
+ "pixel_values": [torch.zeros(1, 1, 4) for _ in range(3)],
+ "tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
+ "num_slices": torch.tensor([1, 1, 1]),
+ }
+
+ result = model._process_vision_input(
+ image_input,
+ use_vit_merger=[
+ torch.tensor([True]),
+ torch.tensor([False]),
+ torch.tensor([True]),
+ ],
+ )
+
+ # Each item keeps its own mode even though they share one call.
+ assert [float(r[0]) for r in result] == [1.0, 0.0, 1.0]
+ # Mixed flags are encoded as separate groups, not one collapsed batch.
+ assert calls == ["16x", "4x"]
+
+ def test_process_vision_input_uses_one_group_for_uniform_flags(self):
+ calls: list[str | None] = []
+
+ def fake_get_vision_hidden_states(data, downsample_mode=None):
+ calls.append(downsample_mode)
+ return [torch.zeros(1) for _ in data["pixel_values"]]
+
+ model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
+ model.get_vision_hidden_states = fake_get_vision_hidden_states
+
+ image_input = {
+ "type": "pixel_values",
+ "pixel_values": [torch.zeros(1, 1, 4) for _ in range(2)],
+ "tgt_sizes": torch.zeros(2, 2, dtype=torch.long),
+ "num_slices": torch.tensor([1, 1]),
+ }
+
+ model._process_vision_input(image_input, use_vit_merger=torch.tensor([True]))
+
+ # The common case still makes a single encoder call.
+ assert calls == ["16x"]
diff --git a/tests/model_executor/test_minicpmv4_7.py b/tests/model_executor/test_minicpmv4_7.py
new file mode 100644
index 0000000000..3d9e8b4678
--- /dev/null
+++ b/tests/model_executor/test_minicpmv4_7.py
@@ -0,0 +1,310 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+
+from vllm.model_executor.models.minicpmv4_6 import _stack_vit_merger_qkv
+from vllm.model_executor.models.minicpmv4_7 import (
+ MiniCPMV4_7MultiModalProcessor,
+ _compute_canvas_single,
+ build_image_bounds,
+ canvas_rope_delta,
+)
+
+pytestmark = pytest.mark.skip_global_cleanup
+
+
+def test_canvas_mrope_sizes_survive_prefix_cache_stripping():
+ """A cached image must keep `tgt_sizes`, or canvas M-RoPE falls back.
+
+ `strip_covered_mm_data` drops the payload of a prefix-cache-covered item and
+ keeps only CPU-side metadata. Canvas M-RoPE places slices from `tgt_sizes`,
+ so 4.7 marks it `keep_on_cpu` while the shared 4.6 config stays as it was.
+ """
+ from vllm.model_executor.models.minicpmv4_6 import _minicpmv4_6_field_config
+
+ shared = _minicpmv4_6_field_config({})
+ assert shared["tgt_sizes"].field.keep_on_cpu is False
+ assert shared["video_tgt_sizes"].field.keep_on_cpu is False
+
+ processor = MiniCPMV4_7MultiModalProcessor.__new__(MiniCPMV4_7MultiModalProcessor)
+ fields = processor._get_mm_fields_config({}, {})
+
+ assert fields["tgt_sizes"].field.keep_on_cpu is True
+ assert fields["video_tgt_sizes"].field.keep_on_cpu is True
+ # Other fields keep their previous setting.
+ assert fields["pixel_values"].field.keep_on_cpu is False
+ assert fields["video_image_sizes"].field.keep_on_cpu is True
+
+
+def test_modality_mm_kwargs_resolution():
+ """Flat keys reach both modalities; scopes stay scoped, and win on conflict.
+
+ `_merge_and_resolve_mm_processor_kwargs` routes a flat key into every scope
+ whose HF processor declares it, so reading is a plain scope lookup. The
+ routing itself is upstream's, tested here against the MiniCPM-V schema to
+ pin the precedence rule (`scoped > flat`).
+ """
+ from vllm.model_executor.models.minicpmv4_6 import (
+ _flat_processor_kwargs,
+ _scoped_mm_kwarg,
+ )
+ from vllm.multimodal.processing.context import _resolve_mm_processor_kwargs
+
+ # What MiniCPMV4_6{Image,Video}ProcessorKwargs declare.
+ processor_keys = {"downsample_mode", "max_slice_nums"}
+ schema = {
+ "text_kwargs": set(),
+ "images_kwargs": set(processor_keys),
+ "videos_kwargs": set(processor_keys),
+ "audio_kwargs": set(),
+ }
+
+ def read(mm_kwargs, modality, key):
+ merged = _resolve_mm_processor_kwargs(dict(mm_kwargs), schema)
+ return _scoped_mm_kwarg(merged, modality, key)
+
+ assert read({}, "image", "downsample_mode") is None
+ # A flat key is routed into both scopes.
+ assert read({"downsample_mode": "4x"}, "image", "downsample_mode") == "4x"
+ assert read({"downsample_mode": "4x"}, "video", "downsample_mode") == "4x"
+ # A scoped key stays in its scope.
+ assert (
+ read({"images_kwargs": {"downsample_mode": "16x"}}, "image", "downsample_mode")
+ == "16x"
+ )
+ assert (
+ read({"images_kwargs": {"downsample_mode": "16x"}}, "video", "downsample_mode")
+ is None
+ )
+ videos_only = {"videos_kwargs": {"max_slice_nums": 1}}
+ assert read(videos_only, "video", "max_slice_nums") == 1
+ # Scoped to videos, so image must not see it.
+ assert read(videos_only, "image", "max_slice_nums") is None
+ # An explicit scope wins over the flat key it is merged with.
+ assert (
+ read(
+ {"downsample_mode": "16x", "images_kwargs": {"downsample_mode": "4x"}},
+ "image",
+ "downsample_mode",
+ )
+ == "4x"
+ )
+ # A non-mapping scope is ignored rather than raising.
+ assert read({"images_kwargs": None}, "image", "downsample_mode") is None
+
+ # `_flat_processor_kwargs` drops the scoped keys and pins the mode.
+ flat = _flat_processor_kwargs(
+ {
+ "downsample_mode": "4x",
+ "images_kwargs": {"a": 1},
+ "videos_kwargs": {"b": 2},
+ "audio_kwargs": {"c": 3},
+ },
+ "16x",
+ )
+ assert flat == {"downsample_mode": "16x"}
+
+
+def test_vit_merger_qkv_is_stacked_before_load():
+ mapped = list(
+ _stack_vit_merger_qkv(
+ [
+ (
+ "model.vision_tower.vit_merger.self_attn.q_proj.weight",
+ torch.zeros(1),
+ ),
+ (
+ "model.vision_tower.vit_merger.self_attn.k_proj.weight",
+ torch.zeros(1),
+ ),
+ (
+ "model.vision_tower.vit_merger.self_attn.v_proj.weight",
+ torch.zeros(1),
+ ),
+ ("vit_merger.self_attn.qkv_proj.weight", torch.zeros(1)),
+ ]
+ )
+ )
+ names = [name for name, _ in mapped]
+ shards = [getattr(tensor, "shard_id", None) for _, tensor in mapped]
+ assert names == [
+ "model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
+ "model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
+ "model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
+ "vit_merger.self_attn.qkv_proj.weight",
+ ]
+ assert shards == ["q", "k", "v", None]
+
+
+# Canvas M-RoPE goldens, cross-checked against the reference implementation in
+# transformers PR 48979 (`modular_minicpmv4_7.py`). Synthetic ids keep the token
+# sequences readable: the canvas math only compares ids for equality.
+_IM_START = 10
+_IM_END = 11
+_SLICE_START = 12
+_SLICE_END = 13
+_NEWLINE = 14
+_IMAGE_PAD = 20
+_VIDEO_PAD = 21
+_TEXT = 100
+
+_CANVAS_IDS = {
+ "im_start_id": _IM_START,
+ "im_end_id": _IM_END,
+ "slice_start_id": _SLICE_START,
+ "slice_end_id": _SLICE_END,
+ "newline_id": _NEWLINE,
+}
+
+
+def _canvas_positions(tokens, target_sizes):
+ input_ids = torch.tensor([tokens], dtype=torch.long)
+ targets = torch.tensor(target_sizes, dtype=torch.long)
+ bounds = build_image_bounds(input_ids[0], _CANVAS_IDS)
+ positions = _compute_canvas_single(
+ input_ids[0],
+ torch.arange(len(tokens), dtype=torch.long),
+ bounds,
+ targets,
+ _CANVAS_IDS,
+ )
+ return positions.tolist(), canvas_rope_delta(positions, len(tokens))
+
+
+def test_canvas_mrope_trailing_newline_stays_inside_the_canvas():
+ """A newline right after `` belongs to the canvas, not to the text.
+
+ The chat template joins content parts with ``\\n``, so an image followed by
+ text always leaves one there. The transformers reference closes a frame over
+ every marker including newlines; leaving it out pushes every later position
+ one step forward and makes `rope_delta` differ by one.
+ """
+ tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _NEWLINE, _TEXT, _TEXT]
+ positions, delta = _canvas_positions(tokens, [[4, 4]])
+
+ assert positions == [
+ [0, 1, 1, 1, 1, 3, 4],
+ [0, 0, 1, 2, 1, 3, 4],
+ [0, 0, 1, 2, 1, 3, 4],
+ ]
+ assert delta == -2
+
+
+def test_canvas_mrope_thumbnail_matches_reference():
+ # PAD ; 16x turns a 4x4 patch grid into 1 token.
+ tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
+ positions, delta = _canvas_positions(tokens, [[4, 4]])
+
+ assert positions == [
+ [0, 1, 1, 1, 3],
+ [0, 0, 1, 2, 3],
+ [0, 0, 1, 2, 3],
+ ]
+ assert delta == -1
+
+
+def test_canvas_mrope_slice_row_matches_reference():
+ # Thumbnail then a 1x2 slice row; the slices tile the thumbnail's canvas.
+ tokens = [
+ _TEXT,
+ _IM_START,
+ _IMAGE_PAD,
+ _IM_END,
+ _SLICE_START,
+ _IMAGE_PAD,
+ _SLICE_END,
+ _SLICE_START,
+ _IMAGE_PAD,
+ _SLICE_END,
+ _TEXT,
+ ]
+ positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4], [4, 4]])
+
+ assert positions == [
+ [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4],
+ [0, 0, 1, 2, 1, 1, 1, 1, 1, 1, 4],
+ [0, 0, 1, 3, 1, 1, 1, 2, 2, 2, 4],
+ ]
+ assert delta == -6
+
+
+def test_canvas_mrope_resamples_thumbnail_over_canvas():
+ # A 8x8 patch grid is 2x2 LLM tokens, so the canvas is resampled (not arange).
+ tokens = [_TEXT, _IM_START] + [_IMAGE_PAD] * 4 + [_IM_END, _TEXT]
+ positions, delta = _canvas_positions(tokens, [[8, 8]])
+
+ assert positions == [
+ [0, 1, 1, 1, 1, 1, 1, 4],
+ [0, 0, 1, 1, 2, 2, 3, 4],
+ [0, 0, 1, 2, 1, 2, 3, 4],
+ ]
+ assert delta == -3
+
+
+def test_canvas_mrope_video_frames_share_one_canvas():
+ tokens = [
+ _TEXT,
+ _IM_START,
+ _VIDEO_PAD,
+ _IM_END,
+ _IM_START,
+ _VIDEO_PAD,
+ _IM_END,
+ _TEXT,
+ ]
+ positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4]])
+
+ assert positions == [
+ [0, 1, 1, 1, 3, 3, 3, 5],
+ [0, 0, 1, 2, 2, 3, 4, 5],
+ [0, 0, 1, 2, 2, 3, 4, 5],
+ ]
+ assert delta == -2
+
+
+def test_canvas_mrope_interleaved_image_and_video():
+ tokens = (
+ [_TEXT, _IM_START]
+ + [_IMAGE_PAD] * 4
+ + [_IM_END, _IM_START]
+ + [_VIDEO_PAD] * 4
+ + [_IM_END, _TEXT]
+ )
+ positions, delta = _canvas_positions(tokens, [[8, 8], [8, 8]])
+
+ assert positions == [
+ [0, 1, 1, 1, 1, 1, 1, 4, 4, 4, 4, 4, 4, 7],
+ [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7],
+ [0, 0, 1, 2, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7],
+ ]
+ assert delta == -6
+
+
+def test_canvas_span_is_shared_by_image_and_video():
+ """A video frame and a same-geometry image must place identical offsets.
+
+ Both paths go through `_assign_canvas_span`; this pins that they cannot
+ drift apart now that the duplicated geometry is gone.
+ """
+ thumb = [_IM_START] + [_IMAGE_PAD] * 4 + [_IM_END]
+
+ img_pos, _ = _canvas_positions([_TEXT, *thumb, _TEXT], [[8, 8]])
+ vid_pos, _ = _canvas_positions([_TEXT, *thumb, *thumb, _TEXT], [[8, 8], [8, 8]])
+
+ def offsets(pos, start):
+ # The halo is one below the canvas base.
+ base = pos[1][start] + 1
+ return [[c[start + k] - base for k in range(3)] for c in pos]
+
+ assert offsets(img_pos, 0) == offsets(vid_pos, 0)
+
+
+def test_canvas_mrope_delta_leaves_room_for_the_first_decoded_token():
+ # First decoded token must land on max_prefill_position + 1, matching
+ # transformers and the other native M-RoPE models.
+ tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
+ positions, delta = _canvas_positions(tokens, [[4, 4]])
+
+ max_position = max(max(channel) for channel in positions)
+ assert delta == max_position + 1 - len(tokens)
diff --git a/tests/models/registry.py b/tests/models/registry.py
index b57f3fcb72..2ed6d4871c 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -1118,6 +1118,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
"openbmb/MiniCPM-V-4_6",
),
+ "MiniCPMV4_7ForConditionalGeneration": _HfExamplesInfo(
+ "openbmb/MiniCPM-V-4_7",
+ min_transformers_version="5.18.0",
+ is_available_online=False,
+ ),
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
"MiniMaxAI/MiniMax-M3",
trust_remote_code=True,
diff --git a/tests/parser/engine/test_minicpmv_reasoning.py b/tests/parser/engine/test_minicpmv_reasoning.py
new file mode 100644
index 0000000000..5edf5bfc5d
--- /dev/null
+++ b/tests/parser/engine/test_minicpmv_reasoning.py
@@ -0,0 +1,459 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the MiniCPM-V family parser."""
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
+from vllm.entrypoints.generate.base.protocol import (
+ DeltaFunctionCall,
+ DeltaMessage,
+ DeltaToolCall,
+)
+from vllm.entrypoints.openai.chat_completion.protocol import (
+ ChatCompletionRequest,
+ ChatCompletionToolsParam,
+)
+from vllm.parser.minicpmv import (
+ EscapedNewlineNormalizer,
+ MiniCPMVOutputNormalizer,
+ recover_newlines,
+)
+from vllm.parser.parser_manager import ParserManager
+from vllm.reasoning import ReasoningParserManager
+from vllm.tool_parsers import ToolParserManager
+
+_VOCAB = {
+ "": 10,
+ "": 11,
+ "<|im_end|>": 12,
+ "": 13,
+ "": 14,
+}
+
+_TOOLS = [
+ ChatCompletionToolsParam(
+ type="function",
+ function={
+ "name": "get_weather",
+ "parameters": {
+ "type": "object",
+ "properties": {"city": {"type": "string"}},
+ },
+ },
+ )
+]
+
+
+@pytest.fixture
+def mock_tokenizer():
+ return make_mock_tokenizer(_VOCAB)
+
+
+@pytest.fixture
+def request_obj():
+ return ChatCompletionRequest(
+ model="test-model",
+ messages=[{"role": "user", "content": "hi"}],
+ )
+
+
+def _make_parser(tokenizer, *, enable_thinking=None, tools=None):
+ """Build the parser the serving stack resolves for MiniCPM-V.
+
+ Everything is built through ``ParserManager`` so the tests exercise the
+ ``DelegatingParser`` the server actually drives, not the parser engine on
+ its own. Tool calling goes through ``qwen3_coder``: MiniCPM-V emits that
+ XML syntax verbatim and has no tool parser of its own.
+ """
+ parser_cls = ParserManager.get_parser(
+ tool_parser_name="qwen3_coder" if tools else None,
+ reasoning_parser_name="minicpmv",
+ enable_auto_tools=bool(tools),
+ )
+ assert parser_cls is not None
+ kwargs = {}
+ if enable_thinking is not None:
+ kwargs["chat_template_kwargs"] = {"enable_thinking": enable_thinking}
+ return parser_cls(tokenizer, tools, **kwargs)
+
+
+def test_reasoning_parser_is_registered():
+ parser_cls = ReasoningParserManager.get_reasoning_parser("minicpmv")
+
+ assert parser_cls.__name__ == "MiniCPMVParserReasoningAdapter"
+
+
+def test_thinking_defaults_to_the_chat_template(mock_tokenizer):
+ """Without `enable_thinking`, the grammar must wait for reasoning.
+
+ The chat template ends the assistant turn with `` unless the flag is
+ explicitly false, so the model reasons by default. This is the flag
+ `serving.py` latches into the structured-output gate, so a non-thinking
+ default both disables the grammar and leaks the thinking into `content`.
+ """
+
+ def reasoning_pending(**kwargs):
+ return not _make_parser(mock_tokenizer, **kwargs).is_reasoning_end([1, 2])
+
+ assert reasoning_pending() is True # default matches the template
+ assert reasoning_pending(enable_thinking=True) is True
+ assert reasoning_pending(enable_thinking=False) is False
+
+
+def test_no_minicpmv_tool_parser_is_registered():
+ """MiniCPM-V deliberately registers no tool parser.
+
+ The family emits the Qwen3-Coder XML syntax verbatim, so
+ ``--tool-call-parser qwen3_coder`` already covers it. Registering a
+ ``minicpmv`` tool parser would duplicate that name for no parsing gain.
+ """
+ with pytest.raises(KeyError):
+ ToolParserManager.get_tool_parser("minicpmv")
+
+
+def test_composes_with_qwen3_coder_tool_parser(mock_tokenizer):
+ """`--tool-call-parser qwen3_coder` must work through the serving Parser."""
+ request = ChatCompletionRequest(
+ model="test-model",
+ messages=[{"role": "user", "content": "hi"}],
+ tools=_TOOLS,
+ )
+ parser_cls = ParserManager.get_parser(
+ tool_parser_name="qwen3_coder",
+ reasoning_parser_name="minicpmv",
+ enable_auto_tools=True,
+ )
+ assert parser_cls is not None
+ parser = parser_cls(mock_tokenizer, request.tools)
+
+ _, _, tool_calls = parser.parse(
+ ""
+ "SF",
+ request,
+ enable_auto_tools=True,
+ )
+
+ assert tool_calls is not None
+ assert tool_calls[0].name == "get_weather"
+ assert "SF" in tool_calls[0].arguments
+
+
+class TestNonThinking:
+ @pytest.fixture
+ def parser(self, mock_tokenizer):
+ return _make_parser(mock_tokenizer, enable_thinking=False)
+
+ def test_plain_content_is_unchanged(self, parser):
+ reasoning, content = parser.extract_reasoning("The answer is 42.", None)
+
+ assert reasoning is None
+ assert content == "The answer is 42."
+
+ def test_reserved_markers_are_removed(self, parser):
+ output = (
+ "reasoning<|reserved_14|>tool call<|reserved_15|>"
+ )
+
+ reasoning, content = parser.extract_reasoning(output, None)
+
+ assert reasoning is None
+ assert content == "reasoningtool call"
+
+ def test_unrelated_reserved_text_is_unchanged(self, parser):
+ reasoning, content = parser.extract_reasoning(
+ "content<|reserved_16|>",
+ None,
+ )
+
+ assert reasoning is None
+ assert content == "content<|reserved_16|>"
+
+ def test_dangling_standard_think_end_is_removed(self, parser):
+ output = "first part \n\nfinal part"
+
+ reasoning, content = parser.extract_reasoning(output, None)
+
+ assert reasoning is None
+ assert content == "first part \n\nfinal part"
+
+ def test_truncated_standard_reasoning_does_not_leak(self, parser):
+ reasoning, content = parser.extract_reasoning(
+ "prefixprivate reasoning",
+ None,
+ )
+
+ assert reasoning is None
+ assert content == "prefix"
+
+ def test_streaming_split_standard_markers(self, parser):
+ reasoning, content = simulate_reasoning_streaming(
+ parser,
+ [
+ "<",
+ "think>",
+ "private reasoning",
+ "",
+ "final answer",
+ ],
+ )
+
+ assert reasoning == ""
+ assert content == "final answer"
+
+ def test_streaming_split_reserved_markers(self, parser):
+ reasoning, content = simulate_reasoning_streaming(
+ parser,
+ [
+ "",
+ "reasoning",
+ "<|reserved",
+ "_13|>",
+ "final answer",
+ ],
+ )
+
+ assert reasoning == ""
+ assert content == "reasoningfinal answer"
+
+ def test_tool_calls_are_parsed(self, mock_tokenizer):
+ """Thinking is off by default, and tool calls must still be parsed."""
+ parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
+ request = ChatCompletionRequest(
+ model="test-model",
+ messages=[{"role": "user", "content": "hi"}],
+ tools=_TOOLS,
+ )
+
+ reasoning, content, tool_calls = parser.parse(
+ ""
+ "SF",
+ request,
+ enable_auto_tools=True,
+ )
+
+ assert reasoning is None
+ assert not content
+ assert tool_calls is not None
+ assert tool_calls[0].name == "get_weather"
+ assert "SF" in tool_calls[0].arguments
+
+ def test_streaming_tool_calls_are_parsed(self, mock_tokenizer):
+ """Regression: tool calls must stream when thinking is explicitly off.
+
+ The reasoning parser used to report ``reasoning_ended=False`` in
+ non-thinking mode, which kept the delegating parser in its reasoning
+ phase and streamed the raw ```` XML to the client as content.
+ """
+ parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
+ request = ChatCompletionRequest(
+ model="test-model",
+ messages=[{"role": "user", "content": "hi"}],
+ tools=_TOOLS,
+ )
+
+ delta = parser.parse_delta(
+ ""
+ "SF",
+ [],
+ request,
+ prompt_token_ids=[],
+ finished=True,
+ )
+
+ assert delta is not None
+ assert delta.tool_calls
+ assert delta.tool_calls[0].function is not None
+ assert delta.tool_calls[0].function.name == "get_weather"
+ assert "SF" in (delta.tool_calls[0].function.arguments or "")
+ assert not delta.content
+
+
+class TestThinking:
+ def test_standard_think_tags_are_parsed(self, mock_tokenizer):
+ parser = _make_parser(mock_tokenizer, enable_thinking=True)
+
+ reasoning, content = parser.extract_reasoning(
+ "private reasoningfinal answer",
+ None,
+ )
+
+ assert reasoning == "private reasoning"
+ assert content == "final answer"
+
+ def test_reserved_markers_are_removed(self, mock_tokenizer):
+ parser = _make_parser(mock_tokenizer, enable_thinking=True)
+
+ reasoning, content = parser.extract_reasoning(
+ "private reasoning<|reserved_13|>final answer",
+ None,
+ )
+
+ assert reasoning == "private reasoning"
+ assert content == "final answer"
+
+ def test_streaming_im_end_is_removed(self):
+ tokenizer = make_mock_tokenizer(_VOCAB, special_tokens=[])
+ parser = _make_parser(tokenizer, enable_thinking=True)
+
+ reasoning, content = simulate_reasoning_streaming(
+ parser,
+ ["Answer: 111", "<|im_end|>"],
+ )
+
+ assert reasoning == "Answer: 111"
+ assert content == ""
+
+
+class TestNewlineRecovery:
+ """The tokenizer writes a newline as the two characters ``\\`` and ``n``."""
+
+ def test_recover_newlines(self):
+ text = (
+ "first\\nsecond\\rthird\\r\\nfourth "
+ "`code\\nvalue` "
+ "```python\\nvalue = '\\\\n'\\n``` "
+ "$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
+ "escaped \\\\n"
+ )
+
+ assert recover_newlines(text) == (
+ "first\nsecond\nthird\nfourth "
+ "`code\\nvalue` "
+ "```python\\nvalue = '\\\\n'\\n``` "
+ "$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
+ "escaped \\\\n"
+ )
+
+ def test_streaming_matches_complete_recovery(self):
+ text = (
+ "first\\nsecond `code\\nvalue` $x\\ny$ ```text\\nvalue\\n``` last\\r\\nline"
+ )
+ expected = recover_newlines(text)
+
+ for split in range(len(text) + 1):
+ normalizer = EscapedNewlineNormalizer()
+ actual = normalizer.feed(text[:split])
+ actual += normalizer.feed(text[split:], final=True)
+ assert actual == expected
+
+ normalizer = EscapedNewlineNormalizer()
+ actual = "".join(normalizer.feed(char) for char in text)
+ actual += normalizer.feed("", final=True)
+ assert actual == expected
+
+ def test_backslash_runs_stay_consistent_across_splits(self):
+ # A run of backslashes can be split over chunks. Its parity decides
+ # whether a following `n` is a newline, so streaming and non-streaming
+ # must agree for every split point.
+ for count in range(1, 7):
+ text = "\\" * count + "n"
+ expected = recover_newlines(text)
+
+ for split in range(len(text) + 1):
+ normalizer = EscapedNewlineNormalizer()
+ actual = normalizer.feed(text[:split])
+ actual += normalizer.feed(text[split:], final=True)
+ assert actual == expected, (count, split)
+
+ def test_streaming_recovers_reasoning_and_content(self):
+ normalizer = MiniCPMVOutputNormalizer()
+
+ first = normalizer.normalize_delta(
+ DeltaMessage(reasoning="reason\\"),
+ finished=False,
+ )
+ second = normalizer.normalize_delta(
+ DeltaMessage(reasoning="nnext\\", content="answer\\"),
+ finished=False,
+ )
+ final = normalizer.normalize_delta(
+ DeltaMessage(content="nend"),
+ finished=True,
+ )
+
+ assert first == DeltaMessage(reasoning="reason")
+ assert second == DeltaMessage(reasoning="\nnext\\", content="answer")
+ assert final == DeltaMessage(content="\nend")
+
+ def test_streaming_does_not_modify_tool_arguments(self):
+ normalizer = MiniCPMVOutputNormalizer()
+ arguments = '{"value":"first\\\\nsecond"}'
+ delta = DeltaMessage(
+ tool_calls=[
+ DeltaToolCall(
+ index=0,
+ function=DeltaFunctionCall(arguments=arguments),
+ )
+ ]
+ )
+
+ result = normalizer.normalize_delta(delta, finished=True)
+
+ assert result is not None
+ assert result.tool_calls[0].function is not None
+ assert result.tool_calls[0].function.arguments == arguments
+
+ def test_parse_recovers_content(self, mock_tokenizer, request_obj):
+ parser = _make_parser(mock_tokenizer, enable_thinking=False)
+
+ reasoning, content, tool_calls = parser.parse(
+ "first\\nsecond",
+ request_obj,
+ )
+
+ assert reasoning is None
+ assert content == "first\nsecond"
+ assert not tool_calls
+
+ def test_parse_delta_recovers_content(self, mock_tokenizer, request_obj):
+ parser = _make_parser(mock_tokenizer, enable_thinking=False)
+
+ chunk = parser.parse_delta(
+ "first\\",
+ [],
+ request_obj,
+ finished=False,
+ )
+ rest = parser.parse_delta(
+ "nsecond",
+ [],
+ request_obj,
+ finished=True,
+ )
+
+ # The trailing half of the escape is held back until it is complete.
+ assert chunk is not None
+ assert chunk.content == "first"
+ assert rest is not None
+ assert rest.content == "\nsecond"
+
+ def test_buffered_delta_is_dropped_not_sent_empty(
+ self, mock_tokenizer, request_obj
+ ):
+ parser = _make_parser(mock_tokenizer, enable_thinking=False)
+ parser.parse_delta("hello", [], request_obj, finished=False)
+
+ # The lone backtick is held back for the next chunk. Reporting it as an
+ # all-None DeltaMessage would reach the client as an empty SSE chunk.
+ assert parser.parse_delta("`", [], request_obj, finished=False) is None
+
+ def test_normalizer_drops_empty_delta(self):
+ normalizer = MiniCPMVOutputNormalizer()
+
+ assert (
+ normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
+ is None
+ )
+
+ def test_normalizer_flushes_buffered_content_when_finished(self):
+ normalizer = MiniCPMVOutputNormalizer()
+ normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
+
+ final = normalizer.normalize_delta(DeltaMessage(), finished=True)
+
+ assert final is not None
+ assert final.content == "`"
diff --git a/vllm/model_executor/models/minicpmv4_6.py b/vllm/model_executor/models/minicpmv4_6.py
index 0a96c2d847..333ab8eedd 100644
--- a/vllm/model_executor/models/minicpmv4_6.py
+++ b/vllm/model_executor/models/minicpmv4_6.py
@@ -45,6 +45,7 @@ from .interfaces import (
HasInnerState,
IsHybrid,
MultiModalEmbeddings,
+ SupportsEagle3,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
@@ -87,35 +88,186 @@ def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]):
fields["use_vit_merger"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
+ if "video_use_vit_merger" in hf_inputs:
+ fields["video_use_vit_merger"] = MultiModalFieldConfig.batched(
+ "video", keep_on_cpu=True
+ )
return fields
+def _expand_flags(value: object, num_items: int) -> list[bool | None]:
+ """Broadcast a batched per-item flag to exactly one entry per item."""
+ if value is None:
+ return [None] * num_items
+
+ raw: list[object]
+ if isinstance(value, torch.Tensor):
+ raw = value.reshape(-1).tolist()
+ elif isinstance(value, list | tuple):
+ raw = []
+ for item in value:
+ if isinstance(item, torch.Tensor):
+ raw.extend(item.reshape(-1).tolist())
+ else:
+ raw.append(item)
+ else:
+ raw = [value]
+
+ if len(raw) == num_items:
+ return [bool(v) for v in raw]
+ if len(raw) == 1:
+ return [bool(raw[0])] * num_items
+ # Unexpected shape: keep one decision for the whole batch instead of
+ # failing the request.
+ return [any(bool(v) for v in raw)] * num_items
+
+
+def _vision_downsample_mode(flag: bool | None) -> str | None:
+ if flag is None:
+ return None
+ return "16x" if flag else "4x"
+
+
+def _select_vision_items(
+ image_input: MiniCPMVImagePixelInputs,
+ item_indices: list[int],
+ num_slices: list[int],
+) -> MiniCPMVImagePixelInputs:
+ """Slice a vision input down to the given items, keeping their order."""
+ offsets = [0]
+ for n in num_slices:
+ offsets.append(offsets[-1] + n)
+
+ pixel_values: list[torch.Tensor] = []
+ tgt_sizes: list[torch.Tensor] = []
+ sizes: list[int] = []
+ for i in item_indices:
+ start, end = offsets[i], offsets[i + 1]
+ pixel_values.extend(image_input["pixel_values"][start:end])
+ tgt_sizes.append(image_input["tgt_sizes"][start:end])
+ sizes.append(end - start)
+
+ return MiniCPMVImagePixelInputs(
+ type="pixel_values",
+ pixel_values=pixel_values,
+ tgt_sizes=torch.cat(tgt_sizes, dim=0),
+ num_slices=torch.tensor(sizes),
+ )
+
+
+# HF-style modality-scoped keys inside `mm_processor_kwargs`. They scope a value
+# to a single modality and must not be forwarded to the processors, which take
+# flat arguments only.
+_MODALITY_SCOPED_MM_KWARGS = ("images_kwargs", "videos_kwargs", "audio_kwargs")
+
+
+def _flat_processor_kwargs(
+ mm_kwargs: Mapping[str, object],
+ downsample_mode: str,
+) -> dict[str, object]:
+ """Drop the modality-scoped nesting and pin the resolved downsample mode.
+
+ The placeholder count and the vision tower must read the same mode, so the
+ value resolved for this modality is passed down explicitly rather than
+ leaving the processor to fall back to its own default.
+ """
+ kwargs = {
+ key: value
+ for key, value in mm_kwargs.items()
+ if key not in _MODALITY_SCOPED_MM_KWARGS
+ }
+ kwargs["downsample_mode"] = downsample_mode
+ return kwargs
+
+
+_MODALITY_TO_MM_KWARGS_SCOPE = {
+ "image": "images_kwargs",
+ "video": "videos_kwargs",
+}
+
+
+def _scoped_mm_kwarg(
+ merged: Mapping[str, object],
+ modality: str,
+ key: str,
+) -> Any:
+ """Read a processor kwarg for one modality.
+
+ ``merged`` is what ``_merge_and_resolve_mm_processor_kwargs`` returns: it
+ routes a flat key into every scope whose processor declares it, and a value
+ scoped to the other modality stays there, so it is not visible here.
+ """
+ scoped = merged.get(_MODALITY_TO_MM_KWARGS_SCOPE[modality])
+ if isinstance(scoped, Mapping):
+ return scoped.get(key)
+ return None
+
+
class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
def _resolve_downsample_mode(
self,
mm_kwargs: Mapping[str, object],
+ modality: str = "image",
) -> str:
- ds = mm_kwargs.get("downsample_mode")
- if ds is not None:
- return str(ds)
+ # A flat `downsample_mode` applies to both modalities; the nested
+ # `images_kwargs` / `videos_kwargs` forms scope it to one, matching
+ # MiniCPMV4_6Processor.
info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
+ merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
+ downsample_mode = _scoped_mm_kwarg(merged, modality, "downsample_mode")
+ if downsample_mode is not None:
+ return str(downsample_mode)
return info._get_downsample_mode()
+ def _resolve_max_slice_nums(
+ self,
+ mm_kwargs: Mapping[str, object],
+ modality: str = "image",
+ ) -> int | None:
+ # Per-request max_slice_nums. None keeps the processor default.
+ # Resolved like `downsample_mode` so the nested per-modality form
+ # (`images_kwargs` / `videos_kwargs`) is honoured too. This must match
+ # what `process_images` passes to the image processor: the placeholder
+ # count and the visual token count are derived from it separately, so
+ # they diverge (and the engine aborts) if the two sides read different
+ # values.
+ info = self.info
+ assert isinstance(info, MiniCPMV4_6ProcessingInfo)
+ merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
+ max_slice = _scoped_mm_kwarg(merged, modality, "max_slice_nums")
+ if max_slice is None:
+ return None
+ return int(max_slice)
+
def get_image_prompt_texts(
self,
image_size,
image_idx: int = 0,
downsample_mode: str | None = None,
+ max_slice_nums: int | None = None,
) -> str:
info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
return info.get_slice_image_placeholder(
image_size,
image_idx=image_idx,
+ max_slice_nums=max_slice_nums,
downsample_mode=downsample_mode,
)
+ def _video_local_id_prefix(self, video_idx: int) -> str:
+ """`{idx}` prefix emitted in front of a video placeholder.
+
+ MiniCPM-V 4.6 numbers each video with a local id, matching
+ MiniCPMV4_6Processor. 4.7 treats a video as one temporal sequence and is
+ trained without these ids, so it overrides this to return an empty string.
+ """
+ tokenizer = self.info.get_tokenizer()
+ id_start = getattr(tokenizer, "image_id_start_token", "")
+ id_end = getattr(tokenizer, "image_id_end_token", "")
+ return f"{id_start}{video_idx}{id_end}"
+
def get_video_prompt_texts(
self,
image_size,
@@ -144,8 +296,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
image_end = getattr(tokenizer, "image_end_token", "")
slice_start = getattr(tokenizer, "slice_start_token", "")
slice_end = getattr(tokenizer, "slice_end_token", "")
- id_start = getattr(tokenizer, "image_id_start_token", "")
- id_end = getattr(tokenizer, "image_id_end_token", "")
per_frame = image_start + video_token * source_tokens + image_end
if grids[0] > 0 and grids[1] > 0 and patch_tokens > 0:
@@ -154,7 +304,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
per_frame += "\n".join(rows)
body = per_frame * num_frames
- return f"{id_start}{video_idx}{id_end}" + body
+ return self._video_local_id_prefix(video_idx) + body
def process_images(
self,
@@ -179,10 +329,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
n_images = len(parsed_images)
image_processor = self.info.get_image_processor()
patch_size = image_processor.patch_size
+ ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="image")
+ image_proc_kwargs = _flat_processor_kwargs(mm_kwargs, ds_mode)
+ # `_flat_processor_kwargs` drops the nested form, so the value resolved
+ # from it has to be pinned back for the processor and the prompt text
+ # to agree on the slice count.
+ image_max_slice = self._resolve_max_slice_nums(mm_kwargs, modality="image")
+ if image_max_slice is not None:
+ image_proc_kwargs["max_slice_nums"] = image_max_slice
per_image_pixel_values: list[torch.Tensor] = []
per_image_tgt_sizes: list[torch.Tensor] = []
for image in parsed_images:
- ip_out = image_processor([image], **mm_kwargs)
+ ip_out = image_processor([image], **image_proc_kwargs)
pv = ip_out["pixel_values"] # (1, C, P, sum_W)
ts = ip_out["target_sizes"] # (n_slices, 2)
if pv.ndim == 4 and pv.shape[0] == 1:
@@ -210,7 +368,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
"tgt_sizes": per_image_tgt_sizes,
}
- ds_mode = self._resolve_downsample_mode(mm_kwargs)
insert_layer_id = getattr(
self.info.get_hf_config(),
"insert_layer_id",
@@ -245,7 +402,11 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
image_processor = self.info.get_image_processor()
patch_size = image_processor.patch_size
video_max_slice = self.info.get_video_max_slice_num()
- video_mm_kwargs = {**mm_kwargs, "max_slice_nums": video_max_slice}
+ video_ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="video")
+ video_mm_kwargs = {
+ **_flat_processor_kwargs(mm_kwargs, video_ds_mode),
+ "max_slice_nums": video_max_slice,
+ }
per_video_pixel_values: list[torch.Tensor] = []
per_video_tgt_sizes: list[torch.Tensor] = []
@@ -312,10 +473,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
if not per_video_pixel_values:
return {}
+ insert_layer_id = getattr(self.info.get_hf_config(), "insert_layer_id", -1)
+ merger_flag = video_ds_mode != "4x" and insert_layer_id >= 0
+
return {
"video_pixel_values": per_video_pixel_values,
"video_tgt_sizes": per_video_tgt_sizes,
"video_image_sizes": per_video_image_sizes,
+ "video_use_vit_merger": [
+ torch.tensor([merger_flag], dtype=torch.bool)
+ for _ in per_video_pixel_values
+ ],
}
def _get_prompt_updates(
@@ -324,7 +492,13 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs,
):
- ds_mode = self._resolve_downsample_mode(hf_processor_mm_kwargs)
+ image_ds_mode = self._resolve_downsample_mode(
+ hf_processor_mm_kwargs, modality="image"
+ )
+ video_ds_mode = self._resolve_downsample_mode(
+ hf_processor_mm_kwargs, modality="video"
+ )
+ image_max_slice = self._resolve_max_slice_nums(hf_processor_mm_kwargs)
placeholders = [
("image", self.info.image_pattern),
@@ -360,7 +534,8 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self.get_image_prompt_texts(
image_size,
item_idx,
- downsample_mode=ds_mode,
+ downsample_mode=image_ds_mode,
+ max_slice_nums=image_max_slice,
),
add_special_tokens=False,
),
@@ -389,7 +564,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self.get_video_prompt_texts(
frame_size,
num_frames,
- downsample_mode=ds_mode,
+ downsample_mode=video_ds_mode,
video_idx=item_idx,
),
add_special_tokens=False,
@@ -409,7 +584,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
self.get_video_prompt_texts(
frame_size,
num_frames,
- downsample_mode=ds_mode,
+ downsample_mode=video_ds_mode,
video_idx=item_idx,
),
add_special_tokens=False,
@@ -437,31 +612,28 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
new_update = super()._recompute_cached_prompt_update(
cached_update, new_item_idx
)
- # MiniCPM-V 4.6 prefixes video placeholders with `{idx}`
- # (the base class only rewrites the image modality).
+ # Video placeholders carry an `{idx}` prefix that
+ # the base class does not rewrite. 4.7 has no prefix, so this is a
+ # no-op there.
if cached_update.modality == "video":
- tokenizer = self.info.get_tokenizer()
- id_start = getattr(tokenizer, "image_id_start_token", "")
- id_end = getattr(tokenizer, "image_id_end_token", "")
- video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
+ old_prefix = self._video_local_id_prefix(cached_update.item_idx)
+ new_prefix = self._video_local_id_prefix(new_item_idx)
+ if old_prefix != new_prefix:
+ tokenizer = self.info.get_tokenizer()
+ video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
- text = tokenizer.decode(cached_update.content.full)
- prev_item_idx = cached_update.item_idx
+ text = tokenizer.decode(cached_update.content.full)
- new_update = new_update.with_content(
- PromptUpdateDetails.select_token_ids(
- cached_encode(
- tokenizer,
- text.replace(
- f"{id_start}{prev_item_idx}{id_end}",
- f"{id_start}{new_item_idx}{id_end}",
- 1,
+ new_update = new_update.with_content(
+ PromptUpdateDetails.select_token_ids(
+ cached_encode(
+ tokenizer,
+ text.replace(old_prefix, new_prefix, 1),
+ add_special_tokens=False,
),
- add_special_tokens=False,
- ),
- cached_encode(tokenizer, video_token, add_special_tokens=False),
+ cached_encode(tokenizer, video_token, add_special_tokens=False),
+ )
)
- )
return new_update
def _get_mm_fields_config(
@@ -676,6 +848,31 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
return source_tokens + grids[0] * grids[1] * patch_tokens
+def _stack_vit_merger_qkv(
+ weights: Iterable[tuple[str, torch.Tensor]],
+) -> Iterable[tuple[str, torch.Tensor]]:
+ """Fold separate vit_merger q/k/v projections into the fused qkv_proj module.
+
+ HF checkpoints store the window-attention projections separately, while
+ vLLM keeps them fused. The shard_id mechanism on the submodule's
+ WeightsMapper cannot do this: AutoWeightsLoader still recurses over every
+ incoming key after calling a submodule's load_weights, so a "k_proj" key
+ with no matching child raises before the mapping is honoured.
+ """
+ pairs = (
+ ("vit_merger.self_attn.q_proj.", "q"),
+ ("vit_merger.self_attn.k_proj.", "k"),
+ ("vit_merger.self_attn.v_proj.", "v"),
+ )
+ for name, tensor in weights:
+ for src, shard in pairs:
+ if src in name:
+ name = name.replace(src, "vit_merger.self_attn.qkv_proj.", 1)
+ tensor.shard_id = shard
+ break
+ yield name, tensor
+
+
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
def __init__(
self,
@@ -957,6 +1154,7 @@ class MiniCPMV4_6ForConditionalGeneration(
SupportsPP,
HasInnerState,
IsHybrid,
+ SupportsEagle3,
SupportsMRoPE,
):
supports_encoder_tp_data = True
@@ -1031,7 +1229,7 @@ class MiniCPMV4_6ForConditionalGeneration(
apply_encoder_attention_mask=True,
prefix=maybe_prefix(prefix, "vpm"),
)
- if config.drop_vision_last_layer:
+ if self._drop_vision_last_layer():
self.vpm.encoder.layers = self.vpm.encoder.layers[:-1]
self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger(
@@ -1062,6 +1260,14 @@ class MiniCPMV4_6ForConditionalGeneration(
self.language_model.make_empty_intermediate_tensors
)
+ def _drop_vision_last_layer(self) -> bool:
+ """Whether the last vision encoder layer is dropped.
+
+ MiniCPM-V 4.6 checkpoints always define this. 4.7's config no longer
+ carries the field, so it overrides this.
+ """
+ return self.config.drop_vision_last_layer
+
# ----- Multimodal parsing -----
def _parse_and_validate_vision_input(
@@ -1174,35 +1380,35 @@ class MiniCPMV4_6ForConditionalGeneration(
if image_input["type"] == "image_embeds":
return image_input["image_embeds"]
- downsample_mode = None
- if use_vit_merger is not None:
- downsample_mode = "16x" if use_vit_merger else "4x"
- image_features = self.get_vision_hidden_states(
- image_input,
- downsample_mode=downsample_mode,
- )
- num_slices = image_input["num_slices"]
- results = []
- idx = 0
- for n in num_slices.tolist():
- group = image_features[idx : idx + n]
- results.append(torch.cat(group, dim=0))
- idx += n
- return results
+ num_slices = image_input["num_slices"].tolist()
+ flags = _expand_flags(use_vit_merger, len(num_slices))
+
+ # The merger flag is per request, but the model runner may batch items
+ # from several concurrent requests. Encoding such a mixed batch would
+ # apply one request's downsample mode to another request's items, so
+ # group by flag and encode each group separately.
+ encoded: dict[int, torch.Tensor] = {}
+ for flag in dict.fromkeys(flags):
+ item_indices = [i for i, f in enumerate(flags) if f == flag]
+ group_input = _select_vision_items(image_input, item_indices, num_slices)
+ group_features = self.get_vision_hidden_states(
+ group_input,
+ downsample_mode=_vision_downsample_mode(flag),
+ )
+
+ idx = 0
+ for i in item_indices:
+ n = num_slices[i]
+ encoded[i] = torch.cat(group_features[idx : idx + n], dim=0)
+ idx += n
+
+ return [encoded[i] for i in range(len(num_slices))]
# ----- Multimodal embedding interface -----
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
- use_vit_merger_tensors = kwargs.pop("use_vit_merger", None)
- use_vit_merger = None
- if use_vit_merger_tensors is not None:
- if isinstance(use_vit_merger_tensors, torch.Tensor):
- use_vit_merger = bool(use_vit_merger_tensors.any().item())
- elif isinstance(use_vit_merger_tensors, list | tuple):
- use_vit_merger = any(
- bool(t.any().item()) if isinstance(t, torch.Tensor) else bool(t)
- for t in use_vit_merger_tensors
- )
+ use_vit_merger = kwargs.pop("use_vit_merger", None)
+ video_use_vit_merger = kwargs.pop("video_use_vit_merger", None)
image_kwargs = {
k: v
@@ -1235,7 +1441,7 @@ class MiniCPMV4_6ForConditionalGeneration(
multimodal_embeddings += tuple(
self._process_vision_input(
video_input,
- use_vit_merger=use_vit_merger,
+ use_vit_merger=video_use_vit_merger,
)
)
@@ -1298,7 +1504,10 @@ class MiniCPMV4_6ForConditionalGeneration(
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
loader = AutoWeightsLoader(self)
- return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+ return loader.load_weights(
+ _stack_vit_merger_qkv(weights),
+ mapper=self.hf_to_vllm_mapper,
+ )
def get_mm_mapping(self) -> MultiModelKeys:
return MultiModelKeys.from_string_field(
diff --git a/vllm/model_executor/models/minicpmv4_7.py b/vllm/model_executor/models/minicpmv4_7.py
new file mode 100644
index 0000000000..3ce8609169
--- /dev/null
+++ b/vllm/model_executor/models/minicpmv4_7.py
@@ -0,0 +1,764 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inference-only MiniCPM-V 4.7 model.
+
+Same vision/LLM stack as MiniCPM-V 4.6, plus canvas 3D M-RoPE.
+"""
+
+import math
+from collections.abc import Mapping
+from typing import Any, NamedTuple
+
+import torch
+
+from vllm.config import VllmConfig
+from vllm.logger import init_logger
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig
+
+from .minicpmv import MiniCPMVDummyInputsBuilder
+from .minicpmv4_6 import (
+ MiniCPMV4_6ForConditionalGeneration,
+ MiniCPMV4_6MultiModalProcessor,
+ MiniCPMV4_6ProcessingInfo,
+ _scoped_mm_kwarg,
+)
+
+logger = init_logger(__name__)
+
+
+class MiniCPMV4_7ProcessingInfo(MiniCPMV4_6ProcessingInfo):
+ def _mm_max_slice_nums(self, **kwargs: object) -> int | None:
+ # No schema resolution here: it would call back into `get_hf_processor`
+ # below, which resolves this same value.
+ merged = self.ctx.get_merged_mm_kwargs(kwargs)
+ max_slice = _scoped_mm_kwarg(merged, "image", "max_slice_nums")
+ if max_slice is None:
+ max_slice = merged.get("max_slice_nums")
+ if max_slice is None:
+ return None
+ return int(max_slice)
+
+ def _configured_max_slice_nums(self) -> int | None:
+ # Configured (server-level) value only: per-request values are passed
+ # explicitly to the image processor call, so they must not be written
+ # back to the shared processor instance.
+ return self._mm_max_slice_nums()
+
+ def get_hf_processor(self, **kwargs: object):
+ hf_processor = super().get_hf_processor(**kwargs)
+ image_processor = getattr(hf_processor, "image_processor", None)
+ configured = self._configured_max_slice_nums()
+ if image_processor is not None and configured is not None:
+ image_processor.max_slice_nums = configured
+ return hf_processor
+
+ def get_image_max_slice_num(self) -> int:
+ max_slice = self._configured_max_slice_nums()
+ if max_slice is not None:
+ return max_slice
+ return super().get_image_max_slice_num()
+
+
+class MiniCPMV4_7MultiModalProcessor(MiniCPMV4_6MultiModalProcessor):
+ def _get_mm_fields_config(
+ self,
+ hf_inputs,
+ hf_processor_mm_kwargs: Mapping[str, object],
+ ) -> Mapping[str, MultiModalFieldConfig]:
+ fields = dict(super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs))
+ # Canvas M-RoPE places slices from `tgt_sizes`. Stripping the payload of
+ # a prefix-cache-covered item only keeps CPU-side metadata, so without
+ # this a cached image loses its target sizes, `image_bounds` and
+ # `target_sizes` disagree, and the whole request silently falls back to
+ # sequential positions.
+ for key, modality in (("tgt_sizes", "image"), ("video_tgt_sizes", "video")):
+ if key in fields:
+ fields[key] = MultiModalFieldConfig.batched(modality, keep_on_cpu=True)
+ return fields
+
+ def _video_local_id_prefix(self, video_idx: int) -> str:
+ # MiniCPMV4_7Processor emits no local `` in front of video
+ # placeholders: a video is one temporal sequence and 4.7 was trained
+ # without them.
+ return ""
+
+ def get_image_prompt_texts(
+ self,
+ image_size,
+ image_idx: int = 0,
+ downsample_mode: str | None = None,
+ max_slice_nums: int | None = None,
+ ) -> str:
+ info = self.info
+ assert isinstance(info, MiniCPMV4_6ProcessingInfo)
+ if max_slice_nums is None:
+ max_slice_nums = info.get_image_max_slice_num()
+ return info.get_slice_image_placeholder(
+ image_size,
+ image_idx=image_idx,
+ max_slice_nums=max_slice_nums,
+ downsample_mode=downsample_mode,
+ )
+
+
+def _as_hw_pairs(tensor: torch.Tensor) -> torch.Tensor:
+ data = tensor.to(torch.long)
+ if data.numel() == 0:
+ return data.reshape(0, 2)
+ if data.shape[-1] != 2:
+ raise ValueError(f"expected last dim 2, got {tuple(data.shape)}")
+ return data.reshape(-1, 2)
+
+
+def _sequential_positions(seq_len: int) -> torch.Tensor:
+ """Positions for the fallback where canvas M-RoPE cannot be computed."""
+ return torch.arange(seq_len).unsqueeze(0).expand(3, -1)
+
+
+@MULTIMODAL_REGISTRY.register_processor(
+ MiniCPMV4_7MultiModalProcessor,
+ info=MiniCPMV4_7ProcessingInfo,
+ dummy_inputs=MiniCPMVDummyInputsBuilder,
+)
+class MiniCPMV4_7ForConditionalGeneration(MiniCPMV4_6ForConditionalGeneration):
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
+ self._init_canvas_mrope(vllm_config)
+
+ def _drop_vision_last_layer(self) -> bool:
+ # MiniCPMV4_7Config no longer carries this field (transformers marks it
+ # removed), and 4.7 keeps every vision encoder layer.
+ return False
+
+ def _init_canvas_mrope(self, vllm_config: VllmConfig) -> None:
+ config = self.config
+ uses_canvas = bool(getattr(config, "uses_mrope_canvas", False))
+ if not uses_canvas:
+ mrope_mode = getattr(config, "mrope_mode", None)
+ uses_canvas = mrope_mode is not None and "canvas" in str(mrope_mode).lower()
+
+ self._uses_canvas_mrope = uses_canvas
+ self._canvas_special_ids: dict[str, int] | None = None
+ if not uses_canvas:
+ return
+
+ # MiniCPMV4_7Config pins the structural ids in config.json; the
+ # conversion script resolves them from the tokenizer. Prefer them so the
+ # canvas matches training, and fall back to the tokenizer for checkpoints
+ # whose config predates these fields.
+ configured_ids: dict[str, int | None] = {
+ "im_start_id": getattr(config, "image_start_id", None),
+ "im_end_id": getattr(config, "image_end_id", None),
+ "slice_start_id": getattr(config, "slice_start_id", None),
+ "slice_end_id": getattr(config, "slice_end_id", None),
+ "newline_id": getattr(config, "newline_id", None),
+ }
+ if all(value is not None for value in configured_ids.values()):
+ self._canvas_special_ids = {
+ name: int(value)
+ for name, value in configured_ids.items()
+ if value is not None
+ }
+ return
+
+ from vllm.tokenizers.registry import get_tokenizer
+
+ model_config = vllm_config.model_config
+ tokenizer = get_tokenizer(
+ model_config.tokenizer,
+ revision=model_config.tokenizer_revision,
+ trust_remote_code=model_config.trust_remote_code,
+ )
+ self._canvas_special_ids = {
+ "im_start_id": tokenizer.convert_tokens_to_ids(
+ getattr(tokenizer, "image_start_token", "")
+ ),
+ "im_end_id": tokenizer.convert_tokens_to_ids(
+ getattr(tokenizer, "image_end_token", "")
+ ),
+ "slice_start_id": tokenizer.convert_tokens_to_ids(
+ getattr(tokenizer, "slice_start_token", "")
+ ),
+ "slice_end_id": tokenizer.convert_tokens_to_ids(
+ getattr(tokenizer, "slice_end_token", "")
+ ),
+ "newline_id": tokenizer.encode("\n", add_special_tokens=False)[0],
+ }
+
+ def get_mrope_input_positions(
+ self,
+ input_tokens: list[int],
+ mm_features: list[MultiModalFeatureSpec],
+ ) -> tuple[torch.Tensor, int]:
+ seq_len = len(input_tokens)
+ if (
+ not getattr(self, "_uses_canvas_mrope", False)
+ or self._canvas_special_ids is None
+ or not mm_features
+ ):
+ return _sequential_positions(seq_len), 0
+
+ input_ids = torch.tensor(input_tokens, dtype=torch.long)
+ image_bounds = build_image_bounds(input_ids, self._canvas_special_ids)
+
+ target_sizes_list: list[torch.Tensor] = []
+ for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
+ if mm_feature.data is None:
+ continue
+ for key in ("tgt_sizes", "video_tgt_sizes"):
+ tgt = mm_feature.data.get(key)
+ if tgt is None or not isinstance(tgt.data, torch.Tensor):
+ continue
+ try:
+ target_sizes_list.append(_as_hw_pairs(tgt.data))
+ except ValueError:
+ # Not an (N, 2) size table. The bounds/sizes match below
+ # turns this into a sequential fallback.
+ continue
+
+ if not target_sizes_list or image_bounds.numel() == 0:
+ logger.warning(
+ "MiniCPM-V 4.7 canvas M-RoPE missing inputs: "
+ "tgt_chunks=%s image_bounds=%s; using sequential positions.",
+ len(target_sizes_list),
+ tuple(image_bounds.shape) if image_bounds is not None else None,
+ )
+ return _sequential_positions(seq_len), 0
+
+ target_sizes = torch.cat(target_sizes_list, dim=0)
+ if target_sizes.ndim != 2 or target_sizes.shape[-1] != 2:
+ logger.warning(
+ "MiniCPM-V 4.7 canvas M-RoPE unexpected tgt_sizes shape %s; "
+ "using sequential positions.",
+ tuple(target_sizes.shape),
+ )
+ return _sequential_positions(seq_len), 0
+ if image_bounds.shape[0] != target_sizes.shape[0]:
+ logger.warning(
+ "MiniCPM-V 4.7 canvas M-RoPE size mismatch: "
+ "image_bounds=%s target_sizes=%s; using sequential positions.",
+ tuple(image_bounds.shape),
+ tuple(target_sizes.shape),
+ )
+ return _sequential_positions(seq_len), 0
+
+ try:
+ pos3d = _compute_canvas_single(
+ input_ids,
+ torch.arange(seq_len, dtype=torch.long),
+ image_bounds,
+ target_sizes,
+ self._canvas_special_ids,
+ )
+ except Exception:
+ logger.warning(
+ "MiniCPM-V 4.7 canvas M-RoPE failed; "
+ "falling back to sequential positions.",
+ exc_info=True,
+ )
+ return _sequential_positions(seq_len), 0
+ return pos3d, canvas_rope_delta(pos3d, seq_len)
+
+
+# Canvas 3D M-RoPE: position ids for the slice canvas.
+def compute_llm_grid(
+ n_tokens: int, vision_grid_height: int, vision_grid_width: int
+) -> tuple[int, int]:
+ if n_tokens <= 0 or vision_grid_height <= 0 or vision_grid_width <= 0:
+ return 0, 0
+ vit_total = vision_grid_height * vision_grid_width
+ if vit_total == n_tokens:
+ return vision_grid_height, vision_grid_width
+ if vit_total < n_tokens:
+ return 0, 0
+ factor = int(round(math.sqrt(vit_total / n_tokens)))
+ if (
+ factor > 0
+ and vision_grid_height % factor == 0
+ and vision_grid_width % factor == 0
+ ):
+ height, width = vision_grid_height // factor, vision_grid_width // factor
+ if height * width == n_tokens:
+ return height, width
+ ratio = vision_grid_height / vision_grid_width
+ width = max(1, int(round(math.sqrt(n_tokens / ratio))))
+ height = n_tokens // width
+ if height * width == n_tokens and height > 0:
+ return height, width
+ return 0, 0
+
+
+def build_image_bounds(
+ input_ids: torch.LongTensor, special_token_ids: dict
+) -> torch.LongTensor:
+ start_ids = [
+ x
+ for x in (
+ special_token_ids.get("im_start_id"),
+ special_token_ids.get("slice_start_id"),
+ )
+ if x is not None
+ ]
+ end_ids = [
+ x
+ for x in (
+ special_token_ids.get("im_end_id"),
+ special_token_ids.get("slice_end_id"),
+ )
+ if x is not None
+ ]
+ if not start_ids or not end_ids:
+ return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
+ start_cond = torch.zeros_like(input_ids, dtype=torch.bool)
+ end_cond = torch.zeros_like(input_ids, dtype=torch.bool)
+ for sid in start_ids:
+ start_cond |= input_ids == sid
+ for eid in end_ids:
+ end_cond |= input_ids == eid
+ starts = torch.where(start_cond)[0] + 1
+ ends = torch.where(end_cond)[0]
+ n = min(len(starts), len(ends))
+ if n == 0:
+ return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device)
+ return torch.stack([starts[:n], ends[:n]], dim=-1)
+
+
+def _compute_canvas_single(
+ input_ids, position_ids_2d, image_bound, target_sizes, special_token_ids
+):
+ # Inference always handles one unpadded sequence. The packed (cu_seqlens)
+ # form below is kept only to mirror the transformers reference
+ # implementation of its canvas position computation.
+ seq_len = input_ids.shape[0]
+ cu_seqlens = torch.tensor([0, seq_len], device=input_ids.device, dtype=torch.long)
+ return _compute_canvas_packed(
+ position_ids_2d.unsqueeze(0),
+ cu_seqlens,
+ image_bound,
+ target_sizes,
+ input_ids.unsqueeze(0),
+ special_token_ids,
+ )[:, 0, :]
+
+
+def _target_hw(target_sizes: torch.Tensor, index: int) -> tuple[int, int]:
+ """Patch-grid height/width recorded for one image or slice."""
+ return int(target_sizes[index, 0]), int(target_sizes[index, 1])
+
+
+class _CanvasLayout(NamedTuple):
+ """Grid of one canvas measured in LLM tokens.
+
+ A canvas is a thumbnail plus the slices cut from it, tiled row-major.
+ """
+
+ thumb_h: int
+ thumb_w: int
+ """Thumbnail grid."""
+ slice_h: int
+ slice_w: int
+ """Per-slice grid. Zero when the canvas has no slices."""
+ cols: int
+ """Slices sharing a canvas row. Zero when the canvas has no slices."""
+ height: int
+ width: int
+ """Assembled canvas."""
+
+
+def _canvas_layout(
+ thumbnail: tuple[int, int, int],
+ slices: list[tuple[int, int, int]],
+ target_sizes: torch.Tensor,
+) -> _CanvasLayout:
+ """Measure the canvas for one ``(thumbnail, slices)`` group.
+
+ ``thumbnail`` and each slice are ``(start, end, target_index)`` token spans.
+ """
+ t_bs, t_be, t_gi = thumbnail
+ thumb_h, thumb_w = compute_llm_grid(t_be - t_bs, *_target_hw(target_sizes, t_gi))
+ if not slices:
+ return _CanvasLayout(
+ thumb_h, thumb_w, 0, 0, 0, max(thumb_h, 1), max(thumb_w, 1)
+ )
+
+ cols = len(slices)
+ for k in range(len(slices) - 1):
+ if slices[k + 1][0] - slices[k][1] > 2:
+ cols = k + 1
+ break
+ rows = len(slices) // cols if cols > 0 else 1
+ if rows * cols != len(slices):
+ rows, cols = 1, len(slices)
+
+ s0_bs, s0_be, s0_gi = slices[0]
+ slice_h, slice_w = compute_llm_grid(s0_be - s0_bs, *_target_hw(target_sizes, s0_gi))
+ return _CanvasLayout(
+ thumb_h, thumb_w, slice_h, slice_w, cols, rows * slice_h, cols * slice_w
+ )
+
+
+def _assign_canvas_span(
+ pos3d: torch.Tensor,
+ *,
+ thumbnail: tuple[int, int, int],
+ span_end: int,
+ slices: list[tuple[int, int, int]],
+ base: int,
+ target_sizes: torch.Tensor,
+) -> int:
+ """Place one canvas at ``base``, writing its 3-D positions into ``pos3d``.
+
+ Shared by the video-frame and single-image paths; they differ only in how
+ the per-canvas base advances, not in the geometry placed here.
+
+ Mutates ``pos3d`` in place, as the packed canvas computation has always
+ done. Returns the distance to the next canvas base, so each caller keeps
+ only its own base advancement.
+ """
+ t_bs, t_be, _ = thumbnail
+ device = pos3d.device
+ span_start = t_bs - 1 # the token
+ n_thumb = t_be - t_bs
+ layout = _canvas_layout(thumbnail, slices, target_sizes)
+ cols, canvas_H, canvas_W = layout.cols, layout.height, layout.width
+
+ # base coat
+ pos3d[0, 0, span_start:span_end] = base
+ pos3d[1, 0, span_start:span_end] = base
+ pos3d[2, 0, span_start:span_end] = base
+
+ # -> halo
+ halo_lo = max(base - 1, 0)
+ pos3d[1, 0, span_start] = halo_lo
+ pos3d[2, 0, span_start] = halo_lo
+
+ # -> halo (right after thumbnail)
+ if t_be < span_end:
+ pos3d[1, 0, t_be] = base + canvas_H
+ pos3d[2, 0, t_be] = base + canvas_W
+
+ # thumbnail visual tokens
+ llm_th, llm_tw = layout.thumb_h, layout.thumb_w
+ if llm_th > 0 and llm_tw > 0 and llm_th * llm_tw == n_thumb:
+ if slices and canvas_H > 0 and canvas_W > 0:
+ h_c = torch.linspace(0, canvas_H - 1, llm_th, device=device).round().long()
+ w_c = torch.linspace(0, canvas_W - 1, llm_tw, device=device).round().long()
+ else:
+ h_c = torch.arange(llm_th, device=device)
+ w_c = torch.arange(llm_tw, device=device)
+ h_idx = h_c.view(-1, 1).expand(-1, llm_tw).reshape(-1)
+ w_idx = w_c.view(1, -1).expand(llm_th, -1).reshape(-1)
+ pos3d[0, 0, t_bs:t_be] = base
+ pos3d[1, 0, t_bs:t_be] = h_idx + base
+ pos3d[2, 0, t_bs:t_be] = w_idx + base
+ else:
+ fallback = torch.arange(n_thumb, device=device, dtype=torch.long) + base
+ for d in range(3):
+ pos3d[d, 0, t_bs:t_be] = fallback
+
+ # slice visual tokens + slice specials
+ for k, (s_bs, s_be, s_gi) in enumerate(slices):
+ n_s = s_be - s_bs
+ sh_k, sw_k = compute_llm_grid(n_s, *_target_hw(target_sizes, s_gi))
+ h_off = (k // cols) * layout.slice_h
+ w_off = (k % cols) * layout.slice_w
+
+ slice_start_pos = s_bs - 1
+ if slice_start_pos >= span_start:
+ pos3d[1, 0, slice_start_pos] = base + h_off
+ pos3d[2, 0, slice_start_pos] = base + w_off
+
+ slice_end_pos = s_be
+ if sh_k > 0 and sw_k > 0:
+ end_h = h_off + sh_k - 1
+ end_w = w_off + sw_k - 1
+ else:
+ end_h = h_off
+ end_w = w_off
+ if slice_end_pos < span_end:
+ pos3d[1, 0, slice_end_pos] = base + end_h
+ pos3d[2, 0, slice_end_pos] = base + end_w
+
+ if sh_k > 0 and sw_k > 0 and sh_k * sw_k == n_s:
+ h_idx = (
+ torch.arange(sh_k, device=device)
+ .view(-1, 1)
+ .expand(-1, sw_k)
+ .reshape(-1)
+ )
+ w_idx = (
+ torch.arange(sw_k, device=device)
+ .view(1, -1)
+ .expand(sh_k, -1)
+ .reshape(-1)
+ )
+ pos3d[0, 0, s_bs:s_be] = base
+ pos3d[1, 0, s_bs:s_be] = h_idx + h_off + base
+ pos3d[2, 0, s_bs:s_be] = w_idx + w_off + base
+ else:
+ fallback = torch.arange(n_s, device=device, dtype=torch.long) + base
+ for d in range(3):
+ pos3d[d, 0, s_bs:s_be] = fallback
+
+ # \n between slice rows -> W = right edge + 1.
+ # H stays within the ended row (does NOT cross to next row): the \n sits at
+ # the right-outside of the row that just ended, at H = that row's last row
+ # and W = one column past the rightmost slice.
+ for k in range(len(slices) - 1):
+ gap_s = slices[k][1]
+ gap_e = slices[k + 1][0]
+ if gap_e - gap_s <= 2:
+ continue
+ boundary_h = ((k // cols) + 1) * layout.slice_h - 1
+ right_edge_w = cols * layout.slice_w
+ for nl_pos in range(gap_s + 1, gap_e - 1):
+ pos3d[1, 0, nl_pos] = base + boundary_h
+ pos3d[2, 0, nl_pos] = base + right_edge_w
+
+ return max(canvas_H, canvas_W) + 1
+
+
+def _compute_canvas_packed(
+ position_ids_2d,
+ cu_seqlens,
+ image_bound,
+ target_sizes,
+ input_ids,
+ special_token_ids,
+):
+ if position_ids_2d.ndim == 1:
+ position_ids_2d = position_ids_2d.unsqueeze(0)
+ if input_ids.ndim == 1:
+ input_ids = input_ids.unsqueeze(0)
+
+ has_images = (
+ isinstance(image_bound, torch.Tensor)
+ and image_bound.numel() > 0
+ and isinstance(target_sizes, torch.Tensor)
+ and target_sizes.numel() > 0
+ )
+ if not has_images:
+ return position_ids_2d.unsqueeze(0).expand(3, -1, -1).contiguous()
+
+ device = position_ids_2d.device
+ pos3d = position_ids_2d.unsqueeze(0).expand(3, -1, -1).clone()
+ ids_flat = input_ids.view(-1)
+
+ im_start_id = special_token_ids.get("im_start_id")
+ im_end_id = special_token_ids.get("im_end_id")
+ slice_start_id = special_token_ids.get("slice_start_id")
+ slice_end_id = special_token_ids.get("slice_end_id")
+
+ structural_ids = {
+ v
+ for v in (im_start_id, im_end_id, slice_start_id, slice_end_id)
+ if v is not None
+ }
+
+ num_seqs = len(cu_seqlens) - 1
+ img_ptr = 0
+ num_imgs = len(image_bound)
+
+ newline_id = special_token_ids.get("newline_id")
+ gap_ok_ids = structural_ids | ({newline_id} if newline_id is not None else set())
+
+ def _group_end_pos(g, s_end):
+ """First position after a group's trailing structural tokens.
+
+ Matches the transformers reference, which closes a span over every
+ marker in ``markers`` -- newlines included. The chat template joins
+ content parts with ``\\n``, so an image followed by text always ends with
+ one; leaving it out moves that newline out of the canvas and pushes
+ every later position one step forward.
+ """
+ le = g["slices"][-1][1] if g["slices"] else g["thumbnail"][1]
+ ge = le
+ while ge < s_end and ids_flat[ge].item() in gap_ok_ids:
+ ge += 1
+ return min(ge, s_end)
+
+ def _is_tight_gap(g_prev, g_next, s_end):
+ """Check if only structural/nl tokens sit between two groups."""
+ ge = _group_end_pos(g_prev, s_end)
+ ns = g_next["thumbnail"][0] - 1 # im_start of next group
+ return all(ids_flat[p].item() in gap_ok_ids for p in range(ge, ns))
+
+ for si in range(num_seqs):
+ s_start = cu_seqlens[si].item()
+ s_end = cu_seqlens[si + 1].item()
+ if s_start >= s_end:
+ continue
+
+ seq_imgs = []
+ while img_ptr < num_imgs:
+ bs_ = image_bound[img_ptr, 0].item()
+ be_ = image_bound[img_ptr, 1].item()
+ if bs_ >= s_start and be_ <= s_end:
+ seq_imgs.append((bs_, be_, img_ptr))
+ img_ptr += 1
+ else:
+ break
+ if not seq_imgs:
+ continue
+
+ # ---- group images: thumbnail + following slices ----
+ # merge consecutive no-slice frames into one video group
+ raw_groups = []
+ cur_group: dict[str, Any] | None = None
+ for bs_, be_, gi in seq_imgs:
+ marker_pos = bs_ - 1
+ is_slice = (
+ slice_start_id is not None
+ and marker_pos >= s_start
+ and ids_flat[marker_pos].item() == slice_start_id
+ )
+ if is_slice and cur_group is not None:
+ cur_group["slices"].append((bs_, be_, gi))
+ else:
+ if cur_group is not None:
+ raw_groups.append(cur_group)
+ cur_group = {"thumbnail": (bs_, be_, gi), "slices": []}
+ if cur_group is not None:
+ raw_groups.append(cur_group)
+
+ # merge consecutive tightly-adjacent groups into video groups.
+ # Two groups are "tightly adjacent" if the gap between them
+ # contains only structural tokens and \n (no real text).
+ # This handles both no-slice frames and frames-with-slices.
+ groups = []
+ i_g = 0
+ while i_g < len(raw_groups):
+ video_frames = [raw_groups[i_g]]
+ j_g = i_g + 1
+ while j_g < len(raw_groups) and _is_tight_gap(
+ raw_groups[j_g - 1], raw_groups[j_g], s_end
+ ):
+ video_frames.append(raw_groups[j_g])
+ j_g += 1
+ if len(video_frames) == 1:
+ groups.append(raw_groups[i_g])
+ else:
+ groups.append({"video_frames": video_frames})
+ i_g = j_g
+
+ # ---- assign 3-D positions ----
+ pos = 0
+ cursor = s_start
+
+ for group in groups:
+ is_video_group = "video_frames" in group
+
+ if is_video_group:
+ # === Video group: multiple frames merged ===
+ # Each frame gets its own base_f so different frames have
+ # distinct spatial positions. Inter-frame gap tokens (\n
+ # etc.) are pure 1D text. After each gap, pos advances
+ # by 1 extra so halo_lo of the next frame does not
+ # collide with the last gap token.
+ frames = group["video_frames"]
+ first_frame = frames[0]
+ last_frame = frames[-1]
+
+ # group_start = of first frame
+ group_start = first_frame["thumbnail"][0] - 1
+ # group_end = after last structural token of last frame
+ group_end = _group_end_pos(last_frame, s_end)
+
+ # text before video group
+ text_len = group_start - cursor
+ if text_len > 0:
+ t_pos = (
+ torch.arange(text_len, device=device, dtype=torch.long) + pos
+ )
+ for d in range(3):
+ pos3d[d, 0, cursor:group_start] = t_pos
+ pos += text_len
+
+ # process each frame with its own per-frame base
+ frame_cursor = group_start
+ for vf in frames:
+ t_bs = vf["thumbnail"][0]
+ vf_slices = vf["slices"]
+ im_start_pos = t_bs - 1
+
+ # --- inter-frame gap: pure 1D text T=H=W=pos ---
+ gap_len = im_start_pos - frame_cursor
+ if gap_len > 0:
+ t_pos = (
+ torch.arange(gap_len, device=device, dtype=torch.long) + pos
+ )
+ for d in range(3):
+ pos3d[d, 0, frame_cursor:im_start_pos] = t_pos
+ pos += gap_len
+ pos += 1 # buffer: prevents halo_lo collision
+
+ # --- per-frame base ---
+ base_f = pos
+ frame_end = _group_end_pos(vf, group_end)
+ pos = base_f + _assign_canvas_span(
+ pos3d,
+ thumbnail=vf["thumbnail"],
+ span_end=frame_end,
+ slices=vf_slices,
+ base=base_f,
+ target_sizes=target_sizes,
+ )
+ frame_cursor = frame_end
+
+ # trailing tokens after last frame (if any)
+ if frame_cursor < group_end:
+ trail_len = group_end - frame_cursor
+ t_pos = (
+ torch.arange(trail_len, device=device, dtype=torch.long) + pos
+ )
+ for d in range(3):
+ pos3d[d, 0, frame_cursor:group_end] = t_pos
+ pos += trail_len
+
+ cursor = group_end
+
+ else:
+ # === Single image group (thumbnail + optional slices) ===
+ slices = group["slices"]
+
+ group_start = group["thumbnail"][0] - 1
+ group_end = _group_end_pos(group, s_end)
+
+ # text before group
+ text_len = group_start - cursor
+ if text_len > 0:
+ t_pos = (
+ torch.arange(text_len, device=device, dtype=torch.long) + pos
+ )
+ for d in range(3):
+ pos3d[d, 0, cursor:group_start] = t_pos
+ pos += text_len
+
+ base = pos
+ pos = base + _assign_canvas_span(
+ pos3d,
+ thumbnail=group["thumbnail"],
+ span_end=group_end,
+ slices=slices,
+ base=base,
+ target_sizes=target_sizes,
+ )
+ cursor = group_end
+
+ # remaining text
+ rem = s_end - cursor
+ if rem > 0:
+ t_pos = torch.arange(rem, device=device, dtype=torch.long) + pos
+ for d in range(3):
+ pos3d[d, 0, cursor:s_end] = t_pos
+
+ return pos3d
+
+
+def canvas_rope_delta(position_ids: torch.Tensor, seq_len: int) -> int:
+ """Offset applied to the 1-D positions of tokens decoded after the prompt.
+
+ The first decoded token sits one past the last prefill position.
+ """
+ return int(position_ids.amax().item()) + 1 - seq_len
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index 875179a88f..fa29244413 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -514,6 +514,10 @@ _MULTIMODAL_MODELS = {
"minicpmv4_6",
"MiniCPMV4_6ForConditionalGeneration",
),
+ "MiniCPMV4_7ForConditionalGeneration": (
+ "minicpmv4_7",
+ "MiniCPMV4_7ForConditionalGeneration",
+ ),
"Mistral3ForConditionalGeneration": (
"mistral3",
"Mistral3ForConditionalGeneration",
diff --git a/vllm/parser/minicpmv.py b/vllm/parser/minicpmv.py
new file mode 100644
index 0000000000..88a6865c40
--- /dev/null
+++ b/vllm/parser/minicpmv.py
@@ -0,0 +1,405 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""MiniCPM-V family parser.
+
+Wraps the Qwen3 grammar (```` reasoning + XML tool calls) with two
+MiniCPM-V specifics:
+
+- ``<|im_end|>`` and the reserved marker tokens are passthrough terminals,
+ so they never leak into ``content``.
+- The tokenizer decodes a newline as the two characters ``\\`` and ``n``
+ (e.g. token id 1639) instead of ``\\n``, so the decoded text is recovered
+ before it reaches the client. Code blocks, inline code and LaTeX are left
+ alone, as are tool call arguments.
+"""
+
+from __future__ import annotations
+
+import functools
+from collections.abc import Sequence
+from dataclasses import replace
+from typing import TYPE_CHECKING
+
+from vllm.entrypoints.generate.base.protocol import DeltaMessage
+from vllm.parser.abstract_parser import DelegatingParser
+from vllm.parser.engine.parser_engine import ParserEngine
+from vllm.parser.engine.parser_engine_config import (
+ ParserEngineConfig,
+ ParserState,
+ Transition,
+)
+from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
+
+if TYPE_CHECKING:
+ from vllm.entrypoints.generate.base.protocol import FunctionCall
+ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
+ from vllm.tokenizers import TokenizerLike
+ from vllm.tool_parsers.abstract_tool_parser import Tool
+
+
+_RESERVED_MARKER_IDS = range(12, 16)
+_IM_END = "<|im_end|>"
+
+
+@functools.cache
+def minicpmv_config(thinking: bool) -> ParserEngineConfig:
+ """MiniCPM-V grammar: the qwen3 grammar plus MiniCPM-V markers.
+
+ Both modes keep the qwen3 tool call terminals and transitions, so
+ ```` output is parsed whether or not thinking is enabled.
+ """
+ name = "minicpmv" if thinking else "minicpmv_no_thinking"
+ base = qwen3_config(thinking=thinking, name=name)
+
+ transitions = dict(base.transitions)
+ if not thinking:
+ # Default mode: the model answers directly, so `` must not open
+ # a reasoning span that would swallow the answer. A stray
+ # `...` in the output is still dropped, not printed.
+ transitions[(ParserState.CONTENT, "THINK_START")] = Transition(
+ ParserState.REASONING
+ )
+ base = replace(base, strip_trailing_reasoning_whitespace=False)
+
+ terminals = dict(base.terminals)
+ token_id_terminals = dict(base.token_id_terminals)
+ terminals["IM_END"] = _IM_END
+ token_id_terminals["IM_END"] = _IM_END
+ for state in (ParserState.CONTENT, ParserState.REASONING):
+ transitions[(state, "IM_END")] = Transition(state)
+
+ for index in _RESERVED_MARKER_IDS:
+ text_name = f"RESERVED_TEXT_{index}"
+ token_name = f"RESERVED_TOKEN_{index}"
+ terminals[text_name] = f""
+ terminals[token_name] = f"<|reserved_{index}|>"
+ token_id_terminals[token_name] = f"<|reserved_{index}|>"
+ for state in (ParserState.CONTENT, ParserState.REASONING):
+ transitions[(state, text_name)] = Transition(state)
+ transitions[(state, token_name)] = Transition(state)
+
+ return replace(
+ base,
+ terminals=terminals,
+ token_id_terminals=token_id_terminals,
+ transitions=transitions,
+ )
+
+
+class EscapedNewlineNormalizer:
+ """Recover literal escape sequences written by the model as real text.
+
+ Scans character by character so that a backslash inside a code block,
+ inline code, LaTeX span or a doubled escape is left untouched. A trailing
+ half of an escape is held in ``_pending`` until the next chunk arrives,
+ which keeps streaming output identical to the non-streaming result.
+ """
+
+ def __init__(self) -> None:
+ self._closing_marker: str | None = None
+ self._pending = ""
+
+ def feed(self, text: str, *, final: bool = False) -> str:
+ text = self._pending + text
+ self._pending = ""
+ output: list[str] = []
+ index = 0
+
+ while index < len(text):
+ if self._closing_marker is not None:
+ marker = self._closing_marker
+ remaining = text[index:]
+ if remaining.startswith(marker):
+ output.append(marker)
+ index += len(marker)
+ self._closing_marker = None
+ continue
+ if not final and marker.startswith(remaining):
+ self._pending = remaining
+ break
+ output.append(text[index])
+ index += 1
+ continue
+
+ remaining = text[index:]
+ if text[index] == "`":
+ if not final and remaining in ("`", "``"):
+ self._pending = remaining
+ break
+ if remaining.startswith("```"):
+ output.append("```")
+ index += 3
+ self._closing_marker = "```"
+ else:
+ output.append("`")
+ index += 1
+ self._closing_marker = "`"
+ continue
+
+ if text[index] == "$":
+ if not final and remaining == "$":
+ self._pending = remaining
+ break
+ if remaining.startswith("$$"):
+ output.append("$$")
+ index += 2
+ self._closing_marker = "$$"
+ else:
+ output.append("$")
+ index += 1
+ self._closing_marker = "$"
+ continue
+
+ if text[index] != "\\":
+ output.append(text[index])
+ index += 1
+ continue
+
+ slash_end = index + 1
+ while slash_end < len(text) and text[slash_end] == "\\":
+ slash_end += 1
+ if slash_end - index > 1:
+ if not final and slash_end == len(text):
+ # The run may still grow, and its parity decides whether a
+ # following `n` is a newline or stays literal. Hold it until
+ # the run is complete.
+ self._pending = text[index:slash_end]
+ break
+ output.append(text[index:slash_end])
+ index = slash_end
+ continue
+
+ if not final and remaining in ("\\", "\\r", "\\r\\"):
+ self._pending = remaining
+ break
+ if remaining.startswith("\\r\\n"):
+ output.append("\n")
+ index += 4
+ elif remaining.startswith(("\\n", "\\r")):
+ output.append("\n")
+ index += 2
+ elif remaining.startswith("\\("):
+ output.append("\\(")
+ index += 2
+ self._closing_marker = "\\)"
+ elif remaining.startswith("\\["):
+ output.append("\\[")
+ index += 2
+ self._closing_marker = "\\]"
+ else:
+ output.append("\\")
+ index += 1
+
+ if final and self._pending:
+ output.append(self._pending)
+ self._pending = ""
+ return "".join(output)
+
+
+def recover_newlines(text: str | None) -> str | None:
+ if text is None or "\\" not in text:
+ return text
+ return EscapedNewlineNormalizer().feed(text, final=True)
+
+
+class MiniCPMVOutputNormalizer:
+ """Apply :class:`EscapedNewlineNormalizer` to a whole stream.
+
+ Reasoning and content are tracked separately so that a half-written escape
+ at the end of a reasoning chunk does not swallow the first characters of
+ the content that follows.
+ """
+
+ def __init__(self) -> None:
+ self._reasoning = EscapedNewlineNormalizer()
+ self._content = EscapedNewlineNormalizer()
+
+ def normalize_delta(
+ self,
+ delta: DeltaMessage | None,
+ *,
+ finished: bool,
+ ) -> DeltaMessage | None:
+ reasoning_finished = finished or (
+ delta is not None and (delta.content is not None or bool(delta.tool_calls))
+ )
+ reasoning = self._reasoning.feed(
+ delta.reasoning if delta and delta.reasoning else "",
+ final=reasoning_finished,
+ )
+ content = self._content.feed(
+ delta.content if delta and delta.content else "",
+ final=finished,
+ )
+
+ if delta is None:
+ if not reasoning and not content:
+ return None
+ return DeltaMessage(
+ reasoning=reasoning or None,
+ content=content or None,
+ )
+
+ if reasoning:
+ delta.reasoning = reasoning
+ elif delta.reasoning is not None:
+ delta.reasoning = None
+ if content:
+ delta.content = content
+ elif delta.content is not None:
+ delta.content = None
+
+ # A chunk that normalizes to nothing must be dropped, not returned as an
+ # all-None DeltaMessage: the engine only skips on `None`, so anything
+ # else reaches the client as an empty delta.
+ if not delta.content and not delta.reasoning and not delta.tool_calls:
+ return None
+ return delta
+
+
+class MiniCPMVParser(Qwen3Parser):
+ """Parse MiniCPM-V family output in thinking and non-thinking modes."""
+
+ CONFIG_NAME = "minicpmv"
+
+ def __init__(
+ self,
+ tokenizer: TokenizerLike,
+ tools: list[Tool] | None = None,
+ **kwargs,
+ ) -> None:
+ chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
+ # The chat template emits `` unless `enable_thinking` is
+ # explicitly false, so the grammar must default to thinking too, or the
+ # reasoning span is misread as content.
+ thinking_enabled = chat_kwargs.get("enable_thinking", True)
+ kwargs.setdefault("parser_engine_config", minicpmv_config(thinking_enabled))
+ super().__init__(tokenizer, tools, **kwargs)
+ # ``Qwen3Parser.__init__`` reads the same key with a default of True,
+ # which does not apply here: MiniCPM-V defaults to thinking disabled.
+ # Keep the attribute in sync with the grammar built above.
+ self.thinking_enabled = thinking_enabled
+
+ @property
+ def reasoning_ended(self) -> bool:
+ # The engine derives this from the presence of THINK terminals, which
+ # both MiniCPM-V configs keep, so the base property stays False in
+ # non-thinking mode. There is no reasoning span to end there.
+ if not self.thinking_enabled:
+ return True
+ return super().reasoning_ended
+
+ def extract_reasoning(
+ self,
+ model_output: str,
+ request: ChatCompletionRequest | ResponsesRequest,
+ ) -> tuple[str | None, str | None]:
+ if self.thinking_enabled:
+ return super().extract_reasoning(model_output, request)
+ _, content = ParserEngine.extract_reasoning(self, model_output, request)
+ return None, content
+
+ def extract_reasoning_streaming(
+ self,
+ previous_text: str,
+ current_text: str,
+ delta_text: str,
+ previous_token_ids: Sequence[int],
+ current_token_ids: Sequence[int],
+ delta_token_ids: Sequence[int],
+ ) -> DeltaMessage | None:
+ if self.thinking_enabled:
+ return super().extract_reasoning_streaming(
+ previous_text,
+ current_text,
+ delta_text,
+ previous_token_ids,
+ current_token_ids,
+ delta_token_ids,
+ )
+
+ delta = ParserEngine.extract_reasoning_streaming(
+ self,
+ previous_text,
+ current_text,
+ delta_text,
+ previous_token_ids,
+ current_token_ids,
+ delta_token_ids,
+ )
+ if delta is None:
+ return None
+ delta.reasoning = None
+ if delta.content is None and not delta.tool_calls:
+ return None
+ return delta
+
+ def get_streaming_fallback_content(
+ self,
+ text: str,
+ request: ChatCompletionRequest | ResponsesRequest,
+ ) -> str | None:
+ if self.thinking_enabled:
+ return super().get_streaming_fallback_content(text, request)
+ delta = ParserEngine.finish_streaming(self)
+ return delta.content if delta is not None else None
+
+
+class MiniCPMVUnifiedParser(DelegatingParser):
+ """MiniCPM-V serving parser: the adapters plus newline recovery.
+
+ Recovery cannot live in :class:`MiniCPMVParser`: the serving stack drives
+ the reasoning/tool adapters, never ``ParserEngine.parse`` /
+ ``ParserEngine.parse_delta``, so a rewrite applied there would only be
+ visible to callers that construct the engine directly.
+ """
+
+ def __init__(
+ self,
+ tokenizer: TokenizerLike,
+ tools: list[Tool] | None = None,
+ *args,
+ **kwargs,
+ ) -> None:
+ super().__init__(tokenizer, tools, *args, **kwargs)
+ self._output_normalizer = MiniCPMVOutputNormalizer()
+
+ def parse(
+ self,
+ model_output: str,
+ request: ChatCompletionRequest | ResponsesRequest,
+ enable_auto_tools: bool = False,
+ model_output_token_ids: Sequence[int] = (),
+ ) -> tuple[str | None, str | None, list[FunctionCall] | None]:
+ reasoning, content, tool_calls = super().parse(
+ model_output,
+ request,
+ enable_auto_tools,
+ model_output_token_ids,
+ )
+ # Tool call arguments are JSON, where a literal `\n` is meaningful and
+ # must survive as written.
+ return (
+ recover_newlines(reasoning),
+ recover_newlines(content),
+ tool_calls,
+ )
+
+ def parse_delta(
+ self,
+ delta_text: str,
+ delta_token_ids: list[int],
+ request: ChatCompletionRequest | ResponsesRequest,
+ prompt_token_ids: list[int] | None = None,
+ *,
+ finished: bool,
+ ) -> DeltaMessage | None:
+ delta = super().parse_delta(
+ delta_text,
+ delta_token_ids,
+ request,
+ prompt_token_ids,
+ finished=finished,
+ )
+ return self._output_normalizer.normalize_delta(delta, finished=finished)
diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py
index cf0be908ed..6766d15e91 100644
--- a/vllm/parser/parser_manager.py
+++ b/vllm/parser/parser_manager.py
@@ -242,6 +242,19 @@ class ParserManager:
return _CohereCommandParser
+ if "minicpmv" in {reasoning_parser_name, tool_parser_name}:
+ from vllm.parser.minicpmv import MiniCPMVUnifiedParser
+
+ r_cls = reasoning_parser_cls
+ t_cls = tool_parser_cls
+
+ class _MiniCPMVParser(MiniCPMVUnifiedParser):
+ reasoning_parser_cls = r_cls
+ tool_parser_cls = t_cls
+ tool_strict_level = strict_level
+
+ return _MiniCPMVParser
+
from vllm.parser.abstract_parser import DelegatingParser
r_cls = reasoning_parser_cls
diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py
index f9ab1dc9a1..9e75e9b77e 100644
--- a/vllm/reasoning/__init__.py
+++ b/vllm/reasoning/__init__.py
@@ -116,6 +116,10 @@ _REASONING_PARSERS_TO_REGISTER = {
"mimo_engine_reasoning_parser",
"MiMoParserReasoningAdapter",
),
+ "minicpmv": (
+ "minicpmv_reasoning_parser",
+ "MiniCPMVParserReasoningAdapter",
+ ),
"minimax_m2": (
"minimax_m2_reasoning_parser",
"MiniMaxM2ReasoningParser",
diff --git a/vllm/reasoning/minicpmv_reasoning_parser.py b/vllm/reasoning/minicpmv_reasoning_parser.py
new file mode 100644
index 0000000000..402c0524ca
--- /dev/null
+++ b/vllm/reasoning/minicpmv_reasoning_parser.py
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from vllm.parser.engine.adapters import ParserEngineReasoningAdapter
+from vllm.parser.minicpmv import MiniCPMVParser
+
+
+class MiniCPMVParserReasoningAdapter(ParserEngineReasoningAdapter):
+ # Reasoning-only on purpose: this family registers no tool parser, because
+ # MiniCPM-V emits the Qwen3-Coder XML syntax verbatim and `qwen3_coder`
+ # already covers it. ``make_adapters`` only builds reasoning/tool pairs, so
+ # the two halves are declared explicitly here.
+ _parser_engine_cls = MiniCPMVParser
+
+
+__all__ = ["MiniCPMVParserReasoningAdapter"]
diff --git a/vllm/transformers_utils/chat_templates/registry.py b/vllm/transformers_utils/chat_templates/registry.py
index 422854ab3d..f902ecbbaa 100644
--- a/vllm/transformers_utils/chat_templates/registry.py
+++ b/vllm/transformers_utils/chat_templates/registry.py
@@ -32,6 +32,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
"deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
"minicpmv": _get_minicpmv_chat_template_fallback,
"minicpmv4_6": _get_minicpmv_chat_template_fallback,
+ "minicpmv4_7": _get_minicpmv_chat_template_fallback,
"paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
"siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
"siglip2": CHAT_TEMPLATES_DIR / "template_basic.jinja",