From c85616efc6f0eaa50ca1f1a09c32368aedffd3d8 Mon Sep 17 00:00:00 2001 From: tc-mb <157115220+tc-mb@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:21:10 +0800 Subject: [PATCH] [Model] Support MiniCPM-V 4.7 (#58674) Signed-off-by: tc-mb Co-authored-by: Cyrus Leung --- docs/models/supported_models.md | 2 +- tests/model_executor/test_minicpmv.py | 114 +++ tests/model_executor/test_minicpmv4_7.py | 310 +++++++ tests/models/registry.py | 5 + .../parser/engine/test_minicpmv_reasoning.py | 459 +++++++++++ vllm/model_executor/models/minicpmv4_6.py | 331 ++++++-- vllm/model_executor/models/minicpmv4_7.py | 764 ++++++++++++++++++ vllm/model_executor/models/registry.py | 4 + vllm/parser/minicpmv.py | 405 ++++++++++ vllm/parser/parser_manager.py | 13 + vllm/reasoning/__init__.py | 4 + vllm/reasoning/minicpmv_reasoning_parser.py | 16 + .../chat_templates/registry.py | 1 + 13 files changed, 2366 insertions(+), 62 deletions(-) create mode 100644 tests/model_executor/test_minicpmv4_7.py create mode 100644 tests/parser/engine/test_minicpmv_reasoning.py create mode 100644 vllm/model_executor/models/minicpmv4_7.py create mode 100644 vllm/parser/minicpmv.py create mode 100644 vllm/reasoning/minicpmv_reasoning_parser.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 221ab877ea..bbcb94f32c 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -592,7 +592,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `MiDashengLMModel` | MiDashengLM | T + A+ | `mispeech/midashenglm-7b` | | ✅︎ | | `MiMoV2OmniForCausalLM` | MiMo-V2.5-Omni | T + IE+ + VE+ + A+ | `XiaomiMiMo/MiMo-V2.5-Omni` | | ✅︎ | | `MiniCPMO` | MiniCPM-O | T + IE+ + VE+ + AE+ | `openbmb/MiniCPM-o-2_6`, etc. | ✅︎ | ✅︎ | -| `MiniCPMV` | MiniCPM-V | T + IE+ + VE+ | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, `openbmb/MiniCPM-V-4_6`, etc. | ✅︎ | | +| `MiniCPMV` | MiniCPM-V | T + IE+ + VE+ | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, `openbmb/MiniCPM-V-4_6`, `openbmb/MiniCPM-V-4_7`, etc. | ✅︎ | | | `MiniMaxM3SparseForConditionalGeneration` | MiniMax-M3 | T + I+ + V+ | `MiniMaxAI/MiniMax-M3`, `MiniMaxAI/MiniMax-M3-MXFP8`, etc. | | ✅︎ | | `MiniMaxVL01ForConditionalGeneration` | MiniMax-VL | T + IE+ | `MiniMaxAI/MiniMax-VL-01`, etc. | | ✅︎ | | `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I+ | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ | diff --git a/tests/model_executor/test_minicpmv.py b/tests/model_executor/test_minicpmv.py index 96b80101b4..cffdea2600 100644 --- a/tests/model_executor/test_minicpmv.py +++ b/tests/model_executor/test_minicpmv.py @@ -6,6 +6,9 @@ import torch from vllm.model_executor.models.minicpmv import MiniCPMV2_6 from vllm.model_executor.models.minicpmv4_6 import ( MiniCPMV4_6ForConditionalGeneration, + _expand_flags, + _select_vision_items, + _vision_downsample_mode, ) pytestmark = pytest.mark.skip_global_cleanup @@ -54,3 +57,114 @@ def test_image_and_video_embeds_stay_in_their_own_modality(): assert torch.equal(modalities["images"]["image_embeds"], image_embeds) assert torch.equal(modalities["videos"]["image_embeds"], video_embeds) + + +class TestPerItemMergerFlags: + """The merger flag is per request but items from several requests can be + batched together, so it has to be applied per item.""" + + def test_expand_flags_broadcasts_a_single_value(self): + assert _expand_flags(None, 3) == [None, None, None] + assert _expand_flags(torch.tensor([True]), 3) == [True, True, True] + assert _expand_flags(True, 2) == [True, True] + + def test_expand_flags_keeps_per_item_values(self): + flags = [torch.tensor([True]), torch.tensor([False])] + + assert _expand_flags(flags, 2) == [True, False] + + def test_expand_flags_falls_back_on_a_shape_mismatch(self): + # Too many values to map onto the items: keep one decision for the + # batch rather than failing the request. + assert _expand_flags([True, False, True], 2) == [True, True] + + def test_vision_downsample_mode(self): + assert _vision_downsample_mode(None) is None + assert _vision_downsample_mode(True) == "16x" + assert _vision_downsample_mode(False) == "4x" + + def test_select_vision_items_slices_by_item(self): + # 3 items holding 2, 1 and 2 slices; one channel, one patch, width 4. + pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(5)] + tgt_sizes = torch.arange(10, dtype=torch.long).reshape(5, 2) + image_input = { + "pixel_values": pixel_values, + "tgt_sizes": tgt_sizes, + "num_slices": torch.tensor([2, 1, 2]), + } + + selected = _select_vision_items(image_input, [2], [2, 1, 2]) + + assert len(selected["pixel_values"]) == 2 + assert bool(selected["pixel_values"][0][0, 0, 0] == 3.0) + assert bool(selected["pixel_values"][1][0, 0, 0] == 4.0) + assert selected["num_slices"].tolist() == [2] + assert torch.equal(selected["tgt_sizes"], tgt_sizes[3:5]) + + def test_select_vision_items_preserves_order(self): + pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(3)] + image_input = { + "pixel_values": pixel_values, + "tgt_sizes": torch.zeros(3, 2, dtype=torch.long), + "num_slices": torch.tensor([1, 1, 1]), + } + + selected = _select_vision_items(image_input, [2, 0], [1, 1, 1]) + + assert selected["num_slices"].tolist() == [1, 1] + assert bool(selected["pixel_values"][0][0, 0, 0] == 2.0) + assert bool(selected["pixel_values"][1][0, 0, 0] == 0.0) + + def test_process_vision_input_applies_flags_per_item(self): + calls: list[str | None] = [] + + def fake_get_vision_hidden_states(data, downsample_mode=None): + calls.append(downsample_mode) + tag = 1.0 if downsample_mode == "16x" else 0.0 + return [torch.full((1,), tag) for _ in data["pixel_values"]] + + model = object.__new__(MiniCPMV4_6ForConditionalGeneration) + model.get_vision_hidden_states = fake_get_vision_hidden_states + + image_input = { + "type": "pixel_values", + "pixel_values": [torch.zeros(1, 1, 4) for _ in range(3)], + "tgt_sizes": torch.zeros(3, 2, dtype=torch.long), + "num_slices": torch.tensor([1, 1, 1]), + } + + result = model._process_vision_input( + image_input, + use_vit_merger=[ + torch.tensor([True]), + torch.tensor([False]), + torch.tensor([True]), + ], + ) + + # Each item keeps its own mode even though they share one call. + assert [float(r[0]) for r in result] == [1.0, 0.0, 1.0] + # Mixed flags are encoded as separate groups, not one collapsed batch. + assert calls == ["16x", "4x"] + + def test_process_vision_input_uses_one_group_for_uniform_flags(self): + calls: list[str | None] = [] + + def fake_get_vision_hidden_states(data, downsample_mode=None): + calls.append(downsample_mode) + return [torch.zeros(1) for _ in data["pixel_values"]] + + model = object.__new__(MiniCPMV4_6ForConditionalGeneration) + model.get_vision_hidden_states = fake_get_vision_hidden_states + + image_input = { + "type": "pixel_values", + "pixel_values": [torch.zeros(1, 1, 4) for _ in range(2)], + "tgt_sizes": torch.zeros(2, 2, dtype=torch.long), + "num_slices": torch.tensor([1, 1]), + } + + model._process_vision_input(image_input, use_vit_merger=torch.tensor([True])) + + # The common case still makes a single encoder call. + assert calls == ["16x"] diff --git a/tests/model_executor/test_minicpmv4_7.py b/tests/model_executor/test_minicpmv4_7.py new file mode 100644 index 0000000000..3d9e8b4678 --- /dev/null +++ b/tests/model_executor/test_minicpmv4_7.py @@ -0,0 +1,310 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest +import torch + +from vllm.model_executor.models.minicpmv4_6 import _stack_vit_merger_qkv +from vllm.model_executor.models.minicpmv4_7 import ( + MiniCPMV4_7MultiModalProcessor, + _compute_canvas_single, + build_image_bounds, + canvas_rope_delta, +) + +pytestmark = pytest.mark.skip_global_cleanup + + +def test_canvas_mrope_sizes_survive_prefix_cache_stripping(): + """A cached image must keep `tgt_sizes`, or canvas M-RoPE falls back. + + `strip_covered_mm_data` drops the payload of a prefix-cache-covered item and + keeps only CPU-side metadata. Canvas M-RoPE places slices from `tgt_sizes`, + so 4.7 marks it `keep_on_cpu` while the shared 4.6 config stays as it was. + """ + from vllm.model_executor.models.minicpmv4_6 import _minicpmv4_6_field_config + + shared = _minicpmv4_6_field_config({}) + assert shared["tgt_sizes"].field.keep_on_cpu is False + assert shared["video_tgt_sizes"].field.keep_on_cpu is False + + processor = MiniCPMV4_7MultiModalProcessor.__new__(MiniCPMV4_7MultiModalProcessor) + fields = processor._get_mm_fields_config({}, {}) + + assert fields["tgt_sizes"].field.keep_on_cpu is True + assert fields["video_tgt_sizes"].field.keep_on_cpu is True + # Other fields keep their previous setting. + assert fields["pixel_values"].field.keep_on_cpu is False + assert fields["video_image_sizes"].field.keep_on_cpu is True + + +def test_modality_mm_kwargs_resolution(): + """Flat keys reach both modalities; scopes stay scoped, and win on conflict. + + `_merge_and_resolve_mm_processor_kwargs` routes a flat key into every scope + whose HF processor declares it, so reading is a plain scope lookup. The + routing itself is upstream's, tested here against the MiniCPM-V schema to + pin the precedence rule (`scoped > flat`). + """ + from vllm.model_executor.models.minicpmv4_6 import ( + _flat_processor_kwargs, + _scoped_mm_kwarg, + ) + from vllm.multimodal.processing.context import _resolve_mm_processor_kwargs + + # What MiniCPMV4_6{Image,Video}ProcessorKwargs declare. + processor_keys = {"downsample_mode", "max_slice_nums"} + schema = { + "text_kwargs": set(), + "images_kwargs": set(processor_keys), + "videos_kwargs": set(processor_keys), + "audio_kwargs": set(), + } + + def read(mm_kwargs, modality, key): + merged = _resolve_mm_processor_kwargs(dict(mm_kwargs), schema) + return _scoped_mm_kwarg(merged, modality, key) + + assert read({}, "image", "downsample_mode") is None + # A flat key is routed into both scopes. + assert read({"downsample_mode": "4x"}, "image", "downsample_mode") == "4x" + assert read({"downsample_mode": "4x"}, "video", "downsample_mode") == "4x" + # A scoped key stays in its scope. + assert ( + read({"images_kwargs": {"downsample_mode": "16x"}}, "image", "downsample_mode") + == "16x" + ) + assert ( + read({"images_kwargs": {"downsample_mode": "16x"}}, "video", "downsample_mode") + is None + ) + videos_only = {"videos_kwargs": {"max_slice_nums": 1}} + assert read(videos_only, "video", "max_slice_nums") == 1 + # Scoped to videos, so image must not see it. + assert read(videos_only, "image", "max_slice_nums") is None + # An explicit scope wins over the flat key it is merged with. + assert ( + read( + {"downsample_mode": "16x", "images_kwargs": {"downsample_mode": "4x"}}, + "image", + "downsample_mode", + ) + == "4x" + ) + # A non-mapping scope is ignored rather than raising. + assert read({"images_kwargs": None}, "image", "downsample_mode") is None + + # `_flat_processor_kwargs` drops the scoped keys and pins the mode. + flat = _flat_processor_kwargs( + { + "downsample_mode": "4x", + "images_kwargs": {"a": 1}, + "videos_kwargs": {"b": 2}, + "audio_kwargs": {"c": 3}, + }, + "16x", + ) + assert flat == {"downsample_mode": "16x"} + + +def test_vit_merger_qkv_is_stacked_before_load(): + mapped = list( + _stack_vit_merger_qkv( + [ + ( + "model.vision_tower.vit_merger.self_attn.q_proj.weight", + torch.zeros(1), + ), + ( + "model.vision_tower.vit_merger.self_attn.k_proj.weight", + torch.zeros(1), + ), + ( + "model.vision_tower.vit_merger.self_attn.v_proj.weight", + torch.zeros(1), + ), + ("vit_merger.self_attn.qkv_proj.weight", torch.zeros(1)), + ] + ) + ) + names = [name for name, _ in mapped] + shards = [getattr(tensor, "shard_id", None) for _, tensor in mapped] + assert names == [ + "model.vision_tower.vit_merger.self_attn.qkv_proj.weight", + "model.vision_tower.vit_merger.self_attn.qkv_proj.weight", + "model.vision_tower.vit_merger.self_attn.qkv_proj.weight", + "vit_merger.self_attn.qkv_proj.weight", + ] + assert shards == ["q", "k", "v", None] + + +# Canvas M-RoPE goldens, cross-checked against the reference implementation in +# transformers PR 48979 (`modular_minicpmv4_7.py`). Synthetic ids keep the token +# sequences readable: the canvas math only compares ids for equality. +_IM_START = 10 +_IM_END = 11 +_SLICE_START = 12 +_SLICE_END = 13 +_NEWLINE = 14 +_IMAGE_PAD = 20 +_VIDEO_PAD = 21 +_TEXT = 100 + +_CANVAS_IDS = { + "im_start_id": _IM_START, + "im_end_id": _IM_END, + "slice_start_id": _SLICE_START, + "slice_end_id": _SLICE_END, + "newline_id": _NEWLINE, +} + + +def _canvas_positions(tokens, target_sizes): + input_ids = torch.tensor([tokens], dtype=torch.long) + targets = torch.tensor(target_sizes, dtype=torch.long) + bounds = build_image_bounds(input_ids[0], _CANVAS_IDS) + positions = _compute_canvas_single( + input_ids[0], + torch.arange(len(tokens), dtype=torch.long), + bounds, + targets, + _CANVAS_IDS, + ) + return positions.tolist(), canvas_rope_delta(positions, len(tokens)) + + +def test_canvas_mrope_trailing_newline_stays_inside_the_canvas(): + """A newline right after `` belongs to the canvas, not to the text. + + The chat template joins content parts with ``\\n``, so an image followed by + text always leaves one there. The transformers reference closes a frame over + every marker including newlines; leaving it out pushes every later position + one step forward and makes `rope_delta` differ by one. + """ + tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _NEWLINE, _TEXT, _TEXT] + positions, delta = _canvas_positions(tokens, [[4, 4]]) + + assert positions == [ + [0, 1, 1, 1, 1, 3, 4], + [0, 0, 1, 2, 1, 3, 4], + [0, 0, 1, 2, 1, 3, 4], + ] + assert delta == -2 + + +def test_canvas_mrope_thumbnail_matches_reference(): + # PAD ; 16x turns a 4x4 patch grid into 1 token. + tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT] + positions, delta = _canvas_positions(tokens, [[4, 4]]) + + assert positions == [ + [0, 1, 1, 1, 3], + [0, 0, 1, 2, 3], + [0, 0, 1, 2, 3], + ] + assert delta == -1 + + +def test_canvas_mrope_slice_row_matches_reference(): + # Thumbnail then a 1x2 slice row; the slices tile the thumbnail's canvas. + tokens = [ + _TEXT, + _IM_START, + _IMAGE_PAD, + _IM_END, + _SLICE_START, + _IMAGE_PAD, + _SLICE_END, + _SLICE_START, + _IMAGE_PAD, + _SLICE_END, + _TEXT, + ] + positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4], [4, 4]]) + + assert positions == [ + [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4], + [0, 0, 1, 2, 1, 1, 1, 1, 1, 1, 4], + [0, 0, 1, 3, 1, 1, 1, 2, 2, 2, 4], + ] + assert delta == -6 + + +def test_canvas_mrope_resamples_thumbnail_over_canvas(): + # A 8x8 patch grid is 2x2 LLM tokens, so the canvas is resampled (not arange). + tokens = [_TEXT, _IM_START] + [_IMAGE_PAD] * 4 + [_IM_END, _TEXT] + positions, delta = _canvas_positions(tokens, [[8, 8]]) + + assert positions == [ + [0, 1, 1, 1, 1, 1, 1, 4], + [0, 0, 1, 1, 2, 2, 3, 4], + [0, 0, 1, 2, 1, 2, 3, 4], + ] + assert delta == -3 + + +def test_canvas_mrope_video_frames_share_one_canvas(): + tokens = [ + _TEXT, + _IM_START, + _VIDEO_PAD, + _IM_END, + _IM_START, + _VIDEO_PAD, + _IM_END, + _TEXT, + ] + positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4]]) + + assert positions == [ + [0, 1, 1, 1, 3, 3, 3, 5], + [0, 0, 1, 2, 2, 3, 4, 5], + [0, 0, 1, 2, 2, 3, 4, 5], + ] + assert delta == -2 + + +def test_canvas_mrope_interleaved_image_and_video(): + tokens = ( + [_TEXT, _IM_START] + + [_IMAGE_PAD] * 4 + + [_IM_END, _IM_START] + + [_VIDEO_PAD] * 4 + + [_IM_END, _TEXT] + ) + positions, delta = _canvas_positions(tokens, [[8, 8], [8, 8]]) + + assert positions == [ + [0, 1, 1, 1, 1, 1, 1, 4, 4, 4, 4, 4, 4, 7], + [0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7], + [0, 0, 1, 2, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7], + ] + assert delta == -6 + + +def test_canvas_span_is_shared_by_image_and_video(): + """A video frame and a same-geometry image must place identical offsets. + + Both paths go through `_assign_canvas_span`; this pins that they cannot + drift apart now that the duplicated geometry is gone. + """ + thumb = [_IM_START] + [_IMAGE_PAD] * 4 + [_IM_END] + + img_pos, _ = _canvas_positions([_TEXT, *thumb, _TEXT], [[8, 8]]) + vid_pos, _ = _canvas_positions([_TEXT, *thumb, *thumb, _TEXT], [[8, 8], [8, 8]]) + + def offsets(pos, start): + # The halo is one below the canvas base. + base = pos[1][start] + 1 + return [[c[start + k] - base for k in range(3)] for c in pos] + + assert offsets(img_pos, 0) == offsets(vid_pos, 0) + + +def test_canvas_mrope_delta_leaves_room_for_the_first_decoded_token(): + # First decoded token must land on max_prefill_position + 1, matching + # transformers and the other native M-RoPE models. + tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT] + positions, delta = _canvas_positions(tokens, [[4, 4]]) + + max_position = max(max(channel) for channel in positions) + assert delta == max_position + 1 - len(tokens) diff --git a/tests/models/registry.py b/tests/models/registry.py index b57f3fcb72..2ed6d4871c 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1118,6 +1118,11 @@ _MULTIMODAL_EXAMPLE_MODELS = { "MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo( "openbmb/MiniCPM-V-4_6", ), + "MiniCPMV4_7ForConditionalGeneration": _HfExamplesInfo( + "openbmb/MiniCPM-V-4_7", + min_transformers_version="5.18.0", + is_available_online=False, + ), "MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo( "MiniMaxAI/MiniMax-M3", trust_remote_code=True, diff --git a/tests/parser/engine/test_minicpmv_reasoning.py b/tests/parser/engine/test_minicpmv_reasoning.py new file mode 100644 index 0000000000..5edf5bfc5d --- /dev/null +++ b/tests/parser/engine/test_minicpmv_reasoning.py @@ -0,0 +1,459 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the MiniCPM-V family parser.""" + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming +from vllm.entrypoints.generate.base.protocol import ( + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, +) +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionToolsParam, +) +from vllm.parser.minicpmv import ( + EscapedNewlineNormalizer, + MiniCPMVOutputNormalizer, + recover_newlines, +) +from vllm.parser.parser_manager import ParserManager +from vllm.reasoning import ReasoningParserManager +from vllm.tool_parsers import ToolParserManager + +_VOCAB = { + "": 10, + "": 11, + "<|im_end|>": 12, + "": 13, + "": 14, +} + +_TOOLS = [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + ) +] + + +@pytest.fixture +def mock_tokenizer(): + return make_mock_tokenizer(_VOCAB) + + +@pytest.fixture +def request_obj(): + return ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + ) + + +def _make_parser(tokenizer, *, enable_thinking=None, tools=None): + """Build the parser the serving stack resolves for MiniCPM-V. + + Everything is built through ``ParserManager`` so the tests exercise the + ``DelegatingParser`` the server actually drives, not the parser engine on + its own. Tool calling goes through ``qwen3_coder``: MiniCPM-V emits that + XML syntax verbatim and has no tool parser of its own. + """ + parser_cls = ParserManager.get_parser( + tool_parser_name="qwen3_coder" if tools else None, + reasoning_parser_name="minicpmv", + enable_auto_tools=bool(tools), + ) + assert parser_cls is not None + kwargs = {} + if enable_thinking is not None: + kwargs["chat_template_kwargs"] = {"enable_thinking": enable_thinking} + return parser_cls(tokenizer, tools, **kwargs) + + +def test_reasoning_parser_is_registered(): + parser_cls = ReasoningParserManager.get_reasoning_parser("minicpmv") + + assert parser_cls.__name__ == "MiniCPMVParserReasoningAdapter" + + +def test_thinking_defaults_to_the_chat_template(mock_tokenizer): + """Without `enable_thinking`, the grammar must wait for reasoning. + + The chat template ends the assistant turn with `` unless the flag is + explicitly false, so the model reasons by default. This is the flag + `serving.py` latches into the structured-output gate, so a non-thinking + default both disables the grammar and leaks the thinking into `content`. + """ + + def reasoning_pending(**kwargs): + return not _make_parser(mock_tokenizer, **kwargs).is_reasoning_end([1, 2]) + + assert reasoning_pending() is True # default matches the template + assert reasoning_pending(enable_thinking=True) is True + assert reasoning_pending(enable_thinking=False) is False + + +def test_no_minicpmv_tool_parser_is_registered(): + """MiniCPM-V deliberately registers no tool parser. + + The family emits the Qwen3-Coder XML syntax verbatim, so + ``--tool-call-parser qwen3_coder`` already covers it. Registering a + ``minicpmv`` tool parser would duplicate that name for no parsing gain. + """ + with pytest.raises(KeyError): + ToolParserManager.get_tool_parser("minicpmv") + + +def test_composes_with_qwen3_coder_tool_parser(mock_tokenizer): + """`--tool-call-parser qwen3_coder` must work through the serving Parser.""" + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + tools=_TOOLS, + ) + parser_cls = ParserManager.get_parser( + tool_parser_name="qwen3_coder", + reasoning_parser_name="minicpmv", + enable_auto_tools=True, + ) + assert parser_cls is not None + parser = parser_cls(mock_tokenizer, request.tools) + + _, _, tool_calls = parser.parse( + "" + "SF", + request, + enable_auto_tools=True, + ) + + assert tool_calls is not None + assert tool_calls[0].name == "get_weather" + assert "SF" in tool_calls[0].arguments + + +class TestNonThinking: + @pytest.fixture + def parser(self, mock_tokenizer): + return _make_parser(mock_tokenizer, enable_thinking=False) + + def test_plain_content_is_unchanged(self, parser): + reasoning, content = parser.extract_reasoning("The answer is 42.", None) + + assert reasoning is None + assert content == "The answer is 42." + + def test_reserved_markers_are_removed(self, parser): + output = ( + "reasoning<|reserved_14|>tool call<|reserved_15|>" + ) + + reasoning, content = parser.extract_reasoning(output, None) + + assert reasoning is None + assert content == "reasoningtool call" + + def test_unrelated_reserved_text_is_unchanged(self, parser): + reasoning, content = parser.extract_reasoning( + "content<|reserved_16|>", + None, + ) + + assert reasoning is None + assert content == "content<|reserved_16|>" + + def test_dangling_standard_think_end_is_removed(self, parser): + output = "first part \n\nfinal part" + + reasoning, content = parser.extract_reasoning(output, None) + + assert reasoning is None + assert content == "first part \n\nfinal part" + + def test_truncated_standard_reasoning_does_not_leak(self, parser): + reasoning, content = parser.extract_reasoning( + "prefixprivate reasoning", + None, + ) + + assert reasoning is None + assert content == "prefix" + + def test_streaming_split_standard_markers(self, parser): + reasoning, content = simulate_reasoning_streaming( + parser, + [ + "<", + "think>", + "private reasoning", + "", + "final answer", + ], + ) + + assert reasoning == "" + assert content == "final answer" + + def test_streaming_split_reserved_markers(self, parser): + reasoning, content = simulate_reasoning_streaming( + parser, + [ + "", + "reasoning", + "<|reserved", + "_13|>", + "final answer", + ], + ) + + assert reasoning == "" + assert content == "reasoningfinal answer" + + def test_tool_calls_are_parsed(self, mock_tokenizer): + """Thinking is off by default, and tool calls must still be parsed.""" + parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS) + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + tools=_TOOLS, + ) + + reasoning, content, tool_calls = parser.parse( + "" + "SF", + request, + enable_auto_tools=True, + ) + + assert reasoning is None + assert not content + assert tool_calls is not None + assert tool_calls[0].name == "get_weather" + assert "SF" in tool_calls[0].arguments + + def test_streaming_tool_calls_are_parsed(self, mock_tokenizer): + """Regression: tool calls must stream when thinking is explicitly off. + + The reasoning parser used to report ``reasoning_ended=False`` in + non-thinking mode, which kept the delegating parser in its reasoning + phase and streamed the raw ```` XML to the client as content. + """ + parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS) + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + tools=_TOOLS, + ) + + delta = parser.parse_delta( + "" + "SF", + [], + request, + prompt_token_ids=[], + finished=True, + ) + + assert delta is not None + assert delta.tool_calls + assert delta.tool_calls[0].function is not None + assert delta.tool_calls[0].function.name == "get_weather" + assert "SF" in (delta.tool_calls[0].function.arguments or "") + assert not delta.content + + +class TestThinking: + def test_standard_think_tags_are_parsed(self, mock_tokenizer): + parser = _make_parser(mock_tokenizer, enable_thinking=True) + + reasoning, content = parser.extract_reasoning( + "private reasoningfinal answer", + None, + ) + + assert reasoning == "private reasoning" + assert content == "final answer" + + def test_reserved_markers_are_removed(self, mock_tokenizer): + parser = _make_parser(mock_tokenizer, enable_thinking=True) + + reasoning, content = parser.extract_reasoning( + "private reasoning<|reserved_13|>final answer", + None, + ) + + assert reasoning == "private reasoning" + assert content == "final answer" + + def test_streaming_im_end_is_removed(self): + tokenizer = make_mock_tokenizer(_VOCAB, special_tokens=[]) + parser = _make_parser(tokenizer, enable_thinking=True) + + reasoning, content = simulate_reasoning_streaming( + parser, + ["Answer: 111", "<|im_end|>"], + ) + + assert reasoning == "Answer: 111" + assert content == "" + + +class TestNewlineRecovery: + """The tokenizer writes a newline as the two characters ``\\`` and ``n``.""" + + def test_recover_newlines(self): + text = ( + "first\\nsecond\\rthird\\r\\nfourth " + "`code\\nvalue` " + "```python\\nvalue = '\\\\n'\\n``` " + "$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] " + "escaped \\\\n" + ) + + assert recover_newlines(text) == ( + "first\nsecond\nthird\nfourth " + "`code\\nvalue` " + "```python\\nvalue = '\\\\n'\\n``` " + "$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] " + "escaped \\\\n" + ) + + def test_streaming_matches_complete_recovery(self): + text = ( + "first\\nsecond `code\\nvalue` $x\\ny$ ```text\\nvalue\\n``` last\\r\\nline" + ) + expected = recover_newlines(text) + + for split in range(len(text) + 1): + normalizer = EscapedNewlineNormalizer() + actual = normalizer.feed(text[:split]) + actual += normalizer.feed(text[split:], final=True) + assert actual == expected + + normalizer = EscapedNewlineNormalizer() + actual = "".join(normalizer.feed(char) for char in text) + actual += normalizer.feed("", final=True) + assert actual == expected + + def test_backslash_runs_stay_consistent_across_splits(self): + # A run of backslashes can be split over chunks. Its parity decides + # whether a following `n` is a newline, so streaming and non-streaming + # must agree for every split point. + for count in range(1, 7): + text = "\\" * count + "n" + expected = recover_newlines(text) + + for split in range(len(text) + 1): + normalizer = EscapedNewlineNormalizer() + actual = normalizer.feed(text[:split]) + actual += normalizer.feed(text[split:], final=True) + assert actual == expected, (count, split) + + def test_streaming_recovers_reasoning_and_content(self): + normalizer = MiniCPMVOutputNormalizer() + + first = normalizer.normalize_delta( + DeltaMessage(reasoning="reason\\"), + finished=False, + ) + second = normalizer.normalize_delta( + DeltaMessage(reasoning="nnext\\", content="answer\\"), + finished=False, + ) + final = normalizer.normalize_delta( + DeltaMessage(content="nend"), + finished=True, + ) + + assert first == DeltaMessage(reasoning="reason") + assert second == DeltaMessage(reasoning="\nnext\\", content="answer") + assert final == DeltaMessage(content="\nend") + + def test_streaming_does_not_modify_tool_arguments(self): + normalizer = MiniCPMVOutputNormalizer() + arguments = '{"value":"first\\\\nsecond"}' + delta = DeltaMessage( + tool_calls=[ + DeltaToolCall( + index=0, + function=DeltaFunctionCall(arguments=arguments), + ) + ] + ) + + result = normalizer.normalize_delta(delta, finished=True) + + assert result is not None + assert result.tool_calls[0].function is not None + assert result.tool_calls[0].function.arguments == arguments + + def test_parse_recovers_content(self, mock_tokenizer, request_obj): + parser = _make_parser(mock_tokenizer, enable_thinking=False) + + reasoning, content, tool_calls = parser.parse( + "first\\nsecond", + request_obj, + ) + + assert reasoning is None + assert content == "first\nsecond" + assert not tool_calls + + def test_parse_delta_recovers_content(self, mock_tokenizer, request_obj): + parser = _make_parser(mock_tokenizer, enable_thinking=False) + + chunk = parser.parse_delta( + "first\\", + [], + request_obj, + finished=False, + ) + rest = parser.parse_delta( + "nsecond", + [], + request_obj, + finished=True, + ) + + # The trailing half of the escape is held back until it is complete. + assert chunk is not None + assert chunk.content == "first" + assert rest is not None + assert rest.content == "\nsecond" + + def test_buffered_delta_is_dropped_not_sent_empty( + self, mock_tokenizer, request_obj + ): + parser = _make_parser(mock_tokenizer, enable_thinking=False) + parser.parse_delta("hello", [], request_obj, finished=False) + + # The lone backtick is held back for the next chunk. Reporting it as an + # all-None DeltaMessage would reach the client as an empty SSE chunk. + assert parser.parse_delta("`", [], request_obj, finished=False) is None + + def test_normalizer_drops_empty_delta(self): + normalizer = MiniCPMVOutputNormalizer() + + assert ( + normalizer.normalize_delta(DeltaMessage(content="`"), finished=False) + is None + ) + + def test_normalizer_flushes_buffered_content_when_finished(self): + normalizer = MiniCPMVOutputNormalizer() + normalizer.normalize_delta(DeltaMessage(content="`"), finished=False) + + final = normalizer.normalize_delta(DeltaMessage(), finished=True) + + assert final is not None + assert final.content == "`" diff --git a/vllm/model_executor/models/minicpmv4_6.py b/vllm/model_executor/models/minicpmv4_6.py index 0a96c2d847..333ab8eedd 100644 --- a/vllm/model_executor/models/minicpmv4_6.py +++ b/vllm/model_executor/models/minicpmv4_6.py @@ -45,6 +45,7 @@ from .interfaces import ( HasInnerState, IsHybrid, MultiModalEmbeddings, + SupportsEagle3, SupportsLoRA, SupportsMRoPE, SupportsMultiModal, @@ -87,35 +88,186 @@ def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]): fields["use_vit_merger"] = MultiModalFieldConfig.batched( "image", keep_on_cpu=True ) + if "video_use_vit_merger" in hf_inputs: + fields["video_use_vit_merger"] = MultiModalFieldConfig.batched( + "video", keep_on_cpu=True + ) return fields +def _expand_flags(value: object, num_items: int) -> list[bool | None]: + """Broadcast a batched per-item flag to exactly one entry per item.""" + if value is None: + return [None] * num_items + + raw: list[object] + if isinstance(value, torch.Tensor): + raw = value.reshape(-1).tolist() + elif isinstance(value, list | tuple): + raw = [] + for item in value: + if isinstance(item, torch.Tensor): + raw.extend(item.reshape(-1).tolist()) + else: + raw.append(item) + else: + raw = [value] + + if len(raw) == num_items: + return [bool(v) for v in raw] + if len(raw) == 1: + return [bool(raw[0])] * num_items + # Unexpected shape: keep one decision for the whole batch instead of + # failing the request. + return [any(bool(v) for v in raw)] * num_items + + +def _vision_downsample_mode(flag: bool | None) -> str | None: + if flag is None: + return None + return "16x" if flag else "4x" + + +def _select_vision_items( + image_input: MiniCPMVImagePixelInputs, + item_indices: list[int], + num_slices: list[int], +) -> MiniCPMVImagePixelInputs: + """Slice a vision input down to the given items, keeping their order.""" + offsets = [0] + for n in num_slices: + offsets.append(offsets[-1] + n) + + pixel_values: list[torch.Tensor] = [] + tgt_sizes: list[torch.Tensor] = [] + sizes: list[int] = [] + for i in item_indices: + start, end = offsets[i], offsets[i + 1] + pixel_values.extend(image_input["pixel_values"][start:end]) + tgt_sizes.append(image_input["tgt_sizes"][start:end]) + sizes.append(end - start) + + return MiniCPMVImagePixelInputs( + type="pixel_values", + pixel_values=pixel_values, + tgt_sizes=torch.cat(tgt_sizes, dim=0), + num_slices=torch.tensor(sizes), + ) + + +# HF-style modality-scoped keys inside `mm_processor_kwargs`. They scope a value +# to a single modality and must not be forwarded to the processors, which take +# flat arguments only. +_MODALITY_SCOPED_MM_KWARGS = ("images_kwargs", "videos_kwargs", "audio_kwargs") + + +def _flat_processor_kwargs( + mm_kwargs: Mapping[str, object], + downsample_mode: str, +) -> dict[str, object]: + """Drop the modality-scoped nesting and pin the resolved downsample mode. + + The placeholder count and the vision tower must read the same mode, so the + value resolved for this modality is passed down explicitly rather than + leaving the processor to fall back to its own default. + """ + kwargs = { + key: value + for key, value in mm_kwargs.items() + if key not in _MODALITY_SCOPED_MM_KWARGS + } + kwargs["downsample_mode"] = downsample_mode + return kwargs + + +_MODALITY_TO_MM_KWARGS_SCOPE = { + "image": "images_kwargs", + "video": "videos_kwargs", +} + + +def _scoped_mm_kwarg( + merged: Mapping[str, object], + modality: str, + key: str, +) -> Any: + """Read a processor kwarg for one modality. + + ``merged`` is what ``_merge_and_resolve_mm_processor_kwargs`` returns: it + routes a flat key into every scope whose processor declares it, and a value + scoped to the other modality stays there, so it is not visible here. + """ + scoped = merged.get(_MODALITY_TO_MM_KWARGS_SCOPE[modality]) + if isinstance(scoped, Mapping): + return scoped.get(key) + return None + + class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): def _resolve_downsample_mode( self, mm_kwargs: Mapping[str, object], + modality: str = "image", ) -> str: - ds = mm_kwargs.get("downsample_mode") - if ds is not None: - return str(ds) + # A flat `downsample_mode` applies to both modalities; the nested + # `images_kwargs` / `videos_kwargs` forms scope it to one, matching + # MiniCPMV4_6Processor. info = self.info assert isinstance(info, MiniCPMV4_6ProcessingInfo) + merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs) + downsample_mode = _scoped_mm_kwarg(merged, modality, "downsample_mode") + if downsample_mode is not None: + return str(downsample_mode) return info._get_downsample_mode() + def _resolve_max_slice_nums( + self, + mm_kwargs: Mapping[str, object], + modality: str = "image", + ) -> int | None: + # Per-request max_slice_nums. None keeps the processor default. + # Resolved like `downsample_mode` so the nested per-modality form + # (`images_kwargs` / `videos_kwargs`) is honoured too. This must match + # what `process_images` passes to the image processor: the placeholder + # count and the visual token count are derived from it separately, so + # they diverge (and the engine aborts) if the two sides read different + # values. + info = self.info + assert isinstance(info, MiniCPMV4_6ProcessingInfo) + merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs) + max_slice = _scoped_mm_kwarg(merged, modality, "max_slice_nums") + if max_slice is None: + return None + return int(max_slice) + def get_image_prompt_texts( self, image_size, image_idx: int = 0, downsample_mode: str | None = None, + max_slice_nums: int | None = None, ) -> str: info = self.info assert isinstance(info, MiniCPMV4_6ProcessingInfo) return info.get_slice_image_placeholder( image_size, image_idx=image_idx, + max_slice_nums=max_slice_nums, downsample_mode=downsample_mode, ) + def _video_local_id_prefix(self, video_idx: int) -> str: + """`{idx}` prefix emitted in front of a video placeholder. + + MiniCPM-V 4.6 numbers each video with a local id, matching + MiniCPMV4_6Processor. 4.7 treats a video as one temporal sequence and is + trained without these ids, so it overrides this to return an empty string. + """ + tokenizer = self.info.get_tokenizer() + id_start = getattr(tokenizer, "image_id_start_token", "") + id_end = getattr(tokenizer, "image_id_end_token", "") + return f"{id_start}{video_idx}{id_end}" + def get_video_prompt_texts( self, image_size, @@ -144,8 +296,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): image_end = getattr(tokenizer, "image_end_token", "") slice_start = getattr(tokenizer, "slice_start_token", "") slice_end = getattr(tokenizer, "slice_end_token", "") - id_start = getattr(tokenizer, "image_id_start_token", "") - id_end = getattr(tokenizer, "image_id_end_token", "") per_frame = image_start + video_token * source_tokens + image_end if grids[0] > 0 and grids[1] > 0 and patch_tokens > 0: @@ -154,7 +304,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): per_frame += "\n".join(rows) body = per_frame * num_frames - return f"{id_start}{video_idx}{id_end}" + body + return self._video_local_id_prefix(video_idx) + body def process_images( self, @@ -179,10 +329,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): n_images = len(parsed_images) image_processor = self.info.get_image_processor() patch_size = image_processor.patch_size + ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="image") + image_proc_kwargs = _flat_processor_kwargs(mm_kwargs, ds_mode) + # `_flat_processor_kwargs` drops the nested form, so the value resolved + # from it has to be pinned back for the processor and the prompt text + # to agree on the slice count. + image_max_slice = self._resolve_max_slice_nums(mm_kwargs, modality="image") + if image_max_slice is not None: + image_proc_kwargs["max_slice_nums"] = image_max_slice per_image_pixel_values: list[torch.Tensor] = [] per_image_tgt_sizes: list[torch.Tensor] = [] for image in parsed_images: - ip_out = image_processor([image], **mm_kwargs) + ip_out = image_processor([image], **image_proc_kwargs) pv = ip_out["pixel_values"] # (1, C, P, sum_W) ts = ip_out["target_sizes"] # (n_slices, 2) if pv.ndim == 4 and pv.shape[0] == 1: @@ -210,7 +368,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): "tgt_sizes": per_image_tgt_sizes, } - ds_mode = self._resolve_downsample_mode(mm_kwargs) insert_layer_id = getattr( self.info.get_hf_config(), "insert_layer_id", @@ -245,7 +402,11 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): image_processor = self.info.get_image_processor() patch_size = image_processor.patch_size video_max_slice = self.info.get_video_max_slice_num() - video_mm_kwargs = {**mm_kwargs, "max_slice_nums": video_max_slice} + video_ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="video") + video_mm_kwargs = { + **_flat_processor_kwargs(mm_kwargs, video_ds_mode), + "max_slice_nums": video_max_slice, + } per_video_pixel_values: list[torch.Tensor] = [] per_video_tgt_sizes: list[torch.Tensor] = [] @@ -312,10 +473,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): if not per_video_pixel_values: return {} + insert_layer_id = getattr(self.info.get_hf_config(), "insert_layer_id", -1) + merger_flag = video_ds_mode != "4x" and insert_layer_id >= 0 + return { "video_pixel_values": per_video_pixel_values, "video_tgt_sizes": per_video_tgt_sizes, "video_image_sizes": per_video_image_sizes, + "video_use_vit_merger": [ + torch.tensor([merger_flag], dtype=torch.bool) + for _ in per_video_pixel_values + ], } def _get_prompt_updates( @@ -324,7 +492,13 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): hf_processor_mm_kwargs: Mapping[str, object], out_mm_kwargs, ): - ds_mode = self._resolve_downsample_mode(hf_processor_mm_kwargs) + image_ds_mode = self._resolve_downsample_mode( + hf_processor_mm_kwargs, modality="image" + ) + video_ds_mode = self._resolve_downsample_mode( + hf_processor_mm_kwargs, modality="video" + ) + image_max_slice = self._resolve_max_slice_nums(hf_processor_mm_kwargs) placeholders = [ ("image", self.info.image_pattern), @@ -360,7 +534,8 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): self.get_image_prompt_texts( image_size, item_idx, - downsample_mode=ds_mode, + downsample_mode=image_ds_mode, + max_slice_nums=image_max_slice, ), add_special_tokens=False, ), @@ -389,7 +564,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): self.get_video_prompt_texts( frame_size, num_frames, - downsample_mode=ds_mode, + downsample_mode=video_ds_mode, video_idx=item_idx, ), add_special_tokens=False, @@ -409,7 +584,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): self.get_video_prompt_texts( frame_size, num_frames, - downsample_mode=ds_mode, + downsample_mode=video_ds_mode, video_idx=item_idx, ), add_special_tokens=False, @@ -437,31 +612,28 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor): new_update = super()._recompute_cached_prompt_update( cached_update, new_item_idx ) - # MiniCPM-V 4.6 prefixes video placeholders with `{idx}` - # (the base class only rewrites the image modality). + # Video placeholders carry an `{idx}` prefix that + # the base class does not rewrite. 4.7 has no prefix, so this is a + # no-op there. if cached_update.modality == "video": - tokenizer = self.info.get_tokenizer() - id_start = getattr(tokenizer, "image_id_start_token", "") - id_end = getattr(tokenizer, "image_id_end_token", "") - video_token = getattr(tokenizer, "video_token", "<|video_pad|>") + old_prefix = self._video_local_id_prefix(cached_update.item_idx) + new_prefix = self._video_local_id_prefix(new_item_idx) + if old_prefix != new_prefix: + tokenizer = self.info.get_tokenizer() + video_token = getattr(tokenizer, "video_token", "<|video_pad|>") - text = tokenizer.decode(cached_update.content.full) - prev_item_idx = cached_update.item_idx + text = tokenizer.decode(cached_update.content.full) - new_update = new_update.with_content( - PromptUpdateDetails.select_token_ids( - cached_encode( - tokenizer, - text.replace( - f"{id_start}{prev_item_idx}{id_end}", - f"{id_start}{new_item_idx}{id_end}", - 1, + new_update = new_update.with_content( + PromptUpdateDetails.select_token_ids( + cached_encode( + tokenizer, + text.replace(old_prefix, new_prefix, 1), + add_special_tokens=False, ), - add_special_tokens=False, - ), - cached_encode(tokenizer, video_token, add_special_tokens=False), + cached_encode(tokenizer, video_token, add_special_tokens=False), + ) ) - ) return new_update def _get_mm_fields_config( @@ -676,6 +848,31 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo): return source_tokens + grids[0] * grids[1] * patch_tokens +def _stack_vit_merger_qkv( + weights: Iterable[tuple[str, torch.Tensor]], +) -> Iterable[tuple[str, torch.Tensor]]: + """Fold separate vit_merger q/k/v projections into the fused qkv_proj module. + + HF checkpoints store the window-attention projections separately, while + vLLM keeps them fused. The shard_id mechanism on the submodule's + WeightsMapper cannot do this: AutoWeightsLoader still recurses over every + incoming key after calling a submodule's load_weights, so a "k_proj" key + with no matching child raises before the mapping is honoured. + """ + pairs = ( + ("vit_merger.self_attn.q_proj.", "q"), + ("vit_merger.self_attn.k_proj.", "k"), + ("vit_merger.self_attn.v_proj.", "v"), + ) + for name, tensor in weights: + for src, shard in pairs: + if src in name: + name = name.replace(src, "vit_merger.self_attn.qkv_proj.", 1) + tensor.shard_id = shard + break + yield name, tensor + + class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module): def __init__( self, @@ -957,6 +1154,7 @@ class MiniCPMV4_6ForConditionalGeneration( SupportsPP, HasInnerState, IsHybrid, + SupportsEagle3, SupportsMRoPE, ): supports_encoder_tp_data = True @@ -1031,7 +1229,7 @@ class MiniCPMV4_6ForConditionalGeneration( apply_encoder_attention_mask=True, prefix=maybe_prefix(prefix, "vpm"), ) - if config.drop_vision_last_layer: + if self._drop_vision_last_layer(): self.vpm.encoder.layers = self.vpm.encoder.layers[:-1] self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger( @@ -1062,6 +1260,14 @@ class MiniCPMV4_6ForConditionalGeneration( self.language_model.make_empty_intermediate_tensors ) + def _drop_vision_last_layer(self) -> bool: + """Whether the last vision encoder layer is dropped. + + MiniCPM-V 4.6 checkpoints always define this. 4.7's config no longer + carries the field, so it overrides this. + """ + return self.config.drop_vision_last_layer + # ----- Multimodal parsing ----- def _parse_and_validate_vision_input( @@ -1174,35 +1380,35 @@ class MiniCPMV4_6ForConditionalGeneration( if image_input["type"] == "image_embeds": return image_input["image_embeds"] - downsample_mode = None - if use_vit_merger is not None: - downsample_mode = "16x" if use_vit_merger else "4x" - image_features = self.get_vision_hidden_states( - image_input, - downsample_mode=downsample_mode, - ) - num_slices = image_input["num_slices"] - results = [] - idx = 0 - for n in num_slices.tolist(): - group = image_features[idx : idx + n] - results.append(torch.cat(group, dim=0)) - idx += n - return results + num_slices = image_input["num_slices"].tolist() + flags = _expand_flags(use_vit_merger, len(num_slices)) + + # The merger flag is per request, but the model runner may batch items + # from several concurrent requests. Encoding such a mixed batch would + # apply one request's downsample mode to another request's items, so + # group by flag and encode each group separately. + encoded: dict[int, torch.Tensor] = {} + for flag in dict.fromkeys(flags): + item_indices = [i for i, f in enumerate(flags) if f == flag] + group_input = _select_vision_items(image_input, item_indices, num_slices) + group_features = self.get_vision_hidden_states( + group_input, + downsample_mode=_vision_downsample_mode(flag), + ) + + idx = 0 + for i in item_indices: + n = num_slices[i] + encoded[i] = torch.cat(group_features[idx : idx + n], dim=0) + idx += n + + return [encoded[i] for i in range(len(num_slices))] # ----- Multimodal embedding interface ----- def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: - use_vit_merger_tensors = kwargs.pop("use_vit_merger", None) - use_vit_merger = None - if use_vit_merger_tensors is not None: - if isinstance(use_vit_merger_tensors, torch.Tensor): - use_vit_merger = bool(use_vit_merger_tensors.any().item()) - elif isinstance(use_vit_merger_tensors, list | tuple): - use_vit_merger = any( - bool(t.any().item()) if isinstance(t, torch.Tensor) else bool(t) - for t in use_vit_merger_tensors - ) + use_vit_merger = kwargs.pop("use_vit_merger", None) + video_use_vit_merger = kwargs.pop("video_use_vit_merger", None) image_kwargs = { k: v @@ -1235,7 +1441,7 @@ class MiniCPMV4_6ForConditionalGeneration( multimodal_embeddings += tuple( self._process_vision_input( video_input, - use_vit_merger=use_vit_merger, + use_vit_merger=video_use_vit_merger, ) ) @@ -1298,7 +1504,10 @@ class MiniCPMV4_6ForConditionalGeneration( weights: Iterable[tuple[str, torch.Tensor]], ) -> set[str]: loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + return loader.load_weights( + _stack_vit_merger_qkv(weights), + mapper=self.hf_to_vllm_mapper, + ) def get_mm_mapping(self) -> MultiModelKeys: return MultiModelKeys.from_string_field( diff --git a/vllm/model_executor/models/minicpmv4_7.py b/vllm/model_executor/models/minicpmv4_7.py new file mode 100644 index 0000000000..3ce8609169 --- /dev/null +++ b/vllm/model_executor/models/minicpmv4_7.py @@ -0,0 +1,764 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only MiniCPM-V 4.7 model. + +Same vision/LLM stack as MiniCPM-V 4.6, plus canvas 3D M-RoPE. +""" + +import math +from collections.abc import Mapping +from typing import Any, NamedTuple + +import torch + +from vllm.config import VllmConfig +from vllm.logger import init_logger +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig + +from .minicpmv import MiniCPMVDummyInputsBuilder +from .minicpmv4_6 import ( + MiniCPMV4_6ForConditionalGeneration, + MiniCPMV4_6MultiModalProcessor, + MiniCPMV4_6ProcessingInfo, + _scoped_mm_kwarg, +) + +logger = init_logger(__name__) + + +class MiniCPMV4_7ProcessingInfo(MiniCPMV4_6ProcessingInfo): + def _mm_max_slice_nums(self, **kwargs: object) -> int | None: + # No schema resolution here: it would call back into `get_hf_processor` + # below, which resolves this same value. + merged = self.ctx.get_merged_mm_kwargs(kwargs) + max_slice = _scoped_mm_kwarg(merged, "image", "max_slice_nums") + if max_slice is None: + max_slice = merged.get("max_slice_nums") + if max_slice is None: + return None + return int(max_slice) + + def _configured_max_slice_nums(self) -> int | None: + # Configured (server-level) value only: per-request values are passed + # explicitly to the image processor call, so they must not be written + # back to the shared processor instance. + return self._mm_max_slice_nums() + + def get_hf_processor(self, **kwargs: object): + hf_processor = super().get_hf_processor(**kwargs) + image_processor = getattr(hf_processor, "image_processor", None) + configured = self._configured_max_slice_nums() + if image_processor is not None and configured is not None: + image_processor.max_slice_nums = configured + return hf_processor + + def get_image_max_slice_num(self) -> int: + max_slice = self._configured_max_slice_nums() + if max_slice is not None: + return max_slice + return super().get_image_max_slice_num() + + +class MiniCPMV4_7MultiModalProcessor(MiniCPMV4_6MultiModalProcessor): + def _get_mm_fields_config( + self, + hf_inputs, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> Mapping[str, MultiModalFieldConfig]: + fields = dict(super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs)) + # Canvas M-RoPE places slices from `tgt_sizes`. Stripping the payload of + # a prefix-cache-covered item only keeps CPU-side metadata, so without + # this a cached image loses its target sizes, `image_bounds` and + # `target_sizes` disagree, and the whole request silently falls back to + # sequential positions. + for key, modality in (("tgt_sizes", "image"), ("video_tgt_sizes", "video")): + if key in fields: + fields[key] = MultiModalFieldConfig.batched(modality, keep_on_cpu=True) + return fields + + def _video_local_id_prefix(self, video_idx: int) -> str: + # MiniCPMV4_7Processor emits no local `` in front of video + # placeholders: a video is one temporal sequence and 4.7 was trained + # without them. + return "" + + def get_image_prompt_texts( + self, + image_size, + image_idx: int = 0, + downsample_mode: str | None = None, + max_slice_nums: int | None = None, + ) -> str: + info = self.info + assert isinstance(info, MiniCPMV4_6ProcessingInfo) + if max_slice_nums is None: + max_slice_nums = info.get_image_max_slice_num() + return info.get_slice_image_placeholder( + image_size, + image_idx=image_idx, + max_slice_nums=max_slice_nums, + downsample_mode=downsample_mode, + ) + + +def _as_hw_pairs(tensor: torch.Tensor) -> torch.Tensor: + data = tensor.to(torch.long) + if data.numel() == 0: + return data.reshape(0, 2) + if data.shape[-1] != 2: + raise ValueError(f"expected last dim 2, got {tuple(data.shape)}") + return data.reshape(-1, 2) + + +def _sequential_positions(seq_len: int) -> torch.Tensor: + """Positions for the fallback where canvas M-RoPE cannot be computed.""" + return torch.arange(seq_len).unsqueeze(0).expand(3, -1) + + +@MULTIMODAL_REGISTRY.register_processor( + MiniCPMV4_7MultiModalProcessor, + info=MiniCPMV4_7ProcessingInfo, + dummy_inputs=MiniCPMVDummyInputsBuilder, +) +class MiniCPMV4_7ForConditionalGeneration(MiniCPMV4_6ForConditionalGeneration): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__(vllm_config=vllm_config, prefix=prefix) + self._init_canvas_mrope(vllm_config) + + def _drop_vision_last_layer(self) -> bool: + # MiniCPMV4_7Config no longer carries this field (transformers marks it + # removed), and 4.7 keeps every vision encoder layer. + return False + + def _init_canvas_mrope(self, vllm_config: VllmConfig) -> None: + config = self.config + uses_canvas = bool(getattr(config, "uses_mrope_canvas", False)) + if not uses_canvas: + mrope_mode = getattr(config, "mrope_mode", None) + uses_canvas = mrope_mode is not None and "canvas" in str(mrope_mode).lower() + + self._uses_canvas_mrope = uses_canvas + self._canvas_special_ids: dict[str, int] | None = None + if not uses_canvas: + return + + # MiniCPMV4_7Config pins the structural ids in config.json; the + # conversion script resolves them from the tokenizer. Prefer them so the + # canvas matches training, and fall back to the tokenizer for checkpoints + # whose config predates these fields. + configured_ids: dict[str, int | None] = { + "im_start_id": getattr(config, "image_start_id", None), + "im_end_id": getattr(config, "image_end_id", None), + "slice_start_id": getattr(config, "slice_start_id", None), + "slice_end_id": getattr(config, "slice_end_id", None), + "newline_id": getattr(config, "newline_id", None), + } + if all(value is not None for value in configured_ids.values()): + self._canvas_special_ids = { + name: int(value) + for name, value in configured_ids.items() + if value is not None + } + return + + from vllm.tokenizers.registry import get_tokenizer + + model_config = vllm_config.model_config + tokenizer = get_tokenizer( + model_config.tokenizer, + revision=model_config.tokenizer_revision, + trust_remote_code=model_config.trust_remote_code, + ) + self._canvas_special_ids = { + "im_start_id": tokenizer.convert_tokens_to_ids( + getattr(tokenizer, "image_start_token", "") + ), + "im_end_id": tokenizer.convert_tokens_to_ids( + getattr(tokenizer, "image_end_token", "") + ), + "slice_start_id": tokenizer.convert_tokens_to_ids( + getattr(tokenizer, "slice_start_token", "") + ), + "slice_end_id": tokenizer.convert_tokens_to_ids( + getattr(tokenizer, "slice_end_token", "") + ), + "newline_id": tokenizer.encode("\n", add_special_tokens=False)[0], + } + + def get_mrope_input_positions( + self, + input_tokens: list[int], + mm_features: list[MultiModalFeatureSpec], + ) -> tuple[torch.Tensor, int]: + seq_len = len(input_tokens) + if ( + not getattr(self, "_uses_canvas_mrope", False) + or self._canvas_special_ids is None + or not mm_features + ): + return _sequential_positions(seq_len), 0 + + input_ids = torch.tensor(input_tokens, dtype=torch.long) + image_bounds = build_image_bounds(input_ids, self._canvas_special_ids) + + target_sizes_list: list[torch.Tensor] = [] + for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): + if mm_feature.data is None: + continue + for key in ("tgt_sizes", "video_tgt_sizes"): + tgt = mm_feature.data.get(key) + if tgt is None or not isinstance(tgt.data, torch.Tensor): + continue + try: + target_sizes_list.append(_as_hw_pairs(tgt.data)) + except ValueError: + # Not an (N, 2) size table. The bounds/sizes match below + # turns this into a sequential fallback. + continue + + if not target_sizes_list or image_bounds.numel() == 0: + logger.warning( + "MiniCPM-V 4.7 canvas M-RoPE missing inputs: " + "tgt_chunks=%s image_bounds=%s; using sequential positions.", + len(target_sizes_list), + tuple(image_bounds.shape) if image_bounds is not None else None, + ) + return _sequential_positions(seq_len), 0 + + target_sizes = torch.cat(target_sizes_list, dim=0) + if target_sizes.ndim != 2 or target_sizes.shape[-1] != 2: + logger.warning( + "MiniCPM-V 4.7 canvas M-RoPE unexpected tgt_sizes shape %s; " + "using sequential positions.", + tuple(target_sizes.shape), + ) + return _sequential_positions(seq_len), 0 + if image_bounds.shape[0] != target_sizes.shape[0]: + logger.warning( + "MiniCPM-V 4.7 canvas M-RoPE size mismatch: " + "image_bounds=%s target_sizes=%s; using sequential positions.", + tuple(image_bounds.shape), + tuple(target_sizes.shape), + ) + return _sequential_positions(seq_len), 0 + + try: + pos3d = _compute_canvas_single( + input_ids, + torch.arange(seq_len, dtype=torch.long), + image_bounds, + target_sizes, + self._canvas_special_ids, + ) + except Exception: + logger.warning( + "MiniCPM-V 4.7 canvas M-RoPE failed; " + "falling back to sequential positions.", + exc_info=True, + ) + return _sequential_positions(seq_len), 0 + return pos3d, canvas_rope_delta(pos3d, seq_len) + + +# Canvas 3D M-RoPE: position ids for the slice canvas. +def compute_llm_grid( + n_tokens: int, vision_grid_height: int, vision_grid_width: int +) -> tuple[int, int]: + if n_tokens <= 0 or vision_grid_height <= 0 or vision_grid_width <= 0: + return 0, 0 + vit_total = vision_grid_height * vision_grid_width + if vit_total == n_tokens: + return vision_grid_height, vision_grid_width + if vit_total < n_tokens: + return 0, 0 + factor = int(round(math.sqrt(vit_total / n_tokens))) + if ( + factor > 0 + and vision_grid_height % factor == 0 + and vision_grid_width % factor == 0 + ): + height, width = vision_grid_height // factor, vision_grid_width // factor + if height * width == n_tokens: + return height, width + ratio = vision_grid_height / vision_grid_width + width = max(1, int(round(math.sqrt(n_tokens / ratio)))) + height = n_tokens // width + if height * width == n_tokens and height > 0: + return height, width + return 0, 0 + + +def build_image_bounds( + input_ids: torch.LongTensor, special_token_ids: dict +) -> torch.LongTensor: + start_ids = [ + x + for x in ( + special_token_ids.get("im_start_id"), + special_token_ids.get("slice_start_id"), + ) + if x is not None + ] + end_ids = [ + x + for x in ( + special_token_ids.get("im_end_id"), + special_token_ids.get("slice_end_id"), + ) + if x is not None + ] + if not start_ids or not end_ids: + return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device) + start_cond = torch.zeros_like(input_ids, dtype=torch.bool) + end_cond = torch.zeros_like(input_ids, dtype=torch.bool) + for sid in start_ids: + start_cond |= input_ids == sid + for eid in end_ids: + end_cond |= input_ids == eid + starts = torch.where(start_cond)[0] + 1 + ends = torch.where(end_cond)[0] + n = min(len(starts), len(ends)) + if n == 0: + return torch.zeros(0, 2, dtype=torch.long, device=input_ids.device) + return torch.stack([starts[:n], ends[:n]], dim=-1) + + +def _compute_canvas_single( + input_ids, position_ids_2d, image_bound, target_sizes, special_token_ids +): + # Inference always handles one unpadded sequence. The packed (cu_seqlens) + # form below is kept only to mirror the transformers reference + # implementation of its canvas position computation. + seq_len = input_ids.shape[0] + cu_seqlens = torch.tensor([0, seq_len], device=input_ids.device, dtype=torch.long) + return _compute_canvas_packed( + position_ids_2d.unsqueeze(0), + cu_seqlens, + image_bound, + target_sizes, + input_ids.unsqueeze(0), + special_token_ids, + )[:, 0, :] + + +def _target_hw(target_sizes: torch.Tensor, index: int) -> tuple[int, int]: + """Patch-grid height/width recorded for one image or slice.""" + return int(target_sizes[index, 0]), int(target_sizes[index, 1]) + + +class _CanvasLayout(NamedTuple): + """Grid of one canvas measured in LLM tokens. + + A canvas is a thumbnail plus the slices cut from it, tiled row-major. + """ + + thumb_h: int + thumb_w: int + """Thumbnail grid.""" + slice_h: int + slice_w: int + """Per-slice grid. Zero when the canvas has no slices.""" + cols: int + """Slices sharing a canvas row. Zero when the canvas has no slices.""" + height: int + width: int + """Assembled canvas.""" + + +def _canvas_layout( + thumbnail: tuple[int, int, int], + slices: list[tuple[int, int, int]], + target_sizes: torch.Tensor, +) -> _CanvasLayout: + """Measure the canvas for one ``(thumbnail, slices)`` group. + + ``thumbnail`` and each slice are ``(start, end, target_index)`` token spans. + """ + t_bs, t_be, t_gi = thumbnail + thumb_h, thumb_w = compute_llm_grid(t_be - t_bs, *_target_hw(target_sizes, t_gi)) + if not slices: + return _CanvasLayout( + thumb_h, thumb_w, 0, 0, 0, max(thumb_h, 1), max(thumb_w, 1) + ) + + cols = len(slices) + for k in range(len(slices) - 1): + if slices[k + 1][0] - slices[k][1] > 2: + cols = k + 1 + break + rows = len(slices) // cols if cols > 0 else 1 + if rows * cols != len(slices): + rows, cols = 1, len(slices) + + s0_bs, s0_be, s0_gi = slices[0] + slice_h, slice_w = compute_llm_grid(s0_be - s0_bs, *_target_hw(target_sizes, s0_gi)) + return _CanvasLayout( + thumb_h, thumb_w, slice_h, slice_w, cols, rows * slice_h, cols * slice_w + ) + + +def _assign_canvas_span( + pos3d: torch.Tensor, + *, + thumbnail: tuple[int, int, int], + span_end: int, + slices: list[tuple[int, int, int]], + base: int, + target_sizes: torch.Tensor, +) -> int: + """Place one canvas at ``base``, writing its 3-D positions into ``pos3d``. + + Shared by the video-frame and single-image paths; they differ only in how + the per-canvas base advances, not in the geometry placed here. + + Mutates ``pos3d`` in place, as the packed canvas computation has always + done. Returns the distance to the next canvas base, so each caller keeps + only its own base advancement. + """ + t_bs, t_be, _ = thumbnail + device = pos3d.device + span_start = t_bs - 1 # the token + n_thumb = t_be - t_bs + layout = _canvas_layout(thumbnail, slices, target_sizes) + cols, canvas_H, canvas_W = layout.cols, layout.height, layout.width + + # base coat + pos3d[0, 0, span_start:span_end] = base + pos3d[1, 0, span_start:span_end] = base + pos3d[2, 0, span_start:span_end] = base + + # -> halo + halo_lo = max(base - 1, 0) + pos3d[1, 0, span_start] = halo_lo + pos3d[2, 0, span_start] = halo_lo + + # -> halo (right after thumbnail) + if t_be < span_end: + pos3d[1, 0, t_be] = base + canvas_H + pos3d[2, 0, t_be] = base + canvas_W + + # thumbnail visual tokens + llm_th, llm_tw = layout.thumb_h, layout.thumb_w + if llm_th > 0 and llm_tw > 0 and llm_th * llm_tw == n_thumb: + if slices and canvas_H > 0 and canvas_W > 0: + h_c = torch.linspace(0, canvas_H - 1, llm_th, device=device).round().long() + w_c = torch.linspace(0, canvas_W - 1, llm_tw, device=device).round().long() + else: + h_c = torch.arange(llm_th, device=device) + w_c = torch.arange(llm_tw, device=device) + h_idx = h_c.view(-1, 1).expand(-1, llm_tw).reshape(-1) + w_idx = w_c.view(1, -1).expand(llm_th, -1).reshape(-1) + pos3d[0, 0, t_bs:t_be] = base + pos3d[1, 0, t_bs:t_be] = h_idx + base + pos3d[2, 0, t_bs:t_be] = w_idx + base + else: + fallback = torch.arange(n_thumb, device=device, dtype=torch.long) + base + for d in range(3): + pos3d[d, 0, t_bs:t_be] = fallback + + # slice visual tokens + slice specials + for k, (s_bs, s_be, s_gi) in enumerate(slices): + n_s = s_be - s_bs + sh_k, sw_k = compute_llm_grid(n_s, *_target_hw(target_sizes, s_gi)) + h_off = (k // cols) * layout.slice_h + w_off = (k % cols) * layout.slice_w + + slice_start_pos = s_bs - 1 + if slice_start_pos >= span_start: + pos3d[1, 0, slice_start_pos] = base + h_off + pos3d[2, 0, slice_start_pos] = base + w_off + + slice_end_pos = s_be + if sh_k > 0 and sw_k > 0: + end_h = h_off + sh_k - 1 + end_w = w_off + sw_k - 1 + else: + end_h = h_off + end_w = w_off + if slice_end_pos < span_end: + pos3d[1, 0, slice_end_pos] = base + end_h + pos3d[2, 0, slice_end_pos] = base + end_w + + if sh_k > 0 and sw_k > 0 and sh_k * sw_k == n_s: + h_idx = ( + torch.arange(sh_k, device=device) + .view(-1, 1) + .expand(-1, sw_k) + .reshape(-1) + ) + w_idx = ( + torch.arange(sw_k, device=device) + .view(1, -1) + .expand(sh_k, -1) + .reshape(-1) + ) + pos3d[0, 0, s_bs:s_be] = base + pos3d[1, 0, s_bs:s_be] = h_idx + h_off + base + pos3d[2, 0, s_bs:s_be] = w_idx + w_off + base + else: + fallback = torch.arange(n_s, device=device, dtype=torch.long) + base + for d in range(3): + pos3d[d, 0, s_bs:s_be] = fallback + + # \n between slice rows -> W = right edge + 1. + # H stays within the ended row (does NOT cross to next row): the \n sits at + # the right-outside of the row that just ended, at H = that row's last row + # and W = one column past the rightmost slice. + for k in range(len(slices) - 1): + gap_s = slices[k][1] + gap_e = slices[k + 1][0] + if gap_e - gap_s <= 2: + continue + boundary_h = ((k // cols) + 1) * layout.slice_h - 1 + right_edge_w = cols * layout.slice_w + for nl_pos in range(gap_s + 1, gap_e - 1): + pos3d[1, 0, nl_pos] = base + boundary_h + pos3d[2, 0, nl_pos] = base + right_edge_w + + return max(canvas_H, canvas_W) + 1 + + +def _compute_canvas_packed( + position_ids_2d, + cu_seqlens, + image_bound, + target_sizes, + input_ids, + special_token_ids, +): + if position_ids_2d.ndim == 1: + position_ids_2d = position_ids_2d.unsqueeze(0) + if input_ids.ndim == 1: + input_ids = input_ids.unsqueeze(0) + + has_images = ( + isinstance(image_bound, torch.Tensor) + and image_bound.numel() > 0 + and isinstance(target_sizes, torch.Tensor) + and target_sizes.numel() > 0 + ) + if not has_images: + return position_ids_2d.unsqueeze(0).expand(3, -1, -1).contiguous() + + device = position_ids_2d.device + pos3d = position_ids_2d.unsqueeze(0).expand(3, -1, -1).clone() + ids_flat = input_ids.view(-1) + + im_start_id = special_token_ids.get("im_start_id") + im_end_id = special_token_ids.get("im_end_id") + slice_start_id = special_token_ids.get("slice_start_id") + slice_end_id = special_token_ids.get("slice_end_id") + + structural_ids = { + v + for v in (im_start_id, im_end_id, slice_start_id, slice_end_id) + if v is not None + } + + num_seqs = len(cu_seqlens) - 1 + img_ptr = 0 + num_imgs = len(image_bound) + + newline_id = special_token_ids.get("newline_id") + gap_ok_ids = structural_ids | ({newline_id} if newline_id is not None else set()) + + def _group_end_pos(g, s_end): + """First position after a group's trailing structural tokens. + + Matches the transformers reference, which closes a span over every + marker in ``markers`` -- newlines included. The chat template joins + content parts with ``\\n``, so an image followed by text always ends with + one; leaving it out moves that newline out of the canvas and pushes + every later position one step forward. + """ + le = g["slices"][-1][1] if g["slices"] else g["thumbnail"][1] + ge = le + while ge < s_end and ids_flat[ge].item() in gap_ok_ids: + ge += 1 + return min(ge, s_end) + + def _is_tight_gap(g_prev, g_next, s_end): + """Check if only structural/nl tokens sit between two groups.""" + ge = _group_end_pos(g_prev, s_end) + ns = g_next["thumbnail"][0] - 1 # im_start of next group + return all(ids_flat[p].item() in gap_ok_ids for p in range(ge, ns)) + + for si in range(num_seqs): + s_start = cu_seqlens[si].item() + s_end = cu_seqlens[si + 1].item() + if s_start >= s_end: + continue + + seq_imgs = [] + while img_ptr < num_imgs: + bs_ = image_bound[img_ptr, 0].item() + be_ = image_bound[img_ptr, 1].item() + if bs_ >= s_start and be_ <= s_end: + seq_imgs.append((bs_, be_, img_ptr)) + img_ptr += 1 + else: + break + if not seq_imgs: + continue + + # ---- group images: thumbnail + following slices ---- + # merge consecutive no-slice frames into one video group + raw_groups = [] + cur_group: dict[str, Any] | None = None + for bs_, be_, gi in seq_imgs: + marker_pos = bs_ - 1 + is_slice = ( + slice_start_id is not None + and marker_pos >= s_start + and ids_flat[marker_pos].item() == slice_start_id + ) + if is_slice and cur_group is not None: + cur_group["slices"].append((bs_, be_, gi)) + else: + if cur_group is not None: + raw_groups.append(cur_group) + cur_group = {"thumbnail": (bs_, be_, gi), "slices": []} + if cur_group is not None: + raw_groups.append(cur_group) + + # merge consecutive tightly-adjacent groups into video groups. + # Two groups are "tightly adjacent" if the gap between them + # contains only structural tokens and \n (no real text). + # This handles both no-slice frames and frames-with-slices. + groups = [] + i_g = 0 + while i_g < len(raw_groups): + video_frames = [raw_groups[i_g]] + j_g = i_g + 1 + while j_g < len(raw_groups) and _is_tight_gap( + raw_groups[j_g - 1], raw_groups[j_g], s_end + ): + video_frames.append(raw_groups[j_g]) + j_g += 1 + if len(video_frames) == 1: + groups.append(raw_groups[i_g]) + else: + groups.append({"video_frames": video_frames}) + i_g = j_g + + # ---- assign 3-D positions ---- + pos = 0 + cursor = s_start + + for group in groups: + is_video_group = "video_frames" in group + + if is_video_group: + # === Video group: multiple frames merged === + # Each frame gets its own base_f so different frames have + # distinct spatial positions. Inter-frame gap tokens (\n + # etc.) are pure 1D text. After each gap, pos advances + # by 1 extra so halo_lo of the next frame does not + # collide with the last gap token. + frames = group["video_frames"] + first_frame = frames[0] + last_frame = frames[-1] + + # group_start = of first frame + group_start = first_frame["thumbnail"][0] - 1 + # group_end = after last structural token of last frame + group_end = _group_end_pos(last_frame, s_end) + + # text before video group + text_len = group_start - cursor + if text_len > 0: + t_pos = ( + torch.arange(text_len, device=device, dtype=torch.long) + pos + ) + for d in range(3): + pos3d[d, 0, cursor:group_start] = t_pos + pos += text_len + + # process each frame with its own per-frame base + frame_cursor = group_start + for vf in frames: + t_bs = vf["thumbnail"][0] + vf_slices = vf["slices"] + im_start_pos = t_bs - 1 + + # --- inter-frame gap: pure 1D text T=H=W=pos --- + gap_len = im_start_pos - frame_cursor + if gap_len > 0: + t_pos = ( + torch.arange(gap_len, device=device, dtype=torch.long) + pos + ) + for d in range(3): + pos3d[d, 0, frame_cursor:im_start_pos] = t_pos + pos += gap_len + pos += 1 # buffer: prevents halo_lo collision + + # --- per-frame base --- + base_f = pos + frame_end = _group_end_pos(vf, group_end) + pos = base_f + _assign_canvas_span( + pos3d, + thumbnail=vf["thumbnail"], + span_end=frame_end, + slices=vf_slices, + base=base_f, + target_sizes=target_sizes, + ) + frame_cursor = frame_end + + # trailing tokens after last frame (if any) + if frame_cursor < group_end: + trail_len = group_end - frame_cursor + t_pos = ( + torch.arange(trail_len, device=device, dtype=torch.long) + pos + ) + for d in range(3): + pos3d[d, 0, frame_cursor:group_end] = t_pos + pos += trail_len + + cursor = group_end + + else: + # === Single image group (thumbnail + optional slices) === + slices = group["slices"] + + group_start = group["thumbnail"][0] - 1 + group_end = _group_end_pos(group, s_end) + + # text before group + text_len = group_start - cursor + if text_len > 0: + t_pos = ( + torch.arange(text_len, device=device, dtype=torch.long) + pos + ) + for d in range(3): + pos3d[d, 0, cursor:group_start] = t_pos + pos += text_len + + base = pos + pos = base + _assign_canvas_span( + pos3d, + thumbnail=group["thumbnail"], + span_end=group_end, + slices=slices, + base=base, + target_sizes=target_sizes, + ) + cursor = group_end + + # remaining text + rem = s_end - cursor + if rem > 0: + t_pos = torch.arange(rem, device=device, dtype=torch.long) + pos + for d in range(3): + pos3d[d, 0, cursor:s_end] = t_pos + + return pos3d + + +def canvas_rope_delta(position_ids: torch.Tensor, seq_len: int) -> int: + """Offset applied to the 1-D positions of tokens decoded after the prompt. + + The first decoded token sits one past the last prefill position. + """ + return int(position_ids.amax().item()) + 1 - seq_len diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 875179a88f..fa29244413 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -514,6 +514,10 @@ _MULTIMODAL_MODELS = { "minicpmv4_6", "MiniCPMV4_6ForConditionalGeneration", ), + "MiniCPMV4_7ForConditionalGeneration": ( + "minicpmv4_7", + "MiniCPMV4_7ForConditionalGeneration", + ), "Mistral3ForConditionalGeneration": ( "mistral3", "Mistral3ForConditionalGeneration", diff --git a/vllm/parser/minicpmv.py b/vllm/parser/minicpmv.py new file mode 100644 index 0000000000..88a6865c40 --- /dev/null +++ b/vllm/parser/minicpmv.py @@ -0,0 +1,405 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MiniCPM-V family parser. + +Wraps the Qwen3 grammar (```` reasoning + XML tool calls) with two +MiniCPM-V specifics: + +- ``<|im_end|>`` and the reserved marker tokens are passthrough terminals, + so they never leak into ``content``. +- The tokenizer decodes a newline as the two characters ``\\`` and ``n`` + (e.g. token id 1639) instead of ``\\n``, so the decoded text is recovered + before it reaches the client. Code blocks, inline code and LaTeX are left + alone, as are tool call arguments. +""" + +from __future__ import annotations + +import functools +from collections.abc import Sequence +from dataclasses import replace +from typing import TYPE_CHECKING + +from vllm.entrypoints.generate.base.protocol import DeltaMessage +from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) +from vllm.parser.qwen3 import Qwen3Parser, qwen3_config + +if TYPE_CHECKING: + from vllm.entrypoints.generate.base.protocol import FunctionCall + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + + +_RESERVED_MARKER_IDS = range(12, 16) +_IM_END = "<|im_end|>" + + +@functools.cache +def minicpmv_config(thinking: bool) -> ParserEngineConfig: + """MiniCPM-V grammar: the qwen3 grammar plus MiniCPM-V markers. + + Both modes keep the qwen3 tool call terminals and transitions, so + ```` output is parsed whether or not thinking is enabled. + """ + name = "minicpmv" if thinking else "minicpmv_no_thinking" + base = qwen3_config(thinking=thinking, name=name) + + transitions = dict(base.transitions) + if not thinking: + # Default mode: the model answers directly, so `` must not open + # a reasoning span that would swallow the answer. A stray + # `...` in the output is still dropped, not printed. + transitions[(ParserState.CONTENT, "THINK_START")] = Transition( + ParserState.REASONING + ) + base = replace(base, strip_trailing_reasoning_whitespace=False) + + terminals = dict(base.terminals) + token_id_terminals = dict(base.token_id_terminals) + terminals["IM_END"] = _IM_END + token_id_terminals["IM_END"] = _IM_END + for state in (ParserState.CONTENT, ParserState.REASONING): + transitions[(state, "IM_END")] = Transition(state) + + for index in _RESERVED_MARKER_IDS: + text_name = f"RESERVED_TEXT_{index}" + token_name = f"RESERVED_TOKEN_{index}" + terminals[text_name] = f"" + terminals[token_name] = f"<|reserved_{index}|>" + token_id_terminals[token_name] = f"<|reserved_{index}|>" + for state in (ParserState.CONTENT, ParserState.REASONING): + transitions[(state, text_name)] = Transition(state) + transitions[(state, token_name)] = Transition(state) + + return replace( + base, + terminals=terminals, + token_id_terminals=token_id_terminals, + transitions=transitions, + ) + + +class EscapedNewlineNormalizer: + """Recover literal escape sequences written by the model as real text. + + Scans character by character so that a backslash inside a code block, + inline code, LaTeX span or a doubled escape is left untouched. A trailing + half of an escape is held in ``_pending`` until the next chunk arrives, + which keeps streaming output identical to the non-streaming result. + """ + + def __init__(self) -> None: + self._closing_marker: str | None = None + self._pending = "" + + def feed(self, text: str, *, final: bool = False) -> str: + text = self._pending + text + self._pending = "" + output: list[str] = [] + index = 0 + + while index < len(text): + if self._closing_marker is not None: + marker = self._closing_marker + remaining = text[index:] + if remaining.startswith(marker): + output.append(marker) + index += len(marker) + self._closing_marker = None + continue + if not final and marker.startswith(remaining): + self._pending = remaining + break + output.append(text[index]) + index += 1 + continue + + remaining = text[index:] + if text[index] == "`": + if not final and remaining in ("`", "``"): + self._pending = remaining + break + if remaining.startswith("```"): + output.append("```") + index += 3 + self._closing_marker = "```" + else: + output.append("`") + index += 1 + self._closing_marker = "`" + continue + + if text[index] == "$": + if not final and remaining == "$": + self._pending = remaining + break + if remaining.startswith("$$"): + output.append("$$") + index += 2 + self._closing_marker = "$$" + else: + output.append("$") + index += 1 + self._closing_marker = "$" + continue + + if text[index] != "\\": + output.append(text[index]) + index += 1 + continue + + slash_end = index + 1 + while slash_end < len(text) and text[slash_end] == "\\": + slash_end += 1 + if slash_end - index > 1: + if not final and slash_end == len(text): + # The run may still grow, and its parity decides whether a + # following `n` is a newline or stays literal. Hold it until + # the run is complete. + self._pending = text[index:slash_end] + break + output.append(text[index:slash_end]) + index = slash_end + continue + + if not final and remaining in ("\\", "\\r", "\\r\\"): + self._pending = remaining + break + if remaining.startswith("\\r\\n"): + output.append("\n") + index += 4 + elif remaining.startswith(("\\n", "\\r")): + output.append("\n") + index += 2 + elif remaining.startswith("\\("): + output.append("\\(") + index += 2 + self._closing_marker = "\\)" + elif remaining.startswith("\\["): + output.append("\\[") + index += 2 + self._closing_marker = "\\]" + else: + output.append("\\") + index += 1 + + if final and self._pending: + output.append(self._pending) + self._pending = "" + return "".join(output) + + +def recover_newlines(text: str | None) -> str | None: + if text is None or "\\" not in text: + return text + return EscapedNewlineNormalizer().feed(text, final=True) + + +class MiniCPMVOutputNormalizer: + """Apply :class:`EscapedNewlineNormalizer` to a whole stream. + + Reasoning and content are tracked separately so that a half-written escape + at the end of a reasoning chunk does not swallow the first characters of + the content that follows. + """ + + def __init__(self) -> None: + self._reasoning = EscapedNewlineNormalizer() + self._content = EscapedNewlineNormalizer() + + def normalize_delta( + self, + delta: DeltaMessage | None, + *, + finished: bool, + ) -> DeltaMessage | None: + reasoning_finished = finished or ( + delta is not None and (delta.content is not None or bool(delta.tool_calls)) + ) + reasoning = self._reasoning.feed( + delta.reasoning if delta and delta.reasoning else "", + final=reasoning_finished, + ) + content = self._content.feed( + delta.content if delta and delta.content else "", + final=finished, + ) + + if delta is None: + if not reasoning and not content: + return None + return DeltaMessage( + reasoning=reasoning or None, + content=content or None, + ) + + if reasoning: + delta.reasoning = reasoning + elif delta.reasoning is not None: + delta.reasoning = None + if content: + delta.content = content + elif delta.content is not None: + delta.content = None + + # A chunk that normalizes to nothing must be dropped, not returned as an + # all-None DeltaMessage: the engine only skips on `None`, so anything + # else reaches the client as an empty delta. + if not delta.content and not delta.reasoning and not delta.tool_calls: + return None + return delta + + +class MiniCPMVParser(Qwen3Parser): + """Parse MiniCPM-V family output in thinking and non-thinking modes.""" + + CONFIG_NAME = "minicpmv" + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} + # The chat template emits `` unless `enable_thinking` is + # explicitly false, so the grammar must default to thinking too, or the + # reasoning span is misread as content. + thinking_enabled = chat_kwargs.get("enable_thinking", True) + kwargs.setdefault("parser_engine_config", minicpmv_config(thinking_enabled)) + super().__init__(tokenizer, tools, **kwargs) + # ``Qwen3Parser.__init__`` reads the same key with a default of True, + # which does not apply here: MiniCPM-V defaults to thinking disabled. + # Keep the attribute in sync with the grammar built above. + self.thinking_enabled = thinking_enabled + + @property + def reasoning_ended(self) -> bool: + # The engine derives this from the presence of THINK terminals, which + # both MiniCPM-V configs keep, so the base property stays False in + # non-thinking mode. There is no reasoning span to end there. + if not self.thinking_enabled: + return True + return super().reasoning_ended + + def extract_reasoning( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> tuple[str | None, str | None]: + if self.thinking_enabled: + return super().extract_reasoning(model_output, request) + _, content = ParserEngine.extract_reasoning(self, model_output, request) + return None, content + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> DeltaMessage | None: + if self.thinking_enabled: + return super().extract_reasoning_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + ) + + delta = ParserEngine.extract_reasoning_streaming( + self, + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + ) + if delta is None: + return None + delta.reasoning = None + if delta.content is None and not delta.tool_calls: + return None + return delta + + def get_streaming_fallback_content( + self, + text: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> str | None: + if self.thinking_enabled: + return super().get_streaming_fallback_content(text, request) + delta = ParserEngine.finish_streaming(self) + return delta.content if delta is not None else None + + +class MiniCPMVUnifiedParser(DelegatingParser): + """MiniCPM-V serving parser: the adapters plus newline recovery. + + Recovery cannot live in :class:`MiniCPMVParser`: the serving stack drives + the reasoning/tool adapters, never ``ParserEngine.parse`` / + ``ParserEngine.parse_delta``, so a rewrite applied there would only be + visible to callers that construct the engine directly. + """ + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + *args, + **kwargs, + ) -> None: + super().__init__(tokenizer, tools, *args, **kwargs) + self._output_normalizer = MiniCPMVOutputNormalizer() + + def parse( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + enable_auto_tools: bool = False, + model_output_token_ids: Sequence[int] = (), + ) -> tuple[str | None, str | None, list[FunctionCall] | None]: + reasoning, content, tool_calls = super().parse( + model_output, + request, + enable_auto_tools, + model_output_token_ids, + ) + # Tool call arguments are JSON, where a literal `\n` is meaningful and + # must survive as written. + return ( + recover_newlines(reasoning), + recover_newlines(content), + tool_calls, + ) + + def parse_delta( + self, + delta_text: str, + delta_token_ids: list[int], + request: ChatCompletionRequest | ResponsesRequest, + prompt_token_ids: list[int] | None = None, + *, + finished: bool, + ) -> DeltaMessage | None: + delta = super().parse_delta( + delta_text, + delta_token_ids, + request, + prompt_token_ids, + finished=finished, + ) + return self._output_normalizer.normalize_delta(delta, finished=finished) diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index cf0be908ed..6766d15e91 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -242,6 +242,19 @@ class ParserManager: return _CohereCommandParser + if "minicpmv" in {reasoning_parser_name, tool_parser_name}: + from vllm.parser.minicpmv import MiniCPMVUnifiedParser + + r_cls = reasoning_parser_cls + t_cls = tool_parser_cls + + class _MiniCPMVParser(MiniCPMVUnifiedParser): + reasoning_parser_cls = r_cls + tool_parser_cls = t_cls + tool_strict_level = strict_level + + return _MiniCPMVParser + from vllm.parser.abstract_parser import DelegatingParser r_cls = reasoning_parser_cls diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index f9ab1dc9a1..9e75e9b77e 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -116,6 +116,10 @@ _REASONING_PARSERS_TO_REGISTER = { "mimo_engine_reasoning_parser", "MiMoParserReasoningAdapter", ), + "minicpmv": ( + "minicpmv_reasoning_parser", + "MiniCPMVParserReasoningAdapter", + ), "minimax_m2": ( "minimax_m2_reasoning_parser", "MiniMaxM2ReasoningParser", diff --git a/vllm/reasoning/minicpmv_reasoning_parser.py b/vllm/reasoning/minicpmv_reasoning_parser.py new file mode 100644 index 0000000000..402c0524ca --- /dev/null +++ b/vllm/reasoning/minicpmv_reasoning_parser.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.adapters import ParserEngineReasoningAdapter +from vllm.parser.minicpmv import MiniCPMVParser + + +class MiniCPMVParserReasoningAdapter(ParserEngineReasoningAdapter): + # Reasoning-only on purpose: this family registers no tool parser, because + # MiniCPM-V emits the Qwen3-Coder XML syntax verbatim and `qwen3_coder` + # already covers it. ``make_adapters`` only builds reasoning/tool pairs, so + # the two halves are declared explicitly here. + _parser_engine_cls = MiniCPMVParser + + +__all__ = ["MiniCPMVParserReasoningAdapter"] diff --git a/vllm/transformers_utils/chat_templates/registry.py b/vllm/transformers_utils/chat_templates/registry.py index 422854ab3d..f902ecbbaa 100644 --- a/vllm/transformers_utils/chat_templates/registry.py +++ b/vllm/transformers_utils/chat_templates/registry.py @@ -32,6 +32,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = { "deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja", "minicpmv": _get_minicpmv_chat_template_fallback, "minicpmv4_6": _get_minicpmv_chat_template_fallback, + "minicpmv4_7": _get_minicpmv_chat_template_fallback, "paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja", "siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja", "siglip2": CHAT_TEMPLATES_DIR / "template_basic.jinja",