[Model] Support MiniCPM-V 4.7 (#58674)
Signed-off-by: tc-mb <tianchi_cai@icloud.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -6,6 +6,9 @@ import torch
|
||||
from vllm.model_executor.models.minicpmv import MiniCPMV2_6
|
||||
from vllm.model_executor.models.minicpmv4_6 import (
|
||||
MiniCPMV4_6ForConditionalGeneration,
|
||||
_expand_flags,
|
||||
_select_vision_items,
|
||||
_vision_downsample_mode,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
@@ -54,3 +57,114 @@ def test_image_and_video_embeds_stay_in_their_own_modality():
|
||||
|
||||
assert torch.equal(modalities["images"]["image_embeds"], image_embeds)
|
||||
assert torch.equal(modalities["videos"]["image_embeds"], video_embeds)
|
||||
|
||||
|
||||
class TestPerItemMergerFlags:
|
||||
"""The merger flag is per request but items from several requests can be
|
||||
batched together, so it has to be applied per item."""
|
||||
|
||||
def test_expand_flags_broadcasts_a_single_value(self):
|
||||
assert _expand_flags(None, 3) == [None, None, None]
|
||||
assert _expand_flags(torch.tensor([True]), 3) == [True, True, True]
|
||||
assert _expand_flags(True, 2) == [True, True]
|
||||
|
||||
def test_expand_flags_keeps_per_item_values(self):
|
||||
flags = [torch.tensor([True]), torch.tensor([False])]
|
||||
|
||||
assert _expand_flags(flags, 2) == [True, False]
|
||||
|
||||
def test_expand_flags_falls_back_on_a_shape_mismatch(self):
|
||||
# Too many values to map onto the items: keep one decision for the
|
||||
# batch rather than failing the request.
|
||||
assert _expand_flags([True, False, True], 2) == [True, True]
|
||||
|
||||
def test_vision_downsample_mode(self):
|
||||
assert _vision_downsample_mode(None) is None
|
||||
assert _vision_downsample_mode(True) == "16x"
|
||||
assert _vision_downsample_mode(False) == "4x"
|
||||
|
||||
def test_select_vision_items_slices_by_item(self):
|
||||
# 3 items holding 2, 1 and 2 slices; one channel, one patch, width 4.
|
||||
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(5)]
|
||||
tgt_sizes = torch.arange(10, dtype=torch.long).reshape(5, 2)
|
||||
image_input = {
|
||||
"pixel_values": pixel_values,
|
||||
"tgt_sizes": tgt_sizes,
|
||||
"num_slices": torch.tensor([2, 1, 2]),
|
||||
}
|
||||
|
||||
selected = _select_vision_items(image_input, [2], [2, 1, 2])
|
||||
|
||||
assert len(selected["pixel_values"]) == 2
|
||||
assert bool(selected["pixel_values"][0][0, 0, 0] == 3.0)
|
||||
assert bool(selected["pixel_values"][1][0, 0, 0] == 4.0)
|
||||
assert selected["num_slices"].tolist() == [2]
|
||||
assert torch.equal(selected["tgt_sizes"], tgt_sizes[3:5])
|
||||
|
||||
def test_select_vision_items_preserves_order(self):
|
||||
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(3)]
|
||||
image_input = {
|
||||
"pixel_values": pixel_values,
|
||||
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1, 1]),
|
||||
}
|
||||
|
||||
selected = _select_vision_items(image_input, [2, 0], [1, 1, 1])
|
||||
|
||||
assert selected["num_slices"].tolist() == [1, 1]
|
||||
assert bool(selected["pixel_values"][0][0, 0, 0] == 2.0)
|
||||
assert bool(selected["pixel_values"][1][0, 0, 0] == 0.0)
|
||||
|
||||
def test_process_vision_input_applies_flags_per_item(self):
|
||||
calls: list[str | None] = []
|
||||
|
||||
def fake_get_vision_hidden_states(data, downsample_mode=None):
|
||||
calls.append(downsample_mode)
|
||||
tag = 1.0 if downsample_mode == "16x" else 0.0
|
||||
return [torch.full((1,), tag) for _ in data["pixel_values"]]
|
||||
|
||||
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
|
||||
model.get_vision_hidden_states = fake_get_vision_hidden_states
|
||||
|
||||
image_input = {
|
||||
"type": "pixel_values",
|
||||
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(3)],
|
||||
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1, 1]),
|
||||
}
|
||||
|
||||
result = model._process_vision_input(
|
||||
image_input,
|
||||
use_vit_merger=[
|
||||
torch.tensor([True]),
|
||||
torch.tensor([False]),
|
||||
torch.tensor([True]),
|
||||
],
|
||||
)
|
||||
|
||||
# Each item keeps its own mode even though they share one call.
|
||||
assert [float(r[0]) for r in result] == [1.0, 0.0, 1.0]
|
||||
# Mixed flags are encoded as separate groups, not one collapsed batch.
|
||||
assert calls == ["16x", "4x"]
|
||||
|
||||
def test_process_vision_input_uses_one_group_for_uniform_flags(self):
|
||||
calls: list[str | None] = []
|
||||
|
||||
def fake_get_vision_hidden_states(data, downsample_mode=None):
|
||||
calls.append(downsample_mode)
|
||||
return [torch.zeros(1) for _ in data["pixel_values"]]
|
||||
|
||||
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
|
||||
model.get_vision_hidden_states = fake_get_vision_hidden_states
|
||||
|
||||
image_input = {
|
||||
"type": "pixel_values",
|
||||
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(2)],
|
||||
"tgt_sizes": torch.zeros(2, 2, dtype=torch.long),
|
||||
"num_slices": torch.tensor([1, 1]),
|
||||
}
|
||||
|
||||
model._process_vision_input(image_input, use_vit_merger=torch.tensor([True]))
|
||||
|
||||
# The common case still makes a single encoder call.
|
||||
assert calls == ["16x"]
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.minicpmv4_6 import _stack_vit_merger_qkv
|
||||
from vllm.model_executor.models.minicpmv4_7 import (
|
||||
MiniCPMV4_7MultiModalProcessor,
|
||||
_compute_canvas_single,
|
||||
build_image_bounds,
|
||||
canvas_rope_delta,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def test_canvas_mrope_sizes_survive_prefix_cache_stripping():
|
||||
"""A cached image must keep `tgt_sizes`, or canvas M-RoPE falls back.
|
||||
|
||||
`strip_covered_mm_data` drops the payload of a prefix-cache-covered item and
|
||||
keeps only CPU-side metadata. Canvas M-RoPE places slices from `tgt_sizes`,
|
||||
so 4.7 marks it `keep_on_cpu` while the shared 4.6 config stays as it was.
|
||||
"""
|
||||
from vllm.model_executor.models.minicpmv4_6 import _minicpmv4_6_field_config
|
||||
|
||||
shared = _minicpmv4_6_field_config({})
|
||||
assert shared["tgt_sizes"].field.keep_on_cpu is False
|
||||
assert shared["video_tgt_sizes"].field.keep_on_cpu is False
|
||||
|
||||
processor = MiniCPMV4_7MultiModalProcessor.__new__(MiniCPMV4_7MultiModalProcessor)
|
||||
fields = processor._get_mm_fields_config({}, {})
|
||||
|
||||
assert fields["tgt_sizes"].field.keep_on_cpu is True
|
||||
assert fields["video_tgt_sizes"].field.keep_on_cpu is True
|
||||
# Other fields keep their previous setting.
|
||||
assert fields["pixel_values"].field.keep_on_cpu is False
|
||||
assert fields["video_image_sizes"].field.keep_on_cpu is True
|
||||
|
||||
|
||||
def test_modality_mm_kwargs_resolution():
|
||||
"""Flat keys reach both modalities; scopes stay scoped, and win on conflict.
|
||||
|
||||
`_merge_and_resolve_mm_processor_kwargs` routes a flat key into every scope
|
||||
whose HF processor declares it, so reading is a plain scope lookup. The
|
||||
routing itself is upstream's, tested here against the MiniCPM-V schema to
|
||||
pin the precedence rule (`scoped > flat`).
|
||||
"""
|
||||
from vllm.model_executor.models.minicpmv4_6 import (
|
||||
_flat_processor_kwargs,
|
||||
_scoped_mm_kwarg,
|
||||
)
|
||||
from vllm.multimodal.processing.context import _resolve_mm_processor_kwargs
|
||||
|
||||
# What MiniCPMV4_6{Image,Video}ProcessorKwargs declare.
|
||||
processor_keys = {"downsample_mode", "max_slice_nums"}
|
||||
schema = {
|
||||
"text_kwargs": set(),
|
||||
"images_kwargs": set(processor_keys),
|
||||
"videos_kwargs": set(processor_keys),
|
||||
"audio_kwargs": set(),
|
||||
}
|
||||
|
||||
def read(mm_kwargs, modality, key):
|
||||
merged = _resolve_mm_processor_kwargs(dict(mm_kwargs), schema)
|
||||
return _scoped_mm_kwarg(merged, modality, key)
|
||||
|
||||
assert read({}, "image", "downsample_mode") is None
|
||||
# A flat key is routed into both scopes.
|
||||
assert read({"downsample_mode": "4x"}, "image", "downsample_mode") == "4x"
|
||||
assert read({"downsample_mode": "4x"}, "video", "downsample_mode") == "4x"
|
||||
# A scoped key stays in its scope.
|
||||
assert (
|
||||
read({"images_kwargs": {"downsample_mode": "16x"}}, "image", "downsample_mode")
|
||||
== "16x"
|
||||
)
|
||||
assert (
|
||||
read({"images_kwargs": {"downsample_mode": "16x"}}, "video", "downsample_mode")
|
||||
is None
|
||||
)
|
||||
videos_only = {"videos_kwargs": {"max_slice_nums": 1}}
|
||||
assert read(videos_only, "video", "max_slice_nums") == 1
|
||||
# Scoped to videos, so image must not see it.
|
||||
assert read(videos_only, "image", "max_slice_nums") is None
|
||||
# An explicit scope wins over the flat key it is merged with.
|
||||
assert (
|
||||
read(
|
||||
{"downsample_mode": "16x", "images_kwargs": {"downsample_mode": "4x"}},
|
||||
"image",
|
||||
"downsample_mode",
|
||||
)
|
||||
== "4x"
|
||||
)
|
||||
# A non-mapping scope is ignored rather than raising.
|
||||
assert read({"images_kwargs": None}, "image", "downsample_mode") is None
|
||||
|
||||
# `_flat_processor_kwargs` drops the scoped keys and pins the mode.
|
||||
flat = _flat_processor_kwargs(
|
||||
{
|
||||
"downsample_mode": "4x",
|
||||
"images_kwargs": {"a": 1},
|
||||
"videos_kwargs": {"b": 2},
|
||||
"audio_kwargs": {"c": 3},
|
||||
},
|
||||
"16x",
|
||||
)
|
||||
assert flat == {"downsample_mode": "16x"}
|
||||
|
||||
|
||||
def test_vit_merger_qkv_is_stacked_before_load():
|
||||
mapped = list(
|
||||
_stack_vit_merger_qkv(
|
||||
[
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.q_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.k_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.vision_tower.vit_merger.self_attn.v_proj.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
("vit_merger.self_attn.qkv_proj.weight", torch.zeros(1)),
|
||||
]
|
||||
)
|
||||
)
|
||||
names = [name for name, _ in mapped]
|
||||
shards = [getattr(tensor, "shard_id", None) for _, tensor in mapped]
|
||||
assert names == [
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
|
||||
"vit_merger.self_attn.qkv_proj.weight",
|
||||
]
|
||||
assert shards == ["q", "k", "v", None]
|
||||
|
||||
|
||||
# Canvas M-RoPE goldens, cross-checked against the reference implementation in
|
||||
# transformers PR 48979 (`modular_minicpmv4_7.py`). Synthetic ids keep the token
|
||||
# sequences readable: the canvas math only compares ids for equality.
|
||||
_IM_START = 10
|
||||
_IM_END = 11
|
||||
_SLICE_START = 12
|
||||
_SLICE_END = 13
|
||||
_NEWLINE = 14
|
||||
_IMAGE_PAD = 20
|
||||
_VIDEO_PAD = 21
|
||||
_TEXT = 100
|
||||
|
||||
_CANVAS_IDS = {
|
||||
"im_start_id": _IM_START,
|
||||
"im_end_id": _IM_END,
|
||||
"slice_start_id": _SLICE_START,
|
||||
"slice_end_id": _SLICE_END,
|
||||
"newline_id": _NEWLINE,
|
||||
}
|
||||
|
||||
|
||||
def _canvas_positions(tokens, target_sizes):
|
||||
input_ids = torch.tensor([tokens], dtype=torch.long)
|
||||
targets = torch.tensor(target_sizes, dtype=torch.long)
|
||||
bounds = build_image_bounds(input_ids[0], _CANVAS_IDS)
|
||||
positions = _compute_canvas_single(
|
||||
input_ids[0],
|
||||
torch.arange(len(tokens), dtype=torch.long),
|
||||
bounds,
|
||||
targets,
|
||||
_CANVAS_IDS,
|
||||
)
|
||||
return positions.tolist(), canvas_rope_delta(positions, len(tokens))
|
||||
|
||||
|
||||
def test_canvas_mrope_trailing_newline_stays_inside_the_canvas():
|
||||
"""A newline right after `</image>` belongs to the canvas, not to the text.
|
||||
|
||||
The chat template joins content parts with ``\\n``, so an image followed by
|
||||
text always leaves one there. The transformers reference closes a frame over
|
||||
every marker including newlines; leaving it out pushes every later position
|
||||
one step forward and makes `rope_delta` differ by one.
|
||||
"""
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _NEWLINE, _TEXT, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 3, 4],
|
||||
[0, 0, 1, 2, 1, 3, 4],
|
||||
[0, 0, 1, 2, 1, 3, 4],
|
||||
]
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_canvas_mrope_thumbnail_matches_reference():
|
||||
# <text> <image> PAD </image> <text>; 16x turns a 4x4 patch grid into 1 token.
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 3],
|
||||
[0, 0, 1, 2, 3],
|
||||
[0, 0, 1, 2, 3],
|
||||
]
|
||||
assert delta == -1
|
||||
|
||||
|
||||
def test_canvas_mrope_slice_row_matches_reference():
|
||||
# Thumbnail then a 1x2 slice row; the slices tile the thumbnail's canvas.
|
||||
tokens = [
|
||||
_TEXT,
|
||||
_IM_START,
|
||||
_IMAGE_PAD,
|
||||
_IM_END,
|
||||
_SLICE_START,
|
||||
_IMAGE_PAD,
|
||||
_SLICE_END,
|
||||
_SLICE_START,
|
||||
_IMAGE_PAD,
|
||||
_SLICE_END,
|
||||
_TEXT,
|
||||
]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4], [4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 2, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 3, 1, 1, 1, 2, 2, 2, 4],
|
||||
]
|
||||
assert delta == -6
|
||||
|
||||
|
||||
def test_canvas_mrope_resamples_thumbnail_over_canvas():
|
||||
# A 8x8 patch grid is 2x2 LLM tokens, so the canvas is resampled (not arange).
|
||||
tokens = [_TEXT, _IM_START] + [_IMAGE_PAD] * 4 + [_IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[8, 8]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 4],
|
||||
[0, 0, 1, 1, 2, 2, 3, 4],
|
||||
[0, 0, 1, 2, 1, 2, 3, 4],
|
||||
]
|
||||
assert delta == -3
|
||||
|
||||
|
||||
def test_canvas_mrope_video_frames_share_one_canvas():
|
||||
tokens = [
|
||||
_TEXT,
|
||||
_IM_START,
|
||||
_VIDEO_PAD,
|
||||
_IM_END,
|
||||
_IM_START,
|
||||
_VIDEO_PAD,
|
||||
_IM_END,
|
||||
_TEXT,
|
||||
]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 3, 3, 3, 5],
|
||||
[0, 0, 1, 2, 2, 3, 4, 5],
|
||||
[0, 0, 1, 2, 2, 3, 4, 5],
|
||||
]
|
||||
assert delta == -2
|
||||
|
||||
|
||||
def test_canvas_mrope_interleaved_image_and_video():
|
||||
tokens = (
|
||||
[_TEXT, _IM_START]
|
||||
+ [_IMAGE_PAD] * 4
|
||||
+ [_IM_END, _IM_START]
|
||||
+ [_VIDEO_PAD] * 4
|
||||
+ [_IM_END, _TEXT]
|
||||
)
|
||||
positions, delta = _canvas_positions(tokens, [[8, 8], [8, 8]])
|
||||
|
||||
assert positions == [
|
||||
[0, 1, 1, 1, 1, 1, 1, 4, 4, 4, 4, 4, 4, 7],
|
||||
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7],
|
||||
[0, 0, 1, 2, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7],
|
||||
]
|
||||
assert delta == -6
|
||||
|
||||
|
||||
def test_canvas_span_is_shared_by_image_and_video():
|
||||
"""A video frame and a same-geometry image must place identical offsets.
|
||||
|
||||
Both paths go through `_assign_canvas_span`; this pins that they cannot
|
||||
drift apart now that the duplicated geometry is gone.
|
||||
"""
|
||||
thumb = [_IM_START] + [_IMAGE_PAD] * 4 + [_IM_END]
|
||||
|
||||
img_pos, _ = _canvas_positions([_TEXT, *thumb, _TEXT], [[8, 8]])
|
||||
vid_pos, _ = _canvas_positions([_TEXT, *thumb, *thumb, _TEXT], [[8, 8], [8, 8]])
|
||||
|
||||
def offsets(pos, start):
|
||||
# The <im_start> halo is one below the canvas base.
|
||||
base = pos[1][start] + 1
|
||||
return [[c[start + k] - base for k in range(3)] for c in pos]
|
||||
|
||||
assert offsets(img_pos, 0) == offsets(vid_pos, 0)
|
||||
|
||||
|
||||
def test_canvas_mrope_delta_leaves_room_for_the_first_decoded_token():
|
||||
# First decoded token must land on max_prefill_position + 1, matching
|
||||
# transformers and the other native M-RoPE models.
|
||||
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
|
||||
positions, delta = _canvas_positions(tokens, [[4, 4]])
|
||||
|
||||
max_position = max(max(channel) for channel in positions)
|
||||
assert delta == max_position + 1 - len(tokens)
|
||||
@@ -1118,6 +1118,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM-V-4_6",
|
||||
),
|
||||
"MiniCPMV4_7ForConditionalGeneration": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM-V-4_7",
|
||||
min_transformers_version="5.18.0",
|
||||
is_available_online=False,
|
||||
),
|
||||
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
|
||||
"MiniMaxAI/MiniMax-M3",
|
||||
trust_remote_code=True,
|
||||
|
||||
@@ -0,0 +1,459 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MiniCPM-V family parser."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.minicpmv import (
|
||||
EscapedNewlineNormalizer,
|
||||
MiniCPMVOutputNormalizer,
|
||||
recover_newlines,
|
||||
)
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
_VOCAB = {
|
||||
"<think>": 10,
|
||||
"</think>": 11,
|
||||
"<|im_end|>": 12,
|
||||
"<tool_call>": 13,
|
||||
"</tool_call>": 14,
|
||||
}
|
||||
|
||||
_TOOLS = [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
return make_mock_tokenizer(_VOCAB)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def request_obj():
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
|
||||
def _make_parser(tokenizer, *, enable_thinking=None, tools=None):
|
||||
"""Build the parser the serving stack resolves for MiniCPM-V.
|
||||
|
||||
Everything is built through ``ParserManager`` so the tests exercise the
|
||||
``DelegatingParser`` the server actually drives, not the parser engine on
|
||||
its own. Tool calling goes through ``qwen3_coder``: MiniCPM-V emits that
|
||||
XML syntax verbatim and has no tool parser of its own.
|
||||
"""
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="qwen3_coder" if tools else None,
|
||||
reasoning_parser_name="minicpmv",
|
||||
enable_auto_tools=bool(tools),
|
||||
)
|
||||
assert parser_cls is not None
|
||||
kwargs = {}
|
||||
if enable_thinking is not None:
|
||||
kwargs["chat_template_kwargs"] = {"enable_thinking": enable_thinking}
|
||||
return parser_cls(tokenizer, tools, **kwargs)
|
||||
|
||||
|
||||
def test_reasoning_parser_is_registered():
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser("minicpmv")
|
||||
|
||||
assert parser_cls.__name__ == "MiniCPMVParserReasoningAdapter"
|
||||
|
||||
|
||||
def test_thinking_defaults_to_the_chat_template(mock_tokenizer):
|
||||
"""Without `enable_thinking`, the grammar must wait for reasoning.
|
||||
|
||||
The chat template ends the assistant turn with `<think>` unless the flag is
|
||||
explicitly false, so the model reasons by default. This is the flag
|
||||
`serving.py` latches into the structured-output gate, so a non-thinking
|
||||
default both disables the grammar and leaks the thinking into `content`.
|
||||
"""
|
||||
|
||||
def reasoning_pending(**kwargs):
|
||||
return not _make_parser(mock_tokenizer, **kwargs).is_reasoning_end([1, 2])
|
||||
|
||||
assert reasoning_pending() is True # default matches the template
|
||||
assert reasoning_pending(enable_thinking=True) is True
|
||||
assert reasoning_pending(enable_thinking=False) is False
|
||||
|
||||
|
||||
def test_no_minicpmv_tool_parser_is_registered():
|
||||
"""MiniCPM-V deliberately registers no tool parser.
|
||||
|
||||
The family emits the Qwen3-Coder XML syntax verbatim, so
|
||||
``--tool-call-parser qwen3_coder`` already covers it. Registering a
|
||||
``minicpmv`` tool parser would duplicate that name for no parsing gain.
|
||||
"""
|
||||
with pytest.raises(KeyError):
|
||||
ToolParserManager.get_tool_parser("minicpmv")
|
||||
|
||||
|
||||
def test_composes_with_qwen3_coder_tool_parser(mock_tokenizer):
|
||||
"""`--tool-call-parser qwen3_coder` must work through the serving Parser."""
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=_TOOLS,
|
||||
)
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="qwen3_coder",
|
||||
reasoning_parser_name="minicpmv",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert parser_cls is not None
|
||||
parser = parser_cls(mock_tokenizer, request.tools)
|
||||
|
||||
_, _, tool_calls = parser.parse(
|
||||
"<tool_call><function=get_weather>"
|
||||
"<parameter=city>SF</parameter></function></tool_call>",
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert "SF" in tool_calls[0].arguments
|
||||
|
||||
|
||||
class TestNonThinking:
|
||||
@pytest.fixture
|
||||
def parser(self, mock_tokenizer):
|
||||
return _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
|
||||
def test_plain_content_is_unchanged(self, parser):
|
||||
reasoning, content = parser.extract_reasoning("The answer is 42.", None)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "The answer is 42."
|
||||
|
||||
def test_reserved_markers_are_removed(self, parser):
|
||||
output = (
|
||||
"<reserved_12>reasoning<reserved_13><|reserved_14|>tool call<|reserved_15|>"
|
||||
)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(output, None)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "reasoningtool call"
|
||||
|
||||
def test_unrelated_reserved_text_is_unchanged(self, parser):
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<reserved_11>content<|reserved_16|>",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "<reserved_11>content<|reserved_16|>"
|
||||
|
||||
def test_dangling_standard_think_end_is_removed(self, parser):
|
||||
output = "first part </think>\n\nfinal part"
|
||||
|
||||
reasoning, content = parser.extract_reasoning(output, None)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "first part \n\nfinal part"
|
||||
|
||||
def test_truncated_standard_reasoning_does_not_leak(self, parser):
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"prefix<think>private reasoning",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "prefix"
|
||||
|
||||
def test_streaming_split_standard_markers(self, parser):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
[
|
||||
"<",
|
||||
"think>",
|
||||
"private reasoning",
|
||||
"</",
|
||||
"think>",
|
||||
"final answer",
|
||||
],
|
||||
)
|
||||
|
||||
assert reasoning == ""
|
||||
assert content == "final answer"
|
||||
|
||||
def test_streaming_split_reserved_markers(self, parser):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
[
|
||||
"<reser",
|
||||
"ved_12>",
|
||||
"reasoning",
|
||||
"<|reserved",
|
||||
"_13|>",
|
||||
"final answer",
|
||||
],
|
||||
)
|
||||
|
||||
assert reasoning == ""
|
||||
assert content == "reasoningfinal answer"
|
||||
|
||||
def test_tool_calls_are_parsed(self, mock_tokenizer):
|
||||
"""Thinking is off by default, and tool calls must still be parsed."""
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=_TOOLS,
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
"<tool_call><function=get_weather>"
|
||||
"<parameter=city>SF</parameter></function></tool_call>",
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert not content
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert "SF" in tool_calls[0].arguments
|
||||
|
||||
def test_streaming_tool_calls_are_parsed(self, mock_tokenizer):
|
||||
"""Regression: tool calls must stream when thinking is explicitly off.
|
||||
|
||||
The reasoning parser used to report ``reasoning_ended=False`` in
|
||||
non-thinking mode, which kept the delegating parser in its reasoning
|
||||
phase and streamed the raw ``<tool_call>`` XML to the client as content.
|
||||
"""
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=_TOOLS,
|
||||
)
|
||||
|
||||
delta = parser.parse_delta(
|
||||
"<tool_call><function=get_weather>"
|
||||
"<parameter=city>SF</parameter></function></tool_call>",
|
||||
[],
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
assert delta is not None
|
||||
assert delta.tool_calls
|
||||
assert delta.tool_calls[0].function is not None
|
||||
assert delta.tool_calls[0].function.name == "get_weather"
|
||||
assert "SF" in (delta.tool_calls[0].function.arguments or "")
|
||||
assert not delta.content
|
||||
|
||||
|
||||
class TestThinking:
|
||||
def test_standard_think_tags_are_parsed(self, mock_tokenizer):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=True)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<think>private reasoning</think>final answer",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning == "private reasoning"
|
||||
assert content == "final answer"
|
||||
|
||||
def test_reserved_markers_are_removed(self, mock_tokenizer):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=True)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<think><reserved_12>private reasoning</think><|reserved_13|>final answer",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning == "private reasoning"
|
||||
assert content == "final answer"
|
||||
|
||||
def test_streaming_im_end_is_removed(self):
|
||||
tokenizer = make_mock_tokenizer(_VOCAB, special_tokens=[])
|
||||
parser = _make_parser(tokenizer, enable_thinking=True)
|
||||
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["Answer: 111", "<|im_end|>"],
|
||||
)
|
||||
|
||||
assert reasoning == "Answer: 111"
|
||||
assert content == ""
|
||||
|
||||
|
||||
class TestNewlineRecovery:
|
||||
"""The tokenizer writes a newline as the two characters ``\\`` and ``n``."""
|
||||
|
||||
def test_recover_newlines(self):
|
||||
text = (
|
||||
"first\\nsecond\\rthird\\r\\nfourth "
|
||||
"`code\\nvalue` "
|
||||
"```python\\nvalue = '\\\\n'\\n``` "
|
||||
"$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
|
||||
"escaped \\\\n"
|
||||
)
|
||||
|
||||
assert recover_newlines(text) == (
|
||||
"first\nsecond\nthird\nfourth "
|
||||
"`code\\nvalue` "
|
||||
"```python\\nvalue = '\\\\n'\\n``` "
|
||||
"$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
|
||||
"escaped \\\\n"
|
||||
)
|
||||
|
||||
def test_streaming_matches_complete_recovery(self):
|
||||
text = (
|
||||
"first\\nsecond `code\\nvalue` $x\\ny$ ```text\\nvalue\\n``` last\\r\\nline"
|
||||
)
|
||||
expected = recover_newlines(text)
|
||||
|
||||
for split in range(len(text) + 1):
|
||||
normalizer = EscapedNewlineNormalizer()
|
||||
actual = normalizer.feed(text[:split])
|
||||
actual += normalizer.feed(text[split:], final=True)
|
||||
assert actual == expected
|
||||
|
||||
normalizer = EscapedNewlineNormalizer()
|
||||
actual = "".join(normalizer.feed(char) for char in text)
|
||||
actual += normalizer.feed("", final=True)
|
||||
assert actual == expected
|
||||
|
||||
def test_backslash_runs_stay_consistent_across_splits(self):
|
||||
# A run of backslashes can be split over chunks. Its parity decides
|
||||
# whether a following `n` is a newline, so streaming and non-streaming
|
||||
# must agree for every split point.
|
||||
for count in range(1, 7):
|
||||
text = "\\" * count + "n"
|
||||
expected = recover_newlines(text)
|
||||
|
||||
for split in range(len(text) + 1):
|
||||
normalizer = EscapedNewlineNormalizer()
|
||||
actual = normalizer.feed(text[:split])
|
||||
actual += normalizer.feed(text[split:], final=True)
|
||||
assert actual == expected, (count, split)
|
||||
|
||||
def test_streaming_recovers_reasoning_and_content(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
|
||||
first = normalizer.normalize_delta(
|
||||
DeltaMessage(reasoning="reason\\"),
|
||||
finished=False,
|
||||
)
|
||||
second = normalizer.normalize_delta(
|
||||
DeltaMessage(reasoning="nnext\\", content="answer\\"),
|
||||
finished=False,
|
||||
)
|
||||
final = normalizer.normalize_delta(
|
||||
DeltaMessage(content="nend"),
|
||||
finished=True,
|
||||
)
|
||||
|
||||
assert first == DeltaMessage(reasoning="reason")
|
||||
assert second == DeltaMessage(reasoning="\nnext\\", content="answer")
|
||||
assert final == DeltaMessage(content="\nend")
|
||||
|
||||
def test_streaming_does_not_modify_tool_arguments(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
arguments = '{"value":"first\\\\nsecond"}'
|
||||
delta = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
index=0,
|
||||
function=DeltaFunctionCall(arguments=arguments),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
result = normalizer.normalize_delta(delta, finished=True)
|
||||
|
||||
assert result is not None
|
||||
assert result.tool_calls[0].function is not None
|
||||
assert result.tool_calls[0].function.arguments == arguments
|
||||
|
||||
def test_parse_recovers_content(self, mock_tokenizer, request_obj):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
"first\\nsecond",
|
||||
request_obj,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "first\nsecond"
|
||||
assert not tool_calls
|
||||
|
||||
def test_parse_delta_recovers_content(self, mock_tokenizer, request_obj):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
|
||||
chunk = parser.parse_delta(
|
||||
"first\\",
|
||||
[],
|
||||
request_obj,
|
||||
finished=False,
|
||||
)
|
||||
rest = parser.parse_delta(
|
||||
"nsecond",
|
||||
[],
|
||||
request_obj,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
# The trailing half of the escape is held back until it is complete.
|
||||
assert chunk is not None
|
||||
assert chunk.content == "first"
|
||||
assert rest is not None
|
||||
assert rest.content == "\nsecond"
|
||||
|
||||
def test_buffered_delta_is_dropped_not_sent_empty(
|
||||
self, mock_tokenizer, request_obj
|
||||
):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
parser.parse_delta("hello", [], request_obj, finished=False)
|
||||
|
||||
# The lone backtick is held back for the next chunk. Reporting it as an
|
||||
# all-None DeltaMessage would reach the client as an empty SSE chunk.
|
||||
assert parser.parse_delta("`", [], request_obj, finished=False) is None
|
||||
|
||||
def test_normalizer_drops_empty_delta(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
|
||||
assert (
|
||||
normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_normalizer_flushes_buffered_content_when_finished(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
|
||||
|
||||
final = normalizer.normalize_delta(DeltaMessage(), finished=True)
|
||||
|
||||
assert final is not None
|
||||
assert final.content == "`"
|
||||
Reference in New Issue
Block a user