[Model] Support MiniCPM-V 4.7 (#58674)

Signed-off-by: tc-mb <tianchi_cai@icloud.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
tc-mb
2026-10-11 10:21:10 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent f425a21208
commit c85616efc6
13 changed files with 2366 additions and 62 deletions
+114
View File
@@ -6,6 +6,9 @@ import torch
from vllm.model_executor.models.minicpmv import MiniCPMV2_6
from vllm.model_executor.models.minicpmv4_6 import (
MiniCPMV4_6ForConditionalGeneration,
_expand_flags,
_select_vision_items,
_vision_downsample_mode,
)
pytestmark = pytest.mark.skip_global_cleanup
@@ -54,3 +57,114 @@ def test_image_and_video_embeds_stay_in_their_own_modality():
assert torch.equal(modalities["images"]["image_embeds"], image_embeds)
assert torch.equal(modalities["videos"]["image_embeds"], video_embeds)
class TestPerItemMergerFlags:
"""The merger flag is per request but items from several requests can be
batched together, so it has to be applied per item."""
def test_expand_flags_broadcasts_a_single_value(self):
assert _expand_flags(None, 3) == [None, None, None]
assert _expand_flags(torch.tensor([True]), 3) == [True, True, True]
assert _expand_flags(True, 2) == [True, True]
def test_expand_flags_keeps_per_item_values(self):
flags = [torch.tensor([True]), torch.tensor([False])]
assert _expand_flags(flags, 2) == [True, False]
def test_expand_flags_falls_back_on_a_shape_mismatch(self):
# Too many values to map onto the items: keep one decision for the
# batch rather than failing the request.
assert _expand_flags([True, False, True], 2) == [True, True]
def test_vision_downsample_mode(self):
assert _vision_downsample_mode(None) is None
assert _vision_downsample_mode(True) == "16x"
assert _vision_downsample_mode(False) == "4x"
def test_select_vision_items_slices_by_item(self):
# 3 items holding 2, 1 and 2 slices; one channel, one patch, width 4.
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(5)]
tgt_sizes = torch.arange(10, dtype=torch.long).reshape(5, 2)
image_input = {
"pixel_values": pixel_values,
"tgt_sizes": tgt_sizes,
"num_slices": torch.tensor([2, 1, 2]),
}
selected = _select_vision_items(image_input, [2], [2, 1, 2])
assert len(selected["pixel_values"]) == 2
assert bool(selected["pixel_values"][0][0, 0, 0] == 3.0)
assert bool(selected["pixel_values"][1][0, 0, 0] == 4.0)
assert selected["num_slices"].tolist() == [2]
assert torch.equal(selected["tgt_sizes"], tgt_sizes[3:5])
def test_select_vision_items_preserves_order(self):
pixel_values = [torch.full((1, 1, 4), float(i)) for i in range(3)]
image_input = {
"pixel_values": pixel_values,
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
"num_slices": torch.tensor([1, 1, 1]),
}
selected = _select_vision_items(image_input, [2, 0], [1, 1, 1])
assert selected["num_slices"].tolist() == [1, 1]
assert bool(selected["pixel_values"][0][0, 0, 0] == 2.0)
assert bool(selected["pixel_values"][1][0, 0, 0] == 0.0)
def test_process_vision_input_applies_flags_per_item(self):
calls: list[str | None] = []
def fake_get_vision_hidden_states(data, downsample_mode=None):
calls.append(downsample_mode)
tag = 1.0 if downsample_mode == "16x" else 0.0
return [torch.full((1,), tag) for _ in data["pixel_values"]]
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
model.get_vision_hidden_states = fake_get_vision_hidden_states
image_input = {
"type": "pixel_values",
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(3)],
"tgt_sizes": torch.zeros(3, 2, dtype=torch.long),
"num_slices": torch.tensor([1, 1, 1]),
}
result = model._process_vision_input(
image_input,
use_vit_merger=[
torch.tensor([True]),
torch.tensor([False]),
torch.tensor([True]),
],
)
# Each item keeps its own mode even though they share one call.
assert [float(r[0]) for r in result] == [1.0, 0.0, 1.0]
# Mixed flags are encoded as separate groups, not one collapsed batch.
assert calls == ["16x", "4x"]
def test_process_vision_input_uses_one_group_for_uniform_flags(self):
calls: list[str | None] = []
def fake_get_vision_hidden_states(data, downsample_mode=None):
calls.append(downsample_mode)
return [torch.zeros(1) for _ in data["pixel_values"]]
model = object.__new__(MiniCPMV4_6ForConditionalGeneration)
model.get_vision_hidden_states = fake_get_vision_hidden_states
image_input = {
"type": "pixel_values",
"pixel_values": [torch.zeros(1, 1, 4) for _ in range(2)],
"tgt_sizes": torch.zeros(2, 2, dtype=torch.long),
"num_slices": torch.tensor([1, 1]),
}
model._process_vision_input(image_input, use_vit_merger=torch.tensor([True]))
# The common case still makes a single encoder call.
assert calls == ["16x"]
+310
View File
@@ -0,0 +1,310 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.models.minicpmv4_6 import _stack_vit_merger_qkv
from vllm.model_executor.models.minicpmv4_7 import (
MiniCPMV4_7MultiModalProcessor,
_compute_canvas_single,
build_image_bounds,
canvas_rope_delta,
)
pytestmark = pytest.mark.skip_global_cleanup
def test_canvas_mrope_sizes_survive_prefix_cache_stripping():
"""A cached image must keep `tgt_sizes`, or canvas M-RoPE falls back.
`strip_covered_mm_data` drops the payload of a prefix-cache-covered item and
keeps only CPU-side metadata. Canvas M-RoPE places slices from `tgt_sizes`,
so 4.7 marks it `keep_on_cpu` while the shared 4.6 config stays as it was.
"""
from vllm.model_executor.models.minicpmv4_6 import _minicpmv4_6_field_config
shared = _minicpmv4_6_field_config({})
assert shared["tgt_sizes"].field.keep_on_cpu is False
assert shared["video_tgt_sizes"].field.keep_on_cpu is False
processor = MiniCPMV4_7MultiModalProcessor.__new__(MiniCPMV4_7MultiModalProcessor)
fields = processor._get_mm_fields_config({}, {})
assert fields["tgt_sizes"].field.keep_on_cpu is True
assert fields["video_tgt_sizes"].field.keep_on_cpu is True
# Other fields keep their previous setting.
assert fields["pixel_values"].field.keep_on_cpu is False
assert fields["video_image_sizes"].field.keep_on_cpu is True
def test_modality_mm_kwargs_resolution():
"""Flat keys reach both modalities; scopes stay scoped, and win on conflict.
`_merge_and_resolve_mm_processor_kwargs` routes a flat key into every scope
whose HF processor declares it, so reading is a plain scope lookup. The
routing itself is upstream's, tested here against the MiniCPM-V schema to
pin the precedence rule (`scoped > flat`).
"""
from vllm.model_executor.models.minicpmv4_6 import (
_flat_processor_kwargs,
_scoped_mm_kwarg,
)
from vllm.multimodal.processing.context import _resolve_mm_processor_kwargs
# What MiniCPMV4_6{Image,Video}ProcessorKwargs declare.
processor_keys = {"downsample_mode", "max_slice_nums"}
schema = {
"text_kwargs": set(),
"images_kwargs": set(processor_keys),
"videos_kwargs": set(processor_keys),
"audio_kwargs": set(),
}
def read(mm_kwargs, modality, key):
merged = _resolve_mm_processor_kwargs(dict(mm_kwargs), schema)
return _scoped_mm_kwarg(merged, modality, key)
assert read({}, "image", "downsample_mode") is None
# A flat key is routed into both scopes.
assert read({"downsample_mode": "4x"}, "image", "downsample_mode") == "4x"
assert read({"downsample_mode": "4x"}, "video", "downsample_mode") == "4x"
# A scoped key stays in its scope.
assert (
read({"images_kwargs": {"downsample_mode": "16x"}}, "image", "downsample_mode")
== "16x"
)
assert (
read({"images_kwargs": {"downsample_mode": "16x"}}, "video", "downsample_mode")
is None
)
videos_only = {"videos_kwargs": {"max_slice_nums": 1}}
assert read(videos_only, "video", "max_slice_nums") == 1
# Scoped to videos, so image must not see it.
assert read(videos_only, "image", "max_slice_nums") is None
# An explicit scope wins over the flat key it is merged with.
assert (
read(
{"downsample_mode": "16x", "images_kwargs": {"downsample_mode": "4x"}},
"image",
"downsample_mode",
)
== "4x"
)
# A non-mapping scope is ignored rather than raising.
assert read({"images_kwargs": None}, "image", "downsample_mode") is None
# `_flat_processor_kwargs` drops the scoped keys and pins the mode.
flat = _flat_processor_kwargs(
{
"downsample_mode": "4x",
"images_kwargs": {"a": 1},
"videos_kwargs": {"b": 2},
"audio_kwargs": {"c": 3},
},
"16x",
)
assert flat == {"downsample_mode": "16x"}
def test_vit_merger_qkv_is_stacked_before_load():
mapped = list(
_stack_vit_merger_qkv(
[
(
"model.vision_tower.vit_merger.self_attn.q_proj.weight",
torch.zeros(1),
),
(
"model.vision_tower.vit_merger.self_attn.k_proj.weight",
torch.zeros(1),
),
(
"model.vision_tower.vit_merger.self_attn.v_proj.weight",
torch.zeros(1),
),
("vit_merger.self_attn.qkv_proj.weight", torch.zeros(1)),
]
)
)
names = [name for name, _ in mapped]
shards = [getattr(tensor, "shard_id", None) for _, tensor in mapped]
assert names == [
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
"model.vision_tower.vit_merger.self_attn.qkv_proj.weight",
"vit_merger.self_attn.qkv_proj.weight",
]
assert shards == ["q", "k", "v", None]
# Canvas M-RoPE goldens, cross-checked against the reference implementation in
# transformers PR 48979 (`modular_minicpmv4_7.py`). Synthetic ids keep the token
# sequences readable: the canvas math only compares ids for equality.
_IM_START = 10
_IM_END = 11
_SLICE_START = 12
_SLICE_END = 13
_NEWLINE = 14
_IMAGE_PAD = 20
_VIDEO_PAD = 21
_TEXT = 100
_CANVAS_IDS = {
"im_start_id": _IM_START,
"im_end_id": _IM_END,
"slice_start_id": _SLICE_START,
"slice_end_id": _SLICE_END,
"newline_id": _NEWLINE,
}
def _canvas_positions(tokens, target_sizes):
input_ids = torch.tensor([tokens], dtype=torch.long)
targets = torch.tensor(target_sizes, dtype=torch.long)
bounds = build_image_bounds(input_ids[0], _CANVAS_IDS)
positions = _compute_canvas_single(
input_ids[0],
torch.arange(len(tokens), dtype=torch.long),
bounds,
targets,
_CANVAS_IDS,
)
return positions.tolist(), canvas_rope_delta(positions, len(tokens))
def test_canvas_mrope_trailing_newline_stays_inside_the_canvas():
"""A newline right after `</image>` belongs to the canvas, not to the text.
The chat template joins content parts with ``\\n``, so an image followed by
text always leaves one there. The transformers reference closes a frame over
every marker including newlines; leaving it out pushes every later position
one step forward and makes `rope_delta` differ by one.
"""
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _NEWLINE, _TEXT, _TEXT]
positions, delta = _canvas_positions(tokens, [[4, 4]])
assert positions == [
[0, 1, 1, 1, 1, 3, 4],
[0, 0, 1, 2, 1, 3, 4],
[0, 0, 1, 2, 1, 3, 4],
]
assert delta == -2
def test_canvas_mrope_thumbnail_matches_reference():
# <text> <image> PAD </image> <text>; 16x turns a 4x4 patch grid into 1 token.
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
positions, delta = _canvas_positions(tokens, [[4, 4]])
assert positions == [
[0, 1, 1, 1, 3],
[0, 0, 1, 2, 3],
[0, 0, 1, 2, 3],
]
assert delta == -1
def test_canvas_mrope_slice_row_matches_reference():
# Thumbnail then a 1x2 slice row; the slices tile the thumbnail's canvas.
tokens = [
_TEXT,
_IM_START,
_IMAGE_PAD,
_IM_END,
_SLICE_START,
_IMAGE_PAD,
_SLICE_END,
_SLICE_START,
_IMAGE_PAD,
_SLICE_END,
_TEXT,
]
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4], [4, 4]])
assert positions == [
[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4],
[0, 0, 1, 2, 1, 1, 1, 1, 1, 1, 4],
[0, 0, 1, 3, 1, 1, 1, 2, 2, 2, 4],
]
assert delta == -6
def test_canvas_mrope_resamples_thumbnail_over_canvas():
# A 8x8 patch grid is 2x2 LLM tokens, so the canvas is resampled (not arange).
tokens = [_TEXT, _IM_START] + [_IMAGE_PAD] * 4 + [_IM_END, _TEXT]
positions, delta = _canvas_positions(tokens, [[8, 8]])
assert positions == [
[0, 1, 1, 1, 1, 1, 1, 4],
[0, 0, 1, 1, 2, 2, 3, 4],
[0, 0, 1, 2, 1, 2, 3, 4],
]
assert delta == -3
def test_canvas_mrope_video_frames_share_one_canvas():
tokens = [
_TEXT,
_IM_START,
_VIDEO_PAD,
_IM_END,
_IM_START,
_VIDEO_PAD,
_IM_END,
_TEXT,
]
positions, delta = _canvas_positions(tokens, [[4, 4], [4, 4]])
assert positions == [
[0, 1, 1, 1, 3, 3, 3, 5],
[0, 0, 1, 2, 2, 3, 4, 5],
[0, 0, 1, 2, 2, 3, 4, 5],
]
assert delta == -2
def test_canvas_mrope_interleaved_image_and_video():
tokens = (
[_TEXT, _IM_START]
+ [_IMAGE_PAD] * 4
+ [_IM_END, _IM_START]
+ [_VIDEO_PAD] * 4
+ [_IM_END, _TEXT]
)
positions, delta = _canvas_positions(tokens, [[8, 8], [8, 8]])
assert positions == [
[0, 1, 1, 1, 1, 1, 1, 4, 4, 4, 4, 4, 4, 7],
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7],
[0, 0, 1, 2, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7],
]
assert delta == -6
def test_canvas_span_is_shared_by_image_and_video():
"""A video frame and a same-geometry image must place identical offsets.
Both paths go through `_assign_canvas_span`; this pins that they cannot
drift apart now that the duplicated geometry is gone.
"""
thumb = [_IM_START] + [_IMAGE_PAD] * 4 + [_IM_END]
img_pos, _ = _canvas_positions([_TEXT, *thumb, _TEXT], [[8, 8]])
vid_pos, _ = _canvas_positions([_TEXT, *thumb, *thumb, _TEXT], [[8, 8], [8, 8]])
def offsets(pos, start):
# The <im_start> halo is one below the canvas base.
base = pos[1][start] + 1
return [[c[start + k] - base for k in range(3)] for c in pos]
assert offsets(img_pos, 0) == offsets(vid_pos, 0)
def test_canvas_mrope_delta_leaves_room_for_the_first_decoded_token():
# First decoded token must land on max_prefill_position + 1, matching
# transformers and the other native M-RoPE models.
tokens = [_TEXT, _IM_START, _IMAGE_PAD, _IM_END, _TEXT]
positions, delta = _canvas_positions(tokens, [[4, 4]])
max_position = max(max(channel) for channel in positions)
assert delta == max_position + 1 - len(tokens)
+5
View File
@@ -1118,6 +1118,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
"openbmb/MiniCPM-V-4_6",
),
"MiniCPMV4_7ForConditionalGeneration": _HfExamplesInfo(
"openbmb/MiniCPM-V-4_7",
min_transformers_version="5.18.0",
is_available_online=False,
),
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
"MiniMaxAI/MiniMax-M3",
trust_remote_code=True,
@@ -0,0 +1,459 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MiniCPM-V family parser."""
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.parser.minicpmv import (
EscapedNewlineNormalizer,
MiniCPMVOutputNormalizer,
recover_newlines,
)
from vllm.parser.parser_manager import ParserManager
from vllm.reasoning import ReasoningParserManager
from vllm.tool_parsers import ToolParserManager
_VOCAB = {
"<think>": 10,
"</think>": 11,
"<|im_end|>": 12,
"<tool_call>": 13,
"</tool_call>": 14,
}
_TOOLS = [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
)
]
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(_VOCAB)
@pytest.fixture
def request_obj():
return ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
)
def _make_parser(tokenizer, *, enable_thinking=None, tools=None):
"""Build the parser the serving stack resolves for MiniCPM-V.
Everything is built through ``ParserManager`` so the tests exercise the
``DelegatingParser`` the server actually drives, not the parser engine on
its own. Tool calling goes through ``qwen3_coder``: MiniCPM-V emits that
XML syntax verbatim and has no tool parser of its own.
"""
parser_cls = ParserManager.get_parser(
tool_parser_name="qwen3_coder" if tools else None,
reasoning_parser_name="minicpmv",
enable_auto_tools=bool(tools),
)
assert parser_cls is not None
kwargs = {}
if enable_thinking is not None:
kwargs["chat_template_kwargs"] = {"enable_thinking": enable_thinking}
return parser_cls(tokenizer, tools, **kwargs)
def test_reasoning_parser_is_registered():
parser_cls = ReasoningParserManager.get_reasoning_parser("minicpmv")
assert parser_cls.__name__ == "MiniCPMVParserReasoningAdapter"
def test_thinking_defaults_to_the_chat_template(mock_tokenizer):
"""Without `enable_thinking`, the grammar must wait for reasoning.
The chat template ends the assistant turn with `<think>` unless the flag is
explicitly false, so the model reasons by default. This is the flag
`serving.py` latches into the structured-output gate, so a non-thinking
default both disables the grammar and leaks the thinking into `content`.
"""
def reasoning_pending(**kwargs):
return not _make_parser(mock_tokenizer, **kwargs).is_reasoning_end([1, 2])
assert reasoning_pending() is True # default matches the template
assert reasoning_pending(enable_thinking=True) is True
assert reasoning_pending(enable_thinking=False) is False
def test_no_minicpmv_tool_parser_is_registered():
"""MiniCPM-V deliberately registers no tool parser.
The family emits the Qwen3-Coder XML syntax verbatim, so
``--tool-call-parser qwen3_coder`` already covers it. Registering a
``minicpmv`` tool parser would duplicate that name for no parsing gain.
"""
with pytest.raises(KeyError):
ToolParserManager.get_tool_parser("minicpmv")
def test_composes_with_qwen3_coder_tool_parser(mock_tokenizer):
"""`--tool-call-parser qwen3_coder` must work through the serving Parser."""
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
tools=_TOOLS,
)
parser_cls = ParserManager.get_parser(
tool_parser_name="qwen3_coder",
reasoning_parser_name="minicpmv",
enable_auto_tools=True,
)
assert parser_cls is not None
parser = parser_cls(mock_tokenizer, request.tools)
_, _, tool_calls = parser.parse(
"<tool_call><function=get_weather>"
"<parameter=city>SF</parameter></function></tool_call>",
request,
enable_auto_tools=True,
)
assert tool_calls is not None
assert tool_calls[0].name == "get_weather"
assert "SF" in tool_calls[0].arguments
class TestNonThinking:
@pytest.fixture
def parser(self, mock_tokenizer):
return _make_parser(mock_tokenizer, enable_thinking=False)
def test_plain_content_is_unchanged(self, parser):
reasoning, content = parser.extract_reasoning("The answer is 42.", None)
assert reasoning is None
assert content == "The answer is 42."
def test_reserved_markers_are_removed(self, parser):
output = (
"<reserved_12>reasoning<reserved_13><|reserved_14|>tool call<|reserved_15|>"
)
reasoning, content = parser.extract_reasoning(output, None)
assert reasoning is None
assert content == "reasoningtool call"
def test_unrelated_reserved_text_is_unchanged(self, parser):
reasoning, content = parser.extract_reasoning(
"<reserved_11>content<|reserved_16|>",
None,
)
assert reasoning is None
assert content == "<reserved_11>content<|reserved_16|>"
def test_dangling_standard_think_end_is_removed(self, parser):
output = "first part </think>\n\nfinal part"
reasoning, content = parser.extract_reasoning(output, None)
assert reasoning is None
assert content == "first part \n\nfinal part"
def test_truncated_standard_reasoning_does_not_leak(self, parser):
reasoning, content = parser.extract_reasoning(
"prefix<think>private reasoning",
None,
)
assert reasoning is None
assert content == "prefix"
def test_streaming_split_standard_markers(self, parser):
reasoning, content = simulate_reasoning_streaming(
parser,
[
"<",
"think>",
"private reasoning",
"</",
"think>",
"final answer",
],
)
assert reasoning == ""
assert content == "final answer"
def test_streaming_split_reserved_markers(self, parser):
reasoning, content = simulate_reasoning_streaming(
parser,
[
"<reser",
"ved_12>",
"reasoning",
"<|reserved",
"_13|>",
"final answer",
],
)
assert reasoning == ""
assert content == "reasoningfinal answer"
def test_tool_calls_are_parsed(self, mock_tokenizer):
"""Thinking is off by default, and tool calls must still be parsed."""
parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
tools=_TOOLS,
)
reasoning, content, tool_calls = parser.parse(
"<tool_call><function=get_weather>"
"<parameter=city>SF</parameter></function></tool_call>",
request,
enable_auto_tools=True,
)
assert reasoning is None
assert not content
assert tool_calls is not None
assert tool_calls[0].name == "get_weather"
assert "SF" in tool_calls[0].arguments
def test_streaming_tool_calls_are_parsed(self, mock_tokenizer):
"""Regression: tool calls must stream when thinking is explicitly off.
The reasoning parser used to report ``reasoning_ended=False`` in
non-thinking mode, which kept the delegating parser in its reasoning
phase and streamed the raw ``<tool_call>`` XML to the client as content.
"""
parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
tools=_TOOLS,
)
delta = parser.parse_delta(
"<tool_call><function=get_weather>"
"<parameter=city>SF</parameter></function></tool_call>",
[],
request,
prompt_token_ids=[],
finished=True,
)
assert delta is not None
assert delta.tool_calls
assert delta.tool_calls[0].function is not None
assert delta.tool_calls[0].function.name == "get_weather"
assert "SF" in (delta.tool_calls[0].function.arguments or "")
assert not delta.content
class TestThinking:
def test_standard_think_tags_are_parsed(self, mock_tokenizer):
parser = _make_parser(mock_tokenizer, enable_thinking=True)
reasoning, content = parser.extract_reasoning(
"<think>private reasoning</think>final answer",
None,
)
assert reasoning == "private reasoning"
assert content == "final answer"
def test_reserved_markers_are_removed(self, mock_tokenizer):
parser = _make_parser(mock_tokenizer, enable_thinking=True)
reasoning, content = parser.extract_reasoning(
"<think><reserved_12>private reasoning</think><|reserved_13|>final answer",
None,
)
assert reasoning == "private reasoning"
assert content == "final answer"
def test_streaming_im_end_is_removed(self):
tokenizer = make_mock_tokenizer(_VOCAB, special_tokens=[])
parser = _make_parser(tokenizer, enable_thinking=True)
reasoning, content = simulate_reasoning_streaming(
parser,
["Answer: 111", "<|im_end|>"],
)
assert reasoning == "Answer: 111"
assert content == ""
class TestNewlineRecovery:
"""The tokenizer writes a newline as the two characters ``\\`` and ``n``."""
def test_recover_newlines(self):
text = (
"first\\nsecond\\rthird\\r\\nfourth "
"`code\\nvalue` "
"```python\\nvalue = '\\\\n'\\n``` "
"$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
"escaped \\\\n"
)
assert recover_newlines(text) == (
"first\nsecond\nthird\nfourth "
"`code\\nvalue` "
"```python\\nvalue = '\\\\n'\\n``` "
"$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
"escaped \\\\n"
)
def test_streaming_matches_complete_recovery(self):
text = (
"first\\nsecond `code\\nvalue` $x\\ny$ ```text\\nvalue\\n``` last\\r\\nline"
)
expected = recover_newlines(text)
for split in range(len(text) + 1):
normalizer = EscapedNewlineNormalizer()
actual = normalizer.feed(text[:split])
actual += normalizer.feed(text[split:], final=True)
assert actual == expected
normalizer = EscapedNewlineNormalizer()
actual = "".join(normalizer.feed(char) for char in text)
actual += normalizer.feed("", final=True)
assert actual == expected
def test_backslash_runs_stay_consistent_across_splits(self):
# A run of backslashes can be split over chunks. Its parity decides
# whether a following `n` is a newline, so streaming and non-streaming
# must agree for every split point.
for count in range(1, 7):
text = "\\" * count + "n"
expected = recover_newlines(text)
for split in range(len(text) + 1):
normalizer = EscapedNewlineNormalizer()
actual = normalizer.feed(text[:split])
actual += normalizer.feed(text[split:], final=True)
assert actual == expected, (count, split)
def test_streaming_recovers_reasoning_and_content(self):
normalizer = MiniCPMVOutputNormalizer()
first = normalizer.normalize_delta(
DeltaMessage(reasoning="reason\\"),
finished=False,
)
second = normalizer.normalize_delta(
DeltaMessage(reasoning="nnext\\", content="answer\\"),
finished=False,
)
final = normalizer.normalize_delta(
DeltaMessage(content="nend"),
finished=True,
)
assert first == DeltaMessage(reasoning="reason")
assert second == DeltaMessage(reasoning="\nnext\\", content="answer")
assert final == DeltaMessage(content="\nend")
def test_streaming_does_not_modify_tool_arguments(self):
normalizer = MiniCPMVOutputNormalizer()
arguments = '{"value":"first\\\\nsecond"}'
delta = DeltaMessage(
tool_calls=[
DeltaToolCall(
index=0,
function=DeltaFunctionCall(arguments=arguments),
)
]
)
result = normalizer.normalize_delta(delta, finished=True)
assert result is not None
assert result.tool_calls[0].function is not None
assert result.tool_calls[0].function.arguments == arguments
def test_parse_recovers_content(self, mock_tokenizer, request_obj):
parser = _make_parser(mock_tokenizer, enable_thinking=False)
reasoning, content, tool_calls = parser.parse(
"first\\nsecond",
request_obj,
)
assert reasoning is None
assert content == "first\nsecond"
assert not tool_calls
def test_parse_delta_recovers_content(self, mock_tokenizer, request_obj):
parser = _make_parser(mock_tokenizer, enable_thinking=False)
chunk = parser.parse_delta(
"first\\",
[],
request_obj,
finished=False,
)
rest = parser.parse_delta(
"nsecond",
[],
request_obj,
finished=True,
)
# The trailing half of the escape is held back until it is complete.
assert chunk is not None
assert chunk.content == "first"
assert rest is not None
assert rest.content == "\nsecond"
def test_buffered_delta_is_dropped_not_sent_empty(
self, mock_tokenizer, request_obj
):
parser = _make_parser(mock_tokenizer, enable_thinking=False)
parser.parse_delta("hello", [], request_obj, finished=False)
# The lone backtick is held back for the next chunk. Reporting it as an
# all-None DeltaMessage would reach the client as an empty SSE chunk.
assert parser.parse_delta("`", [], request_obj, finished=False) is None
def test_normalizer_drops_empty_delta(self):
normalizer = MiniCPMVOutputNormalizer()
assert (
normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
is None
)
def test_normalizer_flushes_buffered_content_when_finished(self):
normalizer = MiniCPMVOutputNormalizer()
normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
final = normalizer.normalize_delta(DeltaMessage(), finished=True)
assert final is not None
assert final.content == "`"