[Model] Support MiniCPM-V 4.7 (#58674)
Signed-off-by: tc-mb <tianchi_cai@icloud.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -0,0 +1,405 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiniCPM-V family parser.
|
||||
|
||||
Wraps the Qwen3 grammar (``<think>`` reasoning + XML tool calls) with two
|
||||
MiniCPM-V specifics:
|
||||
|
||||
- ``<|im_end|>`` and the reserved marker tokens are passthrough terminals,
|
||||
so they never leak into ``content``.
|
||||
- The tokenizer decodes a newline as the two characters ``\\`` and ``n``
|
||||
(e.g. token id 1639) instead of ``\\n``, so the decoded text is recovered
|
||||
before it reaches the client. Code blocks, inline code and LaTeX are left
|
||||
alone, as are tool call arguments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
|
||||
_RESERVED_MARKER_IDS = range(12, 16)
|
||||
_IM_END = "<|im_end|>"
|
||||
|
||||
|
||||
@functools.cache
|
||||
def minicpmv_config(thinking: bool) -> ParserEngineConfig:
|
||||
"""MiniCPM-V grammar: the qwen3 grammar plus MiniCPM-V markers.
|
||||
|
||||
Both modes keep the qwen3 tool call terminals and transitions, so
|
||||
``<tool_call>`` output is parsed whether or not thinking is enabled.
|
||||
"""
|
||||
name = "minicpmv" if thinking else "minicpmv_no_thinking"
|
||||
base = qwen3_config(thinking=thinking, name=name)
|
||||
|
||||
transitions = dict(base.transitions)
|
||||
if not thinking:
|
||||
# Default mode: the model answers directly, so `<think>` must not open
|
||||
# a reasoning span that would swallow the answer. A stray
|
||||
# `<think>...</think>` in the output is still dropped, not printed.
|
||||
transitions[(ParserState.CONTENT, "THINK_START")] = Transition(
|
||||
ParserState.REASONING
|
||||
)
|
||||
base = replace(base, strip_trailing_reasoning_whitespace=False)
|
||||
|
||||
terminals = dict(base.terminals)
|
||||
token_id_terminals = dict(base.token_id_terminals)
|
||||
terminals["IM_END"] = _IM_END
|
||||
token_id_terminals["IM_END"] = _IM_END
|
||||
for state in (ParserState.CONTENT, ParserState.REASONING):
|
||||
transitions[(state, "IM_END")] = Transition(state)
|
||||
|
||||
for index in _RESERVED_MARKER_IDS:
|
||||
text_name = f"RESERVED_TEXT_{index}"
|
||||
token_name = f"RESERVED_TOKEN_{index}"
|
||||
terminals[text_name] = f"<reserved_{index}>"
|
||||
terminals[token_name] = f"<|reserved_{index}|>"
|
||||
token_id_terminals[token_name] = f"<|reserved_{index}|>"
|
||||
for state in (ParserState.CONTENT, ParserState.REASONING):
|
||||
transitions[(state, text_name)] = Transition(state)
|
||||
transitions[(state, token_name)] = Transition(state)
|
||||
|
||||
return replace(
|
||||
base,
|
||||
terminals=terminals,
|
||||
token_id_terminals=token_id_terminals,
|
||||
transitions=transitions,
|
||||
)
|
||||
|
||||
|
||||
class EscapedNewlineNormalizer:
|
||||
"""Recover literal escape sequences written by the model as real text.
|
||||
|
||||
Scans character by character so that a backslash inside a code block,
|
||||
inline code, LaTeX span or a doubled escape is left untouched. A trailing
|
||||
half of an escape is held in ``_pending`` until the next chunk arrives,
|
||||
which keeps streaming output identical to the non-streaming result.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._closing_marker: str | None = None
|
||||
self._pending = ""
|
||||
|
||||
def feed(self, text: str, *, final: bool = False) -> str:
|
||||
text = self._pending + text
|
||||
self._pending = ""
|
||||
output: list[str] = []
|
||||
index = 0
|
||||
|
||||
while index < len(text):
|
||||
if self._closing_marker is not None:
|
||||
marker = self._closing_marker
|
||||
remaining = text[index:]
|
||||
if remaining.startswith(marker):
|
||||
output.append(marker)
|
||||
index += len(marker)
|
||||
self._closing_marker = None
|
||||
continue
|
||||
if not final and marker.startswith(remaining):
|
||||
self._pending = remaining
|
||||
break
|
||||
output.append(text[index])
|
||||
index += 1
|
||||
continue
|
||||
|
||||
remaining = text[index:]
|
||||
if text[index] == "`":
|
||||
if not final and remaining in ("`", "``"):
|
||||
self._pending = remaining
|
||||
break
|
||||
if remaining.startswith("```"):
|
||||
output.append("```")
|
||||
index += 3
|
||||
self._closing_marker = "```"
|
||||
else:
|
||||
output.append("`")
|
||||
index += 1
|
||||
self._closing_marker = "`"
|
||||
continue
|
||||
|
||||
if text[index] == "$":
|
||||
if not final and remaining == "$":
|
||||
self._pending = remaining
|
||||
break
|
||||
if remaining.startswith("$$"):
|
||||
output.append("$$")
|
||||
index += 2
|
||||
self._closing_marker = "$$"
|
||||
else:
|
||||
output.append("$")
|
||||
index += 1
|
||||
self._closing_marker = "$"
|
||||
continue
|
||||
|
||||
if text[index] != "\\":
|
||||
output.append(text[index])
|
||||
index += 1
|
||||
continue
|
||||
|
||||
slash_end = index + 1
|
||||
while slash_end < len(text) and text[slash_end] == "\\":
|
||||
slash_end += 1
|
||||
if slash_end - index > 1:
|
||||
if not final and slash_end == len(text):
|
||||
# The run may still grow, and its parity decides whether a
|
||||
# following `n` is a newline or stays literal. Hold it until
|
||||
# the run is complete.
|
||||
self._pending = text[index:slash_end]
|
||||
break
|
||||
output.append(text[index:slash_end])
|
||||
index = slash_end
|
||||
continue
|
||||
|
||||
if not final and remaining in ("\\", "\\r", "\\r\\"):
|
||||
self._pending = remaining
|
||||
break
|
||||
if remaining.startswith("\\r\\n"):
|
||||
output.append("\n")
|
||||
index += 4
|
||||
elif remaining.startswith(("\\n", "\\r")):
|
||||
output.append("\n")
|
||||
index += 2
|
||||
elif remaining.startswith("\\("):
|
||||
output.append("\\(")
|
||||
index += 2
|
||||
self._closing_marker = "\\)"
|
||||
elif remaining.startswith("\\["):
|
||||
output.append("\\[")
|
||||
index += 2
|
||||
self._closing_marker = "\\]"
|
||||
else:
|
||||
output.append("\\")
|
||||
index += 1
|
||||
|
||||
if final and self._pending:
|
||||
output.append(self._pending)
|
||||
self._pending = ""
|
||||
return "".join(output)
|
||||
|
||||
|
||||
def recover_newlines(text: str | None) -> str | None:
|
||||
if text is None or "\\" not in text:
|
||||
return text
|
||||
return EscapedNewlineNormalizer().feed(text, final=True)
|
||||
|
||||
|
||||
class MiniCPMVOutputNormalizer:
|
||||
"""Apply :class:`EscapedNewlineNormalizer` to a whole stream.
|
||||
|
||||
Reasoning and content are tracked separately so that a half-written escape
|
||||
at the end of a reasoning chunk does not swallow the first characters of
|
||||
the content that follows.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._reasoning = EscapedNewlineNormalizer()
|
||||
self._content = EscapedNewlineNormalizer()
|
||||
|
||||
def normalize_delta(
|
||||
self,
|
||||
delta: DeltaMessage | None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
reasoning_finished = finished or (
|
||||
delta is not None and (delta.content is not None or bool(delta.tool_calls))
|
||||
)
|
||||
reasoning = self._reasoning.feed(
|
||||
delta.reasoning if delta and delta.reasoning else "",
|
||||
final=reasoning_finished,
|
||||
)
|
||||
content = self._content.feed(
|
||||
delta.content if delta and delta.content else "",
|
||||
final=finished,
|
||||
)
|
||||
|
||||
if delta is None:
|
||||
if not reasoning and not content:
|
||||
return None
|
||||
return DeltaMessage(
|
||||
reasoning=reasoning or None,
|
||||
content=content or None,
|
||||
)
|
||||
|
||||
if reasoning:
|
||||
delta.reasoning = reasoning
|
||||
elif delta.reasoning is not None:
|
||||
delta.reasoning = None
|
||||
if content:
|
||||
delta.content = content
|
||||
elif delta.content is not None:
|
||||
delta.content = None
|
||||
|
||||
# A chunk that normalizes to nothing must be dropped, not returned as an
|
||||
# all-None DeltaMessage: the engine only skips on `None`, so anything
|
||||
# else reaches the client as an empty delta.
|
||||
if not delta.content and not delta.reasoning and not delta.tool_calls:
|
||||
return None
|
||||
return delta
|
||||
|
||||
|
||||
class MiniCPMVParser(Qwen3Parser):
|
||||
"""Parse MiniCPM-V family output in thinking and non-thinking modes."""
|
||||
|
||||
CONFIG_NAME = "minicpmv"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
# The chat template emits `<think>` unless `enable_thinking` is
|
||||
# explicitly false, so the grammar must default to thinking too, or the
|
||||
# reasoning span is misread as content.
|
||||
thinking_enabled = chat_kwargs.get("enable_thinking", True)
|
||||
kwargs.setdefault("parser_engine_config", minicpmv_config(thinking_enabled))
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
# ``Qwen3Parser.__init__`` reads the same key with a default of True,
|
||||
# which does not apply here: MiniCPM-V defaults to thinking disabled.
|
||||
# Keep the attribute in sync with the grammar built above.
|
||||
self.thinking_enabled = thinking_enabled
|
||||
|
||||
@property
|
||||
def reasoning_ended(self) -> bool:
|
||||
# The engine derives this from the presence of THINK terminals, which
|
||||
# both MiniCPM-V configs keep, so the base property stays False in
|
||||
# non-thinking mode. There is no reasoning span to end there.
|
||||
if not self.thinking_enabled:
|
||||
return True
|
||||
return super().reasoning_ended
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if self.thinking_enabled:
|
||||
return super().extract_reasoning(model_output, request)
|
||||
_, content = ParserEngine.extract_reasoning(self, model_output, request)
|
||||
return None, content
|
||||
|
||||
def extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> DeltaMessage | None:
|
||||
if self.thinking_enabled:
|
||||
return super().extract_reasoning_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
)
|
||||
|
||||
delta = ParserEngine.extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
)
|
||||
if delta is None:
|
||||
return None
|
||||
delta.reasoning = None
|
||||
if delta.content is None and not delta.tool_calls:
|
||||
return None
|
||||
return delta
|
||||
|
||||
def get_streaming_fallback_content(
|
||||
self,
|
||||
text: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> str | None:
|
||||
if self.thinking_enabled:
|
||||
return super().get_streaming_fallback_content(text, request)
|
||||
delta = ParserEngine.finish_streaming(self)
|
||||
return delta.content if delta is not None else None
|
||||
|
||||
|
||||
class MiniCPMVUnifiedParser(DelegatingParser):
|
||||
"""MiniCPM-V serving parser: the adapters plus newline recovery.
|
||||
|
||||
Recovery cannot live in :class:`MiniCPMVParser`: the serving stack drives
|
||||
the reasoning/tool adapters, never ``ParserEngine.parse`` /
|
||||
``ParserEngine.parse_delta``, so a rewrite applied there would only be
|
||||
visible to callers that construct the engine directly.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(tokenizer, tools, *args, **kwargs)
|
||||
self._output_normalizer = MiniCPMVOutputNormalizer()
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
model_output_token_ids: Sequence[int] = (),
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
reasoning, content, tool_calls = super().parse(
|
||||
model_output,
|
||||
request,
|
||||
enable_auto_tools,
|
||||
model_output_token_ids,
|
||||
)
|
||||
# Tool call arguments are JSON, where a literal `\n` is meaningful and
|
||||
# must survive as written.
|
||||
return (
|
||||
recover_newlines(reasoning),
|
||||
recover_newlines(content),
|
||||
tool_calls,
|
||||
)
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
delta = super().parse_delta(
|
||||
delta_text,
|
||||
delta_token_ids,
|
||||
request,
|
||||
prompt_token_ids,
|
||||
finished=finished,
|
||||
)
|
||||
return self._output_normalizer.normalize_delta(delta, finished=finished)
|
||||
@@ -242,6 +242,19 @@ class ParserManager:
|
||||
|
||||
return _CohereCommandParser
|
||||
|
||||
if "minicpmv" in {reasoning_parser_name, tool_parser_name}:
|
||||
from vllm.parser.minicpmv import MiniCPMVUnifiedParser
|
||||
|
||||
r_cls = reasoning_parser_cls
|
||||
t_cls = tool_parser_cls
|
||||
|
||||
class _MiniCPMVParser(MiniCPMVUnifiedParser):
|
||||
reasoning_parser_cls = r_cls
|
||||
tool_parser_cls = t_cls
|
||||
tool_strict_level = strict_level
|
||||
|
||||
return _MiniCPMVParser
|
||||
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
|
||||
r_cls = reasoning_parser_cls
|
||||
|
||||
Reference in New Issue
Block a user