[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,41 +0,0 @@
|
||||
# vllm/parser
|
||||
|
||||
Model-output parsing for the serving layer. `Parser` (`abstract_parser.py`)
|
||||
unifies reasoning extraction and tool-call extraction behind one object; the
|
||||
serving layer calls `parse` / `parse_delta` and `is_reasoning_end` on it.
|
||||
|
||||
## Layout
|
||||
|
||||
- `abstract_parser.py` — `Parser`, and `DelegatingParser` for wrapping a legacy
|
||||
`ReasoningParser` + `ToolParser` pair.
|
||||
- `parser_manager.py` — resolves parser names to classes.
|
||||
- `engine/` — the declarative parser engine. A `ParserEngineConfig`
|
||||
(`parser_engine_config.py`) is a set of terminals, transitions and content
|
||||
events; `parser_engine.py` and `streaming_parser_engine.py` drive it for the
|
||||
non-streaming and streaming paths; `token_id_scanner.py` pre-lexes
|
||||
special-token ids and `incremental_lexer.py` lexes text; `adapters.py`
|
||||
exposes an engine through the legacy reasoning/tool parser interfaces.
|
||||
- `<model>.py` — one file per model family. Most subclass `ParserEngine`
|
||||
(`deepseek_v4.py`, `qwen3.py`, ...); a few are `DelegatingParser` wrappers
|
||||
over legacy parsers (`harmony.py`, `kimi_k3.py`).
|
||||
|
||||
## Guidelines
|
||||
|
||||
- A new parser either subclasses `ParserEngine` with a declarative config, or
|
||||
subclasses `Parser` and implements `parse` / `parse_delta` directly. Use the
|
||||
engine when the format is expressible as terminals and transitions, and
|
||||
subclass an existing config when it is a variant of one (`seed_oss.py`,
|
||||
`ling3.py`). `DelegatingParser` exists for backward compatibility with the
|
||||
separate `ReasoningParser` / `ToolParser` classes; do not use it for new
|
||||
formats.
|
||||
- Keep a model-specific fix inside that model's `<model>.py` where possible.
|
||||
Change `engine/` or `abstract_parser.py` only when the behavior is wrong for
|
||||
every parser, and say so in the PR.
|
||||
- Text emitted in a state with no `content_events` entry is discarded. Make
|
||||
that deliberate when adding a state.
|
||||
- Special tokens reach the engine as token ids, not text. A text terminal that
|
||||
spans a special token never matches in streaming; list it in
|
||||
`token_id_terminals`.
|
||||
- Comment only where the code alone would leave a reader with a question —
|
||||
a non-obvious transition, a deliberate swallow, a model quirk being worked
|
||||
around. Keep it minimal and concise.
|
||||
@@ -1,262 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Parser for Beam checkpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
from openai.types.responses import ToolChoiceFunction
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.abstract_parser import (
|
||||
_xgrammar_supports,
|
||||
structured_outputs_to_format,
|
||||
)
|
||||
from vllm.parser.chat_parsing import ResponseParser
|
||||
from vllm.parser.chat_parsing.response_templates import ResponseTemplate
|
||||
from vllm.parser.response_template import (
|
||||
CONTENT_FIELD,
|
||||
THINKING_FIELD,
|
||||
TOOL_FIELD,
|
||||
ResponseTemplateParser,
|
||||
ResponseTemplateReasoningParser,
|
||||
ResponseTemplateToolParser,
|
||||
_as_tool_dicts,
|
||||
_is_named_tool_choice,
|
||||
_tool_parsing_enabled,
|
||||
)
|
||||
from vllm.parser.response_template_grammar import (
|
||||
_openers,
|
||||
answer_rest_structural_tag,
|
||||
message_start_anchors,
|
||||
tool_call_structural_tag,
|
||||
)
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _named_tool(request: ChatCompletionRequest | ResponsesRequest) -> str:
|
||||
choice = request.tool_choice
|
||||
if isinstance(choice, ToolChoiceFunction):
|
||||
return choice.name
|
||||
return cast(ChatCompletionNamedToolChoiceParam, choice).function.name
|
||||
|
||||
|
||||
def _starts_message(template: ResponseTemplate, prefix: str) -> bool:
|
||||
"""Whether output after `prefix` starts the assistant message."""
|
||||
ends = [match.end() for match in template.start_anchor_re.finditer(prefix)]
|
||||
return bool(ends) and ends[-1] == len(prefix)
|
||||
|
||||
|
||||
def _open_field(template: ResponseTemplate, prefix: str) -> str | None:
|
||||
"""The field that `prefix` leaves open with its explicit opener, if any."""
|
||||
field = None
|
||||
for event in ResponseParser(template, prefix=prefix).initial_events:
|
||||
if event["type"] == "region_open" and event["end"] > event["start"]:
|
||||
field = event["field"]
|
||||
elif event["type"] in {"region_close", "region_malformed"}:
|
||||
field = None
|
||||
return field
|
||||
|
||||
|
||||
class BeamReasoningParser(ResponseTemplateReasoningParser):
|
||||
"""Selects `BeamParser` as the reasoning parser."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
return BeamParser
|
||||
|
||||
@property
|
||||
def reasoning_start_str(self) -> str | None:
|
||||
if literal := super().reasoning_start_str:
|
||||
return literal
|
||||
thinking = self.response_template.fields.get(THINKING_FIELD)
|
||||
if thinking is None:
|
||||
return None
|
||||
openers = _openers(thinking, message_start_anchors(self.response_template))
|
||||
if not openers:
|
||||
return None
|
||||
# Budget tracking needs one marker for both a prompt continuation
|
||||
# and a complete assistant message in the checkpoint's pattern.
|
||||
marker = min(openers, key=len)
|
||||
return marker if all(opener.endswith(marker) for opener in openers) else None
|
||||
|
||||
|
||||
class BeamToolParser(ResponseTemplateToolParser):
|
||||
"""Selects `BeamParser` as the tool parser."""
|
||||
|
||||
@classmethod
|
||||
def unified_parser_cls(cls) -> type[ResponseTemplateParser]:
|
||||
return BeamParser
|
||||
|
||||
|
||||
class BeamParser(ResponseTemplateParser):
|
||||
"""Parse Beam output with the checkpoint's `response_template` metadata,
|
||||
and constrain it with a grammar derived from that template."""
|
||||
|
||||
reasoning_parser_cls = BeamReasoningParser
|
||||
tool_parser_cls = BeamToolParser
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
chat_template_kwargs = kwargs.get("chat_template_kwargs") or {}
|
||||
# From the message start, the grammar opens the message with non-empty
|
||||
# reasoning, unless the request turns thinking off.
|
||||
self._requires_reasoning = bool(
|
||||
chat_template_kwargs.get("enable_thinking", True)
|
||||
)
|
||||
|
||||
def _constrain_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
message_start = _starts_message(self.response_template, self._prefix)
|
||||
tag = self._template_tag(request, message_start=message_start)
|
||||
if tag is not None:
|
||||
grammar = StructuredOutputsParams(structural_tag=json.dumps(tag))
|
||||
if isinstance(request, ChatCompletionRequest):
|
||||
request.structured_outputs = grammar
|
||||
request.response_format = None
|
||||
else:
|
||||
request._parser_structured_outputs = grammar
|
||||
# From the message start, the tag also covers the reasoning.
|
||||
request._grammar_from_parser = message_start
|
||||
return request
|
||||
|
||||
def _template_tag(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
*,
|
||||
message_start: bool,
|
||||
) -> dict[str, Any] | None:
|
||||
"""The template grammar for `request`, or `None` to leave its output
|
||||
to the request's own constraint, if any."""
|
||||
fields = self.response_template.fields
|
||||
named = _is_named_tool_choice(request)
|
||||
tools_enabled = (
|
||||
self._tool_parser is not None
|
||||
and bool(request.tools)
|
||||
and _tool_parsing_enabled(
|
||||
request, enable_auto_tools=self._enable_auto_tools
|
||||
)
|
||||
)
|
||||
forced = tools_enabled and (request.tool_choice == "required" or named)
|
||||
# The prompt may still list the tools, so only the grammar keeps the
|
||||
# model from calling them.
|
||||
tools_forbidden = (
|
||||
bool(request.tools)
|
||||
and request.tool_choice == "none"
|
||||
and TOOL_FIELD in fields
|
||||
)
|
||||
constrained_tools = None
|
||||
if tools_enabled and (forced or envs.VLLM_ENFORCE_STRICT_TOOL_CALLING):
|
||||
assert self._tool_parser is not None
|
||||
strict_level = self.tool_strict_level
|
||||
if strict_level is None:
|
||||
strict_level = self._tool_parser.default_tool_strict_level
|
||||
constrained_tools = resolve_tool_strictness(
|
||||
request.tools or [], request.tool_choice, strict_level
|
||||
)
|
||||
output = request.extract_structured_outputs()
|
||||
if forced and output is not None:
|
||||
logger.warning_once(
|
||||
"structured outputs are ignored because tool_choice forces tool call."
|
||||
)
|
||||
output = None
|
||||
answer = fields.get(CONTENT_FIELD)
|
||||
if output is not None and (
|
||||
output.structural_tag is not None
|
||||
or answer is None
|
||||
or answer.open_re is None
|
||||
or not _xgrammar_supports(output)
|
||||
):
|
||||
if constrained_tools is not None:
|
||||
logger.warning_once(
|
||||
"response_template parsing cannot enforce strict tools "
|
||||
"alongside this structured output."
|
||||
)
|
||||
return None
|
||||
# Without a tool or structured output constraint, the output still
|
||||
# follows the template.
|
||||
always_constrain = envs.VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT and not forced
|
||||
if (
|
||||
constrained_tools is None
|
||||
and output is None
|
||||
and not tools_forbidden
|
||||
and not always_constrain
|
||||
):
|
||||
return None
|
||||
|
||||
# Inside an open answer, only the always-on grammar applies, finishing a
|
||||
# free answer. Inside open reasoning, the grammar needs the reasoning
|
||||
# gate to defer it to the reasoning's end.
|
||||
open_field = _open_field(self.response_template, self._prefix)
|
||||
reasoning_gated = (
|
||||
self._reasoning_parser is not None and request.include_reasoning
|
||||
)
|
||||
tag = None
|
||||
if open_field == CONTENT_FIELD:
|
||||
if always_constrain and output is None:
|
||||
tag = answer_rest_structural_tag(fields[CONTENT_FIELD])
|
||||
elif open_field != THINKING_FIELD or reasoning_gated:
|
||||
functions = []
|
||||
if tools_enabled:
|
||||
tools = _as_tool_dicts(constrained_tools or request.tools)
|
||||
functions = [tool.get("function", tool) for tool in tools]
|
||||
if named:
|
||||
name = _named_tool(request)
|
||||
functions = [f for f in functions if f.get("name") == name]
|
||||
answer_format = (
|
||||
None if output is None else structured_outputs_to_format(output)
|
||||
)
|
||||
try:
|
||||
tag = tool_call_structural_tag(
|
||||
fields.get(TOOL_FIELD),
|
||||
functions,
|
||||
parallel_tool_calls=(
|
||||
not named and request.parallel_tool_calls is not False
|
||||
),
|
||||
thinking=fields.get(THINKING_FIELD),
|
||||
answer=None if forced else answer,
|
||||
answer_format=(
|
||||
None if answer_format is None else answer_format.model_dump()
|
||||
),
|
||||
message_start=(
|
||||
message_start_anchors(self.response_template)
|
||||
if message_start
|
||||
else None
|
||||
),
|
||||
requires_reasoning=self._requires_reasoning,
|
||||
# Beam drifts from schemas the grammar does not enforce,
|
||||
# and also writes arguments sorted by name.
|
||||
sorted_arguments=True,
|
||||
strict_arguments=True,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise VLLMValidationError(str(exc), parameter="tools") from exc
|
||||
if tag is None and forced:
|
||||
kind = "named" if named else "required"
|
||||
raise VLLMValidationError(
|
||||
f"response_template parsing cannot guarantee {kind} tool choice "
|
||||
"for this request",
|
||||
parameter="tool_choice",
|
||||
)
|
||||
if tag is None and constrained_tools is not None:
|
||||
logger.warning_once(
|
||||
"response_template parsing cannot enforce strict tools for this "
|
||||
"request."
|
||||
)
|
||||
return tag
|
||||
@@ -1,691 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cohere Command parser.
|
||||
|
||||
Uses ``cohere_melody`` as a stateful parser filter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, TypedDict, TypeGuard
|
||||
|
||||
import regex as re
|
||||
import xgrammar as xgr
|
||||
|
||||
try:
|
||||
from cohere_melody import (
|
||||
AccumulatedToolCall,
|
||||
FilterAggregatedResult,
|
||||
PyFilter,
|
||||
PyFilterOptions,
|
||||
)
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"The Cohere Command parser requires the `cohere_melody` package. "
|
||||
"Install it with:\n pip install 'cohere-melody>=0.14.0'"
|
||||
) from e
|
||||
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
Citation,
|
||||
CitationSource,
|
||||
CohereDeltaMessage,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
AnyResponseFormat,
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
FunctionCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.reasoning.cohere_command_reasoning_parser import (
|
||||
BaseCohereCommandReasoningParser,
|
||||
)
|
||||
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
|
||||
from vllm.sampling_params import StructuredOutputsParams, check_json_nesting
|
||||
from vllm.tool_parsers.cohere_command_tool_parser import BaseCohereCommandToolParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Declared in melody's stub only; not exported at runtime.
|
||||
from cohere_melody import FilterCitation, Source
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.utils import Tool
|
||||
|
||||
PositionToSource = Mapping[tuple[int, int], CitationSource]
|
||||
|
||||
|
||||
class CohereTagRegistry(NamedTuple):
|
||||
"""A single ``structural_tag`` trigger / end pair (``begin`` uses ``trigger``)."""
|
||||
|
||||
trigger: str
|
||||
end: str
|
||||
|
||||
|
||||
class CohereTagStyle(NamedTuple):
|
||||
"""The structural tags style for a given model architecture.
|
||||
|
||||
``json_tags`` lists every JSON-schema wrapper the model may emit (MOE uses
|
||||
both response and text delimiters). ``tools`` is the tool-call wrapper.
|
||||
"""
|
||||
|
||||
json_tags: tuple[CohereTagRegistry, ...]
|
||||
tools: CohereTagRegistry
|
||||
|
||||
|
||||
class CohereNormalizedTool(TypedDict):
|
||||
"""A tool definition normalized to the shape ``collect_tool_schema`` expects.
|
||||
|
||||
``parameters`` is a JSON Schema object (possibly empty) describing the tool's
|
||||
call signature.
|
||||
"""
|
||||
|
||||
name: str
|
||||
parameters: dict[str, Any]
|
||||
|
||||
|
||||
COMMAND_A_TOOLS_TAG = CohereTagRegistry(
|
||||
trigger="<|START_ACTION|>",
|
||||
end="<|END_ACTION|>",
|
||||
)
|
||||
COMMAND_A_JSON_TAG = CohereTagRegistry(
|
||||
trigger="<|START_RESPONSE|>",
|
||||
end="<|END_RESPONSE|>",
|
||||
)
|
||||
COMMAND_A_PLUS_JSON_TAG = CohereTagRegistry(
|
||||
trigger="<|START_TEXT|>",
|
||||
end="<|END_TEXT|>",
|
||||
)
|
||||
|
||||
MODEL_TO_TAG_STYLE: dict[str, CohereTagStyle] = {
|
||||
"Cohere2ForCausalLM": CohereTagStyle(
|
||||
json_tags=(COMMAND_A_JSON_TAG,),
|
||||
tools=COMMAND_A_TOOLS_TAG,
|
||||
),
|
||||
"Cohere2VisionForConditionalGeneration": CohereTagStyle(
|
||||
json_tags=(COMMAND_A_JSON_TAG, COMMAND_A_PLUS_JSON_TAG),
|
||||
tools=COMMAND_A_TOOLS_TAG,
|
||||
),
|
||||
"Cohere2MoeForCausalLM": CohereTagStyle(
|
||||
json_tags=(COMMAND_A_JSON_TAG, COMMAND_A_PLUS_JSON_TAG),
|
||||
tools=COMMAND_A_TOOLS_TAG,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str:
|
||||
"""Build an xgrammar EBNF grammar that matches a JSON array of tool calls.
|
||||
|
||||
The grammar shape is architecture-independent; callers are responsible for
|
||||
wrapping it in the correct structural tag (see ``CohereTagStyle.tools``).
|
||||
"""
|
||||
tool_dictionary: dict[str, str] = {}
|
||||
for tool in tool_schema:
|
||||
tool_name = tool["name"]
|
||||
check_json_nesting(tool["parameters"])
|
||||
tool_parameters = dict(tool["parameters"])
|
||||
json_schema: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tool_call_id": {"type": "string", "pattern": "^[0-9]+$"},
|
||||
"tool_name": {"type": "string", "const": tool_name},
|
||||
"parameters": tool_parameters,
|
||||
},
|
||||
}
|
||||
for defs_key in ("$defs", "definitions"):
|
||||
if defs_key in tool_parameters:
|
||||
json_schema[defs_key] = tool_parameters.pop(defs_key)
|
||||
tool_grammar = str(xgr.Grammar.from_json_schema(json.dumps(json_schema)))
|
||||
for match in re.findall(r"\b(\w+)\s*::=", tool_grammar):
|
||||
tool_grammar = re.sub(
|
||||
rf"\b{re.escape(match)}\b", tool_name + match, tool_grammar
|
||||
)
|
||||
tool_dictionary[tool_name] = f"{tool_name} ::= {tool_name}root\n{tool_grammar}"
|
||||
# Emitted grammar shape:
|
||||
# root ::= tools
|
||||
# tools ::= ws "[" ws tool (ws "," ws tool)* ws "]" ws
|
||||
# ws ::= (" " | "\t" | "\n")*
|
||||
# tool ::= <tool_a> | <tool_b> | ... (one alternative per input)
|
||||
# <tool_x> ::= <tool_x>root (per-tool xgrammar rules)
|
||||
# <tool_x>root ::= ... (from xgr.Grammar.from_json_schema)
|
||||
tool_alternatives = "tool ::= " + " | ".join(tool_dictionary.keys())
|
||||
tool_rules = "\n ".join(tool_dictionary.values())
|
||||
grammar = f"""root ::= tools
|
||||
tools ::= ws "[" ws tool (ws "," ws tool)* ws "]" ws
|
||||
ws ::= (" " | "\\t" | "\\n")*
|
||||
{tool_alternatives}
|
||||
{tool_rules}
|
||||
"""
|
||||
return grammar
|
||||
|
||||
|
||||
def _tool_definitions_to_schema_list(
|
||||
tools: str | list[Any],
|
||||
) -> list[CohereNormalizedTool]:
|
||||
"""Build the list of ``CohereNormalizedTool`` dicts expected by
|
||||
``collect_tool_schema``.
|
||||
|
||||
Accepts:
|
||||
- JSON string
|
||||
- list of dicts with top-level ``name`` / ``parameters``
|
||||
- list of Chat Completions-style ``{"type": "function", "function": {...}}``
|
||||
- list of Pydantic models with ``model_dump()``
|
||||
"""
|
||||
if isinstance(tools, str):
|
||||
try:
|
||||
parsed = json.loads(tools)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
if not isinstance(parsed, list):
|
||||
return []
|
||||
else:
|
||||
parsed = list(tools)
|
||||
|
||||
out: list[CohereNormalizedTool] = []
|
||||
for raw in parsed:
|
||||
t = raw.model_dump() if hasattr(raw, "model_dump") else raw
|
||||
if not isinstance(t, dict):
|
||||
continue
|
||||
# Unwrap Chat Completions' ``{"type": "function", "function": {...}}``
|
||||
# shape; otherwise take the dict as-is.
|
||||
if t.get("type") == "function" and isinstance(t.get("function"), dict):
|
||||
t = t["function"]
|
||||
name = t.get("name")
|
||||
if not isinstance(name, str):
|
||||
continue
|
||||
params = t.get("parameters")
|
||||
out.append(
|
||||
CohereNormalizedTool(
|
||||
name=name,
|
||||
parameters=params if isinstance(params, dict) else {},
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _has_effective_tools(
|
||||
tools: str | list[Any] | None,
|
||||
) -> TypeGuard[str | list[Any]]:
|
||||
"""True when ``tools`` contains at least one tool definition to convert.
|
||||
|
||||
``ResponsesRequest`` defaults ``tools`` to ``[]``; ``ChatCompletionRequest``
|
||||
uses ``None``. Both mean "no tools" here. Strings (e.g. a JSON blob) are
|
||||
treated as effective only when non-blank.
|
||||
"""
|
||||
if tools is None:
|
||||
return False
|
||||
if isinstance(tools, str):
|
||||
return bool(tools.strip())
|
||||
return len(tools) > 0
|
||||
|
||||
|
||||
# Builder: produces vLLM response_format in xgrammar's canonical format.
|
||||
# See xgrammar docs: type "structural_tag" with "format" = triggered_tags
|
||||
# and tag content type = json_schema | grammar.
|
||||
def convert_schema_to_structural_tags(
|
||||
schema: dict | None = None,
|
||||
tools: str | list[Any] | None = None,
|
||||
model_architecture: str | None = None,
|
||||
) -> str | None:
|
||||
"""Returns a response_format string accepted by xgrammar's structural tag format.
|
||||
Uses the canonical shape: {"type": "structural_tag", "format": {...}} with
|
||||
format.type "triggered_tags" and tag content type "json_schema" or "grammar".
|
||||
|
||||
Callers that are not on an engine path (e.g. the reasoning parser) must pass
|
||||
``model_architecture`` explicitly.
|
||||
"""
|
||||
if model_architecture is None or model_architecture not in MODEL_TO_TAG_STYLE:
|
||||
return None
|
||||
style = MODEL_TO_TAG_STYLE[model_architecture]
|
||||
|
||||
tags: list[dict] = []
|
||||
triggers: list[str] = []
|
||||
|
||||
def _add_tag(tag: CohereTagRegistry, content: dict) -> None:
|
||||
tags.append({"begin": tag.trigger, "content": content, "end": tag.end})
|
||||
triggers.append(tag.trigger)
|
||||
|
||||
if schema is not None:
|
||||
# One structural tag per JSON wrapper (e.g. MOE: response + text).
|
||||
# Same for schema-only and "tools plus JSON mode" (North: schema when
|
||||
# the model does not call tools).
|
||||
for jt in style.json_tags:
|
||||
_add_tag(jt, {"type": "json_schema", "json_schema": schema})
|
||||
|
||||
if _has_effective_tools(tools):
|
||||
# ``tools`` may be a JSON string (poseidon / RESPONSE_FORMAT_TOOL_DEFINITIONS)
|
||||
# or a list (Chat Completions ``request.tools`` as Pydantic models or dicts).
|
||||
tool_schema_list = _tool_definitions_to_schema_list(tools)
|
||||
if not tool_schema_list:
|
||||
raise ValueError(
|
||||
"No valid tool definitions could be parsed from the request for "
|
||||
"structural tag conversion."
|
||||
)
|
||||
tool_grammar = collect_tool_schema(tool_schema_list)
|
||||
_add_tag(style.tools, {"type": "grammar", "grammar": tool_grammar})
|
||||
|
||||
if not tags:
|
||||
return None
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "structural_tag",
|
||||
"format": {
|
||||
"type": "triggered_tags",
|
||||
"triggers": triggers,
|
||||
"tags": tags,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _response_format_type(
|
||||
response_format: AnyResponseFormat | dict | None,
|
||||
) -> str | None:
|
||||
if response_format is None:
|
||||
return None
|
||||
if isinstance(response_format, dict):
|
||||
t = response_format.get("type")
|
||||
return t if isinstance(t, str) else None
|
||||
return response_format.type
|
||||
|
||||
|
||||
def _maybe_parse_json_dict(value: Any) -> dict | None:
|
||||
"""If value is a JSON string, parse to dict; otherwise require dict."""
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return None
|
||||
return parsed if isinstance(parsed, dict) else None
|
||||
return None
|
||||
|
||||
|
||||
def _unwrap_nested_schema(candidate: Any) -> dict | None:
|
||||
"""Return ``candidate`` as a dict, unwrapping a nested ``schema`` if present.
|
||||
|
||||
Returns ``None`` if ``candidate`` is not (and cannot be parsed into) a dict.
|
||||
"""
|
||||
cand = _maybe_parse_json_dict(candidate)
|
||||
if not isinstance(cand, dict):
|
||||
return None
|
||||
nested = cand.get("schema")
|
||||
return nested if isinstance(nested, dict) else cand
|
||||
|
||||
|
||||
def _schema_from_json_schema_field(js_wr: Any) -> dict | None:
|
||||
"""Extract the JSON Schema object from Chat Completions ``json_schema`` payload.
|
||||
|
||||
Accepts:
|
||||
- ``JsonSchemaResponseFormat`` (Pydantic) with ``schema`` / ``json_schema`` field
|
||||
- dict in OpenAI shape ``{"name": ..., "schema": {...}}``
|
||||
- dict with ``json_schema`` key holding either the schema or a nested wrapper
|
||||
- dict that is already a JSON Schema document (some clients omit the wrapper)
|
||||
- JSON strings for any of the above
|
||||
"""
|
||||
if js_wr is None:
|
||||
return None
|
||||
|
||||
parsed_wr = _maybe_parse_json_dict(js_wr)
|
||||
if parsed_wr is not None:
|
||||
js_wr = parsed_wr
|
||||
|
||||
if hasattr(js_wr, "model_dump"):
|
||||
for by_alias in (True, False):
|
||||
try:
|
||||
data = js_wr.model_dump(by_alias=by_alias, exclude_none=False)
|
||||
except TypeError:
|
||||
data = js_wr.model_dump(by_alias=by_alias)
|
||||
out = _unwrap_nested_schema(data.get("schema") or data.get("json_schema"))
|
||||
if out is not None:
|
||||
return out
|
||||
inner_attr = getattr(js_wr, "json_schema", None)
|
||||
return inner_attr if isinstance(inner_attr, dict) else None
|
||||
|
||||
if isinstance(js_wr, dict):
|
||||
for key in ("schema", "json_schema"):
|
||||
out = _unwrap_nested_schema(js_wr.get(key))
|
||||
if out is not None:
|
||||
return out
|
||||
return js_wr
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _schema_dict_from_chat_response_format(
|
||||
rf: AnyResponseFormat | dict | None,
|
||||
) -> dict | None:
|
||||
"""JSON schema dict from Chat Completions ``request.response_format`` only."""
|
||||
if rf is None:
|
||||
return None
|
||||
rf_type = _response_format_type(rf)
|
||||
if rf_type == "json_object":
|
||||
return {"type": "object"}
|
||||
if rf_type != "json_schema":
|
||||
return None
|
||||
js_wr = (
|
||||
rf.get("json_schema")
|
||||
if isinstance(rf, dict)
|
||||
else getattr(rf, "json_schema", None)
|
||||
)
|
||||
return _schema_from_json_schema_field(js_wr)
|
||||
|
||||
|
||||
def _schema_dict_from_structured_outputs(
|
||||
so: StructuredOutputsParams | None,
|
||||
) -> dict | None:
|
||||
"""Schema dict from ``structured_outputs`` (``json`` / ``json_object``).
|
||||
|
||||
Same unwrapping as ``json_schema``. ``json`` is expected to be ``str`` or
|
||||
``dict`` (enforced by ``StructuredOutputsParams`` / request models); other
|
||||
types raise ``ValueError`` only if a caller bypasses that validation.
|
||||
"""
|
||||
if so is None:
|
||||
return None
|
||||
if so.json_object:
|
||||
return {"type": "object"}
|
||||
raw: Any = so.json
|
||||
if raw is None:
|
||||
return None
|
||||
|
||||
if hasattr(raw, "model_dump"):
|
||||
out = _schema_from_json_schema_field(raw)
|
||||
if out is None:
|
||||
raise ValueError(
|
||||
"structured_outputs.json model has no extractable JSON Schema."
|
||||
)
|
||||
return out
|
||||
|
||||
if isinstance(raw, str):
|
||||
if not raw.strip():
|
||||
raise ValueError("structured_outputs.json cannot be empty.")
|
||||
check_json_nesting(raw)
|
||||
try:
|
||||
raw = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError("structured_outputs.json must be valid JSON.") from e
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError("structured_outputs.json must decode to a JSON object.")
|
||||
|
||||
if isinstance(raw, Mapping):
|
||||
body = raw if isinstance(raw, dict) else dict(raw)
|
||||
return _schema_from_json_schema_field(body) or body
|
||||
|
||||
raise ValueError(
|
||||
f"structured_outputs.json has unsupported type {type(raw).__name__}."
|
||||
)
|
||||
|
||||
|
||||
def _melody_sources_to_vllm(
|
||||
sources: list[Source], position_to_source: PositionToSource | None
|
||||
) -> list[CitationSource]:
|
||||
"""Resolve melody ``Source`` addresses into :class:`CitationSource` objects.
|
||||
|
||||
Melody reports each source as ``(tool_call_index, tool_result_indices)``.
|
||||
``position_to_source`` is built by
|
||||
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._build_position_to_source`
|
||||
and forwarded through ``chat_template_kwargs``; each index fans out to one
|
||||
resolved source, and unresolved positions are skipped. Without a map (a
|
||||
request that did not come through the Cohere v2 handler) every source is
|
||||
dropped and the serving layer discards the citation at wire-coercion time.
|
||||
"""
|
||||
if not position_to_source:
|
||||
return []
|
||||
out: list[CitationSource] = []
|
||||
for source in sources:
|
||||
for idx in source.tool_result_indices:
|
||||
info = position_to_source.get((source.tool_call_index, idx))
|
||||
if info is not None:
|
||||
out.append(info)
|
||||
return out
|
||||
|
||||
|
||||
def _melody_citations_to_vllm(
|
||||
citations: list[FilterCitation], position_to_source: PositionToSource | None
|
||||
) -> list[Citation] | None:
|
||||
"""Convert melody ``FilterCitation`` objects into :class:`Citation`.
|
||||
|
||||
Citations whose sources all fail to resolve are still returned with
|
||||
``sources=[]``; the serving layer drops them so the fail-closed policy
|
||||
lives in one place.
|
||||
"""
|
||||
if not citations:
|
||||
return None
|
||||
return [
|
||||
Citation(
|
||||
start=c.start_index,
|
||||
end=c.end_index,
|
||||
text=c.text,
|
||||
sources=_melody_sources_to_vllm(c.sources, position_to_source),
|
||||
type="THINKING_CONTENT" if c.is_thinking else "TEXT_CONTENT",
|
||||
)
|
||||
for c in citations
|
||||
]
|
||||
|
||||
|
||||
class CohereCommandParser(DelegatingParser):
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
*args,
|
||||
model_config: ModelConfig | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(tokenizer, tools, *args, model_config=model_config, **kwargs)
|
||||
self._model_config = model_config
|
||||
self.start_token_id = tokenizer.convert_tokens_to_ids("<|START_THINKING|>")
|
||||
self.end_token_id = tokenizer.convert_tokens_to_ids("<|END_THINKING|>")
|
||||
shim = self._reasoning_parser or self._tool_parser
|
||||
assert isinstance(
|
||||
shim, (BaseCohereCommandReasoningParser, BaseCohereCommandToolParser)
|
||||
)
|
||||
options = getattr(PyFilterOptions(), shim.melody_preset)()
|
||||
self._streaming = PyFilter(options)
|
||||
self._unary = PyFilter(options)
|
||||
# Citations from the most recent ``parse`` call, surfaced on
|
||||
# ``CohereChatMessage`` by ``CohereServingChatV2``.
|
||||
self.last_unary_citations: list[Citation] | None = None
|
||||
# Melody may emit a tool-call id before the function name; hold it until
|
||||
# the first name delta so clients receive both together.
|
||||
self._pending_tool_ids: dict[int, str] = {}
|
||||
ctk = kwargs.get("chat_template_kwargs") or {}
|
||||
raw_map = ctk.get(POSITION_TO_SOURCE_KEY)
|
||||
self._position_to_source: PositionToSource | None = (
|
||||
raw_map if isinstance(raw_map, Mapping) else None
|
||||
)
|
||||
|
||||
def adjust_request(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
# keep special tokens to ensure melody can correctly identify boundaries.
|
||||
request.skip_special_tokens = False
|
||||
request = self._apply_structural_tags(request)
|
||||
return super().adjust_request(request)
|
||||
|
||||
def _apply_structural_tags(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
"""Fold ``tools`` and any JSON schema into a Cohere ``structural_tag``."""
|
||||
so = request.structured_outputs
|
||||
if so is not None and so.structural_tag:
|
||||
return request
|
||||
# Schema: prefer ``response_format`` (OpenAI Chat Completions), then
|
||||
# ``structured_outputs.json`` / ``json_object`` (vLLM direct). Tools stay
|
||||
# on ``request.tools``.
|
||||
rf = (
|
||||
request.response_format
|
||||
if isinstance(request, ChatCompletionRequest)
|
||||
else None
|
||||
)
|
||||
if rf is not None and _response_format_type(rf) == "structural_tag":
|
||||
return request
|
||||
model_architecture = (
|
||||
self._model_config.architecture if self._model_config is not None else None
|
||||
)
|
||||
tools = request.tools
|
||||
# ``response_format`` wins if both it and ``structured_outputs`` supply JSON.
|
||||
schema = _schema_dict_from_chat_response_format(rf)
|
||||
if schema is None:
|
||||
schema = _schema_dict_from_structured_outputs(so)
|
||||
if schema is None and not _has_effective_tools(tools):
|
||||
return request
|
||||
if model_architecture is None:
|
||||
return request
|
||||
result = convert_schema_to_structural_tags(
|
||||
schema=schema,
|
||||
tools=tools,
|
||||
model_architecture=model_architecture,
|
||||
)
|
||||
if result is None:
|
||||
# Unsupported architectures are not in ``MODEL_TO_TAG_STYLE``.
|
||||
raise ValueError(
|
||||
"Failed to build structural_tag guided decoding constraints from "
|
||||
"this request's JSON schema and/or tools. The configured model "
|
||||
f"architecture ({model_architecture!r}) does not support Cohere "
|
||||
"command structural tags, or the schema cannot be expressed in "
|
||||
"that format."
|
||||
)
|
||||
request.structured_outputs = StructuredOutputsParams(structural_tag=result)
|
||||
# Folded JSON constraints into ``structural_tag``; drop ``response_format``
|
||||
# when it was the source so ``to_sampling_params`` does not also set ``json`` /
|
||||
# ``json_object`` (mutually exclusive in ``StructuredOutputsParams``).
|
||||
if isinstance(request, ChatCompletionRequest) and rf is not None:
|
||||
rf_type = _response_format_type(rf)
|
||||
if rf_type in ("json_schema", "json_object"):
|
||||
request.response_format = None
|
||||
return request
|
||||
|
||||
def _apply_structural_tag(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
# Note(arpera):
|
||||
# No need to go through _apply_structural_tag in abstract_parser.py
|
||||
# since we have already go through _apply_structural_tags here
|
||||
# So, override this method to return request as is
|
||||
return request
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
count = 0
|
||||
depth = 0
|
||||
for token_id in token_ids:
|
||||
if token_id == self.start_token_id:
|
||||
depth += 1
|
||||
continue
|
||||
if token_id == self.end_token_id:
|
||||
if depth > 0:
|
||||
depth -= 1
|
||||
continue
|
||||
if depth > 0:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
model_output_token_ids: Sequence[int] = (),
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
result = self._unary.process_full_text(model_output)
|
||||
self.last_unary_citations = _melody_citations_to_vllm(
|
||||
result.citations, self._position_to_source
|
||||
)
|
||||
tool_calls: list[FunctionCall] | None = None
|
||||
if self._tools_allowed(request):
|
||||
tool_calls = [
|
||||
FunctionCall(id=tc.id, name=tc.name, arguments=tc.arguments)
|
||||
for tc in result.tool_calls
|
||||
] or None
|
||||
return result.reasoning, result.content, tool_calls
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
results = [self._streaming.write_decoded(delta_text)]
|
||||
if finished:
|
||||
results.append(self._streaming.flush_partials())
|
||||
return self._to_delta(results, request)
|
||||
|
||||
def _tools_allowed(self, request: ChatCompletionRequest | ResponsesRequest) -> bool:
|
||||
return self._tool_parser is not None and request.tool_choice != "none"
|
||||
|
||||
def _to_delta(
|
||||
self,
|
||||
results: list[FilterAggregatedResult],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> CohereDeltaMessage | None:
|
||||
content = self._join(r.content for r in results)
|
||||
reasoning = (
|
||||
self._join(r.reasoning for r in results)
|
||||
if request.include_reasoning
|
||||
else None
|
||||
)
|
||||
tool_calls: list[DeltaToolCall] = []
|
||||
citations: list[Citation] = []
|
||||
for result in results:
|
||||
if result.tool_calls and self._tools_allowed(request):
|
||||
tool_calls.extend(self._tool_call_deltas(result.tool_calls))
|
||||
citations.extend(
|
||||
_melody_citations_to_vllm(result.citations, self._position_to_source)
|
||||
or []
|
||||
)
|
||||
if not (content or reasoning or tool_calls or citations):
|
||||
return None
|
||||
# ``CohereDeltaMessage`` carries citations to the wire via
|
||||
# ``SerializeAsAny[DeltaMessage]``; it drops the field when unset so the
|
||||
# shape matches a plain ``DeltaMessage``.
|
||||
message = CohereDeltaMessage()
|
||||
if content is not None:
|
||||
message.content = content
|
||||
if reasoning is not None:
|
||||
message.reasoning = reasoning
|
||||
if tool_calls:
|
||||
message.tool_calls = tool_calls
|
||||
if citations:
|
||||
message.citations = citations
|
||||
return message
|
||||
|
||||
@staticmethod
|
||||
def _join(parts: Any) -> str | None:
|
||||
present = [p for p in parts if p is not None]
|
||||
return "".join(present) if present else None
|
||||
|
||||
def _tool_call_deltas(
|
||||
self, melody_tool_calls: list[AccumulatedToolCall]
|
||||
) -> list[DeltaToolCall]:
|
||||
tool_calls: list[DeltaToolCall] = []
|
||||
for tc in melody_tool_calls:
|
||||
name = tc.name or None
|
||||
arguments = tc.arguments or None
|
||||
if tc.id:
|
||||
self._pending_tool_ids[tc.index] = tc.id
|
||||
# Empty strings are melody streaming placeholders; skip fragments
|
||||
# that carry neither a name nor argument text.
|
||||
if name is None and arguments is None:
|
||||
continue
|
||||
function = DeltaFunctionCall()
|
||||
if name is not None:
|
||||
function.name = name
|
||||
if arguments is not None:
|
||||
function.arguments = arguments
|
||||
delta = DeltaToolCall(index=tc.index, function=function)
|
||||
if name is not None:
|
||||
resolved_id = tc.id or self._pending_tool_ids.pop(tc.index, None)
|
||||
if resolved_id is not None:
|
||||
delta.id = resolved_id
|
||||
delta.type = "function"
|
||||
tool_calls.append(delta)
|
||||
return tool_calls
|
||||
@@ -1,144 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V3.2 parser: DSML tool calls with ``function_calls`` wrapper.
|
||||
|
||||
DeepSeek V3.2 output format::
|
||||
|
||||
<|DSML|function_calls>
|
||||
<|DSML|invoke name="func_name">
|
||||
<|DSML|parameter name="location" string="true">杭州</|DSML|parameter>
|
||||
<|DSML|parameter name="count" string="false">5</|DSML|parameter>
|
||||
</|DSML|invoke>
|
||||
</|DSML|function_calls>
|
||||
|
||||
This is identical to DeepSeek V4 except for the outer wrapper
|
||||
(``function_calls`` instead of ``tool_calls``) and the absence of
|
||||
``<think>``/``</think>`` reasoning tags.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.parser.deepseek_v4 import (
|
||||
DSML_INVOKE_END,
|
||||
DSML_INVOKE_NAME_END,
|
||||
DSML_INVOKE_PREFIX,
|
||||
DSML_PARAM_CLOSE,
|
||||
DSML_PARAM_START,
|
||||
_dsml_arg_converter,
|
||||
_unwrap_wrapper_args,
|
||||
)
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
_DSML = "|DSML|"
|
||||
|
||||
DSML_FUNC_START = f"<{_DSML}function_calls>"
|
||||
DSML_FUNC_END = f"</{_DSML}function_calls>"
|
||||
|
||||
|
||||
@functools.cache
|
||||
def deepseek_v32_config() -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="deepseek_v32",
|
||||
initial_state=ParserState.CONTENT,
|
||||
wait_for_reasoning=False,
|
||||
terminals={
|
||||
"TOOL_START": DSML_FUNC_START,
|
||||
"TOOL_END": DSML_FUNC_END,
|
||||
"INVOKE_PREFIX": DSML_INVOKE_PREFIX,
|
||||
"INVOKE_NAME_END": DSML_INVOKE_NAME_END,
|
||||
"INVOKE_END": DSML_INVOKE_END,
|
||||
"PARAM_START": DSML_PARAM_START,
|
||||
"PARAM_CLOSE": DSML_PARAM_CLOSE,
|
||||
},
|
||||
token_id_terminals={
|
||||
"TOOL_START": DSML_FUNC_START,
|
||||
"TOOL_END": DSML_FUNC_END,
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "INVOKE_PREFIX"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.CONTENT, "INVOKE_PREFIX"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "INVOKE_NAME_END"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "INVOKE_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
# Parallel tool calls
|
||||
(ParserState.TOOL_BETWEEN, "INVOKE_PREFIX"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
# A tool call ends the turn: stay in TOOL_BETWEEN so any text
|
||||
# after the block is dropped.
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(),
|
||||
),
|
||||
},
|
||||
content_events={
|
||||
ParserState.CONTENT: EventType.TEXT_CHUNK,
|
||||
ParserState.TOOL_NAME: EventType.TOOL_NAME,
|
||||
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
|
||||
},
|
||||
arg_converter=_dsml_arg_converter,
|
||||
arg_structural_chars=frozenset(">"),
|
||||
strip_content_whitespace_with_tools=False,
|
||||
tool_args_json=False,
|
||||
)
|
||||
|
||||
|
||||
class DeepSeekV32Parser(ParserEngine):
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
kwargs.pop("chat_template_kwargs", None)
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
tools,
|
||||
parser_engine_config=deepseek_v32_config(),
|
||||
**kwargs,
|
||||
)
|
||||
self._arg_converter = self._convert_args
|
||||
|
||||
def _convert_args(self, raw_args: str, partial: bool) -> str:
|
||||
result = _dsml_arg_converter(raw_args, partial)
|
||||
if not self._tools:
|
||||
return result
|
||||
func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None)
|
||||
return _unwrap_wrapper_args(result, self._tools, func_name)
|
||||
@@ -1,277 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4 parser: ``<think>``/``</think>``
|
||||
reasoning plus DSML tool calls in a single state machine.
|
||||
|
||||
DeepSeek V4 output format::
|
||||
|
||||
<think>
|
||||
...reasoning...
|
||||
</think>
|
||||
<|DSML|tool_calls>
|
||||
<|DSML|invoke name="func_name">
|
||||
<|DSML|parameter name="location" string="true">杭州</|DSML|parameter>
|
||||
<|DSML|parameter name="count" string="false">5</|DSML|parameter>
|
||||
</|DSML|invoke>
|
||||
</|DSML|tool_calls>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.tool_parsers.utils import find_tool_properties
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
_DSML = "|DSML|"
|
||||
|
||||
DSML_THINK_START = "<think>"
|
||||
DSML_THINK_END = "</think>"
|
||||
DSML_TOOL_START = f"<{_DSML}tool_calls>"
|
||||
DSML_TOOL_END = f"</{_DSML}tool_calls>"
|
||||
DSML_INVOKE_PREFIX = f'<{_DSML}invoke name="'
|
||||
DSML_INVOKE_NAME_END = '">'
|
||||
DSML_INVOKE_END = f"</{_DSML}invoke>"
|
||||
DSML_PARAM_START = f"<{_DSML}parameter"
|
||||
DSML_PARAM_CLOSE = f"</{_DSML}parameter>"
|
||||
|
||||
# Spellings variants of ``DSML_TOOL_START`` observed in production.
|
||||
DSML_TOOL_START_VARIANTS: tuple[str, ...] = (
|
||||
f"<{_DSML}toolcalls>",
|
||||
f"<{_DSML}tool>",
|
||||
)
|
||||
|
||||
|
||||
def _param_patterns(
|
||||
param_start: str, param_close: str
|
||||
) -> tuple[re.Pattern, re.Pattern]:
|
||||
"""Complete and trailing-partial parameter regexes for one DSML dialect.
|
||||
|
||||
The ``string`` attribute is optional: the model sometimes omits it, and
|
||||
dropping such a parameter hands the client a tool call with no arguments.
|
||||
"""
|
||||
start, close = re.escape(param_start), re.escape(param_close)
|
||||
head = rf'{start}\s+name="([^"]+)"(?:\s+string="([^"]*)")?\s*>'
|
||||
return (
|
||||
re.compile(rf"{head}(.*?)(?:{close}|(?={start}\s+name=))", re.DOTALL),
|
||||
re.compile(rf"{head}(.*)$", re.DOTALL),
|
||||
)
|
||||
|
||||
|
||||
_PARAM_RE, _PARTIAL_PARAM_RE = _param_patterns(DSML_PARAM_START, DSML_PARAM_CLOSE)
|
||||
|
||||
|
||||
def _dsml_arg_converter(
|
||||
raw_args: str,
|
||||
partial: bool,
|
||||
*,
|
||||
param_re: re.Pattern = _PARAM_RE,
|
||||
partial_param_re: re.Pattern = _PARTIAL_PARAM_RE,
|
||||
) -> str:
|
||||
params: dict[str, object] = {}
|
||||
|
||||
last_end = 0
|
||||
# A missing ``string`` attribute is treated like ``string="false"``: try
|
||||
# JSON, fall back to the literal text.
|
||||
for m in param_re.finditer(raw_args):
|
||||
name, is_str, value = m.group(1), m.group(2), m.group(3)
|
||||
if is_str == "true":
|
||||
params[name] = value
|
||||
else:
|
||||
try:
|
||||
params[name] = json.loads(value)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
params[name] = value
|
||||
last_end = m.end()
|
||||
|
||||
if partial:
|
||||
pm = partial_param_re.search(raw_args, last_end)
|
||||
if pm:
|
||||
name, is_str, value = pm.group(1), pm.group(2), pm.group(3)
|
||||
if is_str == "true":
|
||||
params[name] = value
|
||||
else:
|
||||
with contextlib.suppress(json.JSONDecodeError, ValueError):
|
||||
params[name] = json.loads(value)
|
||||
|
||||
return json.dumps(params, ensure_ascii=False)
|
||||
|
||||
|
||||
def _unwrap_wrapper_args(
|
||||
args_json: str,
|
||||
tools: list[Tool] | None,
|
||||
func_name: str | None,
|
||||
) -> str:
|
||||
if not tools or not func_name:
|
||||
return args_json
|
||||
try:
|
||||
args = json.loads(args_json)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return args_json
|
||||
if not isinstance(args, dict):
|
||||
return args_json
|
||||
properties = find_tool_properties(tools, func_name)
|
||||
if not properties:
|
||||
return args_json
|
||||
allowed = set(properties.keys())
|
||||
for wrapper in ("arguments", "input"):
|
||||
if set(args.keys()) != {wrapper} or wrapper in allowed:
|
||||
continue
|
||||
inner = args[wrapper]
|
||||
if isinstance(inner, str):
|
||||
try:
|
||||
inner = json.loads(inner)
|
||||
except json.JSONDecodeError:
|
||||
return args_json
|
||||
if isinstance(inner, dict) and set(inner.keys()).issubset(allowed):
|
||||
return json.dumps(inner, ensure_ascii=False)
|
||||
return args_json
|
||||
|
||||
|
||||
@functools.cache
|
||||
def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="deepseek_v4",
|
||||
initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
|
||||
wait_for_reasoning=thinking,
|
||||
terminals={
|
||||
"THINK_START": DSML_THINK_START,
|
||||
"THINK_END": DSML_THINK_END,
|
||||
"TOOL_START": (DSML_TOOL_START, *DSML_TOOL_START_VARIANTS),
|
||||
"TOOL_END": DSML_TOOL_END,
|
||||
"INVOKE_PREFIX": DSML_INVOKE_PREFIX,
|
||||
"INVOKE_NAME_END": DSML_INVOKE_NAME_END,
|
||||
"INVOKE_END": DSML_INVOKE_END,
|
||||
"PARAM_START": DSML_PARAM_START,
|
||||
"PARAM_CLOSE": DSML_PARAM_CLOSE,
|
||||
},
|
||||
token_id_terminals={
|
||||
"THINK_START": DSML_THINK_START,
|
||||
"THINK_END": DSML_THINK_END,
|
||||
"TOOL_START": DSML_TOOL_START,
|
||||
"TOOL_END": DSML_TOOL_END,
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
# Absorb a bare </think> with no prior <think>
|
||||
(ParserState.CONTENT, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
# Absorb a duplicate <think> while already reasoning
|
||||
(ParserState.REASONING, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
# Tool call beginning while still inside <think>
|
||||
(ParserState.REASONING, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "INVOKE_PREFIX"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.CONTENT, "INVOKE_PREFIX"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "INVOKE_NAME_END"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "INVOKE_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
# Parallel tool calls
|
||||
(ParserState.TOOL_BETWEEN, "INVOKE_PREFIX"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
# A tool call ends the turn: stay in TOOL_BETWEEN so any text
|
||||
# after the block is dropped.
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(),
|
||||
),
|
||||
},
|
||||
content_events={
|
||||
ParserState.CONTENT: EventType.TEXT_CHUNK,
|
||||
ParserState.REASONING: EventType.REASONING_CHUNK,
|
||||
ParserState.TOOL_NAME: EventType.TOOL_NAME,
|
||||
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
|
||||
},
|
||||
arg_converter=_dsml_arg_converter,
|
||||
arg_structural_chars=frozenset(">"),
|
||||
strip_content_whitespace_with_tools=False,
|
||||
tool_args_json=False,
|
||||
)
|
||||
|
||||
|
||||
class DeepSeekV4Parser(ParserEngine):
|
||||
parser_config = staticmethod(deepseek_v4_config)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.pop("chat_template_kwargs", None) or {}
|
||||
thinking = bool(
|
||||
chat_kwargs.get("thinking") or chat_kwargs.get("enable_thinking")
|
||||
)
|
||||
if "thinking" not in chat_kwargs and "enable_thinking" not in chat_kwargs:
|
||||
thinking = True
|
||||
thinking = thinking and chat_kwargs.get("reasoning_effort") != "none"
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
tools,
|
||||
parser_engine_config=self.parser_config(thinking=thinking),
|
||||
**kwargs,
|
||||
)
|
||||
self._arg_converter = self._convert_args
|
||||
|
||||
def _convert_args(self, raw_args: str, partial: bool) -> str:
|
||||
converter = self.parser_engine_config.arg_converter
|
||||
assert converter is not None
|
||||
result = converter(raw_args, partial)
|
||||
if not self._tools:
|
||||
return result
|
||||
func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None)
|
||||
return _unwrap_wrapper_args(result, self._tools, func_name)
|
||||
@@ -1,54 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4.1 reasoning and spaced DSML tool calls."""
|
||||
|
||||
import functools
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm.parser.deepseek_v4 import (
|
||||
DeepSeekV4Parser,
|
||||
_dsml_arg_converter,
|
||||
_param_patterns,
|
||||
deepseek_v4_config,
|
||||
)
|
||||
from vllm.parser.engine.parser_engine_config import ParserEngineConfig
|
||||
|
||||
DSML_TOOL_START = "<|DSML| calls>"
|
||||
DSML_TOOL_END = "</|DSML| calls>"
|
||||
DSML_INVOKE_PREFIX = '<|DSML| invoke name="'
|
||||
DSML_INVOKE_END = "</|DSML| invoke>"
|
||||
DSML_PARAM_START = "<|DSML| parameter"
|
||||
DSML_PARAM_CLOSE = "</|DSML| parameter>"
|
||||
|
||||
_PARAM_RE, _PARTIAL_PARAM_RE = _param_patterns(DSML_PARAM_START, DSML_PARAM_CLOSE)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def deepseek_v41_config(thinking: bool = False) -> ParserEngineConfig:
|
||||
config = deepseek_v4_config(thinking=thinking)
|
||||
terminal_overrides = {
|
||||
"TOOL_START": DSML_TOOL_START,
|
||||
"TOOL_END": DSML_TOOL_END,
|
||||
"INVOKE_PREFIX": DSML_INVOKE_PREFIX,
|
||||
"INVOKE_END": DSML_INVOKE_END,
|
||||
"PARAM_START": DSML_PARAM_START,
|
||||
"PARAM_CLOSE": DSML_PARAM_CLOSE,
|
||||
}
|
||||
return replace(
|
||||
config,
|
||||
name="deepseek_v41",
|
||||
terminals={**config.terminals, **terminal_overrides},
|
||||
token_id_terminals={
|
||||
key: terminal_overrides.get(key, value)
|
||||
for key, value in config.token_id_terminals.items()
|
||||
},
|
||||
arg_converter=functools.partial(
|
||||
_dsml_arg_converter,
|
||||
param_re=_PARAM_RE,
|
||||
partial_param_re=_PARTIAL_PARAM_RE,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class DeepSeekV41Parser(DeepSeekV4Parser):
|
||||
parser_config = staticmethod(deepseek_v41_config)
|
||||
@@ -7,112 +7,10 @@ names so that :class:`ReasoningParserManager` and
|
||||
:class:`ToolParserManager` can load them lazily.
|
||||
"""
|
||||
|
||||
from vllm.parser.deepseek_v4 import DeepSeekV4Parser
|
||||
from vllm.parser.deepseek_v32 import DeepSeekV32Parser
|
||||
from vllm.parser.deepseek_v41 import DeepSeekV41Parser
|
||||
from vllm.parser.engine.adapters import make_adapters
|
||||
from vllm.parser.gemma4 import Gemma4Parser
|
||||
from vllm.parser.glm47_moe import Glm47MoeParser
|
||||
from vllm.parser.granite import GraniteParser
|
||||
from vllm.parser.granite_thinking import GraniteThinkingParser
|
||||
from vllm.parser.inkling import InklingParser
|
||||
from vllm.parser.kimi_k2 import KimiK2Parser
|
||||
from vllm.parser.mimo import MiMoParser
|
||||
from vllm.parser.minimax_m2 import MinimaxM2Parser
|
||||
from vllm.parser.minimax_m3 import MinimaxM3Parser
|
||||
from vllm.parser.mistral import MistralParser
|
||||
from vllm.parser.nemotron_v3 import NemotronV3Parser
|
||||
from vllm.parser.plamo3 import Plamo3Parser
|
||||
from vllm.parser.qwen3 import Qwen3Parser
|
||||
from vllm.parser.seed_oss import SeedOssParser
|
||||
from vllm.parser.step3p5 import Step3p5Parser
|
||||
|
||||
(
|
||||
DeepSeekV32ParserReasoningAdapter,
|
||||
DeepSeekV32ParserToolAdapter,
|
||||
) = make_adapters(DeepSeekV32Parser)
|
||||
|
||||
(
|
||||
DeepSeekV4ParserReasoningAdapter,
|
||||
DeepSeekV4ParserToolAdapter,
|
||||
) = make_adapters(DeepSeekV4Parser)
|
||||
|
||||
(
|
||||
DeepSeekV41ParserReasoningAdapter,
|
||||
DeepSeekV41ParserToolAdapter,
|
||||
) = make_adapters(DeepSeekV41Parser)
|
||||
|
||||
(
|
||||
MinimaxM2ParserReasoningAdapter,
|
||||
MinimaxM2ParserToolAdapter,
|
||||
) = make_adapters(MinimaxM2Parser)
|
||||
|
||||
(
|
||||
MinimaxM3ParserReasoningAdapter,
|
||||
MinimaxM3ParserToolAdapter,
|
||||
) = make_adapters(MinimaxM3Parser)
|
||||
|
||||
(
|
||||
Gemma4ParserReasoningAdapter,
|
||||
Gemma4ParserToolAdapter,
|
||||
) = make_adapters(Gemma4Parser)
|
||||
|
||||
(
|
||||
GraniteParserReasoningAdapter,
|
||||
GraniteParserToolAdapter,
|
||||
) = make_adapters(GraniteParser)
|
||||
|
||||
(
|
||||
GraniteThinkingParserReasoningAdapter,
|
||||
GraniteThinkingParserToolAdapter,
|
||||
) = make_adapters(GraniteThinkingParser)
|
||||
|
||||
(
|
||||
NemotronV3ParserReasoningAdapter,
|
||||
NemotronV3ParserToolAdapter,
|
||||
) = make_adapters(NemotronV3Parser)
|
||||
|
||||
(
|
||||
Qwen3ParserReasoningAdapter,
|
||||
Qwen3ParserToolAdapter,
|
||||
) = make_adapters(Qwen3Parser)
|
||||
|
||||
(
|
||||
MiMoParserReasoningAdapter,
|
||||
MiMoParserToolAdapter,
|
||||
) = make_adapters(MiMoParser)
|
||||
|
||||
(
|
||||
SeedOssParserReasoningAdapter,
|
||||
SeedOssParserToolAdapter,
|
||||
) = make_adapters(SeedOssParser)
|
||||
|
||||
(
|
||||
Step3p5ParserReasoningAdapter,
|
||||
Step3p5ParserToolAdapter,
|
||||
) = make_adapters(Step3p5Parser)
|
||||
|
||||
(
|
||||
Glm47MoeParserReasoningAdapter,
|
||||
Glm47MoeParserToolAdapter,
|
||||
) = make_adapters(Glm47MoeParser)
|
||||
|
||||
(
|
||||
KimiK2ParserReasoningAdapter,
|
||||
KimiK2ParserToolAdapter,
|
||||
) = make_adapters(KimiK2Parser)
|
||||
|
||||
(
|
||||
InklingParserReasoningAdapter,
|
||||
InklingParserToolAdapter,
|
||||
) = make_adapters(InklingParser)
|
||||
|
||||
(
|
||||
MistralParserReasoningAdapter,
|
||||
MistralParserToolAdapter,
|
||||
) = make_adapters(MistralParser)
|
||||
|
||||
(
|
||||
Plamo3ParserReasoningAdapter,
|
||||
Plamo3ParserToolAdapter,
|
||||
) = make_adapters(Plamo3Parser)
|
||||
|
||||
@@ -1,636 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Gemma4 parser.
|
||||
|
||||
Handles channel-based reasoning plus custom tool call format in a single
|
||||
state machine::
|
||||
|
||||
<|channel>thought
|
||||
...reasoning...<channel|>
|
||||
<|tool_call>call:func_name{key:<|"|>value<|"|>,num:42}<tool_call|>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ArgValueExtender,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
CHANNEL_START = "<|channel>"
|
||||
CHANNEL_END = "<channel|>"
|
||||
TOOL_CALL_START = "<|tool_call>"
|
||||
TOOL_CALL_END = "<tool_call|>"
|
||||
STRING_DELIM = '<|"|>'
|
||||
_DELIM_LEN = len(STRING_DELIM)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gemma4 argument parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PARTIAL_DELIM_SUFFIXES = tuple(
|
||||
STRING_DELIM[:k] for k in range(len(STRING_DELIM), 0, -1)
|
||||
)
|
||||
|
||||
|
||||
def _strip_partial_delim(value: str) -> str:
|
||||
"""Strip a trailing partial ``STRING_DELIM`` prefix from *value*.
|
||||
|
||||
Prevents partial delimiters from leaking into the streamed JSON diff.
|
||||
"""
|
||||
for suffix in _PARTIAL_DELIM_SUFFIXES:
|
||||
if value.endswith(suffix):
|
||||
return value[: -len(suffix)]
|
||||
return value
|
||||
|
||||
|
||||
def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
|
||||
return _parse_gemma4_fields(args_str, partial=partial)[0]
|
||||
|
||||
|
||||
def _parse_gemma4_fields(
|
||||
args_str: str, *, partial: bool = False
|
||||
) -> tuple[dict, str | None]:
|
||||
"""Parse Gemma4's custom key:value format into a Python dict.
|
||||
|
||||
Format examples::
|
||||
|
||||
location:<|"|>Tokyo<|"|>
|
||||
location:<|"|>San Francisco<|"|>,unit:<|"|>celsius<|"|>
|
||||
count:42,flag:true
|
||||
nested:{inner_key:<|"|>val<|"|>}
|
||||
items:[<|"|>a<|"|>,<|"|>b<|"|>]
|
||||
|
||||
Args:
|
||||
args_str: The raw Gemma4 argument string.
|
||||
partial: When True (streaming), bare values at end of string are
|
||||
omitted because they may be incomplete and type-unstable
|
||||
(e.g. partial boolean parsed as bare string).
|
||||
|
||||
Returns a dict ready for ``json.dumps()`` and the key of the trailing
|
||||
top-level ``STRING_DELIM`` value left unterminated, if any.
|
||||
|
||||
"""
|
||||
if not args_str or not args_str.strip():
|
||||
return {}, None
|
||||
|
||||
result: dict = {}
|
||||
open_key: str | None = None
|
||||
i = 0
|
||||
n = len(args_str)
|
||||
|
||||
while i < n:
|
||||
while i < n and args_str[i] in (" ", ",", "\n", "\t"):
|
||||
i += 1
|
||||
if i >= n:
|
||||
break
|
||||
|
||||
key_start = i
|
||||
while i < n and args_str[i] != ":":
|
||||
i += 1
|
||||
if i >= n:
|
||||
break
|
||||
key = args_str[key_start:i].strip()
|
||||
if key.startswith(STRING_DELIM) and key.endswith(STRING_DELIM):
|
||||
key = key[_DELIM_LEN:-_DELIM_LEN]
|
||||
i += 1
|
||||
|
||||
if i >= n:
|
||||
if not partial:
|
||||
result[key] = ""
|
||||
break
|
||||
|
||||
while i < n and args_str[i] in (" ", "\n", "\t"):
|
||||
i += 1
|
||||
if i >= n:
|
||||
if not partial:
|
||||
result[key] = ""
|
||||
break
|
||||
|
||||
if args_str.startswith(STRING_DELIM, i):
|
||||
i += _DELIM_LEN
|
||||
val_start = i
|
||||
end_pos = args_str.find(STRING_DELIM, i)
|
||||
if end_pos == -1:
|
||||
# Unterminated string — take rest, strip partial delimiter.
|
||||
value = args_str[val_start:]
|
||||
if partial:
|
||||
value = _strip_partial_delim(value)
|
||||
result[key] = value
|
||||
open_key = key
|
||||
break
|
||||
result[key] = args_str[val_start:end_pos]
|
||||
i = end_pos + _DELIM_LEN
|
||||
|
||||
elif args_str[i] == "{":
|
||||
depth = 1
|
||||
obj_start = i + 1
|
||||
i += 1
|
||||
while i < n and depth > 0:
|
||||
if args_str.startswith(STRING_DELIM, i):
|
||||
# Skip over string contents to avoid counting { inside strings
|
||||
i += _DELIM_LEN
|
||||
next_delim = args_str.find(STRING_DELIM, i)
|
||||
i = n if next_delim == -1 else next_delim + _DELIM_LEN
|
||||
continue
|
||||
if args_str[i] == "{":
|
||||
depth += 1
|
||||
elif args_str[i] == "}":
|
||||
depth -= 1
|
||||
i += 1
|
||||
if depth > 0:
|
||||
# Incomplete nested object — use i (not i-1) to avoid
|
||||
# dropping the last char, and recurse as partial.
|
||||
result[key] = _parse_gemma4_args(args_str[obj_start:i], partial=True)
|
||||
else:
|
||||
result[key] = _parse_gemma4_args(args_str[obj_start : i - 1])
|
||||
|
||||
elif args_str[i] == "[":
|
||||
depth = 1
|
||||
arr_start = i + 1
|
||||
i += 1
|
||||
while i < n and depth > 0:
|
||||
if args_str.startswith(STRING_DELIM, i):
|
||||
i += _DELIM_LEN
|
||||
next_delim = args_str.find(STRING_DELIM, i)
|
||||
i = n if next_delim == -1 else next_delim + _DELIM_LEN
|
||||
continue
|
||||
if args_str[i] == "[":
|
||||
depth += 1
|
||||
elif args_str[i] == "]":
|
||||
depth -= 1
|
||||
i += 1
|
||||
if depth > 0:
|
||||
result[key] = _parse_gemma4_array(args_str[arr_start:i], partial=True)
|
||||
else:
|
||||
result[key] = _parse_gemma4_array(args_str[arr_start : i - 1])
|
||||
|
||||
else:
|
||||
val_start = i
|
||||
while i < n and args_str[i] not in (",", "}", "]"):
|
||||
i += 1
|
||||
if partial and i >= n:
|
||||
# Value may be incomplete (e.g. partial boolean) —
|
||||
# withhold to avoid type instability during streaming.
|
||||
break
|
||||
if i == val_start:
|
||||
logger.warning(
|
||||
"Gemma4 args parser made no progress at position %d; "
|
||||
"aborting on malformed input.",
|
||||
i,
|
||||
)
|
||||
break
|
||||
raw_val = args_str[val_start:i].strip()
|
||||
if partial and raw_val.endswith("."):
|
||||
# Digits may still arrive (e.g. "108." -> "108.2");
|
||||
# withhold to avoid corrupting the streaming diff.
|
||||
break
|
||||
result[key] = raw_val
|
||||
|
||||
return result, open_key
|
||||
|
||||
|
||||
def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list:
|
||||
items: list = []
|
||||
i = 0
|
||||
n = len(arr_str)
|
||||
|
||||
while i < n:
|
||||
while i < n and arr_str[i] in (" ", ",", "\n", "\t"):
|
||||
i += 1
|
||||
if i >= n:
|
||||
break
|
||||
|
||||
if arr_str.startswith(STRING_DELIM, i):
|
||||
i += _DELIM_LEN
|
||||
end_pos = arr_str.find(STRING_DELIM, i)
|
||||
if end_pos == -1:
|
||||
items.append(arr_str[i:])
|
||||
break
|
||||
items.append(arr_str[i:end_pos])
|
||||
i = end_pos + _DELIM_LEN
|
||||
|
||||
elif arr_str[i] == "{":
|
||||
depth = 1
|
||||
obj_start = i + 1
|
||||
i += 1
|
||||
while i < n and depth > 0:
|
||||
if arr_str.startswith(STRING_DELIM, i):
|
||||
i += _DELIM_LEN
|
||||
nd = arr_str.find(STRING_DELIM, i)
|
||||
i = nd + _DELIM_LEN if nd != -1 else n
|
||||
continue
|
||||
if arr_str[i] == "{":
|
||||
depth += 1
|
||||
elif arr_str[i] == "}":
|
||||
depth -= 1
|
||||
i += 1
|
||||
if depth > 0:
|
||||
items.append(_parse_gemma4_args(arr_str[obj_start:i], partial=True))
|
||||
else:
|
||||
items.append(_parse_gemma4_args(arr_str[obj_start : i - 1]))
|
||||
|
||||
elif arr_str[i] == "[":
|
||||
depth = 1
|
||||
sub_start = i + 1
|
||||
i += 1
|
||||
while i < n and depth > 0:
|
||||
if arr_str.startswith(STRING_DELIM, i):
|
||||
i += _DELIM_LEN
|
||||
nd = arr_str.find(STRING_DELIM, i)
|
||||
i = nd + _DELIM_LEN if nd != -1 else n
|
||||
continue
|
||||
if arr_str[i] == "[":
|
||||
depth += 1
|
||||
elif arr_str[i] == "]":
|
||||
depth -= 1
|
||||
i += 1
|
||||
if depth > 0:
|
||||
items.append(_parse_gemma4_array(arr_str[sub_start:i], partial=True))
|
||||
else:
|
||||
items.append(_parse_gemma4_array(arr_str[sub_start : i - 1]))
|
||||
|
||||
else:
|
||||
val_start = i
|
||||
while i < n and arr_str[i] not in (",", "]"):
|
||||
i += 1
|
||||
if partial and i >= n:
|
||||
break
|
||||
if i == val_start:
|
||||
logger.warning(
|
||||
"Gemma4 array parser made no progress at position %d; "
|
||||
"aborting on malformed input.",
|
||||
i,
|
||||
)
|
||||
break
|
||||
raw_val = arr_str[val_start:i].strip()
|
||||
if partial and raw_val.endswith("."):
|
||||
break
|
||||
items.append(raw_val)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _drop_call_closer(text: str, paren_excess: bool) -> str:
|
||||
"""Drop a trailing ``}`` or, with more ``)`` than ``(``, a trailing ``)``.
|
||||
|
||||
The closer of the call body is dropped before parsing, so a value that
|
||||
itself ends in one loses that character until more text arrives.
|
||||
"""
|
||||
if text.endswith("}") or (text.endswith(")") and paren_excess):
|
||||
return text[:-1]
|
||||
return text
|
||||
|
||||
|
||||
def _gemma4_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
"""Convert Gemma4 custom arg format to a JSON string."""
|
||||
text = raw_args.strip()
|
||||
text = _drop_call_closer(text, text.count("(") < text.count(")"))
|
||||
parsed = _parse_gemma4_args(text, partial=partial)
|
||||
return json.dumps(parsed, ensure_ascii=False)
|
||||
|
||||
|
||||
def _gemma4_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
|
||||
"""Extend the open top-level ``STRING_DELIM`` value at the end of
|
||||
*raw_args*, if it is the last value in the converted JSON.
|
||||
|
||||
The extender streams exactly what ``_gemma4_arg_converter`` would, so it
|
||||
holds back the same tail: trailing whitespace, a dropped closer and a
|
||||
partial ``STRING_DELIM``. ``STRING_DELIM`` is a special token but not a
|
||||
lexer terminal, so unlike a tag it may reach the extender split across
|
||||
deltas; the held tail doubles as the window that detects the close.
|
||||
"""
|
||||
text = raw_args.strip()
|
||||
opens, closes = text.count("("), text.count(")")
|
||||
body = _drop_call_closer(text, opens < closes)
|
||||
params, open_key = _parse_gemma4_fields(body, partial=True)
|
||||
if open_key is None or next(reversed(params)) != open_key:
|
||||
return None
|
||||
|
||||
held_len = (
|
||||
len(raw_args)
|
||||
- len(raw_args.rstrip())
|
||||
+ len(text)
|
||||
- len(_strip_partial_delim(body))
|
||||
)
|
||||
held = raw_args[len(raw_args) - held_len :] if held_len else ""
|
||||
|
||||
def extend(delta: str) -> str | None:
|
||||
nonlocal held, opens, closes
|
||||
text = held + delta
|
||||
if STRING_DELIM in text:
|
||||
return None
|
||||
opens += delta.count("(")
|
||||
closes += delta.count(")")
|
||||
visible = _strip_partial_delim(_drop_call_closer(text.rstrip(), opens < closes))
|
||||
held = text[len(visible) :]
|
||||
return json.dumps(visible, ensure_ascii=False)[1:-1]
|
||||
|
||||
return extend
|
||||
|
||||
|
||||
@functools.cache
|
||||
def gemma4_config(thinking: bool = False) -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="gemma4",
|
||||
initial_state=ParserState.CONTENT,
|
||||
# Decides a silent tail: the E-series HF templates write no marker
|
||||
# with thinking off, and no template does after a tool response.
|
||||
wait_for_reasoning=thinking,
|
||||
turn_boundary_tokens=frozenset({"<|turn>", "<|tool_response>"}),
|
||||
terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
"CALL_PREFIX": "call:",
|
||||
"COLON": ":",
|
||||
"OPEN_BRACE": "{",
|
||||
"OPEN_PAREN": "(",
|
||||
},
|
||||
token_id_terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
},
|
||||
transitions={
|
||||
# -- Reasoning transitions --
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
# No-op: if we pre-initialised the engine to REASONING from the
|
||||
# prompt (see ``adjust_initial_state_from_prompt``) but the model
|
||||
# still emits its own ``<|channel>`` opener, swallow it instead
|
||||
# of leaking it as TEXT_CHUNK.
|
||||
(ParserState.REASONING, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
# Tool call directly from reasoning (no explicit <channel|>)
|
||||
(ParserState.REASONING, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.REASONING_END, EventType.TOOL_CALL_START),
|
||||
),
|
||||
# -- Tool call transitions --
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.REASONING_END, EventType.TOOL_CALL_START),
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "CALL_PREFIX"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(),
|
||||
),
|
||||
# Bare opener: some checkpoints emit "<|tool_call>:name{...}"
|
||||
# without the "call" prefix. Match the colon directly so the
|
||||
# tool call is not silently dropped.
|
||||
(ParserState.TOOL_PREAMBLE, "COLON"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "OPEN_BRACE"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "OPEN_PAREN"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
# Back-to-back tool calls
|
||||
(ParserState.CONTENT, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
# Absorb a bare <channel|> that arrives after we already
|
||||
# returned to CONTENT; prevents leaking it as TEXT_CHUNK.
|
||||
(ParserState.CONTENT, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
},
|
||||
content_events={
|
||||
ParserState.CONTENT: EventType.TEXT_CHUNK,
|
||||
ParserState.REASONING: EventType.REASONING_CHUNK,
|
||||
ParserState.TOOL_NAME: EventType.TOOL_NAME,
|
||||
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
|
||||
},
|
||||
arg_converter=_gemma4_arg_converter,
|
||||
arg_value_extender=_gemma4_arg_value_extender,
|
||||
tool_args_json=False,
|
||||
arg_structural_chars=frozenset(",:{}[]<"),
|
||||
preserve_tokens=frozenset({STRING_DELIM}),
|
||||
)
|
||||
|
||||
|
||||
_GEMMA4_THOUGHT_PREFIX = "thought\n"
|
||||
_GEMMA4_THOUGHT_TOKEN = "thought"
|
||||
|
||||
|
||||
class Gemma4Parser(ParserEngine):
|
||||
"""Gemma4 parser: ``<|channel>`` reasoning + ``<|tool_call>``
|
||||
tool calls in a single engine.
|
||||
|
||||
- Strips the ``thought\\n`` prefix from reasoning content
|
||||
- Sets ``skip_special_tokens=False`` so boundary tokens are visible
|
||||
- Detects ``<|tool_call>`` token as implicit reasoning end
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
thinking = chat_kwargs.get("enable_thinking", False)
|
||||
kwargs.setdefault("parser_engine_config", gemma4_config(thinking=thinking))
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
vocab = self.vocab
|
||||
self._tool_call_token_id: int | None = vocab.get("<|tool_call>")
|
||||
self._new_turn_token_id: int | None = vocab.get("<|turn>")
|
||||
self._tool_response_token_id: int | None = vocab.get("<|tool_response>")
|
||||
self._reasoning_text: str = ""
|
||||
self._prefix_stripped: bool = False
|
||||
self._is_first_feed: bool = True
|
||||
|
||||
def _reset(self, initial_state=None) -> None:
|
||||
super()._reset(initial_state=initial_state)
|
||||
self._reasoning_text = ""
|
||||
self._prefix_stripped = False
|
||||
self._is_first_feed = True
|
||||
|
||||
def _preprocess_feed(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> tuple[str, Sequence[int]]:
|
||||
if not self._is_first_feed:
|
||||
return delta_text, delta_token_ids
|
||||
self._is_first_feed = False
|
||||
|
||||
if (
|
||||
not delta_text
|
||||
or self._engine.state != ParserState.CONTENT
|
||||
or self._reasoning_start_token_id is None
|
||||
or self._reasoning_end_token_id is None
|
||||
):
|
||||
return delta_text, delta_token_ids
|
||||
|
||||
if CHANNEL_START in delta_text:
|
||||
return delta_text, delta_token_ids
|
||||
|
||||
needs_injection = (
|
||||
CHANNEL_END in delta_text
|
||||
or delta_text.startswith(_GEMMA4_THOUGHT_PREFIX)
|
||||
or delta_text == _GEMMA4_THOUGHT_TOKEN
|
||||
)
|
||||
if not needs_injection:
|
||||
return delta_text, delta_token_ids
|
||||
|
||||
delta_text = CHANNEL_START + delta_text
|
||||
if delta_token_ids:
|
||||
delta_token_ids = [self._reasoning_start_token_id, *delta_token_ids]
|
||||
|
||||
return delta_text, delta_token_ids
|
||||
|
||||
def _prompt_ends_in_open_reasoning(self, prompt_token_ids: Sequence[int]) -> bool:
|
||||
"""Whether the prompt tail is inside an open ``<|channel>`` block.
|
||||
|
||||
Scans backwards: a ``<|channel>`` start token seen before any
|
||||
closing or turn-boundary token means the block is still open.
|
||||
"""
|
||||
start_id = self._reasoning_start_token_id
|
||||
if start_id is None:
|
||||
return False
|
||||
boundary_ids = {
|
||||
tid
|
||||
for tid in (
|
||||
self._reasoning_end_token_id,
|
||||
self._tool_call_token_id,
|
||||
self._new_turn_token_id,
|
||||
self._tool_response_token_id,
|
||||
)
|
||||
if tid is not None
|
||||
}
|
||||
for tid in reversed(prompt_token_ids):
|
||||
if tid == start_id:
|
||||
return True
|
||||
if tid in boundary_ids:
|
||||
return False
|
||||
return False
|
||||
|
||||
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
"""Pre-initialise the engine to ``REASONING`` when the prompt ends
|
||||
inside an open ``<|channel>`` block.
|
||||
|
||||
This covers the post-tool-response continuation case where the chat
|
||||
template leaves the prompt ending with ``<|channel>thought\n``
|
||||
(issue #45834). A prompt that merely starts a new model turn must
|
||||
not pre-initialise reasoning: the model may answer directly without
|
||||
emitting any channel markers, and the non-streaming path classifies
|
||||
such output as content (issue #48217). When the model does open its
|
||||
own ``<|channel>``, the ``(CONTENT, THINK_START)`` transition
|
||||
handles it, and the ``thought\n`` prefix in the first reasoning
|
||||
chunk is stripped by ``_events_to_delta`` as in the default flow.
|
||||
"""
|
||||
if not self._prompt_ends_in_open_reasoning(prompt_token_ids):
|
||||
return
|
||||
self._engine.reset(initial_state=ParserState.REASONING)
|
||||
# Prevent a later default ``initialize_streaming()`` (e.g. from
|
||||
# ``ParserEngineReasoningAdapter.extract_reasoning_streaming``) from
|
||||
# clobbering this with ``CONTENT``.
|
||||
self._streaming_initialized = True
|
||||
|
||||
def _events_to_delta(
|
||||
self,
|
||||
events: list[SemanticEvent],
|
||||
finished: bool = False,
|
||||
) -> DeltaMessage | None:
|
||||
delta = super()._events_to_delta(events, finished=finished)
|
||||
if delta is None or delta.reasoning is None:
|
||||
return delta
|
||||
|
||||
if self._prefix_stripped:
|
||||
return delta
|
||||
self._reasoning_text += delta.reasoning
|
||||
|
||||
if self._reasoning_text.startswith(_GEMMA4_THOUGHT_PREFIX):
|
||||
prefix_len = len(_GEMMA4_THOUGHT_PREFIX)
|
||||
prev_reasoning_len = len(self._reasoning_text) - len(delta.reasoning)
|
||||
if prev_reasoning_len >= prefix_len:
|
||||
self._prefix_stripped = True
|
||||
return delta
|
||||
chars_of_prefix_in_delta = prefix_len - prev_reasoning_len
|
||||
stripped = delta.reasoning[chars_of_prefix_in_delta:]
|
||||
if stripped:
|
||||
self._prefix_stripped = True
|
||||
delta.reasoning = stripped
|
||||
return delta
|
||||
if len(self._reasoning_text) >= prefix_len:
|
||||
self._prefix_stripped = True
|
||||
delta.reasoning = None
|
||||
if delta.content is not None or delta.tool_calls:
|
||||
return delta
|
||||
return None
|
||||
return None
|
||||
|
||||
if _GEMMA4_THOUGHT_PREFIX.startswith(self._reasoning_text):
|
||||
if finished:
|
||||
self._prefix_stripped = True
|
||||
return None
|
||||
|
||||
self._prefix_stripped = True
|
||||
delta.reasoning = self._reasoning_text
|
||||
return delta
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
reasoning, content = super().extract_reasoning(model_output, request)
|
||||
if reasoning:
|
||||
if reasoning.startswith(_GEMMA4_THOUGHT_PREFIX):
|
||||
reasoning = reasoning[len(_GEMMA4_THOUGHT_PREFIX) :]
|
||||
elif reasoning == _GEMMA4_THOUGHT_PREFIX.rstrip():
|
||||
reasoning = None
|
||||
return reasoning or None, content
|
||||
@@ -1,286 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-4.7 parser for reasoning and tool calls.
|
||||
|
||||
GLM-4.7 uses XML-like tool calls::
|
||||
|
||||
<tool_call>func_name<arg_key>key</arg_key><arg_value>value</arg_value></tool_call>
|
||||
|
||||
The function name can be followed directly by the first ``<arg_key>`` tag,
|
||||
and tool calls may have no arguments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ArgValueExtender,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
THINK_START = "<think>"
|
||||
THINK_END = "</think>"
|
||||
TOOL_CALL_START = "<tool_call>"
|
||||
TOOL_CALL_END = "</tool_call>"
|
||||
ARG_KEY_START = "<arg_key>"
|
||||
ARG_KEY_END = "</arg_key>"
|
||||
ARG_VALUE_START = "<arg_value>"
|
||||
ARG_VALUE_END = "</arg_value>"
|
||||
|
||||
# Special tokens that delimit conversation turns in a rendered GLM prompt.
|
||||
# Reasoning markers belonging to earlier turns must not be mistaken for the
|
||||
# state of the turn currently being generated.
|
||||
GLM_TURN_BOUNDARIES = frozenset(
|
||||
("<|system|>", "<|user|>", "<|assistant|>", "<|observation|>")
|
||||
)
|
||||
|
||||
_ARG_RE = re.compile(
|
||||
r"<arg_key>(?P<key>.*?)</arg_key>\s*"
|
||||
r"<arg_value>(?P<value>.*?)</arg_value>",
|
||||
re.DOTALL,
|
||||
)
|
||||
_PARTIAL_ARG_RE = re.compile(
|
||||
r"<arg_key>(?P<key>.*?)</arg_key>\s*"
|
||||
r"<arg_value>(?P<value>.*)$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def _glm47_parse_args(
|
||||
raw_args: str, partial: bool
|
||||
) -> tuple[dict[str, object], str | None]:
|
||||
"""Return the arguments and the key of the open trailing value, if any."""
|
||||
params: dict[str, object] = {}
|
||||
|
||||
for match in _ARG_RE.finditer(raw_args):
|
||||
params[match.group("key").strip()] = match.group("value")
|
||||
|
||||
open_key = None
|
||||
if partial:
|
||||
remaining = _ARG_RE.sub("", raw_args)
|
||||
match = _PARTIAL_ARG_RE.search(remaining)
|
||||
if match:
|
||||
key = match.group("key").strip()
|
||||
if key:
|
||||
params[key] = match.group("value")
|
||||
open_key = key
|
||||
|
||||
return params, open_key
|
||||
|
||||
|
||||
def _glm47_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
return json.dumps(_glm47_parse_args(raw_args, partial)[0], ensure_ascii=False)
|
||||
|
||||
|
||||
def _glm47_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
|
||||
# The open value runs to the end of raw_args and is the last JSON value,
|
||||
# so a delta appends its JSON-escaped text unless it closes the value.
|
||||
params, open_key = _glm47_parse_args(raw_args, partial=True)
|
||||
if open_key is None or next(reversed(params)) != open_key:
|
||||
return None
|
||||
|
||||
keep = len(ARG_VALUE_END) - 1
|
||||
tail = raw_args[-keep:]
|
||||
|
||||
def extend(delta: str) -> str | None:
|
||||
nonlocal tail
|
||||
text = tail + delta
|
||||
if ARG_VALUE_END in text:
|
||||
return None
|
||||
tail = text[-keep:]
|
||||
return json.dumps(delta, ensure_ascii=False)[1:-1]
|
||||
|
||||
return extend
|
||||
|
||||
|
||||
def _glm53_always_thinks(tokenizer: TokenizerLike) -> bool:
|
||||
# GLM-5.3 keeps the GLM-4.x prompt and tool-call format but replaces the
|
||||
# enable_thinking toggle with an always-on "Reasoning Effort:" header, so
|
||||
# its generation prompt always opens <think> and the model output carries
|
||||
# no opening tag.
|
||||
template = getattr(tokenizer, "chat_template", None)
|
||||
return (
|
||||
isinstance(template, str)
|
||||
and "[gMASK]<sop>" in template
|
||||
and "Reasoning Effort:" in template
|
||||
and "enable_thinking" not in template
|
||||
and "<tool_call>" in template
|
||||
and "<arg_key>" in template
|
||||
and "<arg_value>" in template
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig:
|
||||
arg_tag_transitions = {
|
||||
(ParserState.TOOL_ARGS, terminal): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.ARG_VALUE_CHUNK,),
|
||||
)
|
||||
for terminal in (
|
||||
"ARG_KEY_START",
|
||||
"ARG_KEY_END",
|
||||
"ARG_VALUE_START",
|
||||
"ARG_VALUE_END",
|
||||
)
|
||||
}
|
||||
|
||||
reasoning_terminals = (
|
||||
{
|
||||
"THINK_START": THINK_START,
|
||||
"THINK_END": THINK_END,
|
||||
}
|
||||
if thinking
|
||||
else {}
|
||||
)
|
||||
reasoning_token_id_terminals = (
|
||||
{
|
||||
"THINK_START": THINK_START,
|
||||
"THINK_END": THINK_END,
|
||||
}
|
||||
if thinking
|
||||
else {}
|
||||
)
|
||||
reasoning_transitions = (
|
||||
{
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
}
|
||||
if thinking
|
||||
else {}
|
||||
)
|
||||
|
||||
return ParserEngineConfig(
|
||||
name="glm47_moe",
|
||||
initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
|
||||
terminals={
|
||||
**reasoning_terminals,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
"ARG_KEY_START": ARG_KEY_START,
|
||||
"ARG_KEY_END": ARG_KEY_END,
|
||||
"ARG_VALUE_START": ARG_VALUE_START,
|
||||
"ARG_VALUE_END": ARG_VALUE_END,
|
||||
},
|
||||
token_id_terminals={
|
||||
**reasoning_token_id_terminals,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
},
|
||||
transitions={
|
||||
**reasoning_transitions,
|
||||
(ParserState.REASONING, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(),
|
||||
),
|
||||
(ParserState.REASONING, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.REASONING_END, EventType.TOOL_CALL_START),
|
||||
),
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "ARG_KEY_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.ARG_VALUE_CHUNK,),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
**arg_tag_transitions,
|
||||
},
|
||||
turn_boundary_tokens=GLM_TURN_BOUNDARIES,
|
||||
arg_converter=_glm47_arg_converter,
|
||||
arg_value_extender=_glm47_arg_value_extender,
|
||||
stream_arg_deltas=True,
|
||||
tool_args_json=False,
|
||||
validate_tool_names=True,
|
||||
)
|
||||
|
||||
|
||||
class Glm47MoeParser(ParserEngine):
|
||||
"""GLM-4.7 parser backed by the declarative parser engine."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
thinking = chat_kwargs.get("thinking", None)
|
||||
enable_thinking = chat_kwargs.get("enable_thinking", None)
|
||||
if (thinking is False or enable_thinking is False) and _glm53_always_thinks(
|
||||
tokenizer
|
||||
):
|
||||
logger.warning_once(
|
||||
"Ignoring enable_thinking/thinking: the GLM-5.3 chat template "
|
||||
"has no thinking switch, so reasoning is always on and "
|
||||
"disabling extraction would leak it into the content."
|
||||
)
|
||||
thinking = None
|
||||
enable_thinking = None
|
||||
self.thinking_enabled = (
|
||||
True
|
||||
if thinking is None and enable_thinking is None
|
||||
else bool(thinking) or bool(enable_thinking)
|
||||
)
|
||||
kwargs.setdefault(
|
||||
"parser_engine_config",
|
||||
glm47_moe_config(thinking=self.thinking_enabled),
|
||||
)
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
|
||||
def _emit_name_delta(self, idx: int, deltas, name: str | None) -> None:
|
||||
if name is not None:
|
||||
name = name.strip()
|
||||
super()._emit_name_delta(idx, deltas, name)
|
||||
|
||||
def _handle_tool_end(self, event, deltas) -> None:
|
||||
idx = event.tool_index
|
||||
if 0 <= idx < len(self._tool_slots):
|
||||
self._tool_slots[idx].name = self._tool_slots[idx].name.strip()
|
||||
super()._handle_tool_end(event, deltas)
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if not self.thinking_enabled:
|
||||
return None, model_output
|
||||
return super().extract_reasoning(model_output, request)
|
||||
@@ -1,88 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Granite parser for JSON-array tool calls.
|
||||
|
||||
Granite 3.0/3.1 emit tool calls as a JSON array following a marker::
|
||||
|
||||
<|tool_call|> [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.0
|
||||
<tool_call> [{"name": "get_weather", "arguments": {"city": "SF"}}] # 3.1
|
||||
|
||||
``<|tool_call|>`` is a single special token; ``<tool_call>`` is plain text.
|
||||
The array may hold several calls, has no closing marker (the ``]`` ends it),
|
||||
and any surrounding prose is content. The engine's ``tool_call_body_array``
|
||||
mode splits the array into one call per element and extracts each name via the
|
||||
name-from-args path; ``_granite_arg_converter`` carves the ``arguments`` object
|
||||
out of each ``{"name":..,"arguments":{..}}`` element. Granite has no reasoning.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.utils import extract_json_member_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
TOOL_TOKEN = "<|tool_call|>" # granite 3.0 (single special token)
|
||||
TOOL_STRING = "<tool_call>" # granite 3.1 (plain text)
|
||||
|
||||
|
||||
def _granite_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
"""Carve the ``arguments`` object out of a ``{"name":..,"arguments":{..}}``
|
||||
array element (see :func:`extract_json_member_object`)."""
|
||||
span = extract_json_member_object(raw_args, "arguments")
|
||||
if span is None:
|
||||
return "" if partial else "{}"
|
||||
return span
|
||||
|
||||
|
||||
@functools.cache
|
||||
def granite_config() -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="granite",
|
||||
initial_state=ParserState.CONTENT,
|
||||
terminals={
|
||||
"TOOL_TOKEN": TOOL_TOKEN,
|
||||
"TOOL_STRING": TOOL_STRING,
|
||||
},
|
||||
token_id_terminals={
|
||||
"TOOL_TOKEN": TOOL_TOKEN,
|
||||
},
|
||||
transitions={
|
||||
# The marker only switches into the tool region; the array splitter
|
||||
# emits TOOL_CALL_START per element (empty events here).
|
||||
(ParserState.CONTENT, "TOOL_TOKEN"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
),
|
||||
(ParserState.CONTENT, "TOOL_STRING"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
),
|
||||
},
|
||||
arg_converter=_granite_arg_converter,
|
||||
tool_call_body_array=True,
|
||||
stream_arg_deltas=True,
|
||||
)
|
||||
|
||||
|
||||
class GraniteParser(ParserEngine):
|
||||
CONFIG_NAME = "granite"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
kwargs.setdefault("parser_engine_config", granite_config())
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
@@ -1,131 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Granite 4.2 thinking parser.
|
||||
|
||||
Granite 4.2 uses the same tool call and reasoning format as Nemotron V3
|
||||
(``<think>``/``</think>`` + ``<tool_call>`` XML). This config reuses
|
||||
:class:`NemotronV3Parser` with two Granite-specific behaviors:
|
||||
|
||||
1. Strip leading newlines from content — the Granite chat template writes
|
||||
``\\n</think>\\n``, so the first content character is always ``\\n``.
|
||||
2. Inherit the Nemotron V3 reasoning-to-content swap for
|
||||
``enable_thinking=False`` / ``force_nonempty_content=True``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import functools
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.parser.nemotron_v3 import NemotronV3Parser, nemotron_v3_config
|
||||
from vllm.parser.qwen3 import Qwen3Parser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.engine.parser_engine import SemanticEvent
|
||||
from vllm.parser.engine.parser_engine_config import ParserEngineConfig
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
|
||||
@functools.cache
|
||||
def granite_thinking_config(thinking: bool = True) -> ParserEngineConfig:
|
||||
return dataclasses.replace(
|
||||
nemotron_v3_config(thinking=thinking),
|
||||
name="granite_thinking_parser",
|
||||
)
|
||||
|
||||
|
||||
class GraniteThinkingParser(NemotronV3Parser):
|
||||
"""Granite 4.2 parser: same format as Nemotron V3, with
|
||||
leading-newline stripping on content after ``</think>``.
|
||||
|
||||
The ``reasoning_ended`` property is overridden to delay the
|
||||
reasoning-to-content transition reported to
|
||||
:class:`~vllm.parser.abstract_parser.DelegatingParser` until the
|
||||
template's leading newline has been consumed. Without this delay,
|
||||
``DelegatingParser.parse_delta`` passes newline tokens through a
|
||||
raw pass-through path that bypasses the engine's
|
||||
``_events_to_delta`` (and therefore our stripping logic).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
thinking = chat_kwargs.get("enable_thinking", True)
|
||||
kwargs.pop("parser_engine_config", None)
|
||||
Qwen3Parser.__init__(
|
||||
self,
|
||||
tokenizer,
|
||||
tools,
|
||||
parser_engine_config=granite_thinking_config(thinking=thinking),
|
||||
**kwargs,
|
||||
)
|
||||
self._streamed_reasoning: list[str] = []
|
||||
self._content_started = False
|
||||
|
||||
@property
|
||||
def reasoning_ended(self) -> bool:
|
||||
# Internal engine logic uses _reasoning_ended directly; this
|
||||
# property is only read by DelegatingParser (via the adapter's
|
||||
# has_engine_confirmed_reasoning_end) to decide when to stop
|
||||
# routing deltas through extract_reasoning_streaming.
|
||||
if not self._reasoning_ended:
|
||||
return False
|
||||
return self._content_started
|
||||
|
||||
def _reset(self, initial_state=None) -> None:
|
||||
super()._reset(initial_state=initial_state)
|
||||
self._content_started = False
|
||||
|
||||
def _events_to_delta(
|
||||
self,
|
||||
events: list[SemanticEvent],
|
||||
finished: bool = False,
|
||||
) -> DeltaMessage | None:
|
||||
delta = super()._events_to_delta(events, finished=finished)
|
||||
if (
|
||||
delta is not None
|
||||
and delta.content is not None
|
||||
and not self._content_started
|
||||
):
|
||||
stripped = delta.content.lstrip("\n")
|
||||
if not stripped:
|
||||
delta.content = None
|
||||
if delta.reasoning is None and not delta.tool_calls:
|
||||
return None
|
||||
else:
|
||||
self._content_started = True
|
||||
delta.content = stripped
|
||||
return delta
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
reasoning, content = Qwen3Parser.extract_reasoning(self, model_output, request)
|
||||
|
||||
# Track whether content was truly absent (None) vs present
|
||||
# but whitespace-only — the swap should only fire for absent.
|
||||
content_was_absent = content is None
|
||||
|
||||
if content is not None:
|
||||
content = content.lstrip("\n") or None
|
||||
|
||||
# Only swap when content was truly absent from model output
|
||||
# (truncated reasoning or max_tokens cut at </think>), not
|
||||
# when it was present but stripped to empty (newlines-only).
|
||||
if self._should_force_content(request) and content_was_absent:
|
||||
reasoning, content = content, reasoning
|
||||
|
||||
return reasoning, content
|
||||
@@ -1,332 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inkling parser: typed content blocks parsed by a single state machine.
|
||||
|
||||
Inkling output format (every marker is a dedicated special token)::
|
||||
|
||||
<|message_model|><|content_thinking|>...reasoning...<|end_message|>
|
||||
<|message_model|><|content_text|>...visible text...<|end_message|>
|
||||
<|message_model|><|content_invoke_tool_json|>
|
||||
{"name":"get_weather","args":{"city":"SF"}}<|end_message|>
|
||||
|
||||
Blocks are self-describing and may repeat in any order; sampling may
|
||||
also end a block with the standalone ``<|content_model_end_sampling|>``
|
||||
token. The tool-call payload is a single JSON object whose ``name`` is
|
||||
extracted by the engine's name-from-args path and whose ``args`` object
|
||||
is carved out of the wrapper by :func:`_inkling_arg_converter`.
|
||||
|
||||
Note the terminal *labels*: ``THINK_START``/``THINK_END`` are what the
|
||||
engine keys its reasoning plumbing on (``is_reasoning_end``,
|
||||
``count_reasoning_tokens``, initial-state seeding), so ``<|end_message|>``
|
||||
is labelled ``THINK_END`` here even though it ends every block kind —
|
||||
the transition table, not the label, carries the semantics.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import ExtractedToolCallInformation
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.utils import extract_json_member_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
MESSAGE_MODEL = "<|message_model|>"
|
||||
CONTENT_TEXT = "<|content_text|>"
|
||||
CONTENT_THINKING = "<|content_thinking|>"
|
||||
CONTENT_INVOKE_TOOL_JSON = "<|content_invoke_tool_json|>"
|
||||
CONTENT_INVOKE_TOOL_TEXT = "<|content_invoke_tool_text|>"
|
||||
CONTENT_TOOL_ERROR = "<|content_tool_error|>"
|
||||
CONTENT_MODEL_END_SAMPLING = "<|content_model_end_sampling|>"
|
||||
END_MESSAGE = "<|end_message|>"
|
||||
|
||||
INKLING_SPECIAL_TOKENS = (
|
||||
MESSAGE_MODEL,
|
||||
CONTENT_TEXT,
|
||||
CONTENT_THINKING,
|
||||
CONTENT_INVOKE_TOOL_JSON,
|
||||
CONTENT_INVOKE_TOOL_TEXT,
|
||||
CONTENT_TOOL_ERROR,
|
||||
CONTENT_MODEL_END_SAMPLING,
|
||||
END_MESSAGE,
|
||||
)
|
||||
|
||||
|
||||
def _inkling_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
"""Carve the ``args`` object out of the tool-call JSON wrapper.
|
||||
|
||||
Why a converter at all: the engine's ``tool_args_json`` machinery
|
||||
treats the *entire* TOOL_ARGS text as the tool arguments, but Inkling's
|
||||
payload is the ``{"name":...,"args":{...}}`` wrapper — without a
|
||||
converter, ``_compute_arg_delta`` streams the wrapper verbatim into
|
||||
the OpenAI ``arguments`` field (``converter is None -> raw delta``,
|
||||
unconditionally; ``stream_arg_deltas=False`` does not stop it).
|
||||
:func:`extract_json_member_object` returns a prefix-stable verbatim
|
||||
substring, which the engine's argument-delta diffing requires.
|
||||
"""
|
||||
span = extract_json_member_object(raw_args, "args")
|
||||
if span is None:
|
||||
# No args value yet (streaming) or none at all (treat as empty).
|
||||
return "" if partial else "{}"
|
||||
return span
|
||||
|
||||
|
||||
@functools.cache
|
||||
def inkling_config() -> ParserEngineConfig:
|
||||
terminals = {
|
||||
"MSG_MODEL": MESSAGE_MODEL,
|
||||
"TEXT_START": CONTENT_TEXT,
|
||||
"THINK_START": CONTENT_THINKING,
|
||||
"THINK_END": END_MESSAGE,
|
||||
"END_SAMPLING": CONTENT_MODEL_END_SAMPLING,
|
||||
"TOOL_START": CONTENT_INVOKE_TOOL_JSON,
|
||||
"TOOL_TEXT": CONTENT_INVOKE_TOOL_TEXT,
|
||||
"TOOL_ERROR": CONTENT_TOOL_ERROR,
|
||||
}
|
||||
transitions: dict[tuple[ParserState, str], Transition] = {
|
||||
# ── Between blocks / inside a text block ──────────────────────
|
||||
(ParserState.CONTENT, "MSG_MODEL"): Transition(
|
||||
ParserState.MESSAGE_HEADER,
|
||||
(),
|
||||
),
|
||||
# Opening a block that renders as visible content proves no reasoning
|
||||
# is still open. Confirming that here is what lets DelegatingParser
|
||||
# hand the following blocks to the tool pass: the reasoning pass only
|
||||
# leaves its reasoning phase on an explicit reasoning-end event, and a
|
||||
# response with no thinking block never emits one otherwise.
|
||||
(ParserState.CONTENT, "TEXT_START"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
# A tool block confirms the same boundary, and is the one opener that
|
||||
# can start a turn with no visible block ahead of it.
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.REASONING_END, EventType.TOOL_CALL_START),
|
||||
),
|
||||
# Raw / error tool blocks render as visible text.
|
||||
(ParserState.CONTENT, "TOOL_TEXT"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "TOOL_ERROR"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
# The optional function name between the model-role and content-kind
|
||||
# markers is metadata, not visible assistant content.
|
||||
(ParserState.MESSAGE_HEADER, "MSG_MODEL"): Transition(
|
||||
ParserState.MESSAGE_HEADER,
|
||||
(),
|
||||
),
|
||||
(ParserState.MESSAGE_HEADER, "TEXT_START"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.MESSAGE_HEADER, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.MESSAGE_HEADER, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.REASONING_END, EventType.TOOL_CALL_START),
|
||||
),
|
||||
(ParserState.MESSAGE_HEADER, "TOOL_TEXT"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.MESSAGE_HEADER, "TOOL_ERROR"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
# ── Inside a thinking block ───────────────────────────────────
|
||||
(ParserState.REASONING, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(),
|
||||
),
|
||||
# Defensive: tool call opening while a thinking block is still
|
||||
# unclosed.
|
||||
(ParserState.REASONING, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.REASONING_END, EventType.TOOL_CALL_START),
|
||||
),
|
||||
}
|
||||
# A closed tool block returns to CONTENT (Inkling has no section wrapper;
|
||||
# blocks of any kind may follow), which also keeps the block-kind
|
||||
# and role tokens out of the engine's tool-terminal set so the
|
||||
# skip_tool_parsing reasoning pass still classifies reasoning.
|
||||
# In a reasoning block, THINK_END defers REASONING_END to the next block
|
||||
# opener so the function-name header is not emitted as content.
|
||||
for end in ("THINK_END", "END_SAMPLING"):
|
||||
transitions[(ParserState.CONTENT, end)] = Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
)
|
||||
transitions[(ParserState.REASONING, end)] = Transition(
|
||||
ParserState.CONTENT,
|
||||
() if end == "THINK_END" else (EventType.REASONING_END,),
|
||||
)
|
||||
transitions[(ParserState.TOOL_ARGS, end)] = Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
)
|
||||
transitions[(ParserState.MESSAGE_HEADER, end)] = Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TEXT_CHUNK,),
|
||||
)
|
||||
|
||||
return ParserEngineConfig(
|
||||
name="inkling",
|
||||
# Normal generation continues after a prompt-prefilled
|
||||
# `<|message_model|>`. Non-streaming parsing receives only the generated
|
||||
# suffix, so begin in the corresponding message-header state as well.
|
||||
initial_state=ParserState.MESSAGE_HEADER,
|
||||
wait_for_reasoning=True,
|
||||
terminals=terminals,
|
||||
# Inkling content-kind markers are the grammar. When the engine is
|
||||
# used through DelegatingParser, the reasoning pass can hand the tool
|
||||
# pass reconstructed text whose token-id slice no longer contains the
|
||||
# content-kind marker that starts the tool block, so keep Inkling on
|
||||
# the text grammar instead of token-id-only terminal matching.
|
||||
token_id_terminals={},
|
||||
transitions=transitions,
|
||||
arg_converter=_inkling_arg_converter,
|
||||
stream_arg_deltas=True,
|
||||
tool_args_json=True,
|
||||
strip_trailing_reasoning_whitespace=True,
|
||||
drop_whitespace_only_content_before_tools=True,
|
||||
strip_content_whitespace_with_tools=False,
|
||||
validate_tool_names=False,
|
||||
)
|
||||
|
||||
|
||||
class InklingParser(ParserEngine):
|
||||
CONFIG_NAME = "inkling"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
kwargs.setdefault("parser_engine_config", inkling_config())
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
|
||||
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
"""Seed the initial parsing state from the prompt tail.
|
||||
|
||||
Mirrors the Rust parser's ``initialize()``: scanning the prompt
|
||||
backwards, the last relevant special token decides whether generation
|
||||
continues inside a thinking block, a text block, a model message
|
||||
header, or between blocks.
|
||||
"""
|
||||
vocab = self.vocab
|
||||
thinking_id = vocab.get(CONTENT_THINKING)
|
||||
text_id = vocab.get(CONTENT_TEXT)
|
||||
model_id = vocab.get(MESSAGE_MODEL)
|
||||
special_ids = {vocab[text] for text in INKLING_SPECIAL_TOKENS if text in vocab}
|
||||
for token_id in reversed(prompt_token_ids):
|
||||
if token_id == thinking_id:
|
||||
self._engine.reset(initial_state=ParserState.REASONING)
|
||||
self._streaming_initialized = True
|
||||
return
|
||||
if token_id == text_id:
|
||||
self._engine.reset(initial_state=ParserState.CONTENT)
|
||||
self._streaming_initialized = True
|
||||
return
|
||||
if token_id == model_id:
|
||||
self._engine.reset(initial_state=ParserState.MESSAGE_HEADER)
|
||||
self._streaming_initialized = True
|
||||
return
|
||||
if token_id in special_ids:
|
||||
break
|
||||
|
||||
def is_reasoning_end(self, input_ids: list[int]) -> bool:
|
||||
vocab = self.vocab
|
||||
thinking_id = vocab.get(CONTENT_THINKING)
|
||||
text_id = vocab.get(CONTENT_TEXT)
|
||||
model_id = vocab.get(MESSAGE_MODEL)
|
||||
end_sampling_id = vocab.get(CONTENT_MODEL_END_SAMPLING)
|
||||
for token_id in reversed(input_ids):
|
||||
if token_id in (thinking_id, model_id):
|
||||
return False
|
||||
if token_id in (text_id, end_sampling_id):
|
||||
return True
|
||||
return False
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
vocab = self.vocab
|
||||
thinking_id = vocab.get(CONTENT_THINKING)
|
||||
end_ids = {
|
||||
token_id
|
||||
for token_id in (
|
||||
vocab.get(END_MESSAGE),
|
||||
vocab.get(CONTENT_MODEL_END_SAMPLING),
|
||||
)
|
||||
if token_id is not None
|
||||
}
|
||||
in_reasoning = False
|
||||
count = 0
|
||||
for token_id in token_ids:
|
||||
if token_id == thinking_id:
|
||||
in_reasoning = True
|
||||
continue
|
||||
if token_id in end_ids:
|
||||
in_reasoning = False
|
||||
continue
|
||||
if in_reasoning:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def _single_pass_parse(
|
||||
self,
|
||||
text: str,
|
||||
token_ids: Sequence[int],
|
||||
initial_state: ParserState | None = None,
|
||||
) -> tuple[str | None, str | None, ExtractedToolCallInformation]:
|
||||
reasoning, content, tool_call_info = super()._single_pass_parse(
|
||||
text, token_ids, initial_state=initial_state
|
||||
)
|
||||
# The engine defers content that follows tool-call events within a
|
||||
# single pass; Inkling allows text blocks after tool-call blocks, so
|
||||
# flush the trailing text (matching the Rust unified parser).
|
||||
if self._deferred_content:
|
||||
trailing = self._deferred_content
|
||||
self._deferred_content = ""
|
||||
content = self._strip_content_whitespace(
|
||||
(content or "") + trailing,
|
||||
tool_call_info.tools_called,
|
||||
)
|
||||
tool_call_info = ExtractedToolCallInformation(
|
||||
tools_called=tool_call_info.tools_called,
|
||||
tool_calls=tool_call_info.tool_calls,
|
||||
content=content,
|
||||
)
|
||||
return reasoning, content, tool_call_info
|
||||
|
||||
@staticmethod
|
||||
def _extract_args_value(parsed: dict) -> str | None:
|
||||
# Inkling wraps arguments under "args" rather than "arguments".
|
||||
for key in ("args", "arguments", "parameters"):
|
||||
if key in parsed:
|
||||
val = parsed[key]
|
||||
if isinstance(val, str):
|
||||
return val
|
||||
return json.dumps(val, ensure_ascii=False)
|
||||
return None
|
||||
@@ -1,268 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi K2 parser for reasoning and tool calls.
|
||||
|
||||
Kimi K2 tool call format::
|
||||
|
||||
<|tool_calls_section_begin|>
|
||||
<|tool_call_begin|>functions.get_weather:0
|
||||
<|tool_call_argument_begin|>{"city": "Tokyo"}<|tool_call_end|>
|
||||
<|tool_calls_section_end|>
|
||||
|
||||
The header before ``<|tool_call_argument_begin|>`` is Kimi's native tool
|
||||
call id. The function name is the final component before ``:N``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaFunctionCall, DeltaToolCall
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
THINK_START = "<think>"
|
||||
THINK_END = "</think>"
|
||||
TOOL_SECTION_START = "<|tool_calls_section_begin|>"
|
||||
TOOL_SECTION_END = "<|tool_calls_section_end|>"
|
||||
TOOL_CALL_START = "<|tool_call_begin|>"
|
||||
TOOL_CALL_END = "<|tool_call_end|>"
|
||||
TOOL_ARG_START = "<|tool_call_argument_begin|>"
|
||||
|
||||
_TOOL_ID_RE = re.compile(r"(?P<id>.+:\d+)")
|
||||
|
||||
|
||||
@functools.cache
|
||||
def kimi_k2_config(thinking: bool = True) -> ParserEngineConfig:
|
||||
reasoning_terminals = (
|
||||
{
|
||||
"THINK_START": THINK_START,
|
||||
"THINK_END": THINK_END,
|
||||
}
|
||||
if thinking
|
||||
else {}
|
||||
)
|
||||
reasoning_transitions = (
|
||||
{
|
||||
(ParserState.REASONING, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
}
|
||||
if thinking
|
||||
else {}
|
||||
)
|
||||
|
||||
return ParserEngineConfig(
|
||||
name="kimi_k2",
|
||||
initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
|
||||
wait_for_reasoning=thinking,
|
||||
terminals={
|
||||
**reasoning_terminals,
|
||||
"TOOL_SECTION_START": TOOL_SECTION_START,
|
||||
"TOOL_SECTION_END": TOOL_SECTION_END,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
"ARG_START": TOOL_ARG_START,
|
||||
},
|
||||
token_id_terminals={
|
||||
**reasoning_terminals,
|
||||
"TOOL_SECTION_START": TOOL_SECTION_START,
|
||||
"TOOL_SECTION_END": TOOL_SECTION_END,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
"ARG_START": TOOL_ARG_START,
|
||||
},
|
||||
transitions={
|
||||
**reasoning_transitions,
|
||||
(ParserState.REASONING, "TOOL_SECTION_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "TOOL_SECTION_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "ARG_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_SECTION_END"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
# Keep the parser in a tool state after the section closes so
|
||||
# trailing model text after native tool calls is suppressed.
|
||||
(ParserState.TOOL_PREAMBLE, "TOOL_SECTION_END"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_SECTION_END"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(),
|
||||
),
|
||||
},
|
||||
stream_arg_deltas=True,
|
||||
tool_args_json=True,
|
||||
strip_trailing_reasoning_whitespace=True,
|
||||
drop_whitespace_only_content_before_tools=True,
|
||||
strip_content_whitespace_with_tools=False,
|
||||
validate_tool_names=False,
|
||||
)
|
||||
|
||||
|
||||
class KimiK2Parser(ParserEngine):
|
||||
"""Kimi K2 parser backed by the declarative parser engine."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
thinking = chat_kwargs.get("thinking", None)
|
||||
enable_thinking = chat_kwargs.get("enable_thinking", None)
|
||||
self.thinking_enabled = (
|
||||
True
|
||||
if thinking is None and enable_thinking is None
|
||||
else bool(thinking) or bool(enable_thinking)
|
||||
)
|
||||
kwargs.setdefault(
|
||||
"parser_engine_config",
|
||||
kimi_k2_config(thinking=self.thinking_enabled),
|
||||
)
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
|
||||
vocab = self.vocab
|
||||
self._start_token_id = vocab.get(THINK_START)
|
||||
self._end_token_id = vocab.get(THINK_END)
|
||||
self._tool_section_start_token_id = vocab.get(TOOL_SECTION_START)
|
||||
|
||||
@staticmethod
|
||||
def _extract_tool_id_and_name(header: str | None) -> tuple[str | None, str | None]:
|
||||
if header is None:
|
||||
return None, None
|
||||
match = _TOOL_ID_RE.match(header.strip())
|
||||
if not match:
|
||||
return None, None
|
||||
|
||||
tool_id = match.group("id").strip()
|
||||
tool_name = tool_id.split(":")[0].removeprefix("functions.")
|
||||
return tool_id, tool_name
|
||||
|
||||
def _emit_name_delta(
|
||||
self,
|
||||
idx: int,
|
||||
deltas: list[DeltaToolCall],
|
||||
name: str | None,
|
||||
) -> None:
|
||||
tool_id, tool_name = self._extract_tool_id_and_name(name)
|
||||
if not tool_name:
|
||||
if 0 <= idx < len(self._tool_slots):
|
||||
self._tool_slots[idx].name = ""
|
||||
return
|
||||
|
||||
slot = self._tool_slots[idx]
|
||||
slot.id = tool_id or ""
|
||||
super()._emit_name_delta(idx, deltas, tool_name)
|
||||
|
||||
def _handle_tool_end(self, event, deltas) -> None:
|
||||
idx = event.tool_index
|
||||
if 0 <= idx < len(self._tool_slots) and not self._tool_slots[idx].name_sent:
|
||||
tool_id, tool_name = self._extract_tool_id_and_name(
|
||||
self._tool_slots[idx].name
|
||||
)
|
||||
if tool_name:
|
||||
self._tool_slots[idx].id = tool_id or ""
|
||||
self._tool_slots[idx].name = tool_name
|
||||
super()._handle_tool_end(event, deltas)
|
||||
|
||||
def _handle_arg_chunk(self, event, deltas) -> None:
|
||||
idx = event.tool_index
|
||||
name_sent_before = (
|
||||
0 <= idx < len(self._tool_slots) and self._tool_slots[idx].name_sent
|
||||
)
|
||||
super()._handle_arg_chunk(event, deltas)
|
||||
if (
|
||||
event.value
|
||||
and not name_sent_before
|
||||
and 0 <= idx < len(self._tool_slots)
|
||||
and self._tool_slots[idx].name_sent
|
||||
):
|
||||
deltas.append(
|
||||
DeltaToolCall(
|
||||
index=idx,
|
||||
function=DeltaFunctionCall(arguments=event.value),
|
||||
)
|
||||
)
|
||||
|
||||
def _extract_args_json(self, raw_args: str, func_name: str) -> str:
|
||||
return raw_args.strip() or "{}"
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
if not self.thinking_enabled:
|
||||
return input_ids
|
||||
|
||||
end_id = self._end_token_id
|
||||
if end_id is not None and end_id in input_ids:
|
||||
end_idx = len(input_ids) - 1 - input_ids[::-1].index(end_id)
|
||||
return input_ids[end_idx + 1 :]
|
||||
|
||||
tool_section_id = self._tool_section_start_token_id
|
||||
if tool_section_id is not None and tool_section_id in input_ids:
|
||||
section_idx = len(input_ids) - 1 - input_ids[::-1].index(tool_section_id)
|
||||
return input_ids[section_idx:]
|
||||
|
||||
return []
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if not self.thinking_enabled:
|
||||
return None, model_output
|
||||
return super().extract_reasoning(model_output, request)
|
||||
|
||||
def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
|
||||
if not self.thinking_enabled:
|
||||
return 0
|
||||
return super().count_reasoning_tokens(token_ids)
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
FunctionCall,
|
||||
)
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
|
||||
|
||||
class KimiK3Parser(DelegatingParser):
|
||||
"""Compose the Kimi K3 reasoning and tool parsers for XTML output."""
|
||||
|
||||
# TODO: Switch Kimi K3 to the parser engine once its XTML reasoning/tool
|
||||
# path is covered there.
|
||||
def _extract_tool_calls(
|
||||
self,
|
||||
content: str | None,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
) -> tuple[list[FunctionCall] | None, str | None]:
|
||||
if self._tool_parser is None or not enable_auto_tools:
|
||||
return super()._extract_tool_calls(content, request, enable_auto_tools)
|
||||
|
||||
tool_call_info = self.extract_tool_calls(content or "", request=request)
|
||||
if request.tool_choice == "none":
|
||||
return [], tool_call_info.content
|
||||
if not tool_call_info.tools_called:
|
||||
return None, tool_call_info.content
|
||||
|
||||
tool_calls = [
|
||||
FunctionCall(
|
||||
id=tool_call.id,
|
||||
name=tool_call.function.name,
|
||||
arguments=tool_call.function.arguments,
|
||||
)
|
||||
for tool_call in tool_call_info.tool_calls
|
||||
]
|
||||
parsed_content = tool_call_info.content
|
||||
if parsed_content and parsed_content.strip() == "":
|
||||
parsed_content = None
|
||||
return tool_calls, parsed_content
|
||||
|
||||
def _extract_tool_calls_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
tool_call_idx: int | None = None,
|
||||
tool_call_id_type: str = "random",
|
||||
function_name_returned: bool = False,
|
||||
) -> tuple[DeltaMessage | None, bool]:
|
||||
if request.tool_choice != "none":
|
||||
return super()._extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request,
|
||||
tool_call_idx=tool_call_idx,
|
||||
tool_call_id_type=tool_call_id_type,
|
||||
function_name_returned=function_name_returned,
|
||||
)
|
||||
|
||||
delta_message = self.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request,
|
||||
)
|
||||
if delta_message is not None:
|
||||
delta_message.tool_calls = []
|
||||
return delta_message, False
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
state = self._stream_state
|
||||
previous_content = state.previous_text if state.reasoning_ended else ""
|
||||
delta_message = super().parse_delta(
|
||||
delta_text,
|
||||
delta_token_ids,
|
||||
request,
|
||||
prompt_token_ids,
|
||||
finished=finished,
|
||||
)
|
||||
|
||||
if (
|
||||
self._tool_parser is not None
|
||||
or not isinstance(self._reasoning_parser, KimiK3ReasoningParser)
|
||||
or not state.reasoning_ended
|
||||
or delta_message is None
|
||||
):
|
||||
return delta_message
|
||||
|
||||
stripped = self._reasoning_parser.strip_content_streaming(
|
||||
previous_text=previous_content,
|
||||
current_text=state.previous_text,
|
||||
)
|
||||
delta_message.content = stripped.content if stripped is not None else None
|
||||
if (
|
||||
delta_message.role is None
|
||||
and delta_message.content is None
|
||||
and delta_message.reasoning is None
|
||||
and not delta_message.tool_calls
|
||||
):
|
||||
return None
|
||||
return delta_message
|
||||
@@ -1,80 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Ling3 parser for reasoning and tool calls.
|
||||
|
||||
Ling3 uses the same XML tool-call format as GLM-4.7 and defaults thinking on.
|
||||
It follows the GLM-style ``<think>`` / ``</think>`` reasoning format and
|
||||
treats ``<tool_call>`` as an implicit reasoning terminator.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.engine.adapters import make_adapters
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.glm47_moe import (
|
||||
THINK_END,
|
||||
THINK_START,
|
||||
Glm47MoeParser,
|
||||
glm47_moe_config,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
|
||||
class Ling3Parser(Glm47MoeParser):
|
||||
"""Ling3 parser backed by the GLM XML parser engine."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
thinking = chat_kwargs.get("thinking", None)
|
||||
enable_thinking = chat_kwargs.get("enable_thinking", None)
|
||||
self.thinking_enabled = (
|
||||
True
|
||||
if thinking is None and enable_thinking is None
|
||||
else bool(thinking) or bool(enable_thinking)
|
||||
)
|
||||
parser_config = replace(
|
||||
glm47_moe_config(thinking=self.thinking_enabled),
|
||||
name="ling3",
|
||||
)
|
||||
kwargs.setdefault(
|
||||
"parser_engine_config",
|
||||
parser_config,
|
||||
)
|
||||
ParserEngine.__init__(self, tokenizer, tools, **kwargs)
|
||||
|
||||
@property
|
||||
def reasoning_start_str(self) -> str:
|
||||
return THINK_START
|
||||
|
||||
@property
|
||||
def reasoning_end_str(self) -> str:
|
||||
return THINK_END
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if not self.thinking_enabled:
|
||||
return None, model_output
|
||||
|
||||
return super().extract_reasoning(model_output, request)
|
||||
|
||||
|
||||
(
|
||||
Ling3ParserReasoningAdapter,
|
||||
Ling3ParserToolAdapter,
|
||||
) = make_adapters(Ling3Parser)
|
||||
@@ -1,47 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiMo's compact XML parameters contain verbatim string values."""
|
||||
|
||||
import functools
|
||||
import json
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm.parser.qwen3 import (
|
||||
_PARAM_RE,
|
||||
_PARTIAL_PARAM_RE,
|
||||
CHATML_TURN_BOUNDARIES,
|
||||
Qwen3Parser,
|
||||
_param_value_extender,
|
||||
qwen3_config,
|
||||
)
|
||||
|
||||
|
||||
def _mimo_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
params = {m.group(1): m.group(2) for m in _PARAM_RE.finditer(raw_args)}
|
||||
if partial:
|
||||
match = _PARTIAL_PARAM_RE.search(_PARAM_RE.sub("", raw_args))
|
||||
if match and match.group(1):
|
||||
params[match.group(1)] = match.group(2)
|
||||
return json.dumps(params, ensure_ascii=False)
|
||||
|
||||
|
||||
class MiMoParser(Qwen3Parser):
|
||||
def __init__(self, tokenizer, tools=None, **kwargs):
|
||||
thinking = (kwargs.get("chat_template_kwargs") or {}).get(
|
||||
"enable_thinking", True
|
||||
)
|
||||
kwargs.setdefault(
|
||||
"parser_engine_config",
|
||||
replace(
|
||||
qwen3_config(
|
||||
thinking=thinking,
|
||||
name="mimo",
|
||||
turn_boundary_tokens=CHATML_TURN_BOUNDARIES,
|
||||
),
|
||||
arg_converter=_mimo_arg_converter,
|
||||
arg_value_extender=functools.partial(
|
||||
_param_value_extender, trim_newlines=False
|
||||
),
|
||||
),
|
||||
)
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
@@ -1,405 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiniCPM-V family parser.
|
||||
|
||||
Wraps the Qwen3 grammar (``<think>`` reasoning + XML tool calls) with two
|
||||
MiniCPM-V specifics:
|
||||
|
||||
- ``<|im_end|>`` and the reserved marker tokens are passthrough terminals,
|
||||
so they never leak into ``content``.
|
||||
- The tokenizer decodes a newline as the two characters ``\\`` and ``n``
|
||||
(e.g. token id 1639) instead of ``\\n``, so the decoded text is recovered
|
||||
before it reaches the client. Code blocks, inline code and LaTeX are left
|
||||
alone, as are tool call arguments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
|
||||
_RESERVED_MARKER_IDS = range(12, 16)
|
||||
_IM_END = "<|im_end|>"
|
||||
|
||||
|
||||
@functools.cache
|
||||
def minicpmv_config(thinking: bool) -> ParserEngineConfig:
|
||||
"""MiniCPM-V grammar: the qwen3 grammar plus MiniCPM-V markers.
|
||||
|
||||
Both modes keep the qwen3 tool call terminals and transitions, so
|
||||
``<tool_call>`` output is parsed whether or not thinking is enabled.
|
||||
"""
|
||||
name = "minicpmv" if thinking else "minicpmv_no_thinking"
|
||||
base = qwen3_config(thinking=thinking, name=name)
|
||||
|
||||
transitions = dict(base.transitions)
|
||||
if not thinking:
|
||||
# Default mode: the model answers directly, so `<think>` must not open
|
||||
# a reasoning span that would swallow the answer. A stray
|
||||
# `<think>...</think>` in the output is still dropped, not printed.
|
||||
transitions[(ParserState.CONTENT, "THINK_START")] = Transition(
|
||||
ParserState.REASONING
|
||||
)
|
||||
base = replace(base, strip_trailing_reasoning_whitespace=False)
|
||||
|
||||
terminals = dict(base.terminals)
|
||||
token_id_terminals = dict(base.token_id_terminals)
|
||||
terminals["IM_END"] = _IM_END
|
||||
token_id_terminals["IM_END"] = _IM_END
|
||||
for state in (ParserState.CONTENT, ParserState.REASONING):
|
||||
transitions[(state, "IM_END")] = Transition(state)
|
||||
|
||||
for index in _RESERVED_MARKER_IDS:
|
||||
text_name = f"RESERVED_TEXT_{index}"
|
||||
token_name = f"RESERVED_TOKEN_{index}"
|
||||
terminals[text_name] = f"<reserved_{index}>"
|
||||
terminals[token_name] = f"<|reserved_{index}|>"
|
||||
token_id_terminals[token_name] = f"<|reserved_{index}|>"
|
||||
for state in (ParserState.CONTENT, ParserState.REASONING):
|
||||
transitions[(state, text_name)] = Transition(state)
|
||||
transitions[(state, token_name)] = Transition(state)
|
||||
|
||||
return replace(
|
||||
base,
|
||||
terminals=terminals,
|
||||
token_id_terminals=token_id_terminals,
|
||||
transitions=transitions,
|
||||
)
|
||||
|
||||
|
||||
class EscapedNewlineNormalizer:
|
||||
"""Recover literal escape sequences written by the model as real text.
|
||||
|
||||
Scans character by character so that a backslash inside a code block,
|
||||
inline code, LaTeX span or a doubled escape is left untouched. A trailing
|
||||
half of an escape is held in ``_pending`` until the next chunk arrives,
|
||||
which keeps streaming output identical to the non-streaming result.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._closing_marker: str | None = None
|
||||
self._pending = ""
|
||||
|
||||
def feed(self, text: str, *, final: bool = False) -> str:
|
||||
text = self._pending + text
|
||||
self._pending = ""
|
||||
output: list[str] = []
|
||||
index = 0
|
||||
|
||||
while index < len(text):
|
||||
if self._closing_marker is not None:
|
||||
marker = self._closing_marker
|
||||
remaining = text[index:]
|
||||
if remaining.startswith(marker):
|
||||
output.append(marker)
|
||||
index += len(marker)
|
||||
self._closing_marker = None
|
||||
continue
|
||||
if not final and marker.startswith(remaining):
|
||||
self._pending = remaining
|
||||
break
|
||||
output.append(text[index])
|
||||
index += 1
|
||||
continue
|
||||
|
||||
remaining = text[index:]
|
||||
if text[index] == "`":
|
||||
if not final and remaining in ("`", "``"):
|
||||
self._pending = remaining
|
||||
break
|
||||
if remaining.startswith("```"):
|
||||
output.append("```")
|
||||
index += 3
|
||||
self._closing_marker = "```"
|
||||
else:
|
||||
output.append("`")
|
||||
index += 1
|
||||
self._closing_marker = "`"
|
||||
continue
|
||||
|
||||
if text[index] == "$":
|
||||
if not final and remaining == "$":
|
||||
self._pending = remaining
|
||||
break
|
||||
if remaining.startswith("$$"):
|
||||
output.append("$$")
|
||||
index += 2
|
||||
self._closing_marker = "$$"
|
||||
else:
|
||||
output.append("$")
|
||||
index += 1
|
||||
self._closing_marker = "$"
|
||||
continue
|
||||
|
||||
if text[index] != "\\":
|
||||
output.append(text[index])
|
||||
index += 1
|
||||
continue
|
||||
|
||||
slash_end = index + 1
|
||||
while slash_end < len(text) and text[slash_end] == "\\":
|
||||
slash_end += 1
|
||||
if slash_end - index > 1:
|
||||
if not final and slash_end == len(text):
|
||||
# The run may still grow, and its parity decides whether a
|
||||
# following `n` is a newline or stays literal. Hold it until
|
||||
# the run is complete.
|
||||
self._pending = text[index:slash_end]
|
||||
break
|
||||
output.append(text[index:slash_end])
|
||||
index = slash_end
|
||||
continue
|
||||
|
||||
if not final and remaining in ("\\", "\\r", "\\r\\"):
|
||||
self._pending = remaining
|
||||
break
|
||||
if remaining.startswith("\\r\\n"):
|
||||
output.append("\n")
|
||||
index += 4
|
||||
elif remaining.startswith(("\\n", "\\r")):
|
||||
output.append("\n")
|
||||
index += 2
|
||||
elif remaining.startswith("\\("):
|
||||
output.append("\\(")
|
||||
index += 2
|
||||
self._closing_marker = "\\)"
|
||||
elif remaining.startswith("\\["):
|
||||
output.append("\\[")
|
||||
index += 2
|
||||
self._closing_marker = "\\]"
|
||||
else:
|
||||
output.append("\\")
|
||||
index += 1
|
||||
|
||||
if final and self._pending:
|
||||
output.append(self._pending)
|
||||
self._pending = ""
|
||||
return "".join(output)
|
||||
|
||||
|
||||
def recover_newlines(text: str | None) -> str | None:
|
||||
if text is None or "\\" not in text:
|
||||
return text
|
||||
return EscapedNewlineNormalizer().feed(text, final=True)
|
||||
|
||||
|
||||
class MiniCPMVOutputNormalizer:
|
||||
"""Apply :class:`EscapedNewlineNormalizer` to a whole stream.
|
||||
|
||||
Reasoning and content are tracked separately so that a half-written escape
|
||||
at the end of a reasoning chunk does not swallow the first characters of
|
||||
the content that follows.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._reasoning = EscapedNewlineNormalizer()
|
||||
self._content = EscapedNewlineNormalizer()
|
||||
|
||||
def normalize_delta(
|
||||
self,
|
||||
delta: DeltaMessage | None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
reasoning_finished = finished or (
|
||||
delta is not None and (delta.content is not None or bool(delta.tool_calls))
|
||||
)
|
||||
reasoning = self._reasoning.feed(
|
||||
delta.reasoning if delta and delta.reasoning else "",
|
||||
final=reasoning_finished,
|
||||
)
|
||||
content = self._content.feed(
|
||||
delta.content if delta and delta.content else "",
|
||||
final=finished,
|
||||
)
|
||||
|
||||
if delta is None:
|
||||
if not reasoning and not content:
|
||||
return None
|
||||
return DeltaMessage(
|
||||
reasoning=reasoning or None,
|
||||
content=content or None,
|
||||
)
|
||||
|
||||
if reasoning:
|
||||
delta.reasoning = reasoning
|
||||
elif delta.reasoning is not None:
|
||||
delta.reasoning = None
|
||||
if content:
|
||||
delta.content = content
|
||||
elif delta.content is not None:
|
||||
delta.content = None
|
||||
|
||||
# A chunk that normalizes to nothing must be dropped, not returned as an
|
||||
# all-None DeltaMessage: the engine only skips on `None`, so anything
|
||||
# else reaches the client as an empty delta.
|
||||
if not delta.content and not delta.reasoning and not delta.tool_calls:
|
||||
return None
|
||||
return delta
|
||||
|
||||
|
||||
class MiniCPMVParser(Qwen3Parser):
|
||||
"""Parse MiniCPM-V family output in thinking and non-thinking modes."""
|
||||
|
||||
CONFIG_NAME = "minicpmv"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
# The chat template emits `<think>` unless `enable_thinking` is
|
||||
# explicitly false, so the grammar must default to thinking too, or the
|
||||
# reasoning span is misread as content.
|
||||
thinking_enabled = chat_kwargs.get("enable_thinking", True)
|
||||
kwargs.setdefault("parser_engine_config", minicpmv_config(thinking_enabled))
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
# ``Qwen3Parser.__init__`` reads the same key with a default of True,
|
||||
# which does not apply here: MiniCPM-V defaults to thinking disabled.
|
||||
# Keep the attribute in sync with the grammar built above.
|
||||
self.thinking_enabled = thinking_enabled
|
||||
|
||||
@property
|
||||
def reasoning_ended(self) -> bool:
|
||||
# The engine derives this from the presence of THINK terminals, which
|
||||
# both MiniCPM-V configs keep, so the base property stays False in
|
||||
# non-thinking mode. There is no reasoning span to end there.
|
||||
if not self.thinking_enabled:
|
||||
return True
|
||||
return super().reasoning_ended
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if self.thinking_enabled:
|
||||
return super().extract_reasoning(model_output, request)
|
||||
_, content = ParserEngine.extract_reasoning(self, model_output, request)
|
||||
return None, content
|
||||
|
||||
def extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> DeltaMessage | None:
|
||||
if self.thinking_enabled:
|
||||
return super().extract_reasoning_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
)
|
||||
|
||||
delta = ParserEngine.extract_reasoning_streaming(
|
||||
self,
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
)
|
||||
if delta is None:
|
||||
return None
|
||||
delta.reasoning = None
|
||||
if delta.content is None and not delta.tool_calls:
|
||||
return None
|
||||
return delta
|
||||
|
||||
def get_streaming_fallback_content(
|
||||
self,
|
||||
text: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> str | None:
|
||||
if self.thinking_enabled:
|
||||
return super().get_streaming_fallback_content(text, request)
|
||||
delta = ParserEngine.finish_streaming(self)
|
||||
return delta.content if delta is not None else None
|
||||
|
||||
|
||||
class MiniCPMVUnifiedParser(DelegatingParser):
|
||||
"""MiniCPM-V serving parser: the adapters plus newline recovery.
|
||||
|
||||
Recovery cannot live in :class:`MiniCPMVParser`: the serving stack drives
|
||||
the reasoning/tool adapters, never ``ParserEngine.parse`` /
|
||||
``ParserEngine.parse_delta``, so a rewrite applied there would only be
|
||||
visible to callers that construct the engine directly.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(tokenizer, tools, *args, **kwargs)
|
||||
self._output_normalizer = MiniCPMVOutputNormalizer()
|
||||
|
||||
def parse(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
enable_auto_tools: bool = False,
|
||||
model_output_token_ids: Sequence[int] = (),
|
||||
) -> tuple[str | None, str | None, list[FunctionCall] | None]:
|
||||
reasoning, content, tool_calls = super().parse(
|
||||
model_output,
|
||||
request,
|
||||
enable_auto_tools,
|
||||
model_output_token_ids,
|
||||
)
|
||||
# Tool call arguments are JSON, where a literal `\n` is meaningful and
|
||||
# must survive as written.
|
||||
return (
|
||||
recover_newlines(reasoning),
|
||||
recover_newlines(content),
|
||||
tool_calls,
|
||||
)
|
||||
|
||||
def parse_delta(
|
||||
self,
|
||||
delta_text: str,
|
||||
delta_token_ids: list[int],
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
*,
|
||||
finished: bool,
|
||||
) -> DeltaMessage | None:
|
||||
delta = super().parse_delta(
|
||||
delta_text,
|
||||
delta_token_ids,
|
||||
request,
|
||||
prompt_token_ids,
|
||||
finished=finished,
|
||||
)
|
||||
return self._output_normalizer.normalize_delta(delta, finished=finished)
|
||||
@@ -1,265 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiniMax M2 parser for XML-style tool calls.
|
||||
|
||||
MiniMax M2 tool call format::
|
||||
|
||||
<minimax:tool_call><invoke name="get_weather">
|
||||
<parameter name="city">Seattle</parameter>
|
||||
</invoke></minimax:tool_call>
|
||||
|
||||
Each ``<invoke>`` block becomes one tool call. The argument body consists
|
||||
of ``<parameter name="...">...</parameter>`` tags.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ArgValueExtender,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
|
||||
TOOL_CALL_START = "<minimax:tool_call>"
|
||||
TOOL_CALL_END = "</minimax:tool_call>"
|
||||
THINK_START = "<think>"
|
||||
THINK_END = "</think>"
|
||||
INVOKE_PREFIX_DQ = '<invoke name="'
|
||||
INVOKE_PREFIX_SQ = "<invoke name='"
|
||||
INVOKE_PREFIX_UNQUOTED = "<invoke name="
|
||||
INVOKE_END = "</invoke>"
|
||||
NAME_END_DQ = '">'
|
||||
NAME_END_SQ = "'>"
|
||||
NAME_END_UNQUOTED = ">"
|
||||
PARAM_START = "<parameter name="
|
||||
PARAM_END = "</parameter>"
|
||||
_PARAM_RE = re.compile(
|
||||
r"<\s*parameter\s+name\s*=\s*"
|
||||
r"(?:\"(?P<dq_name>[^\"]*)\"|'(?P<sq_name>[^']*)'|(?P<bare_name>[^>\s]+))"
|
||||
r"\s*>"
|
||||
r"(?P<value>.*?)"
|
||||
r"(?:<\s*/\s*parameter\s*>|(?=<\s*parameter\s+name\s*=))",
|
||||
re.DOTALL,
|
||||
)
|
||||
_PARTIAL_PARAM_RE = re.compile(
|
||||
r"<\s*parameter\s+name\s*=\s*"
|
||||
r"(?:\"(?P<dq_name>[^\"]*)\"|'(?P<sq_name>[^']*)'|(?P<bare_name>[^>\s]+))"
|
||||
r"\s*>"
|
||||
r"(?P<value>.*)$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
# What ends an open value in ``_PARAM_RE``: a closing tag or the next opener.
|
||||
_PARAM_VALUE_END_RE = re.compile(r"<\s*/\s*parameter\s*>|<\s*parameter\s+name\s*=")
|
||||
|
||||
|
||||
def _param_name(match: re.Match) -> str:
|
||||
return (
|
||||
match.group("dq_name")
|
||||
or match.group("sq_name")
|
||||
or match.group("bare_name")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
|
||||
def _minimax_m2_parse_args(
|
||||
raw_args: str, partial: bool
|
||||
) -> tuple[dict[str, object], re.Match | None]:
|
||||
"""Return the arguments and the match of the open trailing value, if any."""
|
||||
params: dict[str, object] = {}
|
||||
|
||||
for match in _PARAM_RE.finditer(raw_args):
|
||||
name = _param_name(match)
|
||||
if not name:
|
||||
continue
|
||||
# Keep the value verbatim (like the glm47 converter): it is inline
|
||||
# between `>` and `</parameter>`, so surrounding whitespace is data.
|
||||
params[name] = match.group("value")
|
||||
|
||||
open_match = None
|
||||
if partial:
|
||||
remaining = _PARAM_RE.sub("", raw_args)
|
||||
match = _PARTIAL_PARAM_RE.search(remaining)
|
||||
if match:
|
||||
name = _param_name(match)
|
||||
if name:
|
||||
# Verbatim, same as the complete-match loop above.
|
||||
params[name] = match.group("value")
|
||||
open_match = match
|
||||
|
||||
return params, open_match
|
||||
|
||||
|
||||
def _minimax_m2_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
return json.dumps(_minimax_m2_parse_args(raw_args, partial)[0], ensure_ascii=False)
|
||||
|
||||
|
||||
def _value_end_window(text: str) -> str:
|
||||
"""Return the suffix of *text* that may still grow into a value end."""
|
||||
start = text.rfind("<")
|
||||
if start == -1:
|
||||
return ""
|
||||
tail = text[start:]
|
||||
return tail if _PARAM_VALUE_END_RE.fullmatch(tail, partial=True) else ""
|
||||
|
||||
|
||||
def _minimax_m2_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
|
||||
# The open value runs to the end of raw_args and is the last JSON value,
|
||||
# so a delta appends its JSON-escaped text unless it closes the value.
|
||||
params, match = _minimax_m2_parse_args(raw_args, partial=True)
|
||||
if match is None or next(reversed(params)) != _param_name(match):
|
||||
return None
|
||||
# Later text could still re-parse the opener: an earlier unmatched one, or
|
||||
# a bare name that starts a quote, may close into a different name.
|
||||
bare_name = match.group("bare_name")
|
||||
if _PARAM_VALUE_END_RE.search(match.string, 0, match.start()) or (
|
||||
bare_name is not None and bare_name[0] in "\"'"
|
||||
):
|
||||
return None
|
||||
|
||||
window = _value_end_window(match.group("value"))
|
||||
|
||||
def extend(delta: str) -> str | None:
|
||||
nonlocal window
|
||||
text = window + delta
|
||||
if _PARAM_VALUE_END_RE.search(text):
|
||||
return None
|
||||
window = _value_end_window(text)
|
||||
return json.dumps(delta, ensure_ascii=False)[1:-1]
|
||||
|
||||
return extend
|
||||
|
||||
|
||||
@functools.cache
|
||||
def minimax_m2_config() -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="minimax_m2",
|
||||
initial_state=ParserState.REASONING,
|
||||
wait_for_reasoning=True,
|
||||
terminals={
|
||||
"THINK_START": THINK_START,
|
||||
"THINK_END": THINK_END,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"PARAM_START": PARAM_START,
|
||||
"PARAM_END": PARAM_END,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
"INVOKE_PREFIX_DQ": INVOKE_PREFIX_DQ,
|
||||
"INVOKE_PREFIX_SQ": INVOKE_PREFIX_SQ,
|
||||
"INVOKE_PREFIX_UNQUOTED": INVOKE_PREFIX_UNQUOTED,
|
||||
"INVOKE_END": INVOKE_END,
|
||||
"NAME_END_DQ": NAME_END_DQ,
|
||||
"NAME_END_SQ": NAME_END_SQ,
|
||||
"NAME_END_UNQUOTED": NAME_END_UNQUOTED,
|
||||
},
|
||||
token_id_terminals={
|
||||
"THINK_START": THINK_START,
|
||||
"THINK_END": THINK_END,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
},
|
||||
transitions={
|
||||
(ParserState.REASONING, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
(ParserState.REASONING, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "PARAM_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.ARG_VALUE_CHUNK,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "PARAM_END"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.ARG_VALUE_CHUNK,),
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
(ParserState.CONTENT, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "INVOKE_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
**{
|
||||
(state, terminal): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
)
|
||||
for state in (
|
||||
ParserState.CONTENT,
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
ParserState.TOOL_BETWEEN,
|
||||
)
|
||||
for terminal in (
|
||||
"INVOKE_PREFIX_DQ",
|
||||
"INVOKE_PREFIX_SQ",
|
||||
"INVOKE_PREFIX_UNQUOTED",
|
||||
)
|
||||
},
|
||||
**{
|
||||
(ParserState.TOOL_NAME, terminal): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
)
|
||||
for terminal in (
|
||||
"NAME_END_DQ",
|
||||
"NAME_END_SQ",
|
||||
"NAME_END_UNQUOTED",
|
||||
)
|
||||
},
|
||||
},
|
||||
arg_converter=_minimax_m2_arg_converter,
|
||||
arg_value_extender=_minimax_m2_arg_value_extender,
|
||||
stream_arg_deltas=True,
|
||||
tool_args_json=False,
|
||||
validate_tool_names=True,
|
||||
)
|
||||
|
||||
|
||||
class MinimaxM2Parser(ParserEngine):
|
||||
"""MiniMax M2 parser backed by the declarative parser engine."""
|
||||
|
||||
def __init__(self, tokenizer, tools=None, **kwargs) -> None:
|
||||
kwargs.setdefault("parser_engine_config", minimax_m2_config())
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
self._think_end_token_id = self.vocab.get(THINK_END)
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
end_id = self._think_end_token_id
|
||||
if end_id is None:
|
||||
return []
|
||||
for i in range(len(input_ids) - 1, -1, -1):
|
||||
if input_ids[i] == end_id:
|
||||
return input_ids[i + 1 :]
|
||||
return []
|
||||
@@ -1,293 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiniMax M3 parser for namespace-delimited XML-style tool calls.
|
||||
|
||||
MiniMax M3 prefixes every structural tag with the ``]<]minimax[>[`` marker::
|
||||
|
||||
]<]minimax[>[<tool_call>
|
||||
]<]minimax[>[<invoke name="create_order">
|
||||
]<]minimax[>[<user_id>42]<]minimax[>[</user_id>
|
||||
]<]minimax[>[<items>
|
||||
]<]minimax[>[<item>]<]minimax[>[<sku>book-001]<]minimax[>[</sku>]<]minimax[>[</item>
|
||||
]<]minimax[>[</items>
|
||||
]<]minimax[>[</invoke>
|
||||
]<]minimax[>[</tool_call>
|
||||
|
||||
Each ``<invoke>`` becomes one tool call. Parameters are elements named after
|
||||
the argument and may nest: an element holding child elements becomes an
|
||||
object, or an array when its schema says so (the chat template renders list
|
||||
items as ``<item>``). Arguments are emitted once the whole ``<invoke>`` is
|
||||
parsed. Reasoning is handled by ``MiniMaxM3ReasoningParser``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.tool_parsers.utils import (
|
||||
coerce_to_schema_type,
|
||||
extract_types_from_schema,
|
||||
find_tool_properties,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
NAMESPACE = "]<]minimax[>["
|
||||
TOOL_CALL_START = f"{NAMESPACE}<tool_call>"
|
||||
TOOL_CALL_END = f"{NAMESPACE}</tool_call>"
|
||||
INVOKE_PREFIX_DQ = f'{NAMESPACE}<invoke name="'
|
||||
INVOKE_PREFIX_SQ = f"{NAMESPACE}<invoke name='"
|
||||
INVOKE_PREFIX_UNQUOTED = f"{NAMESPACE}<invoke name="
|
||||
INVOKE_END = f"{NAMESPACE}</invoke>"
|
||||
NAME_END_DQ = '">'
|
||||
NAME_END_SQ = "'>"
|
||||
NAME_END_UNQUOTED = ">"
|
||||
ELEMENT_START = f"{NAMESPACE}<"
|
||||
ELEMENT_END_START = f"{NAMESPACE}</"
|
||||
MIXED_TEXT_FIELD = "$text"
|
||||
MAX_ELEMENT_DEPTH = 128
|
||||
|
||||
# An element value: its text, or its ``(name, value)`` children in order.
|
||||
_Element = str | list[tuple[str, "_Element"]]
|
||||
|
||||
|
||||
def _parse_element(text: str, pos: int, depth: int) -> tuple[str, _Element, int] | None:
|
||||
"""Parse the element starting at ``pos``; ``None`` if malformed."""
|
||||
if depth > MAX_ELEMENT_DEPTH:
|
||||
return None
|
||||
name_start = pos + len(ELEMENT_START)
|
||||
name_end = text.find(">", name_start)
|
||||
if name_end <= name_start:
|
||||
return None
|
||||
name = text[name_start:name_end]
|
||||
if name.startswith("/") or not name.strip():
|
||||
return None
|
||||
|
||||
close_tag = f"{ELEMENT_END_START}{name}>"
|
||||
pos = name_end + 1
|
||||
text_parts: list[str] = []
|
||||
children: list[tuple[str, _Element]] = []
|
||||
while True:
|
||||
marker = text.find(NAMESPACE, pos)
|
||||
if marker < 0:
|
||||
return None
|
||||
text_parts.append(text[pos:marker])
|
||||
if text.startswith(close_tag, marker):
|
||||
pos = marker + len(close_tag)
|
||||
break
|
||||
if not text.startswith(ELEMENT_START, marker):
|
||||
return None
|
||||
child = _parse_element(text, marker, depth + 1)
|
||||
if child is None:
|
||||
return None
|
||||
child_name, child_value, pos = child
|
||||
children.append((child_name, child_value))
|
||||
|
||||
body_text = "".join(text_parts)
|
||||
if not children:
|
||||
return name, body_text, pos
|
||||
if body_text.strip():
|
||||
# Keep mixed text under a reserved field that no child uses.
|
||||
field = MIXED_TEXT_FIELD
|
||||
while any(child_name == field for child_name, _ in children):
|
||||
field = "$" + field
|
||||
children.append((field, body_text))
|
||||
return name, children, pos
|
||||
|
||||
|
||||
def _parse_invoke_params(raw_args: str) -> list[tuple[str, _Element]]:
|
||||
"""Parse the parameter elements of one ``<invoke>`` body.
|
||||
|
||||
Parsing stops at the first malformed or unterminated element, keeping the
|
||||
parameters before it.
|
||||
"""
|
||||
params: list[tuple[str, _Element]] = []
|
||||
pos = 0
|
||||
while True:
|
||||
while pos < len(raw_args) and raw_args[pos].isspace():
|
||||
pos += 1
|
||||
if not raw_args.startswith(ELEMENT_START, pos):
|
||||
return params
|
||||
element = _parse_element(raw_args, pos, depth=1)
|
||||
if element is None:
|
||||
return params
|
||||
name, value, pos = element
|
||||
params.append((name, value))
|
||||
|
||||
|
||||
def _container_schema(schema: dict[str, Any]) -> tuple[str, dict[str, Any]] | None:
|
||||
"""Return the first ``array`` or ``object`` alternative of ``schema``."""
|
||||
types = schema.get("type")
|
||||
for kind in ("array", "object"):
|
||||
if types == kind or (isinstance(types, list) and kind in types):
|
||||
return kind, schema
|
||||
if types is None:
|
||||
if "items" in schema:
|
||||
return "array", schema
|
||||
if "properties" in schema or "additionalProperties" in schema:
|
||||
return "object", schema
|
||||
for choice_field in ("anyOf", "oneOf"):
|
||||
for choice in schema.get(choice_field) or ():
|
||||
if isinstance(choice, dict) and (found := _container_schema(choice)):
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _convert_element(value: _Element, schema: Any) -> Any:
|
||||
if not isinstance(schema, dict):
|
||||
schema = None
|
||||
container = _container_schema(schema) if schema is not None else None
|
||||
|
||||
if isinstance(value, str):
|
||||
if schema is None:
|
||||
return value
|
||||
types = extract_types_from_schema(schema)
|
||||
# The template renders an empty list or dict as an empty element.
|
||||
if container is not None and "string" not in types and not value.strip():
|
||||
return [] if container[0] == "array" else {}
|
||||
return coerce_to_schema_type(value, types)
|
||||
|
||||
if container is not None and container[0] == "array":
|
||||
items = container[1].get("items")
|
||||
return [_convert_element(child, items) for _, child in value]
|
||||
return _convert_object(value, container[1] if container is not None else None)
|
||||
|
||||
|
||||
def _convert_object(
|
||||
elements: list[tuple[str, _Element]], schema: dict[str, Any] | None
|
||||
) -> dict[str, Any]:
|
||||
"""Convert named elements to an object; repeated names collect into a list."""
|
||||
properties = (schema or {}).get("properties") or {}
|
||||
additional = (schema or {}).get("additionalProperties")
|
||||
result: dict[str, Any] = {}
|
||||
repeated: set[str] = set()
|
||||
for name, element in elements:
|
||||
converted = _convert_element(element, properties.get(name, additional))
|
||||
if name not in result:
|
||||
result[name] = converted
|
||||
elif name in repeated:
|
||||
result[name].append(converted)
|
||||
else:
|
||||
result[name] = [result[name], converted]
|
||||
repeated.add(name)
|
||||
return result
|
||||
|
||||
|
||||
def _minimax_m3_arg_converter(
|
||||
raw_args: str,
|
||||
partial: bool,
|
||||
properties: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
params = _parse_invoke_params(raw_args)
|
||||
arguments = _convert_object(params, {"properties": properties or {}})
|
||||
return json.dumps(arguments, ensure_ascii=False)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def minimax_m3_config() -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="minimax_m3",
|
||||
initial_state=ParserState.CONTENT,
|
||||
terminals={
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
"INVOKE_PREFIX_DQ": INVOKE_PREFIX_DQ,
|
||||
"INVOKE_PREFIX_SQ": INVOKE_PREFIX_SQ,
|
||||
"INVOKE_PREFIX_UNQUOTED": INVOKE_PREFIX_UNQUOTED,
|
||||
"INVOKE_END": INVOKE_END,
|
||||
"NAME_END_DQ": NAME_END_DQ,
|
||||
"NAME_END_SQ": NAME_END_SQ,
|
||||
"NAME_END_UNQUOTED": NAME_END_UNQUOTED,
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(),
|
||||
),
|
||||
**{
|
||||
(state, terminal): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
)
|
||||
for state in (ParserState.TOOL_PREAMBLE, ParserState.TOOL_BETWEEN)
|
||||
for terminal in (
|
||||
"INVOKE_PREFIX_DQ",
|
||||
"INVOKE_PREFIX_SQ",
|
||||
"INVOKE_PREFIX_UNQUOTED",
|
||||
)
|
||||
},
|
||||
**{
|
||||
(ParserState.TOOL_NAME, terminal): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
)
|
||||
for terminal in ("NAME_END_DQ", "NAME_END_SQ", "NAME_END_UNQUOTED")
|
||||
},
|
||||
# Nested parameter elements are argument text; only the invoke
|
||||
# closer ends the call.
|
||||
(ParserState.TOOL_ARGS, "INVOKE_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
# The tool block ends the message: stay in TOOL_BETWEEN so any
|
||||
# text after it is dropped.
|
||||
(ParserState.TOOL_PREAMBLE, "TOOL_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(),
|
||||
),
|
||||
},
|
||||
arg_converter=_minimax_m3_arg_converter,
|
||||
# Emit arguments once per `<invoke>`: converted values are not
|
||||
# prefix-stable while elements arrive, e.g. a repeated parameter name
|
||||
# turns an already streamed scalar into a list.
|
||||
stream_arg_deltas=False,
|
||||
tool_args_json=False,
|
||||
strip_content_whitespace_with_tools=False,
|
||||
)
|
||||
|
||||
|
||||
class MinimaxM3Parser(ParserEngine):
|
||||
"""MiniMax M3 tool-call parser backed by the declarative parser engine."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
kwargs.setdefault("parser_engine_config", minimax_m3_config())
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
self._arg_converter = self._convert_args
|
||||
|
||||
def adjust_request(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
# The M3 markers are non-special added tokens, so they decode with
|
||||
# special tokens skipped.
|
||||
return request
|
||||
|
||||
def _convert_args(self, raw_args: str, partial: bool) -> str:
|
||||
func_name = next(
|
||||
(slot.name for slot in self._tool_slots if slot.args == raw_args), ""
|
||||
)
|
||||
properties = find_tool_properties(self._tools, func_name.strip())
|
||||
return _minimax_m3_arg_converter(raw_args, partial, properties)
|
||||
@@ -1,113 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Nemotron V3 parser.
|
||||
|
||||
The Nemotron 3 Super model uses the same tool call and reasoning
|
||||
format as Qwen3 (``<think>``/``</think>`` + ``<tool_call>`` XML).
|
||||
This config reuses :func:`qwen3_config` with a distinct name.
|
||||
|
||||
When ``enable_thinking=False`` or ``force_nonempty_content=True`` and
|
||||
content is empty, reasoning and content are swapped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import functools
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.engine.parser_engine import SemanticEvent
|
||||
from vllm.parser.engine.parser_engine_config import ParserEngineConfig
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
|
||||
@functools.cache
|
||||
def nemotron_v3_config(thinking: bool = True) -> ParserEngineConfig:
|
||||
return dataclasses.replace(
|
||||
qwen3_config(thinking=thinking, turn_boundary_tokens=CHATML_TURN_BOUNDARIES),
|
||||
name="nemotron_v3",
|
||||
strip_trailing_reasoning_whitespace=True,
|
||||
)
|
||||
|
||||
|
||||
class NemotronV3Parser(Qwen3Parser):
|
||||
"""Nemotron V3 parser: same format as Qwen3, with Nemotron-specific
|
||||
behavior: when ``enable_thinking=False`` or
|
||||
``force_nonempty_content=True`` and content is empty, swaps
|
||||
reasoning and content.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
thinking = chat_kwargs.get("enable_thinking", True)
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
tools,
|
||||
parser_engine_config=nemotron_v3_config(thinking=thinking),
|
||||
**kwargs,
|
||||
)
|
||||
self._streamed_reasoning: list[str] = []
|
||||
|
||||
def _reset(self, initial_state=None) -> None:
|
||||
super()._reset(initial_state=initial_state)
|
||||
self._streamed_reasoning = []
|
||||
|
||||
def _events_to_delta(
|
||||
self,
|
||||
events: list[SemanticEvent],
|
||||
finished: bool = False,
|
||||
) -> DeltaMessage | None:
|
||||
delta = super()._events_to_delta(events, finished=finished)
|
||||
if delta is not None and delta.reasoning is not None:
|
||||
self._streamed_reasoning.append(delta.reasoning)
|
||||
return delta
|
||||
|
||||
@staticmethod
|
||||
def _should_force_content(
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> bool:
|
||||
chat_template_kwargs = getattr(request, "chat_template_kwargs", None)
|
||||
return bool(
|
||||
chat_template_kwargs
|
||||
and (
|
||||
chat_template_kwargs.get("enable_thinking") is False
|
||||
or chat_template_kwargs.get("force_nonempty_content") is True
|
||||
)
|
||||
)
|
||||
|
||||
def get_streaming_fallback_content(
|
||||
self,
|
||||
text: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> str | None:
|
||||
if not self._should_force_content(request):
|
||||
return None
|
||||
return "".join(self._streamed_reasoning) or None
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
reasoning, content = super().extract_reasoning(model_output, request)
|
||||
|
||||
if self._should_force_content(request) and (
|
||||
content is None or not content.strip()
|
||||
):
|
||||
reasoning, content = content, reasoning
|
||||
|
||||
return reasoning, content
|
||||
@@ -1,318 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from itertools import groupby
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
)
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
BEGIN_TOOL_REQUESTS = "<|plamo:begin_tool_requests:plamo|>"
|
||||
END_TOOL_REQUESTS = "<|plamo:end_tool_requests:plamo|>"
|
||||
BEGIN_TOOL_REQUEST = "<|plamo:begin_tool_request:plamo|>"
|
||||
END_TOOL_REQUEST = "<|plamo:end_tool_request:plamo|>"
|
||||
BEGIN_TOOL_NAME = "<|plamo:begin_tool_name:plamo|>"
|
||||
END_TOOL_NAME = "<|plamo:end_tool_name:plamo|>"
|
||||
BEGIN_TOOL_ARGUMENTS = (
|
||||
"<|plamo:begin_tool_arguments:plamo|><|plamo:constrain|>json<|plamo:msg|>"
|
||||
)
|
||||
END_TOOL_ARGUMENTS = "<|plamo:end_tool_arguments:plamo|>"
|
||||
BEGIN_THINK = "<|plamo:begin_think:plamo|>"
|
||||
END_THINK = "<|plamo:end_think:plamo|>"
|
||||
EOT = "<|plamo:tag|>"
|
||||
|
||||
# PLaMo3 control markers are split into multiple tokenizer tokens. These
|
||||
# atomic pieces must not be treated as unrelated special tokens by the
|
||||
# parser-engine drop-token handling.
|
||||
PLAMO_MARKER_TOKENS = frozenset(
|
||||
{
|
||||
"<|plamo:begin_",
|
||||
"<|plamo:end_",
|
||||
":plamo|>",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def plamo3_config(thinking: bool = True) -> ParserEngineConfig:
|
||||
if thinking:
|
||||
reasoning_terminals = {
|
||||
"THINK_START": BEGIN_THINK,
|
||||
"THINK_END": END_THINK,
|
||||
}
|
||||
reasoning_transitions = {
|
||||
(ParserState.REASONING, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
),
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
(ParserState.REASONING, "TOOL_REQUESTS_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
}
|
||||
initial_state = ParserState.REASONING
|
||||
else:
|
||||
reasoning_terminals = {}
|
||||
reasoning_transitions = {}
|
||||
initial_state = ParserState.CONTENT
|
||||
|
||||
return ParserEngineConfig(
|
||||
name="plamo3",
|
||||
initial_state=initial_state,
|
||||
terminals={
|
||||
**reasoning_terminals,
|
||||
"TOOL_REQUESTS_START": BEGIN_TOOL_REQUESTS,
|
||||
"TOOL_REQUESTS_END": END_TOOL_REQUESTS,
|
||||
"TOOL_REQUEST_START": BEGIN_TOOL_REQUEST,
|
||||
"TOOL_REQUEST_END": END_TOOL_REQUEST,
|
||||
"TOOL_NAME_START": BEGIN_TOOL_NAME,
|
||||
"TOOL_NAME_END": END_TOOL_NAME,
|
||||
"TOOL_ARGS_START": BEGIN_TOOL_ARGUMENTS,
|
||||
"TOOL_ARGS_END": END_TOOL_ARGUMENTS,
|
||||
"EOT": EOT,
|
||||
},
|
||||
transitions={
|
||||
**reasoning_transitions,
|
||||
(ParserState.CONTENT, "TOOL_REQUESTS_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "TOOL_REQUEST_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "TOOL_NAME_START"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "TOOL_NAME_END"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "TOOL_ARGS_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_ARGS_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_REQUEST_END"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "TOOL_REQUESTS_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
),
|
||||
# EOT is normally consumed by generation stop handling. Keep it
|
||||
# in the grammar as well so text-only parsing and explicitly
|
||||
# supplied stop tokens cannot leak it into user-visible output.
|
||||
(ParserState.REASONING, "EOT"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "EOT"): Transition(ParserState.CONTENT),
|
||||
(ParserState.TOOL_PREAMBLE, "EOT"): Transition(ParserState.CONTENT),
|
||||
(ParserState.TOOL_NAME, "EOT"): Transition(ParserState.CONTENT),
|
||||
(ParserState.TOOL_ARGS, "EOT"): Transition(ParserState.CONTENT),
|
||||
(ParserState.TOOL_BETWEEN, "EOT"): Transition(ParserState.CONTENT),
|
||||
},
|
||||
token_id_terminals={"EOT": EOT, **reasoning_terminals},
|
||||
drop_whitespace_only_content_before_tools=True,
|
||||
stream_arg_deltas=True,
|
||||
strip_content_whitespace_with_tools=False,
|
||||
strip_trailing_reasoning_whitespace=False,
|
||||
preserve_tokens=PLAMO_MARKER_TOKENS,
|
||||
tool_args_json=True,
|
||||
)
|
||||
|
||||
|
||||
class Plamo3Parser(ParserEngine):
|
||||
"""PLaMo3 reasoning and tool-call parser backed by ParserEngine."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
# Configure PLaMo parsing and cache its multi-token reasoning markers.
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
self.thinking_enabled = chat_kwargs.get("enable_thinking", True)
|
||||
kwargs.setdefault(
|
||||
"parser_engine_config",
|
||||
plamo3_config(thinking=self.thinking_enabled),
|
||||
)
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
|
||||
self._reasoning_start_token_id_sequence: list[int] = list(
|
||||
tokenizer.encode(BEGIN_THINK, add_special_tokens=False)
|
||||
)
|
||||
self._reasoning_end_token_id_sequence: list[int] = list(
|
||||
tokenizer.encode(END_THINK, add_special_tokens=False)
|
||||
)
|
||||
# Token-ID sequences of every terminal literal, for token-level
|
||||
# unfinished-marker detection.
|
||||
self._terminal_token_id_sequences: dict[str, tuple[int, ...]] = {
|
||||
literal: tuple(tokenizer.encode(literal, add_special_tokens=False))
|
||||
for literal in self.parser_engine_config.terminal_literals
|
||||
}
|
||||
|
||||
def is_reasoning_end(self, input_ids: list[int]) -> bool:
|
||||
# Detect PLaMo reasoning boundaries that span multiple token IDs.
|
||||
if not self.thinking_enabled:
|
||||
return True
|
||||
|
||||
start_ids = self._reasoning_start_token_id_sequence
|
||||
end_ids = self._reasoning_end_token_id_sequence
|
||||
for i in range(len(input_ids) - 1, -1, -1):
|
||||
if input_ids[i : i + len(end_ids)] == end_ids:
|
||||
return True
|
||||
if input_ids[i : i + len(start_ids)] == start_ids:
|
||||
return False
|
||||
if input_ids[i] == self.vocab.get(EOT):
|
||||
return False
|
||||
return False
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
# Extract content after PLaMo's multi-token reasoning end marker.
|
||||
if not self.thinking_enabled:
|
||||
return input_ids
|
||||
end_ids = self._reasoning_end_token_id_sequence
|
||||
for i in range(len(input_ids) - len(end_ids), -1, -1):
|
||||
if input_ids[i : i + len(end_ids)] == end_ids:
|
||||
return input_ids[i + len(end_ids) :]
|
||||
return input_ids
|
||||
|
||||
def _strip_unfinished_marker(self, value: str | None) -> str | None:
|
||||
if not value:
|
||||
return value
|
||||
marker_start = value.rfind("<|plamo:")
|
||||
if marker_start < 0:
|
||||
return value or None
|
||||
suffix = value[marker_start:]
|
||||
# Strip a trailing partial marker at the token-ID level.
|
||||
suffix_ids = tuple(
|
||||
self.model_tokenizer.encode(suffix, add_special_tokens=False)
|
||||
)
|
||||
if any(
|
||||
len(seq) > len(suffix_ids) and seq[: len(suffix_ids)] == suffix_ids
|
||||
for seq in self._terminal_token_id_sequences.values()
|
||||
):
|
||||
return value[:marker_start] or None
|
||||
return value or None
|
||||
|
||||
def _coalesce_finished_tool_events(
|
||||
self, events: list[SemanticEvent]
|
||||
) -> list[SemanticEvent]:
|
||||
combined = []
|
||||
for (kind, _), group in groupby(
|
||||
events, key=lambda event: (event.type, event.tool_index)
|
||||
):
|
||||
chunks = list(group)
|
||||
if kind not in (EventType.TOOL_NAME, EventType.ARG_VALUE_CHUNK):
|
||||
combined.extend(chunks)
|
||||
continue
|
||||
if kind == EventType.TOOL_NAME and chunks[0].tool_index < 0:
|
||||
continue
|
||||
value = "".join(chunk.value for chunk in chunks)
|
||||
if kind == EventType.TOOL_NAME:
|
||||
value = self._strip_unfinished_marker(value) or ""
|
||||
if value:
|
||||
combined.append(replace(chunks[0], value=value))
|
||||
return combined
|
||||
|
||||
def _events_to_delta(
|
||||
self,
|
||||
events: list[SemanticEvent],
|
||||
finished: bool = False,
|
||||
) -> DeltaMessage | None:
|
||||
# Coalesce flushed tool fragments before stripping truncated markers.
|
||||
if not finished:
|
||||
return super()._events_to_delta(events)
|
||||
|
||||
events = self._coalesce_finished_tool_events(events)
|
||||
delta = super()._events_to_delta(events, finished=True)
|
||||
if delta is None:
|
||||
return None
|
||||
|
||||
delta.reasoning = self._strip_unfinished_marker(delta.reasoning)
|
||||
if not self.skip_tool_parsing:
|
||||
delta.content = self._strip_unfinished_marker(delta.content)
|
||||
# The flush emits the remaining (never-streamed) tool-call
|
||||
# arguments; an empty remainder must match non-streaming
|
||||
# extraction (_extract_args_json), which normalizes it to '{}'.
|
||||
for tc in delta.tool_calls or []:
|
||||
if tc.function is not None and not (tc.function.arguments or "").strip():
|
||||
tc.function.arguments = "{}"
|
||||
if delta.reasoning is None and delta.content is None and not delta.tool_calls:
|
||||
return None
|
||||
return delta
|
||||
|
||||
def _handle_arg_chunk(
|
||||
self,
|
||||
event: SemanticEvent,
|
||||
deltas: list[DeltaToolCall],
|
||||
) -> None:
|
||||
# Strip truncated marker suffixes and preserve the first argument delta.
|
||||
idx = event.tool_index
|
||||
slot = self._tool_slots[idx]
|
||||
if (marker_pos := event.value.rfind("<|plamo:")) >= 0:
|
||||
stripped = event.value[:marker_pos]
|
||||
try:
|
||||
json.loads(slot.args + stripped)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
event = replace(event, value=stripped)
|
||||
name_sent_before = slot.name_sent
|
||||
super()._handle_arg_chunk(event, deltas)
|
||||
if event.value and not name_sent_before and slot.name_sent:
|
||||
deltas.append(
|
||||
DeltaToolCall(
|
||||
index=idx,
|
||||
function=DeltaFunctionCall(arguments=event.value),
|
||||
)
|
||||
)
|
||||
|
||||
def _extract_args_json(self, raw_args: str, func_name: str) -> str:
|
||||
# Return the raw arguments as JSON
|
||||
# since PLaMo3 generates function names and arguments separately.
|
||||
return raw_args.strip() or "{}"
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
reasoning, content = super().extract_reasoning(model_output, request)
|
||||
return self._strip_unfinished_marker(reasoning), content
|
||||
@@ -1,31 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""seed_oss parser for tool calls and reasoning.
|
||||
|
||||
seed_oss shares the Qwen3 XML grammar exactly; only the wrapper token
|
||||
strings and the turn-boundary tokens differ::
|
||||
|
||||
<think> -> <seed:think>
|
||||
</think> -> </seed:think>
|
||||
<tool_call> -> <seed:tool_call>
|
||||
</tool_call> -> </seed:tool_call>
|
||||
<|im_start|> -> <seed:bos>
|
||||
<|im_end|> -> <seed:eos>
|
||||
|
||||
``<function=...>`` and ``<parameter=...>`` are byte-identical, so the
|
||||
entire transition table and ``_qwen3_arg_converter`` are inherited from
|
||||
:class:`Qwen3Parser` unchanged.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from vllm.parser.qwen3 import Qwen3Parser
|
||||
|
||||
|
||||
class SeedOssParser(Qwen3Parser):
|
||||
CONFIG_NAME = "seed_oss"
|
||||
THINK_START = "<seed:think>"
|
||||
THINK_END = "</seed:think>"
|
||||
TOOL_START = "<seed:tool_call>"
|
||||
TOOL_END = "</seed:tool_call>"
|
||||
TURN_BOUNDARIES = frozenset(("<seed:bos>", "<seed:eos>"))
|
||||
@@ -1,49 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Step-3.5 / Step-3.7 parser for tool calls and reasoning.
|
||||
|
||||
Both models use the Qwen3 XML tool-call grammar and ``<think>`` reasoning
|
||||
unchanged. Two things differ from :class:`Qwen3Parser`:
|
||||
|
||||
* The chat template always prefills ``<think>\\n`` and has no switch to turn
|
||||
thinking off, so ``enable_thinking`` is ignored.
|
||||
* The model closes reasoning with ``\\n</think>\\n``, so trailing reasoning
|
||||
whitespace is stripped.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.parser.engine.parser_engine_config import ParserEngineConfig
|
||||
from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
|
||||
@functools.cache
|
||||
def step3p5_config() -> ParserEngineConfig:
|
||||
return replace(
|
||||
qwen3_config(
|
||||
thinking=True,
|
||||
name="step3p5",
|
||||
turn_boundary_tokens=CHATML_TURN_BOUNDARIES,
|
||||
),
|
||||
strip_trailing_reasoning_whitespace=True,
|
||||
)
|
||||
|
||||
|
||||
class Step3p5Parser(Qwen3Parser):
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
kwargs.setdefault("parser_engine_config", step3p5_config())
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
self.thinking_enabled = True
|
||||
Reference in New Issue
Block a user