[Bugfix] Tolerate misspelled DSML tool_calls wrapper (#56141)
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -30,6 +30,7 @@ from vllm.parser.deepseek_v4 import (
|
||||
DSML_THINK_START,
|
||||
DSML_TOOL_END,
|
||||
DSML_TOOL_START,
|
||||
DSML_TOOL_START_VARIANTS,
|
||||
DeepSeekV4Parser,
|
||||
_dsml_arg_converter,
|
||||
_unwrap_wrapper_args,
|
||||
@@ -1229,3 +1230,164 @@ class TestDelegatingParserLargeDelta:
|
||||
assert eos_text not in output.reasoning
|
||||
assert output.content == ""
|
||||
assert output.tool_calls == []
|
||||
|
||||
|
||||
# ── Malformed DSML markup must not leak into content (#51914) ─────────
|
||||
|
||||
_WRAPPER_VARIANTS = list(DSML_TOOL_START_VARIANTS)
|
||||
|
||||
|
||||
class TestMalformedDsmlNoise:
|
||||
"""DeepSeek-V4-Flash intermittently misspells the tool-call opener, e.g.
|
||||
``<|DSML|toolcalls>``. Known corrupted openers open a tool block like
|
||||
the real wrapper so the markup never surfaces as assistant content.
|
||||
"""
|
||||
|
||||
_INNER = (
|
||||
f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n"
|
||||
f"{_param('city', 'true', 'Seoul')}\n"
|
||||
f"{DSML_INVOKE_END}\n"
|
||||
f"{DSML_TOOL_END}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parser(mock_tokenizer, thinking: bool = False):
|
||||
return DeepSeekV4Parser(
|
||||
mock_tokenizer, chat_template_kwargs={"thinking": thinking}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("wrapper", _WRAPPER_VARIANTS)
|
||||
@pytest.mark.parametrize("preamble", ["", "Let me check.\n"])
|
||||
def test_corrupted_wrapper_behaves_like_real_one(
|
||||
self, mock_tokenizer, mock_request, wrapper, preamble
|
||||
):
|
||||
text = f"{preamble}{wrapper}\n{self._INNER}"
|
||||
reference = f"{preamble}{DSML_TOOL_START}\n{self._INNER}"
|
||||
|
||||
result = self._parser(mock_tokenizer).extract_tool_calls(text, mock_request)
|
||||
expected = self._parser(mock_tokenizer).extract_tool_calls(
|
||||
reference, mock_request
|
||||
)
|
||||
assert [tc.function.name for tc in result.tool_calls] == ["get_weather"]
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Seoul"}
|
||||
assert result.content == expected.content
|
||||
assert "DSML" not in (result.content or "")
|
||||
|
||||
results = simulate_tool_streaming(
|
||||
self._parser(mock_tokenizer), mock_request, list(text)
|
||||
)
|
||||
assert collect_function_name(results) == "get_weather"
|
||||
assert json.loads(collect_tool_arguments(results)) == {"city": "Seoul"}
|
||||
assert collect_content(results) == (expected.content or "")
|
||||
|
||||
@pytest.mark.parametrize("wrapper", _WRAPPER_VARIANTS)
|
||||
def test_corrupted_wrapper_ends_reasoning(
|
||||
self, mock_tokenizer, mock_request, wrapper
|
||||
):
|
||||
"""Like the real wrapper, a corrupted one inside ``<think>`` closes
|
||||
the reasoning block and starts the tool call."""
|
||||
text = f"thinking{wrapper}\n{self._INNER}"
|
||||
parser = self._parser(mock_tokenizer, thinking=True)
|
||||
reasoning, content = parser.extract_reasoning(text, mock_request)
|
||||
assert reasoning == "thinking"
|
||||
assert "DSML" not in (content or "")
|
||||
|
||||
result = self._parser(mock_tokenizer, thinking=True).extract_tool_calls(
|
||||
text, mock_request
|
||||
)
|
||||
assert [tc.function.name for tc in result.tool_calls] == ["get_weather"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
# Fullwidth bars, DSML namespace, but not a registered spelling.
|
||||
"see <|DSML|foo> and <|DSML|tool_call> here",
|
||||
# Unclosed prefix.
|
||||
"see <|DSML| here\nnext line",
|
||||
# ASCII pipes are not the DSML sigil.
|
||||
"see <|DSML|tool> and <|DSML|toolcalls> there",
|
||||
],
|
||||
)
|
||||
def test_unregistered_markup_is_ordinary_content(
|
||||
self, mock_tokenizer, mock_request, text
|
||||
):
|
||||
"""Only the registered spellings are treated as wrappers."""
|
||||
result = self._parser(mock_tokenizer).extract_tool_calls(text, mock_request)
|
||||
assert result.tools_called is False
|
||||
assert result.content == text
|
||||
|
||||
results = simulate_tool_streaming(
|
||||
self._parser(mock_tokenizer), mock_request, list(text)
|
||||
)
|
||||
assert collect_content(results) == text
|
||||
|
||||
@pytest.mark.parametrize("wrapper", _WRAPPER_VARIANTS)
|
||||
def test_prose_mention_matches_canonical_wrapper(
|
||||
self, mock_tokenizer, mock_request, wrapper
|
||||
):
|
||||
"""A registered spelling mentioned in prose behaves exactly like a
|
||||
prose mention of the real wrapper: the opener starts a tool block and
|
||||
the rest of the message is dropped. That is pre-existing behavior of
|
||||
``<|DSML|tool_calls>``; the variants must not differ from it either
|
||||
way."""
|
||||
template = "The opener looks like {} and then params follow. Done."
|
||||
text = template.format(wrapper)
|
||||
reference = template.format(DSML_TOOL_START)
|
||||
|
||||
result = self._parser(mock_tokenizer).extract_tool_calls(text, mock_request)
|
||||
expected = self._parser(mock_tokenizer).extract_tool_calls(
|
||||
reference, mock_request
|
||||
)
|
||||
assert result.tools_called == expected.tools_called
|
||||
assert result.content == expected.content
|
||||
|
||||
results = simulate_tool_streaming(
|
||||
self._parser(mock_tokenizer), mock_request, list(text)
|
||||
)
|
||||
expected_results = simulate_tool_streaming(
|
||||
self._parser(mock_tokenizer), mock_request, list(reference)
|
||||
)
|
||||
assert collect_content(results) == collect_content(expected_results)
|
||||
|
||||
@pytest.mark.parametrize("wrapper", _WRAPPER_VARIANTS)
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, None], ids=lambda c: f"chunk={c}")
|
||||
def test_delegating_parser_corrupted_wrapper(self, wrapper, chunk_size):
|
||||
"""The serving-layer shape: reasoning and tool adapters on separate
|
||||
engines, with the corrupted opener arriving as plain text."""
|
||||
tokens = _dsv4_tokens(
|
||||
reasoning="Checking the weather.",
|
||||
tool_name="get_weather",
|
||||
params=[("city", "true", "Seoul")],
|
||||
)
|
||||
start_id = _DSV4_FULL_VOCAB[DSML_TOOL_START]
|
||||
tokens = [
|
||||
(900, wrapper) if tid == start_id else (tid, text) for tid, text in tokens
|
||||
]
|
||||
# Only the think markers are single tokens in the real
|
||||
# DeepSeek-V4-Flash vocab; the DSML wrappers are ordinary text and
|
||||
# must be matched by the lexer. Mirror that here.
|
||||
vocab = {
|
||||
DSML_THINK_START: _DSV4_FULL_VOCAB[DSML_THINK_START],
|
||||
DSML_THINK_END: _DSV4_FULL_VOCAB[DSML_THINK_END],
|
||||
}
|
||||
tokenizer = MockTokenizer(vocab=vocab, tokens=tokens)
|
||||
parser = _DeepSeekV4Delegating(
|
||||
tokenizer, chat_template_kwargs={"thinking": True}
|
||||
)
|
||||
|
||||
deltas = replay_streaming(
|
||||
parser,
|
||||
tokens,
|
||||
chunk_size=chunk_size,
|
||||
finished_on_last=True,
|
||||
tools=DUMMY_TOOLS,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
|
||||
assert "Checking" in output.reasoning
|
||||
assert len(output.tool_calls) == 1, (
|
||||
f"reasoning={output.reasoning!r}, content={output.content!r}"
|
||||
)
|
||||
assert output.tool_calls[0]["name"] == "get_weather"
|
||||
assert json.loads(output.tool_calls[0]["arguments"]) == {"city": "Seoul"}
|
||||
assert "DSML" not in (output.content or "")
|
||||
|
||||
@@ -197,7 +197,7 @@ def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample):
|
||||
cfg = parser._tool_parser._parser_engine.parser_engine_config
|
||||
terminals = sorted(
|
||||
v
|
||||
for v in set(cfg.terminals.values()) | set(cfg.token_id_terminals.values())
|
||||
for v in cfg.terminal_literals | set(cfg.token_id_terminals.values())
|
||||
if len(v) > 1
|
||||
)
|
||||
assert_no_terminal_leakage(
|
||||
|
||||
@@ -953,3 +953,61 @@ class TestSkipToolParsingFromMessageHeader:
|
||||
assert types[0] == EventType.TEXT_CHUNK
|
||||
assert events[0].value == "<tool_call>"
|
||||
assert engine._message_header_buffer == ""
|
||||
|
||||
|
||||
# ── Terminals with several spellings ──────────────────────────────────
|
||||
|
||||
|
||||
def _alias_config() -> ParserEngineConfig:
|
||||
"""``TOOL_START`` has a canonical spelling and one corrupted alias."""
|
||||
return ParserEngineConfig(
|
||||
name="alias_test",
|
||||
terminals={"TOOL_START": ("<tc>", "<tcx>"), "TOOL_END": "</tc>"},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
},
|
||||
content_events={
|
||||
ParserState.CONTENT: EventType.TEXT_CHUNK,
|
||||
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
|
||||
},
|
||||
tool_args_json=False,
|
||||
)
|
||||
|
||||
|
||||
class TestTerminalAliases:
|
||||
def test_terminal_defs_expand_every_spelling(self):
|
||||
defs = terminals_from_literals({"A": ("x", "y"), "B": "z"})
|
||||
assert [(d.name, d.literal) for d in defs] == [
|
||||
("A", "x"),
|
||||
("A", "y"),
|
||||
("B", "z"),
|
||||
]
|
||||
|
||||
def test_terminal_literal_is_canonical_spelling(self):
|
||||
cfg = _alias_config()
|
||||
assert cfg.terminal_literal("TOOL_START") == "<tc>"
|
||||
assert cfg.terminal_literal("TOOL_END") == "</tc>"
|
||||
assert cfg.terminal_literal("MISSING") is None
|
||||
assert cfg.terminal_literals == {"<tc>", "<tcx>", "</tc>"}
|
||||
|
||||
@pytest.mark.parametrize("spelling", ["<tc>", "<tcx>"])
|
||||
@pytest.mark.parametrize("char_by_char", [False, True])
|
||||
def test_every_spelling_shares_transitions(self, spelling, char_by_char):
|
||||
engine = StreamingParserEngine(_alias_config(), None)
|
||||
text = f"{spelling}a</tc>"
|
||||
events = []
|
||||
for chunk in list(text) if char_by_char else [text]:
|
||||
events.extend(engine.feed(chunk, []))
|
||||
events.extend(engine.finish())
|
||||
assert [e.type for e in events] == [
|
||||
EventType.TOOL_CALL_START,
|
||||
EventType.ARG_VALUE_CHUNK,
|
||||
EventType.TOOL_CALL_END,
|
||||
]
|
||||
|
||||
@@ -83,7 +83,7 @@ def _discover_parsers() -> list[_ParserInfo]:
|
||||
raise RuntimeError(
|
||||
f"{obj.__name__} config missing 'TOOL_END' in token_id_terminals"
|
||||
)
|
||||
all_vals = set(cfg.terminals.values()) | set(cfg.token_id_terminals.values())
|
||||
all_vals = cfg.terminal_literals | set(cfg.token_id_terminals.values())
|
||||
found.append(
|
||||
_ParserInfo(
|
||||
parser_cls=obj,
|
||||
@@ -91,12 +91,13 @@ def _discover_parsers() -> list[_ParserInfo]:
|
||||
samples=build_samples(cfg.name),
|
||||
terminals=sorted(v for v in all_vals if len(v) > 1),
|
||||
tool_end=tool_end,
|
||||
think_end=cfg.terminals.get("THINK_END", ""),
|
||||
think_end=cfg.terminal_literal("THINK_END") or "",
|
||||
tool_start=(
|
||||
cfg.terminals["TOOL_SECTION_START"]
|
||||
cfg.terminal_literal("TOOL_SECTION_START")
|
||||
if (ParserState.CONTENT, "TOOL_SECTION_START") in cfg.transitions
|
||||
else cfg.terminals.get("TOOL_START", "")
|
||||
),
|
||||
else cfg.terminal_literal("TOOL_START")
|
||||
)
|
||||
or "",
|
||||
)
|
||||
)
|
||||
if missing_builders:
|
||||
|
||||
@@ -50,6 +50,12 @@ DSML_INVOKE_END = f"</{_DSML}invoke>"
|
||||
DSML_PARAM_START = f"<{_DSML}parameter"
|
||||
DSML_PARAM_CLOSE = f"</{_DSML}parameter>"
|
||||
|
||||
# Spellings variants of ``DSML_TOOL_START`` observed in production.
|
||||
DSML_TOOL_START_VARIANTS: tuple[str, ...] = (
|
||||
f"<{_DSML}toolcalls>",
|
||||
f"<{_DSML}tool>",
|
||||
)
|
||||
|
||||
_ESCAPED_DSML = re.escape(_DSML)
|
||||
_PARAM_RE = re.compile(
|
||||
rf'<{_ESCAPED_DSML}parameter\s+name="([^"]+)"\s+string="(true|false)">'
|
||||
@@ -131,7 +137,7 @@ def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig:
|
||||
terminals={
|
||||
"THINK_START": DSML_THINK_START,
|
||||
"THINK_END": DSML_THINK_END,
|
||||
"TOOL_START": DSML_TOOL_START,
|
||||
"TOOL_START": (DSML_TOOL_START, *DSML_TOOL_START_VARIANTS),
|
||||
"TOOL_END": DSML_TOOL_END,
|
||||
"INVOKE_PREFIX": DSML_INVOKE_PREFIX,
|
||||
"INVOKE_NAME_END": DSML_INVOKE_NAME_END,
|
||||
|
||||
@@ -5,6 +5,7 @@ tokens, with prefix-match buffering for ambiguous boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
import regex as re
|
||||
@@ -284,7 +285,9 @@ class IncrementalLexer:
|
||||
return n
|
||||
|
||||
|
||||
def terminals_from_literals(literals: dict[str, str]) -> list[TerminalDef]:
|
||||
def terminals_from_literals(
|
||||
literals: Mapping[str, str | tuple[str, ...]],
|
||||
) -> list[TerminalDef]:
|
||||
return [
|
||||
TerminalDef(
|
||||
name=name,
|
||||
@@ -292,5 +295,6 @@ def terminals_from_literals(literals: dict[str, str]) -> list[TerminalDef]:
|
||||
is_literal=True,
|
||||
literal=lit,
|
||||
)
|
||||
for name, lit in literals.items()
|
||||
for name, value in literals.items()
|
||||
for lit in ((value,) if isinstance(value, str) else value)
|
||||
]
|
||||
|
||||
@@ -158,11 +158,11 @@ class ParserEngine(Parser):
|
||||
|
||||
@property
|
||||
def reasoning_start_str(self) -> str | None:
|
||||
return self.parser_engine_config.terminals.get("THINK_START")
|
||||
return self.parser_engine_config.terminal_literal("THINK_START")
|
||||
|
||||
@property
|
||||
def reasoning_end_str(self) -> str | None:
|
||||
return self.parser_engine_config.terminals.get("THINK_END")
|
||||
return self.parser_engine_config.terminal_literal("THINK_END")
|
||||
|
||||
@cached_property
|
||||
def vocab(self) -> dict[str, int]:
|
||||
|
||||
@@ -17,7 +17,7 @@ Each model format is described by a :class:`ParserEngineConfig` that specifies:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum, auto
|
||||
from functools import cached_property
|
||||
@@ -54,7 +54,8 @@ class ParserEngineConfig:
|
||||
|
||||
name: str
|
||||
|
||||
terminals: dict[str, str] = field(default_factory=dict)
|
||||
# A terminal may carry several spellings; the first spelling is the canonical one.
|
||||
terminals: Mapping[str, str | tuple[str, ...]] = field(default_factory=dict)
|
||||
|
||||
token_id_terminals: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
@@ -100,6 +101,22 @@ class ParserEngineConfig:
|
||||
# Reject tool calls whose names are absent from the request tools.
|
||||
validate_tool_names: bool = False
|
||||
|
||||
def terminal_literal(self, name: str) -> str | None:
|
||||
"""Canonical spelling of terminal *name*, or ``None`` if undeclared."""
|
||||
value = self.terminals.get(name)
|
||||
if isinstance(value, tuple):
|
||||
return value[0] if value else None
|
||||
return value
|
||||
|
||||
@property
|
||||
def terminal_literals(self) -> frozenset[str]:
|
||||
"""Every declared spelling across all terminals."""
|
||||
return frozenset(
|
||||
lit
|
||||
for value in self.terminals.values()
|
||||
for lit in ((value,) if isinstance(value, str) else value)
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def terminal_defs(self):
|
||||
from vllm.parser.engine.incremental_lexer import terminals_from_literals
|
||||
|
||||
@@ -51,7 +51,7 @@ def _build_drop_info(
|
||||
|
||||
configured_texts = (
|
||||
set(config.token_id_terminals.values())
|
||||
| set(config.terminals.values())
|
||||
| config.terminal_literals
|
||||
| config.preserve_tokens
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user