diff --git a/tests/parser/engine/test_deepseek_v4.py b/tests/parser/engine/test_deepseek_v4.py index 7bea4f913d..1c1029b51a 100644 --- a/tests/parser/engine/test_deepseek_v4.py +++ b/tests/parser/engine/test_deepseek_v4.py @@ -30,6 +30,7 @@ from vllm.parser.deepseek_v4 import ( DSML_THINK_START, DSML_TOOL_END, DSML_TOOL_START, + DSML_TOOL_START_VARIANTS, DeepSeekV4Parser, _dsml_arg_converter, _unwrap_wrapper_args, @@ -1229,3 +1230,164 @@ class TestDelegatingParserLargeDelta: assert eos_text not in output.reasoning assert output.content == "" assert output.tool_calls == [] + + +# ── Malformed DSML markup must not leak into content (#51914) ───────── + +_WRAPPER_VARIANTS = list(DSML_TOOL_START_VARIANTS) + + +class TestMalformedDsmlNoise: + """DeepSeek-V4-Flash intermittently misspells the tool-call opener, e.g. + ``<|DSML|toolcalls>``. Known corrupted openers open a tool block like + the real wrapper so the markup never surfaces as assistant content. + """ + + _INNER = ( + f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n" + f"{_param('city', 'true', 'Seoul')}\n" + f"{DSML_INVOKE_END}\n" + f"{DSML_TOOL_END}" + ) + + @staticmethod + def _parser(mock_tokenizer, thinking: bool = False): + return DeepSeekV4Parser( + mock_tokenizer, chat_template_kwargs={"thinking": thinking} + ) + + @pytest.mark.parametrize("wrapper", _WRAPPER_VARIANTS) + @pytest.mark.parametrize("preamble", ["", "Let me check.\n"]) + def test_corrupted_wrapper_behaves_like_real_one( + self, mock_tokenizer, mock_request, wrapper, preamble + ): + text = f"{preamble}{wrapper}\n{self._INNER}" + reference = f"{preamble}{DSML_TOOL_START}\n{self._INNER}" + + result = self._parser(mock_tokenizer).extract_tool_calls(text, mock_request) + expected = self._parser(mock_tokenizer).extract_tool_calls( + reference, mock_request + ) + assert [tc.function.name for tc in result.tool_calls] == ["get_weather"] + assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Seoul"} + assert result.content == expected.content + assert "DSML" not in (result.content or "") + + results = simulate_tool_streaming( + self._parser(mock_tokenizer), mock_request, list(text) + ) + assert collect_function_name(results) == "get_weather" + assert json.loads(collect_tool_arguments(results)) == {"city": "Seoul"} + assert collect_content(results) == (expected.content or "") + + @pytest.mark.parametrize("wrapper", _WRAPPER_VARIANTS) + def test_corrupted_wrapper_ends_reasoning( + self, mock_tokenizer, mock_request, wrapper + ): + """Like the real wrapper, a corrupted one inside ```` closes + the reasoning block and starts the tool call.""" + text = f"thinking{wrapper}\n{self._INNER}" + parser = self._parser(mock_tokenizer, thinking=True) + reasoning, content = parser.extract_reasoning(text, mock_request) + assert reasoning == "thinking" + assert "DSML" not in (content or "") + + result = self._parser(mock_tokenizer, thinking=True).extract_tool_calls( + text, mock_request + ) + assert [tc.function.name for tc in result.tool_calls] == ["get_weather"] + + @pytest.mark.parametrize( + "text", + [ + # Fullwidth bars, DSML namespace, but not a registered spelling. + "see <|DSML|foo> and <|DSML|tool_call> here", + # Unclosed prefix. + "see <|DSML| here\nnext line", + # ASCII pipes are not the DSML sigil. + "see <|DSML|tool> and <|DSML|toolcalls> there", + ], + ) + def test_unregistered_markup_is_ordinary_content( + self, mock_tokenizer, mock_request, text + ): + """Only the registered spellings are treated as wrappers.""" + result = self._parser(mock_tokenizer).extract_tool_calls(text, mock_request) + assert result.tools_called is False + assert result.content == text + + results = simulate_tool_streaming( + self._parser(mock_tokenizer), mock_request, list(text) + ) + assert collect_content(results) == text + + @pytest.mark.parametrize("wrapper", _WRAPPER_VARIANTS) + def test_prose_mention_matches_canonical_wrapper( + self, mock_tokenizer, mock_request, wrapper + ): + """A registered spelling mentioned in prose behaves exactly like a + prose mention of the real wrapper: the opener starts a tool block and + the rest of the message is dropped. That is pre-existing behavior of + ``<|DSML|tool_calls>``; the variants must not differ from it either + way.""" + template = "The opener looks like {} and then params follow. Done." + text = template.format(wrapper) + reference = template.format(DSML_TOOL_START) + + result = self._parser(mock_tokenizer).extract_tool_calls(text, mock_request) + expected = self._parser(mock_tokenizer).extract_tool_calls( + reference, mock_request + ) + assert result.tools_called == expected.tools_called + assert result.content == expected.content + + results = simulate_tool_streaming( + self._parser(mock_tokenizer), mock_request, list(text) + ) + expected_results = simulate_tool_streaming( + self._parser(mock_tokenizer), mock_request, list(reference) + ) + assert collect_content(results) == collect_content(expected_results) + + @pytest.mark.parametrize("wrapper", _WRAPPER_VARIANTS) + @pytest.mark.parametrize("chunk_size", [1, 3, None], ids=lambda c: f"chunk={c}") + def test_delegating_parser_corrupted_wrapper(self, wrapper, chunk_size): + """The serving-layer shape: reasoning and tool adapters on separate + engines, with the corrupted opener arriving as plain text.""" + tokens = _dsv4_tokens( + reasoning="Checking the weather.", + tool_name="get_weather", + params=[("city", "true", "Seoul")], + ) + start_id = _DSV4_FULL_VOCAB[DSML_TOOL_START] + tokens = [ + (900, wrapper) if tid == start_id else (tid, text) for tid, text in tokens + ] + # Only the think markers are single tokens in the real + # DeepSeek-V4-Flash vocab; the DSML wrappers are ordinary text and + # must be matched by the lexer. Mirror that here. + vocab = { + DSML_THINK_START: _DSV4_FULL_VOCAB[DSML_THINK_START], + DSML_THINK_END: _DSV4_FULL_VOCAB[DSML_THINK_END], + } + tokenizer = MockTokenizer(vocab=vocab, tokens=tokens) + parser = _DeepSeekV4Delegating( + tokenizer, chat_template_kwargs={"thinking": True} + ) + + deltas = replay_streaming( + parser, + tokens, + chunk_size=chunk_size, + finished_on_last=True, + tools=DUMMY_TOOLS, + ) + output = collect_output(deltas) + + assert "Checking" in output.reasoning + assert len(output.tool_calls) == 1, ( + f"reasoning={output.reasoning!r}, content={output.content!r}" + ) + assert output.tool_calls[0]["name"] == "get_weather" + assert json.loads(output.tool_calls[0]["arguments"]) == {"city": "Seoul"} + assert "DSML" not in (output.content or "") diff --git a/tests/parser/engine/test_delegating_replay.py b/tests/parser/engine/test_delegating_replay.py index 50c41a8ea6..2bc4338785 100644 --- a/tests/parser/engine/test_delegating_replay.py +++ b/tests/parser/engine/test_delegating_replay.py @@ -197,7 +197,7 @@ def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample): cfg = parser._tool_parser._parser_engine.parser_engine_config terminals = sorted( v - for v in set(cfg.terminals.values()) | set(cfg.token_id_terminals.values()) + for v in cfg.terminal_literals | set(cfg.token_id_terminals.values()) if len(v) > 1 ) assert_no_terminal_leakage( diff --git a/tests/parser/engine/test_engine.py b/tests/parser/engine/test_engine.py index 07f5455718..36fa1ad8d0 100644 --- a/tests/parser/engine/test_engine.py +++ b/tests/parser/engine/test_engine.py @@ -953,3 +953,61 @@ class TestSkipToolParsingFromMessageHeader: assert types[0] == EventType.TEXT_CHUNK assert events[0].value == "" assert engine._message_header_buffer == "" + + +# ── Terminals with several spellings ────────────────────────────────── + + +def _alias_config() -> ParserEngineConfig: + """``TOOL_START`` has a canonical spelling and one corrupted alias.""" + return ParserEngineConfig( + name="alias_test", + terminals={"TOOL_START": ("", ""), "TOOL_END": ""}, + transitions={ + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + tool_args_json=False, + ) + + +class TestTerminalAliases: + def test_terminal_defs_expand_every_spelling(self): + defs = terminals_from_literals({"A": ("x", "y"), "B": "z"}) + assert [(d.name, d.literal) for d in defs] == [ + ("A", "x"), + ("A", "y"), + ("B", "z"), + ] + + def test_terminal_literal_is_canonical_spelling(self): + cfg = _alias_config() + assert cfg.terminal_literal("TOOL_START") == "" + assert cfg.terminal_literal("TOOL_END") == "" + assert cfg.terminal_literal("MISSING") is None + assert cfg.terminal_literals == {"", "", ""} + + @pytest.mark.parametrize("spelling", ["", ""]) + @pytest.mark.parametrize("char_by_char", [False, True]) + def test_every_spelling_shares_transitions(self, spelling, char_by_char): + engine = StreamingParserEngine(_alias_config(), None) + text = f"{spelling}a" + events = [] + for chunk in list(text) if char_by_char else [text]: + events.extend(engine.feed(chunk, [])) + events.extend(engine.finish()) + assert [e.type for e in events] == [ + EventType.TOOL_CALL_START, + EventType.ARG_VALUE_CHUNK, + EventType.TOOL_CALL_END, + ] diff --git a/tests/parser/engine/test_replay.py b/tests/parser/engine/test_replay.py index 5bba7f7eaf..fee74f8d2c 100644 --- a/tests/parser/engine/test_replay.py +++ b/tests/parser/engine/test_replay.py @@ -83,7 +83,7 @@ def _discover_parsers() -> list[_ParserInfo]: raise RuntimeError( f"{obj.__name__} config missing 'TOOL_END' in token_id_terminals" ) - all_vals = set(cfg.terminals.values()) | set(cfg.token_id_terminals.values()) + all_vals = cfg.terminal_literals | set(cfg.token_id_terminals.values()) found.append( _ParserInfo( parser_cls=obj, @@ -91,12 +91,13 @@ def _discover_parsers() -> list[_ParserInfo]: samples=build_samples(cfg.name), terminals=sorted(v for v in all_vals if len(v) > 1), tool_end=tool_end, - think_end=cfg.terminals.get("THINK_END", ""), + think_end=cfg.terminal_literal("THINK_END") or "", tool_start=( - cfg.terminals["TOOL_SECTION_START"] + cfg.terminal_literal("TOOL_SECTION_START") if (ParserState.CONTENT, "TOOL_SECTION_START") in cfg.transitions - else cfg.terminals.get("TOOL_START", "") - ), + else cfg.terminal_literal("TOOL_START") + ) + or "", ) ) if missing_builders: diff --git a/vllm/parser/deepseek_v4.py b/vllm/parser/deepseek_v4.py index 8159924f93..2508064b34 100644 --- a/vllm/parser/deepseek_v4.py +++ b/vllm/parser/deepseek_v4.py @@ -50,6 +50,12 @@ DSML_INVOKE_END = f"" DSML_PARAM_START = f"<{_DSML}parameter" DSML_PARAM_CLOSE = f"" +# Spellings variants of ``DSML_TOOL_START`` observed in production. +DSML_TOOL_START_VARIANTS: tuple[str, ...] = ( + f"<{_DSML}toolcalls>", + f"<{_DSML}tool>", +) + _ESCAPED_DSML = re.escape(_DSML) _PARAM_RE = re.compile( rf'<{_ESCAPED_DSML}parameter\s+name="([^"]+)"\s+string="(true|false)">' @@ -131,7 +137,7 @@ def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig: terminals={ "THINK_START": DSML_THINK_START, "THINK_END": DSML_THINK_END, - "TOOL_START": DSML_TOOL_START, + "TOOL_START": (DSML_TOOL_START, *DSML_TOOL_START_VARIANTS), "TOOL_END": DSML_TOOL_END, "INVOKE_PREFIX": DSML_INVOKE_PREFIX, "INVOKE_NAME_END": DSML_INVOKE_NAME_END, diff --git a/vllm/parser/engine/incremental_lexer.py b/vllm/parser/engine/incremental_lexer.py index 34345fe1d8..8c3820b59a 100644 --- a/vllm/parser/engine/incremental_lexer.py +++ b/vllm/parser/engine/incremental_lexer.py @@ -5,6 +5,7 @@ tokens, with prefix-match buffering for ambiguous boundaries.""" from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass import regex as re @@ -284,7 +285,9 @@ class IncrementalLexer: return n -def terminals_from_literals(literals: dict[str, str]) -> list[TerminalDef]: +def terminals_from_literals( + literals: Mapping[str, str | tuple[str, ...]], +) -> list[TerminalDef]: return [ TerminalDef( name=name, @@ -292,5 +295,6 @@ def terminals_from_literals(literals: dict[str, str]) -> list[TerminalDef]: is_literal=True, literal=lit, ) - for name, lit in literals.items() + for name, value in literals.items() + for lit in ((value,) if isinstance(value, str) else value) ] diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 36bb4d0cb5..4817dedb46 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -158,11 +158,11 @@ class ParserEngine(Parser): @property def reasoning_start_str(self) -> str | None: - return self.parser_engine_config.terminals.get("THINK_START") + return self.parser_engine_config.terminal_literal("THINK_START") @property def reasoning_end_str(self) -> str | None: - return self.parser_engine_config.terminals.get("THINK_END") + return self.parser_engine_config.terminal_literal("THINK_END") @cached_property def vocab(self) -> dict[str, int]: diff --git a/vllm/parser/engine/parser_engine_config.py b/vllm/parser/engine/parser_engine_config.py index 4799604dab..5c5cd2b467 100644 --- a/vllm/parser/engine/parser_engine_config.py +++ b/vllm/parser/engine/parser_engine_config.py @@ -17,7 +17,7 @@ Each model format is described by a :class:`ParserEngineConfig` that specifies: from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from enum import Enum, auto from functools import cached_property @@ -54,7 +54,8 @@ class ParserEngineConfig: name: str - terminals: dict[str, str] = field(default_factory=dict) + # A terminal may carry several spellings; the first spelling is the canonical one. + terminals: Mapping[str, str | tuple[str, ...]] = field(default_factory=dict) token_id_terminals: dict[str, str] = field(default_factory=dict) @@ -100,6 +101,22 @@ class ParserEngineConfig: # Reject tool calls whose names are absent from the request tools. validate_tool_names: bool = False + def terminal_literal(self, name: str) -> str | None: + """Canonical spelling of terminal *name*, or ``None`` if undeclared.""" + value = self.terminals.get(name) + if isinstance(value, tuple): + return value[0] if value else None + return value + + @property + def terminal_literals(self) -> frozenset[str]: + """Every declared spelling across all terminals.""" + return frozenset( + lit + for value in self.terminals.values() + for lit in ((value,) if isinstance(value, str) else value) + ) + @cached_property def terminal_defs(self): from vllm.parser.engine.incremental_lexer import terminals_from_literals diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py index 46dfb79c23..21d053ef28 100644 --- a/vllm/parser/engine/streaming_parser_engine.py +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -51,7 +51,7 @@ def _build_drop_info( configured_texts = ( set(config.token_id_terminals.values()) - | set(config.terminals.values()) + | config.terminal_literals | config.preserve_tokens )