[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ben Browning
2026-06-30 13:46:53 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 27d5f78b63
commit 25671cb520
14 changed files with 783 additions and 219 deletions
+14 -2
View File
@@ -15,8 +15,17 @@ def should_do_global_cleanup_after_test() -> bool:
return False
def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
"""Create a mock tokenizer with the given special-token vocabulary.
def make_mock_tokenizer(
vocab: dict[str, int],
special_tokens: list[str] | None = None,
) -> MagicMock:
"""Create a mock tokenizer with the given vocabulary.
Args:
vocab: Mapping of token text to token ID.
special_tokens: Which tokens to mark as special. When ``None``
(the default), every key in *vocab* is treated as special —
convenient when the vocab only contains delimiter tokens.
The returned mock supports get_vocab(), encode(), and decode().
decode() maps known token IDs back to their text and falls back to
@@ -29,6 +38,9 @@ def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
tokenizer.decode.side_effect = lambda ids: "".join(
id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
)
st = special_tokens if special_tokens is not None else list(vocab.keys())
tokenizer.all_special_tokens = st
tokenizer.all_special_ids = [vocab[t] for t in st if t in vocab]
return tokenizer
+41
View File
@@ -62,6 +62,7 @@ class MockTokenizer:
"eos_token_id",
"bos_token_id",
"pad_token_id",
"_all_special_tokens",
)
def __init__(
@@ -73,6 +74,7 @@ class MockTokenizer:
self._token_ids = [tid for tid, _ in tokens]
self._token_decode_map = {tid: text for tid, text in tokens}
self._special_ids = set(vocab.values())
self._all_special_tokens = list(vocab.keys())
self.eos_token_id = None
self.bos_token_id = None
self.pad_token_id = None
@@ -83,6 +85,14 @@ class MockTokenizer:
def get_vocab(self) -> dict[str, int]:
return self._vocab
@property
def all_special_tokens(self) -> list[str]:
return self._all_special_tokens
@property
def all_special_ids(self) -> list[int]:
return [self._vocab[t] for t in self._all_special_tokens if t in self._vocab]
def encode(self, text: str, **kwargs) -> list[int]:
return self._token_ids
@@ -117,6 +127,37 @@ def _test_request(
)
DUMMY_TOOLS = [
{
"type": "function",
"function": {"name": "stub", "parameters": {"type": "object"}},
},
]
def parse_non_streaming(
parser,
sample: Sample,
request: ChatCompletionRequest,
) -> ParseOutput:
"""Run ``parser.parse()`` and return a :class:`ParseOutput`."""
full_text = "".join(text for _, text in sample.tokens)
reasoning, content, tool_calls = parser.parse(
full_text,
request,
enable_auto_tools=True,
)
tc_list: list[dict] = []
if tool_calls:
for tc in tool_calls:
tc_list.append({"name": tc.name, "arguments": tc.arguments})
return ParseOutput(
reasoning=reasoning or "",
content=content or "",
tool_calls=tc_list,
)
def replay_streaming(
parser,
tokens: list[tuple[int, str]],
@@ -20,10 +20,14 @@ from pydantic import TypeAdapter
from tests.parser.engine.replay_harness import (
CHUNK_SIZES,
DUMMY_TOOLS,
MockTokenizer,
_test_request,
assert_no_terminal_leakage,
assert_parse_output,
collect_output,
make_mock_tokenizer,
parse_non_streaming,
replay_streaming,
)
from tests.parser.engine.trace_builder import _BUILDERS, build_samples
@@ -142,3 +146,51 @@ def test_delegating_replay(parser_cls, sample, chunk_size):
)
output = collect_output(deltas)
assert_parse_output(output, sample)
_TOOL_CALL_SAMPLES = [
(p.parser_cls, p.name, s)
for p in _PAIRINGS
for s in p.samples
if s.expected_tool_calls
]
@pytest.mark.parametrize(
"parser_cls,parser_name,sample",
_TOOL_CALL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else "",
)
def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample):
"""Non-streaming parse() with tool_choice='none' via DelegatingParser
must not leak special tokens into content."""
tokenizer = make_mock_tokenizer(sample)
validated_tools = (
_TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
)
parser = parser_cls(
tokenizer,
validated_tools,
chat_template_kwargs=sample.chat_template_kwargs,
)
request = _test_request(tools=DUMMY_TOOLS)
request.tool_choice = "none"
output = parse_non_streaming(parser, sample, request)
assert output.tool_calls == [], (
f"Expected no tool calls but got {output.tool_calls}"
)
cfg = parser._tool_parser._parser_engine.parser_engine_config
terminals = sorted(
v
for v in set(cfg.terminals.values()) | set(cfg.token_id_terminals.values())
if len(v) > 1
)
assert_no_terminal_leakage(
output,
terminals,
context=f"parser={parser_name}",
)
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import (
collect_content,
collect_function_name,
@@ -57,6 +58,8 @@ def _make_tokenizer(sequence: list[tuple[int, str]]) -> MagicMock:
return "".join(parts)
tokenizer.decode.side_effect = decode
tokenizer.all_special_tokens = list(SPECIAL_TOKEN_MAP.values())
tokenizer.all_special_ids = list(SPECIAL_TOKEN_MAP.keys())
return tokenizer
@@ -657,19 +660,15 @@ class TestGemma4ReasoningTruncationWithHoldback:
@pytest.fixture
def tool_call_tokenizer():
"""Mock tokenizer with Gemma4 special token vocab."""
tokenizer = MagicMock()
tokenizer.encode.return_value = [1, 2, 3]
tokenizer.get_vocab.return_value = {
"<|tool_call>": TOOL_CALL_START_ID,
"<tool_call|>": TOOL_CALL_END_ID,
"<|channel>": CHANNEL_START_ID,
"<channel|>": CHANNEL_END_ID,
'<|"|>': QUOTED_ID,
}
tokenizer.decode.side_effect = lambda ids: "".join(
SPECIAL_TOKEN_MAP.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
return make_mock_tokenizer(
vocab={
"<|tool_call>": TOOL_CALL_START_ID,
"<tool_call|>": TOOL_CALL_END_ID,
"<|channel>": CHANNEL_START_ID,
"<channel|>": CHANNEL_END_ID,
'<|"|>': QUOTED_ID,
},
)
return tokenizer
@pytest.fixture
@@ -1407,3 +1406,133 @@ class TestBareThoughtWithoutChannelOpener:
assert reasoning == ""
assert content == ""
assert len(tool_calls) == 0
# ── Regression: commas inside <|"|>-delimited string values ─────────
#
# _make_tokenizer sets all_special_tokens, which activates the auto-drop
# mechanism in _build_drop_info. If <|"|> is not in configured_texts,
# it gets silently dropped and commas inside string values become field
# separators, e.g. "San Francisco, CA" → {"location": "San Francisco"}.
COMMA_TOKEN_SEQUENCE: list[tuple[int, str]] = [
(TOOL_CALL_START_ID, "<|tool_call>"),
(4000, "call"),
(4001, ":"),
(4002, "get_weather"),
(4003, "{"),
(4004, "location"),
(4005, ":"),
(QUOTED_ID, '<|"|>'),
(4006, "San Francisco"),
(4007, ", CA"),
(QUOTED_ID, '<|"|>'),
(4008, ","),
(4009, "unit"),
(4010, ":"),
(QUOTED_ID, '<|"|>'),
(4011, "celsius"),
(QUOTED_ID, '<|"|>'),
(4012, "}"),
(TOOL_CALL_END_ID, "<tool_call|>"),
]
MULTI_COMMA_TOKEN_SEQUENCE: list[tuple[int, str]] = [
(TOOL_CALL_START_ID, "<|tool_call>"),
(4000, "call"),
(4001, ":"),
(4020, "send_message"),
(4003, "{"),
(4021, "destination"),
(4005, ":"),
(QUOTED_ID, '<|"|>'),
(4022, "456 Oakwood Avenue"),
(4023, ", Rivermist"),
(4024, ", 83214"),
(QUOTED_ID, '<|"|>'),
(4012, "}"),
(TOOL_CALL_END_ID, "<tool_call|>"),
]
class TestCommaInStringValueRegression:
"""Regression: <|"|> delimiters must not be auto-dropped.
When _build_drop_info discovers <|"|> as a special token and it is
not in configured_texts, the delimiter is silently removed. Without
it, _parse_gemma4_args treats commas inside string values as field
separators.
"""
@pytest.fixture
def comma_tokenizer(self):
return _make_tokenizer(COMMA_TOKEN_SEQUENCE)
@pytest.fixture
def comma_parser(self, comma_tokenizer):
return Gemma4Parser(comma_tokenizer)
@pytest.fixture
def multi_comma_tokenizer(self):
return _make_tokenizer(MULTI_COMMA_TOKEN_SEQUENCE)
@pytest.fixture
def multi_comma_parser(self, multi_comma_tokenizer):
return Gemma4Parser(multi_comma_tokenizer)
def test_batched_streaming_comma_in_value(
self, comma_parser, comma_tokenizer, request_obj
):
results = _stream_tokens_batched(
comma_parser,
comma_tokenizer,
request_obj,
batch_size=1,
prompt_token_ids=[],
)
_, _, tool_calls = _collect_fields(results)
assert len(tool_calls) > 0
args_text = "".join(
tc.function.arguments
for tc in tool_calls
if tc.function and tc.function.arguments
)
parsed = json.loads(args_text)
assert parsed["location"] == "San Francisco, CA"
assert parsed["unit"] == "celsius"
def test_batched_streaming_multiple_commas(
self, multi_comma_parser, multi_comma_tokenizer, request_obj
):
results = _stream_tokens_batched(
multi_comma_parser,
multi_comma_tokenizer,
request_obj,
batch_size=1,
prompt_token_ids=[],
)
_, _, tool_calls = _collect_fields(results)
assert len(tool_calls) > 0
args_text = "".join(
tc.function.arguments
for tc in tool_calls
if tc.function and tc.function.arguments
)
parsed = json.loads(args_text)
assert parsed["destination"] == "456 Oakwood Avenue, Rivermist, 83214"
def test_non_streaming_comma_in_value(self, comma_parser, request_obj):
text = "".join(text for _, text in COMMA_TOKEN_SEQUENCE)
result = comma_parser.extract_tool_calls(text, request_obj)
assert result.tools_called is True
args = json.loads(result.tool_calls[0].function.arguments)
assert args["location"] == "San Francisco, CA"
assert args["unit"] == "celsius"
def test_non_streaming_multiple_commas(self, multi_comma_parser, request_obj):
text = "".join(text for _, text in MULTI_COMMA_TOKEN_SEQUENCE)
result = multi_comma_parser.extract_tool_calls(text, request_obj)
assert result.tools_called is True
args = json.loads(result.tool_calls[0].function.arguments)
assert args["destination"] == "456 Oakwood Avenue, Rivermist, 83214"
+182 -1
View File
@@ -119,8 +119,10 @@ def _hermes_config() -> ParserEngineConfig:
def _make_engine(
config: ParserEngineConfig | None = None,
tools: list | None = None,
vocab: dict[str, int] | None = None,
special_tokens: list[str] | None = None,
) -> ParserEngine:
tokenizer = make_mock_tokenizer(_VOCAB)
tokenizer = make_mock_tokenizer(vocab or _VOCAB, special_tokens=special_tokens)
cfg = config or _combined_config()
return ParserEngine(
tokenizer,
@@ -843,6 +845,7 @@ class TestParseTokenIdPassthrough:
assert content is not None
assert "<tool_call>" in content
assert content == "Use <tool_call> to call tools."
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "f"
@@ -1490,3 +1493,181 @@ class TestCoercionInstabilityRegression:
assert parsed["val"] == "4e"
assert isinstance(parsed["val"], str)
assert parsed["extra"] == "ok"
_DROP_VOCAB: dict[str, int] = {
**_VOCAB,
"<bos>": 204,
"<eos>": 205,
}
class TestDropSpecialTokens:
"""Special token dropping via the __DROP__ terminal mechanism."""
def test_drops_special_token_by_id_from_content(self):
"""A special token (not a configured terminal) is dropped when
it arrives as its actual token ID."""
config = ParserEngineConfig(
name="drop_content_test",
terminals={},
token_id_terminals={},
transitions={},
initial_state=ParserState.CONTENT,
content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
)
engine = _make_engine(
config=config,
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = engine._engine.feed("hello<bos>world", [72, 204, 73])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<bos>" not in delta.content
assert delta.content == "helloworld"
def test_drops_special_token_by_id_from_reasoning(self):
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = engine._engine.feed("thinking<eos>more", [72, 205, 73])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<eos>" not in delta.reasoning
assert delta.reasoning == "thinkingmore"
def test_drops_via_text_fallback_when_no_token_ids(self):
"""When no token IDs are provided, text-based lexer catches
drop tokens as a fallback."""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = engine._engine.feed("hello<bos>world", [])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<bos>" not in delta.reasoning
def test_regular_tokens_spelling_special_survive(self):
"""Regular tokens that spell out a drop-token string survive
when token IDs prove they are not the special token."""
vocab = {**_DROP_VOCAB, "h": 72, "<": 73, "bos": 74, ">": 75, "w": 76}
engine = _make_engine(
vocab=vocab,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
# Feed regular token IDs (73, 74, 75) that spell <bos>,
# not the special token ID 204.
events = engine._engine.feed("h<bos>w", [72, 73, 74, 75, 76])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<bos>" in delta.reasoning
def test_configured_terminal_not_treated_as_drop(self):
"""Tokens already in config.terminals (like <think>) are handled
by the state machine, not the drop mechanism."""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
# Start from CONTENT so <think> triggers REASONING_START
engine._engine.reset(initial_state=ParserState.CONTENT)
events = engine._engine.feed("<think>", [200])
has_reasoning_start = any(e.type == EventType.REASONING_START for e in events)
assert has_reasoning_start
def test_tool_name_not_affected_by_drops(self):
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = [
SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0),
SemanticEvent(EventType.ARG_VALUE_CHUNK, '{"city": "NYC"}', tool_index=0),
SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
]
delta = engine._events_to_delta(events)
assert delta is not None
assert delta.tool_calls is not None
names = [
tc.function.name
for tc in delta.tool_calls
if tc.function and tc.function.name
]
assert "get_weather" in names
def test_adjacent_drop_tokens_by_id(self):
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = engine._engine.feed("<bos><eos>", [204, 205])
delta = engine._events_to_delta(events)
assert delta is None
def test_no_special_tokens_means_no_drops(self):
"""Engine with no special tokens passes all text through."""
engine = _make_engine(special_tokens=[])
engine._engine.reset()
events = engine._engine.feed("hello<bos>world", [])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<bos>" in delta.reasoning
def test_drops_suppressed_with_skip_tool_parsing(self):
"""When skip_tool_parsing is active, drop tokens are preserved
as content so a later tool-call pass can see them."""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.skip_tool_parsing = True
engine._engine.reset()
events = engine._engine.feed("hello<bos>world", [72, 204, 73])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<bos>" in delta.reasoning
def test_drops_in_tool_args_state(self):
"""Drop tokens in TOOL_ARGS state are silently discarded."""
vocab = {**_DROP_VOCAB, '{"a":1}': 300}
engine = _make_engine(
vocab=vocab,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset(initial_state=ParserState.CONTENT)
events = engine._engine.feed(
'<tool_call>{"a":1}<bos></tool_call>',
[202, 300, 204, 203],
)
arg_chunks = [e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK]
combined = "".join(arg_chunks)
assert "<bos>" not in combined
assert '{"a":1}' in combined
def test_mixed_configured_and_drop_terminals(self):
"""Configured terminals trigger transitions while drop terminals
are silently removed in the same stream."""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = engine._engine.feed("thought<bos></think>answer", [72, 204, 201, 73])
types = [e.type for e in events]
assert EventType.REASONING_CHUNK in types
assert EventType.REASONING_END in types
assert EventType.TEXT_CHUNK in types
reasoning_text = "".join(
e.value for e in events if e.type == EventType.REASONING_CHUNK
)
assert "<bos>" not in reasoning_text
+217 -29
View File
@@ -18,18 +18,21 @@ from typing import NamedTuple
import pytest
from tests.parser.engine.replay_harness import (
DUMMY_TOOLS,
MockTokenizer,
_test_request,
assert_no_terminal_leakage,
assert_parse_output,
collect_output,
make_mock_tokenizer,
parse_non_streaming,
replay_streaming,
replay_with_text_holdback,
)
from tests.parser.engine.trace_builder import _BUILDERS, build_samples
from vllm.parser.engine import registered_adapters as _adapters_mod
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine_config import ParserState
# ── Parser discovery ─────────────────────────────────────────────────
@@ -78,7 +81,11 @@ def _discover_parsers() -> list[_ParserInfo]:
terminals=sorted(v for v in all_vals if len(v) > 1),
tool_end=tool_end,
think_end=cfg.terminals.get("THINK_END", ""),
tool_start=cfg.terminals.get("TOOL_START", ""),
tool_start=(
cfg.terminals["TOOL_SECTION_START"]
if (ParserState.CONTENT, "TOOL_SECTION_START") in cfg.transitions
else cfg.terminals.get("TOOL_START", "")
),
)
)
if missing_builders:
@@ -287,16 +294,18 @@ _TOOL_CALL_SAMPLES = [
]
def _suppressed_expectations(
sample, think_end: str, tool_start: str
def _tool_suppression_expectations(
sample, think_end: str, tool_start: str, *, include_tool_block: bool
) -> tuple[str, str]:
"""Compute expected (reasoning, content) when tools are suppressed.
"""Expected (reasoning, content) when tool calls are not extracted.
When an explicit reasoning-end delimiter is present, reasoning ends
there and the tool call block becomes content. When reasoning ends
implicitly (the tool-start token triggers both REASONING_END and
TOOL_CALL_START), reasoning still ends at the tool start and the raw
tool call block becomes content text.
With ``include_tool_block=True`` (skip_tool_parsing / reasoning
adapter first pass), tool terminal text is preserved as content so
a second-pass parser can see it.
With ``include_tool_block=False`` (_suppress_tool_calls /
tool_choice='none'), the state machine consumes tool blocks and
only non-tool content survives.
"""
full_text = "".join(text for _, text in sample.tokens)
reasoning = sample.expected_reasoning
@@ -307,36 +316,42 @@ def _suppressed_expectations(
if think_end:
pos = after_reasoning.find(think_end)
if pos >= 0:
return (reasoning, after_reasoning[pos + len(think_end) :])
if include_tool_block:
return (reasoning, after_reasoning[pos + len(think_end) :])
after_reasoning = after_reasoning[pos + len(think_end) :]
if tool_start:
pos = after_reasoning.find(tool_start)
if pos >= 0:
return (reasoning, after_reasoning[pos:])
return (full_text, "")
_DUMMY_TOOLS = [
{
"type": "function",
"function": {"name": "stub", "parameters": {"type": "object"}},
}
]
if include_tool_block:
return (reasoning, after_reasoning[pos:])
return (reasoning, after_reasoning[:pos])
if include_tool_block:
return (full_text, "")
return (reasoning, after_reasoning)
@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")
@pytest.mark.parametrize(
"mode",
["skip_tool_parsing", "suppress_tool_calls"],
ids=["skip_tool_parsing", "suppress_tool_calls"],
)
@pytest.mark.parametrize(
"parser_cls,sample,think_end,tool_start",
_TOOL_CALL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
)
class TestSkipToolParsingReplay:
"""Replay with skip_tool_parsing=True (tool_choice='none').
class TestToolCallFilteringReplay:
"""Replay with tool calls not extracted, in both filtering modes.
Verifies that reasoning is extracted normally and the raw tool call
block appears as content text with no tool calls parsed.
``skip_tool_parsing`` (reasoning adapter first pass): tool terminal
text is preserved as content for a second-pass tool parser.
``suppress_tool_calls`` (tool_choice='none'): tool call blocks are
consumed by the state machine and do not leak into content.
"""
def test_replay(self, parser_cls, sample, think_end, tool_start, chunk_size):
def test_replay(self, parser_cls, sample, think_end, tool_start, mode, chunk_size):
tokenizer = make_mock_tokenizer(sample)
kwargs = {}
if sample.chat_template_kwargs:
@@ -344,8 +359,11 @@ class TestSkipToolParsingReplay:
parser = parser_cls(tokenizer, **kwargs)
request = _test_request()
request.tool_choice = "none"
request.tools = _DUMMY_TOOLS
request.tools = DUMMY_TOOLS
if mode == "skip_tool_parsing":
parser.skip_tool_parsing = True
else:
request.tool_choice = "none"
all_ids = [tid for tid, _ in sample.tokens]
all_texts = [text for _, text in sample.tokens]
@@ -370,10 +388,51 @@ class TestSkipToolParsingReplay:
output = collect_output(results)
expected_reasoning, expected_content = _suppressed_expectations(
sample, think_end, tool_start
include_block = mode == "skip_tool_parsing"
expected_reasoning, expected_content = _tool_suppression_expectations(
sample, think_end, tool_start, include_tool_block=include_block
)
assert output.reasoning == expected_reasoning, (
f"Reasoning mismatch (mode={mode}):\n"
f" expected: {expected_reasoning!r}\n"
f" actual: {output.reasoning!r}"
)
assert output.tool_calls == [], (
f"Expected no tool calls (mode={mode}) but got {output.tool_calls}"
)
assert output.content == expected_content, (
f"Content mismatch (mode={mode}):\n"
f" expected: {expected_content!r}\n"
f" actual: {output.content!r}"
)
@pytest.mark.parametrize(
"parser_cls,sample,think_end,tool_start",
_TOOL_CALL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
)
class TestToolCallFilteringNonStreaming:
"""Non-streaming parse() with tool_choice='none' must suppress tool
calls and not leak special tokens into content."""
def test_parse(self, parser_cls, sample, think_end, tool_start):
tokenizer = make_mock_tokenizer(sample)
kwargs = {}
if sample.chat_template_kwargs:
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
parser = parser_cls(tokenizer, **kwargs)
request = _test_request()
request.tools = DUMMY_TOOLS
request.tool_choice = "none"
output = parse_non_streaming(parser, sample, request)
expected_reasoning, expected_content = _tool_suppression_expectations(
sample, think_end, tool_start, include_tool_block=False
)
assert output.reasoning == expected_reasoning, (
f"Reasoning mismatch:\n"
f" expected: {expected_reasoning!r}\n"
@@ -389,6 +448,135 @@ class TestSkipToolParsingReplay:
)
_WS_TOOL_SAMPLES = [(t[0], t[1]) for t in _TOOL_CALL_SAMPLES if "whitespace" in t[1].id]
@pytest.mark.parametrize(
"parser_cls,sample",
_WS_TOOL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
)
class TestToolChoiceNoneStreamingParity:
"""Streaming and non-streaming must return the same content
when tool_choice='none' suppresses tool calls."""
def test_content_matches(self, parser_cls, sample):
tokenizer = make_mock_tokenizer(sample)
kwargs = {}
if sample.chat_template_kwargs:
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
request = _test_request()
request.tools = DUMMY_TOOLS
request.tool_choice = "none"
ns_output = parse_non_streaming(
parser_cls(tokenizer, **kwargs),
sample,
request,
)
s_parser = parser_cls(tokenizer, **kwargs)
results = []
for i, (tid, text) in enumerate(sample.tokens):
is_last = i == len(sample.tokens) - 1
results.append(
s_parser.parse_delta(
text,
[tid],
request,
prompt_token_ids=(sample.prompt_token_ids or [])
if i == 0
else None,
finished=is_last,
)
)
s_output = collect_output(results)
assert ns_output.content == s_output.content, (
f"Streaming/non-streaming content mismatch:\n"
f" streaming: {s_output.content!r}\n"
f" non-streaming: {ns_output.content!r}"
)
_DROP_TOKENS = {"<bos>": 99990, "<eos>": 99991}
def _inject_drop_tokens(sample):
"""Insert <bos> at stream start and <eos> between the first two tokens."""
new_vocab = {**sample.vocab, **_DROP_TOKENS}
tokens = list(sample.tokens)
tokens.insert(0, (99990, "<bos>"))
if len(tokens) >= 3:
tokens.insert(2, (99991, "<eos>"))
else:
tokens.append((99991, "<eos>"))
return dataclasses.replace(sample, vocab=new_vocab, tokens=tokens)
class TestDropTokenReplay:
"""Verify unconfigured special tokens are silently dropped across
all parsers and chunk sizes."""
@pytest.mark.parametrize(
"parser_info",
_PARSERS,
ids=[p.name for p in _PARSERS],
)
@pytest.mark.parametrize("chunk_size", [1, 3, None])
def test_drop_tokens_removed_from_output(self, parser_info, chunk_size):
for sample in parser_info.samples:
injected = _inject_drop_tokens(sample)
tokenizer = make_mock_tokenizer(injected)
parser = parser_info.parser_cls(
tokenizer,
tools=sample.tools,
)
results = replay_streaming(
parser,
injected.tokens,
chunk_size=chunk_size,
tools=sample.tools,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(results)
assert_no_terminal_leakage(
output,
list(_DROP_TOKENS.keys()),
context=f"parser={parser_info.name}, chunk={chunk_size}",
)
assert_parse_output(output, sample)
class TestDropTokenNonStreaming:
"""Non-streaming parse() must also strip unconfigured special tokens."""
@pytest.mark.parametrize(
"parser_info",
_PARSERS,
ids=[p.name for p in _PARSERS],
)
def test_drop_tokens_removed_from_output(self, parser_info):
for sample in parser_info.samples:
injected = _inject_drop_tokens(sample)
tokenizer = make_mock_tokenizer(injected)
parser = parser_info.parser_cls(
tokenizer,
tools=sample.tools,
)
request = _test_request(tools=sample.tools)
output = parse_non_streaming(parser, injected, request)
assert_no_terminal_leakage(
output,
list(_DROP_TOKENS.keys()),
context=f"parser={parser_info.name}",
)
class TestAdapterReferences:
"""Verify make_adapters sets reasoning/tool parser class refs on parser engine
parser classes so the serving layer finds them and calls adjust_request."""
+1 -38
View File
@@ -57,7 +57,7 @@ class TestJoinDecodedTextReturnsStr:
@pytest.fixture
def bare_scanner(self):
return TokenIDScanner({}, tokenizer=None, drop_token_ids=set())
return TokenIDScanner({}, tokenizer=None)
def test_mixed_items(self, bare_scanner):
items = [
@@ -265,43 +265,6 @@ class TestHoldbackTextRecovery:
assert "reasoning end." in combined
class TestDropTokens:
def test_drop_token_with_holdback(self, tokenizer):
"""Drop tokens stripped; hold-back text preserved."""
drop_id = 300
tokenizer.decode.side_effect = lambda ids: {
CHANNEL_END_ID: CHANNEL_END,
drop_id: "<eos>",
}.get(ids[0], "?")
scanner = TokenIDScanner(
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
tokenizer=tokenizer,
drop_token_ids={drop_id},
)
result = scanner.scan(
delta_text="holdback<eos>",
delta_token_ids=[drop_id, CHANNEL_END_ID],
)
assert len(result) == 0
result2 = scanner.scan(
delta_text="<channel|>content",
delta_token_ids=[20],
)
pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
assert len(pre_lexed) == 1
assert pre_lexed[0].terminal == "THINK_END"
texts = [r.text for r in result2 if isinstance(r, TextChunk)]
combined = "".join(texts)
assert "holdback" in combined
assert "<eos>" not in combined
assert len(scanner.flush_pending()) == 0
class TestEndToEndReasoningHoldback:
"""End-to-end engine tests with detokenizer hold-back."""
@@ -100,14 +100,24 @@ def _build_chat_request(
class _StubTokenizer:
"""Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``."""
_VOCAB: dict[str, int] = {
"<|tool_call>": 256_000,
"<tool_call|>": 256_001,
'<|"|>': 52,
"<|channel>": 256_002,
"<channel|>": 256_003,
}
def get_vocab(self) -> dict[str, int]:
return {
"<|tool_call>": 256_000,
"<tool_call|>": 256_001,
'<|"|>': 52,
"<|channel>": 256_002,
"<channel|>": 256_003,
}
return dict(self._VOCAB)
@property
def all_special_tokens(self) -> list[str]:
return list(self._VOCAB.keys())
@property
def all_special_ids(self) -> list[int]:
return list(self._VOCAB.values())
def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None:
@@ -235,10 +245,10 @@ def test_gemma4_keeps_special_tokens_with_tools_thinking_disabled() -> None:
assert request.skip_special_tokens is False
def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:
"""No active tools + thinking disabled: keep the default
(``skip_special_tokens=True``) so stray delimiters do not leak into
content.
def test_gemma4_keeps_skip_special_tokens_false_when_nothing_to_preserve() -> None:
"""No active tools + thinking disabled: ``skip_special_tokens`` stays
``False`` because the parser engine's ``__DROP__`` terminal mechanism
strips unconfigured special tokens automatically.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_chat_request(
@@ -247,4 +257,4 @@ def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:
parser.adjust_request(request)
assert request.skip_special_tokens is True
assert request.skip_special_tokens is False
+6
View File
@@ -424,6 +424,12 @@ class DelegatingParser(Parser):
if tool_parser is None:
return [], content
if request.tool_choice == "none":
if self._engine_based:
result = self.extract_tool_calls(content or "", request=request)
return [], result.content
return [], content
supports_required_and_named = tool_parser.supports_required_and_named
is_named_tool_choice = request.tool_choice and isinstance(
request.tool_choice,
+17 -11
View File
@@ -116,6 +116,7 @@ class ParserEngine(Parser):
self._deferred_content: str = ""
self._deferred_reasoning: str = ""
self._content_has_nonws: bool = False
self._suppress_tool_calls: bool = False
self._arg_converter = parser_engine_config.arg_converter
self._arg_structural_chars = parser_engine_config.arg_structural_chars
@@ -398,10 +399,10 @@ class ParserEngine(Parser):
tools = getattr(request, "tools", None)
if tools:
self._tools = tools
if not self.skip_tool_parsing:
if not self.skip_tool_parsing and not self._suppress_tool_calls:
tool_choice = getattr(request, "tool_choice", None)
if tool_choice == "none" and tools:
self.skip_tool_parsing = True
self._suppress_tool_calls = True
def _strip_content_whitespace(
self,
@@ -642,7 +643,7 @@ class ParserEngine(Parser):
events = self._feed(text, token_ids)
events.extend(self._engine.finish())
delta = self._events_to_delta(events)
delta = self._events_to_delta(events, finished=True)
tool_call_info = self._build_extracted_result()
reasoning = delta.reasoning if delta else None
@@ -701,6 +702,7 @@ class ParserEngine(Parser):
reasoning_parts: list[str] = []
seen_tool_event = False
suppress = self._suppress_tool_calls
for event in events:
match event.type:
case EventType.TEXT_CHUNK:
@@ -713,17 +715,21 @@ class ParserEngine(Parser):
case EventType.REASONING_END:
self._reasoning_ended = True
case EventType.TOOL_CALL_START:
seen_tool_event = True
self._ensure_slot(event.tool_index)
if not suppress:
seen_tool_event = True
self._ensure_slot(event.tool_index)
case EventType.TOOL_NAME:
seen_tool_event = True
self._handle_tool_name(event)
if not suppress:
seen_tool_event = True
self._handle_tool_name(event)
case EventType.ARG_VALUE_CHUNK:
seen_tool_event = True
self._handle_arg_chunk(event, tool_call_deltas)
if not suppress:
seen_tool_event = True
self._handle_arg_chunk(event, tool_call_deltas)
case EventType.TOOL_CALL_END:
seen_tool_event = True
self._handle_tool_end(event, tool_call_deltas)
if not suppress:
seen_tool_event = True
self._handle_tool_end(event, tool_call_deltas)
case EventType.REASONING_START:
pass # no delta-level effect
+3 -12
View File
@@ -24,16 +24,6 @@ from functools import cached_property
from vllm.parser.engine.events import EventType
STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset(
{
"<eos>",
"<bos>",
"<pad>",
"<unk>",
"<mask>",
}
)
class ParserState(Enum):
CONTENT = auto()
@@ -90,6 +80,9 @@ class ParserEngineConfig:
arg_structural_chars: frozenset[str] | None = None
# Special tokens exempt from auto-drop but not state-machine terminals.
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
# Prevents trailing-whitespace accumulation across multi-turn conversations.
strip_trailing_reasoning_whitespace: bool = True
@@ -102,8 +95,6 @@ class ParserEngineConfig:
# Reject tool calls whose names are absent from the request tools.
validate_tool_names: bool = False
drop_tokens: frozenset[str] = field(default_factory=frozenset)
@cached_property
def terminal_defs(self):
from vllm.parser.engine.incremental_lexer import terminals_from_literals
+82 -17
View File
@@ -6,15 +6,17 @@ incremental lexing, and state-machine-driven semantic event emission."""
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
from vllm.parser.engine.events import EventType, SemanticEvent
from vllm.parser.engine.incremental_lexer import (
CONTENT_TERMINAL,
IncrementalLexer,
LexerShape,
LexToken,
TerminalDef,
)
from vllm.parser.engine.parser_engine_config import (
STRUCTURAL_DROP_TOKENS,
ParserEngineConfig,
ParserState,
Transition,
@@ -26,6 +28,64 @@ from vllm.parser.engine.token_id_scanner import (
TokenIDScanner,
)
DROP_TERMINAL = "__DROP__"
@dataclass(slots=True)
class _DropInfo:
lexer_shape: LexerShape
extra_token_ids: dict[int, str]
def _build_drop_info(
config: ParserEngineConfig,
tokenizer,
) -> _DropInfo | None:
try:
special_tokens: list[str] = list(tokenizer.all_special_tokens)
special_ids: list[int] = list(tokenizer.all_special_ids)
except (AttributeError, NotImplementedError):
return None
if not special_tokens:
return None
configured_texts = (
set(config.token_id_terminals.values())
| set(config.terminals.values())
| config.preserve_tokens
)
extra_token_ids: dict[int, str] = {}
drop_texts: set[str] = set()
for text, tid in zip(special_tokens, special_ids):
if text not in configured_texts:
extra_token_ids[tid] = DROP_TERMINAL
drop_texts.add(text)
if not drop_texts:
return None
import regex as re
drop_terminal_defs = [
TerminalDef(
name=DROP_TERMINAL,
pattern=re.compile(re.escape(text)),
is_literal=True,
literal=text,
)
for text in drop_texts
]
all_terminal_defs = list(config.terminal_defs) + drop_terminal_defs
lexer_shape = LexerShape(all_terminal_defs)
return _DropInfo(
lexer_shape=lexer_shape,
extra_token_ids=extra_token_ids,
)
class StreamingParserEngine:
"""Consumes ``(delta_text, delta_token_ids)`` pairs and produces a
@@ -60,7 +120,6 @@ class StreamingParserEngine:
self.config = config
resolved_token_ids: dict[int, str] = {}
drop_token_ids: set[int] = set()
if tokenizer is not None:
if vocab is None:
vocab = tokenizer.get_vocab()
@@ -69,32 +128,29 @@ class StreamingParserEngine:
tid = vocab.get(token_text)
if tid is not None:
resolved_token_ids[tid] = terminal_name
all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS
for token_text in all_drop:
tid = vocab.get(token_text)
if tid is not None:
drop_token_ids.add(tid)
for attr in ("eos_token_id", "bos_token_id", "pad_token_id"):
tid = getattr(tokenizer, attr, None)
if tid is not None:
drop_token_ids.add(tid)
drop_info: _DropInfo | None = None
if tokenizer is not None:
drop_info = _build_drop_info(config, tokenizer)
lexer_shape = config.lexer_shape
if drop_info is not None:
resolved_token_ids.update(drop_info.extra_token_ids)
lexer_shape = drop_info.lexer_shape
self._resolved_token_ids = resolved_token_ids
self._drop_token_ids = drop_token_ids
self._has_drops = drop_info is not None
self._scanner = TokenIDScanner(
resolved_token_ids,
tokenizer,
drop_token_ids,
)
self._token_id_terminal_names: frozenset[str] = frozenset(
resolved_token_ids.values()
)
self._lexer = IncrementalLexer(
config.lexer_shape, content_terminal=CONTENT_TERMINAL
)
self._lexer = IncrementalLexer(lexer_shape, content_terminal=CONTENT_TERMINAL)
self._tool_terminals: frozenset[str] = frozenset(
terminal
@@ -150,7 +206,7 @@ class StreamingParserEngine:
):
has_special = False
for tid in delta_token_ids:
if tid in self._resolved_token_ids or tid in self._drop_token_ids:
if tid in self._resolved_token_ids:
has_special = True
break
if not has_special:
@@ -247,6 +303,15 @@ class StreamingParserEngine:
transition = self.config.transitions.get(key)
if transition is None:
if (
self._has_drops
and terminal == DROP_TERMINAL
# Preserve drop tokens when skip_tool_parsing is active so
# the reasoning pass doesn't silently remove tokens that a
# later tool-call pass might need to see.
and not self.skip_tool_parsing
):
return []
return self._emit_for_state(value)
if self.skip_tool_parsing and terminal in self._tool_terminals:
+4 -27
View File
@@ -42,12 +42,10 @@ class TokenIDScanner:
self,
token_id_to_terminal: dict[int, str],
tokenizer,
drop_token_ids: set[int] | None = None,
) -> None:
self.token_id_to_terminal = token_id_to_terminal
self.tokenizer = tokenizer
self._token_text_cache: dict[int, str] = {}
self._drop_token_ids = drop_token_ids or set()
self._deferred_terminals: list[PreLexedTerminal] = []
self._deferred_post_text: str = ""
@@ -74,22 +72,19 @@ class TokenIDScanner:
if self._deferred_terminals:
prefix_items, effective_text = self._resolve_deferred(delta_text)
if not self.token_id_to_terminal and not self._drop_token_ids:
if not self.token_id_to_terminal:
if effective_text:
prefix_items.append(TextChunk(effective_text))
return prefix_items
has_special = False
has_drop = False
token_id_to_terminal = self.token_id_to_terminal
drop_token_ids = self._drop_token_ids
for tid in delta_token_ids:
if tid in token_id_to_terminal:
has_special = True
if tid in drop_token_ids:
has_drop = True
break
if not has_special and not has_drop:
if not has_special:
if effective_text:
if not prefix_items:
return [TextChunk(effective_text)]
@@ -102,8 +97,6 @@ class TokenIDScanner:
text_accum: list[str] = []
for idx, tid in enumerate(delta_token_ids):
if tid in self._drop_token_ids:
continue
terminal = self.token_id_to_terminal.get(tid)
if terminal is not None:
if text_accum:
@@ -121,23 +114,7 @@ class TokenIDScanner:
results.append(TextChunk(joined))
if effective_text:
if has_drop:
clean_delta = effective_text
for idx, tid in enumerate(delta_token_ids):
if tid in self._drop_token_ids:
dropped = token_texts[idx]
pos = clean_delta.find(dropped)
if pos >= 0:
clean_delta = (
clean_delta[:pos] + clean_delta[pos + len(dropped) :]
)
if clean_delta:
if results:
results = self._recover_holdback_text(clean_delta, results)
else:
results = [TextChunk(clean_delta)]
else:
results = self._recover_holdback_text(effective_text, results)
results = self._recover_holdback_text(effective_text, results)
else:
# No detokenizer text to validate against — individually-decoded
# TextChunks are unreliable (context-dependent decoding).
+1 -58
View File
@@ -35,34 +35,6 @@ if TYPE_CHECKING:
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool
# Tokens the model generates that must not leak into response content.
_GEMMA4_MODEL_DROP_TOKENS: set[str] = {
# Turn boundaries
"<|turn>",
"<turn|>",
# Channel / reasoning
"<|channel>",
"<channel|>",
# Tool protocol tokens
"<|tool>",
"<tool|>",
"<|tool_call>",
"<tool_call|>",
"<|tool_response>",
"<tool_response|>",
'<|"|>',
# Thinking
"<|think|>",
# Multi-modal (defensive — not expected during text completion)
"<|image>",
"<|image|>",
"<image|>",
"<|audio>",
"<|audio|>",
"<audio|>",
"<|video|>",
}
CHANNEL_START = "<|channel>"
CHANNEL_END = "<channel|>"
TOOL_CALL_START = "<|tool_call>"
@@ -322,14 +294,6 @@ def _gemma4_arg_converter(raw_args: str, partial: bool) -> str:
@functools.cache
def gemma4_config() -> ParserEngineConfig:
used_tokens = {
CHANNEL_START,
CHANNEL_END,
TOOL_CALL_START,
TOOL_CALL_END,
'<|"|>',
}
return ParserEngineConfig(
name="gemma4",
initial_state=ParserState.CONTENT,
@@ -412,7 +376,7 @@ def gemma4_config() -> ParserEngineConfig:
arg_converter=_gemma4_arg_converter,
tool_args_json=False,
arg_structural_chars=frozenset(",:{}[]<"),
drop_tokens=frozenset(_GEMMA4_MODEL_DROP_TOKENS - used_tokens),
preserve_tokens=frozenset({STRING_DELIM}),
)
@@ -451,27 +415,6 @@ class Gemma4Parser(ParserEngine):
self._prefix_stripped: bool = False
self._is_first_feed: bool = True
def adjust_request(
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
"""Keep special tokens when thinking or tool calls need them.
``skip_special_tokens`` must stay ``False`` when there is something to
preserve: reasoning channel tokens (thinking enabled) or tool-call
delimiters (tools active). Otherwise keep the default so stray
delimiters do not leak into content (e.g. ``tool_choice="none"`` with
thinking disabled).
"""
request = super().adjust_request(request)
chat_template_kwargs = getattr(request, "chat_template_kwargs", None) or {}
enable_thinking = chat_template_kwargs.get("enable_thinking", True)
has_tools = bool(getattr(request, "tools", None))
tools_active = has_tools and request.tool_choice != "none"
if not enable_thinking and not tools_active:
request.skip_special_tokens = True
return request
def _reset(self, initial_state=None) -> None:
super()._reset(initial_state=initial_state)
self._reasoning_text = ""