[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ben Browning
2026-06-30 13:46:53 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 27d5f78b63
commit 25671cb520
14 changed files with 783 additions and 219 deletions
+182 -1
View File
@@ -119,8 +119,10 @@ def _hermes_config() -> ParserEngineConfig:
def _make_engine(
config: ParserEngineConfig | None = None,
tools: list | None = None,
vocab: dict[str, int] | None = None,
special_tokens: list[str] | None = None,
) -> ParserEngine:
tokenizer = make_mock_tokenizer(_VOCAB)
tokenizer = make_mock_tokenizer(vocab or _VOCAB, special_tokens=special_tokens)
cfg = config or _combined_config()
return ParserEngine(
tokenizer,
@@ -843,6 +845,7 @@ class TestParseTokenIdPassthrough:
assert content is not None
assert "<tool_call>" in content
assert content == "Use <tool_call> to call tools."
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "f"
@@ -1490,3 +1493,181 @@ class TestCoercionInstabilityRegression:
assert parsed["val"] == "4e"
assert isinstance(parsed["val"], str)
assert parsed["extra"] == "ok"
_DROP_VOCAB: dict[str, int] = {
**_VOCAB,
"<bos>": 204,
"<eos>": 205,
}
class TestDropSpecialTokens:
"""Special token dropping via the __DROP__ terminal mechanism."""
def test_drops_special_token_by_id_from_content(self):
"""A special token (not a configured terminal) is dropped when
it arrives as its actual token ID."""
config = ParserEngineConfig(
name="drop_content_test",
terminals={},
token_id_terminals={},
transitions={},
initial_state=ParserState.CONTENT,
content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
)
engine = _make_engine(
config=config,
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = engine._engine.feed("hello<bos>world", [72, 204, 73])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<bos>" not in delta.content
assert delta.content == "helloworld"
def test_drops_special_token_by_id_from_reasoning(self):
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = engine._engine.feed("thinking<eos>more", [72, 205, 73])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<eos>" not in delta.reasoning
assert delta.reasoning == "thinkingmore"
def test_drops_via_text_fallback_when_no_token_ids(self):
"""When no token IDs are provided, text-based lexer catches
drop tokens as a fallback."""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = engine._engine.feed("hello<bos>world", [])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<bos>" not in delta.reasoning
def test_regular_tokens_spelling_special_survive(self):
"""Regular tokens that spell out a drop-token string survive
when token IDs prove they are not the special token."""
vocab = {**_DROP_VOCAB, "h": 72, "<": 73, "bos": 74, ">": 75, "w": 76}
engine = _make_engine(
vocab=vocab,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
# Feed regular token IDs (73, 74, 75) that spell <bos>,
# not the special token ID 204.
events = engine._engine.feed("h<bos>w", [72, 73, 74, 75, 76])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<bos>" in delta.reasoning
def test_configured_terminal_not_treated_as_drop(self):
"""Tokens already in config.terminals (like <think>) are handled
by the state machine, not the drop mechanism."""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
# Start from CONTENT so <think> triggers REASONING_START
engine._engine.reset(initial_state=ParserState.CONTENT)
events = engine._engine.feed("<think>", [200])
has_reasoning_start = any(e.type == EventType.REASONING_START for e in events)
assert has_reasoning_start
def test_tool_name_not_affected_by_drops(self):
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = [
SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0),
SemanticEvent(EventType.ARG_VALUE_CHUNK, '{"city": "NYC"}', tool_index=0),
SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
]
delta = engine._events_to_delta(events)
assert delta is not None
assert delta.tool_calls is not None
names = [
tc.function.name
for tc in delta.tool_calls
if tc.function and tc.function.name
]
assert "get_weather" in names
def test_adjacent_drop_tokens_by_id(self):
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = engine._engine.feed("<bos><eos>", [204, 205])
delta = engine._events_to_delta(events)
assert delta is None
def test_no_special_tokens_means_no_drops(self):
"""Engine with no special tokens passes all text through."""
engine = _make_engine(special_tokens=[])
engine._engine.reset()
events = engine._engine.feed("hello<bos>world", [])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<bos>" in delta.reasoning
def test_drops_suppressed_with_skip_tool_parsing(self):
"""When skip_tool_parsing is active, drop tokens are preserved
as content so a later tool-call pass can see them."""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.skip_tool_parsing = True
engine._engine.reset()
events = engine._engine.feed("hello<bos>world", [72, 204, 73])
delta = engine._events_to_delta(events)
assert delta is not None
assert "<bos>" in delta.reasoning
def test_drops_in_tool_args_state(self):
"""Drop tokens in TOOL_ARGS state are silently discarded."""
vocab = {**_DROP_VOCAB, '{"a":1}': 300}
engine = _make_engine(
vocab=vocab,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset(initial_state=ParserState.CONTENT)
events = engine._engine.feed(
'<tool_call>{"a":1}<bos></tool_call>',
[202, 300, 204, 203],
)
arg_chunks = [e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK]
combined = "".join(arg_chunks)
assert "<bos>" not in combined
assert '{"a":1}' in combined
def test_mixed_configured_and_drop_terminals(self):
"""Configured terminals trigger transitions while drop terminals
are silently removed in the same stream."""
engine = _make_engine(
vocab=_DROP_VOCAB,
special_tokens=list(_DROP_VOCAB.keys()),
)
engine._engine.reset()
events = engine._engine.feed("thought<bos></think>answer", [72, 204, 201, 73])
types = [e.type for e in events]
assert EventType.REASONING_CHUNK in types
assert EventType.REASONING_END in types
assert EventType.TEXT_CHUNK in types
reasoning_text = "".join(
e.value for e in events if e.type == EventType.REASONING_CHUNK
)
assert "<bos>" not in reasoning_text