[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ben Browning
2026-06-30 13:46:53 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 27d5f78b63
commit 25671cb520
14 changed files with 783 additions and 219 deletions
+17 -11
View File
@@ -116,6 +116,7 @@ class ParserEngine(Parser):
self._deferred_content: str = ""
self._deferred_reasoning: str = ""
self._content_has_nonws: bool = False
self._suppress_tool_calls: bool = False
self._arg_converter = parser_engine_config.arg_converter
self._arg_structural_chars = parser_engine_config.arg_structural_chars
@@ -398,10 +399,10 @@ class ParserEngine(Parser):
tools = getattr(request, "tools", None)
if tools:
self._tools = tools
if not self.skip_tool_parsing:
if not self.skip_tool_parsing and not self._suppress_tool_calls:
tool_choice = getattr(request, "tool_choice", None)
if tool_choice == "none" and tools:
self.skip_tool_parsing = True
self._suppress_tool_calls = True
def _strip_content_whitespace(
self,
@@ -642,7 +643,7 @@ class ParserEngine(Parser):
events = self._feed(text, token_ids)
events.extend(self._engine.finish())
delta = self._events_to_delta(events)
delta = self._events_to_delta(events, finished=True)
tool_call_info = self._build_extracted_result()
reasoning = delta.reasoning if delta else None
@@ -701,6 +702,7 @@ class ParserEngine(Parser):
reasoning_parts: list[str] = []
seen_tool_event = False
suppress = self._suppress_tool_calls
for event in events:
match event.type:
case EventType.TEXT_CHUNK:
@@ -713,17 +715,21 @@ class ParserEngine(Parser):
case EventType.REASONING_END:
self._reasoning_ended = True
case EventType.TOOL_CALL_START:
seen_tool_event = True
self._ensure_slot(event.tool_index)
if not suppress:
seen_tool_event = True
self._ensure_slot(event.tool_index)
case EventType.TOOL_NAME:
seen_tool_event = True
self._handle_tool_name(event)
if not suppress:
seen_tool_event = True
self._handle_tool_name(event)
case EventType.ARG_VALUE_CHUNK:
seen_tool_event = True
self._handle_arg_chunk(event, tool_call_deltas)
if not suppress:
seen_tool_event = True
self._handle_arg_chunk(event, tool_call_deltas)
case EventType.TOOL_CALL_END:
seen_tool_event = True
self._handle_tool_end(event, tool_call_deltas)
if not suppress:
seen_tool_event = True
self._handle_tool_end(event, tool_call_deltas)
case EventType.REASONING_START:
pass # no delta-level effect
+3 -12
View File
@@ -24,16 +24,6 @@ from functools import cached_property
from vllm.parser.engine.events import EventType
STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset(
{
"<eos>",
"<bos>",
"<pad>",
"<unk>",
"<mask>",
}
)
class ParserState(Enum):
CONTENT = auto()
@@ -90,6 +80,9 @@ class ParserEngineConfig:
arg_structural_chars: frozenset[str] | None = None
# Special tokens exempt from auto-drop but not state-machine terminals.
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
# Prevents trailing-whitespace accumulation across multi-turn conversations.
strip_trailing_reasoning_whitespace: bool = True
@@ -102,8 +95,6 @@ class ParserEngineConfig:
# Reject tool calls whose names are absent from the request tools.
validate_tool_names: bool = False
drop_tokens: frozenset[str] = field(default_factory=frozenset)
@cached_property
def terminal_defs(self):
from vllm.parser.engine.incremental_lexer import terminals_from_literals
+82 -17
View File
@@ -6,15 +6,17 @@ incremental lexing, and state-machine-driven semantic event emission."""
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
from vllm.parser.engine.events import EventType, SemanticEvent
from vllm.parser.engine.incremental_lexer import (
CONTENT_TERMINAL,
IncrementalLexer,
LexerShape,
LexToken,
TerminalDef,
)
from vllm.parser.engine.parser_engine_config import (
STRUCTURAL_DROP_TOKENS,
ParserEngineConfig,
ParserState,
Transition,
@@ -26,6 +28,64 @@ from vllm.parser.engine.token_id_scanner import (
TokenIDScanner,
)
DROP_TERMINAL = "__DROP__"
@dataclass(slots=True)
class _DropInfo:
lexer_shape: LexerShape
extra_token_ids: dict[int, str]
def _build_drop_info(
config: ParserEngineConfig,
tokenizer,
) -> _DropInfo | None:
try:
special_tokens: list[str] = list(tokenizer.all_special_tokens)
special_ids: list[int] = list(tokenizer.all_special_ids)
except (AttributeError, NotImplementedError):
return None
if not special_tokens:
return None
configured_texts = (
set(config.token_id_terminals.values())
| set(config.terminals.values())
| config.preserve_tokens
)
extra_token_ids: dict[int, str] = {}
drop_texts: set[str] = set()
for text, tid in zip(special_tokens, special_ids):
if text not in configured_texts:
extra_token_ids[tid] = DROP_TERMINAL
drop_texts.add(text)
if not drop_texts:
return None
import regex as re
drop_terminal_defs = [
TerminalDef(
name=DROP_TERMINAL,
pattern=re.compile(re.escape(text)),
is_literal=True,
literal=text,
)
for text in drop_texts
]
all_terminal_defs = list(config.terminal_defs) + drop_terminal_defs
lexer_shape = LexerShape(all_terminal_defs)
return _DropInfo(
lexer_shape=lexer_shape,
extra_token_ids=extra_token_ids,
)
class StreamingParserEngine:
"""Consumes ``(delta_text, delta_token_ids)`` pairs and produces a
@@ -60,7 +120,6 @@ class StreamingParserEngine:
self.config = config
resolved_token_ids: dict[int, str] = {}
drop_token_ids: set[int] = set()
if tokenizer is not None:
if vocab is None:
vocab = tokenizer.get_vocab()
@@ -69,32 +128,29 @@ class StreamingParserEngine:
tid = vocab.get(token_text)
if tid is not None:
resolved_token_ids[tid] = terminal_name
all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS
for token_text in all_drop:
tid = vocab.get(token_text)
if tid is not None:
drop_token_ids.add(tid)
for attr in ("eos_token_id", "bos_token_id", "pad_token_id"):
tid = getattr(tokenizer, attr, None)
if tid is not None:
drop_token_ids.add(tid)
drop_info: _DropInfo | None = None
if tokenizer is not None:
drop_info = _build_drop_info(config, tokenizer)
lexer_shape = config.lexer_shape
if drop_info is not None:
resolved_token_ids.update(drop_info.extra_token_ids)
lexer_shape = drop_info.lexer_shape
self._resolved_token_ids = resolved_token_ids
self._drop_token_ids = drop_token_ids
self._has_drops = drop_info is not None
self._scanner = TokenIDScanner(
resolved_token_ids,
tokenizer,
drop_token_ids,
)
self._token_id_terminal_names: frozenset[str] = frozenset(
resolved_token_ids.values()
)
self._lexer = IncrementalLexer(
config.lexer_shape, content_terminal=CONTENT_TERMINAL
)
self._lexer = IncrementalLexer(lexer_shape, content_terminal=CONTENT_TERMINAL)
self._tool_terminals: frozenset[str] = frozenset(
terminal
@@ -150,7 +206,7 @@ class StreamingParserEngine:
):
has_special = False
for tid in delta_token_ids:
if tid in self._resolved_token_ids or tid in self._drop_token_ids:
if tid in self._resolved_token_ids:
has_special = True
break
if not has_special:
@@ -247,6 +303,15 @@ class StreamingParserEngine:
transition = self.config.transitions.get(key)
if transition is None:
if (
self._has_drops
and terminal == DROP_TERMINAL
# Preserve drop tokens when skip_tool_parsing is active so
# the reasoning pass doesn't silently remove tokens that a
# later tool-call pass might need to see.
and not self.skip_tool_parsing
):
return []
return self._emit_for_state(value)
if self.skip_tool_parsing and terminal in self._tool_terminals:
+4 -27
View File
@@ -42,12 +42,10 @@ class TokenIDScanner:
self,
token_id_to_terminal: dict[int, str],
tokenizer,
drop_token_ids: set[int] | None = None,
) -> None:
self.token_id_to_terminal = token_id_to_terminal
self.tokenizer = tokenizer
self._token_text_cache: dict[int, str] = {}
self._drop_token_ids = drop_token_ids or set()
self._deferred_terminals: list[PreLexedTerminal] = []
self._deferred_post_text: str = ""
@@ -74,22 +72,19 @@ class TokenIDScanner:
if self._deferred_terminals:
prefix_items, effective_text = self._resolve_deferred(delta_text)
if not self.token_id_to_terminal and not self._drop_token_ids:
if not self.token_id_to_terminal:
if effective_text:
prefix_items.append(TextChunk(effective_text))
return prefix_items
has_special = False
has_drop = False
token_id_to_terminal = self.token_id_to_terminal
drop_token_ids = self._drop_token_ids
for tid in delta_token_ids:
if tid in token_id_to_terminal:
has_special = True
if tid in drop_token_ids:
has_drop = True
break
if not has_special and not has_drop:
if not has_special:
if effective_text:
if not prefix_items:
return [TextChunk(effective_text)]
@@ -102,8 +97,6 @@ class TokenIDScanner:
text_accum: list[str] = []
for idx, tid in enumerate(delta_token_ids):
if tid in self._drop_token_ids:
continue
terminal = self.token_id_to_terminal.get(tid)
if terminal is not None:
if text_accum:
@@ -121,23 +114,7 @@ class TokenIDScanner:
results.append(TextChunk(joined))
if effective_text:
if has_drop:
clean_delta = effective_text
for idx, tid in enumerate(delta_token_ids):
if tid in self._drop_token_ids:
dropped = token_texts[idx]
pos = clean_delta.find(dropped)
if pos >= 0:
clean_delta = (
clean_delta[:pos] + clean_delta[pos + len(dropped) :]
)
if clean_delta:
if results:
results = self._recover_holdback_text(clean_delta, results)
else:
results = [TextChunk(clean_delta)]
else:
results = self._recover_holdback_text(effective_text, results)
results = self._recover_holdback_text(effective_text, results)
else:
# No detokenizer text to validate against — individually-decoded
# TextChunks are unreliable (context-dependent decoding).