[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ben Browning
2026-06-30 13:46:53 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 27d5f78b63
commit 25671cb520
14 changed files with 783 additions and 219 deletions
+6
View File
@@ -424,6 +424,12 @@ class DelegatingParser(Parser):
if tool_parser is None:
return [], content
if request.tool_choice == "none":
if self._engine_based:
result = self.extract_tool_calls(content or "", request=request)
return [], result.content
return [], content
supports_required_and_named = tool_parser.supports_required_and_named
is_named_tool_choice = request.tool_choice and isinstance(
request.tool_choice,
+17 -11
View File
@@ -116,6 +116,7 @@ class ParserEngine(Parser):
self._deferred_content: str = ""
self._deferred_reasoning: str = ""
self._content_has_nonws: bool = False
self._suppress_tool_calls: bool = False
self._arg_converter = parser_engine_config.arg_converter
self._arg_structural_chars = parser_engine_config.arg_structural_chars
@@ -398,10 +399,10 @@ class ParserEngine(Parser):
tools = getattr(request, "tools", None)
if tools:
self._tools = tools
if not self.skip_tool_parsing:
if not self.skip_tool_parsing and not self._suppress_tool_calls:
tool_choice = getattr(request, "tool_choice", None)
if tool_choice == "none" and tools:
self.skip_tool_parsing = True
self._suppress_tool_calls = True
def _strip_content_whitespace(
self,
@@ -642,7 +643,7 @@ class ParserEngine(Parser):
events = self._feed(text, token_ids)
events.extend(self._engine.finish())
delta = self._events_to_delta(events)
delta = self._events_to_delta(events, finished=True)
tool_call_info = self._build_extracted_result()
reasoning = delta.reasoning if delta else None
@@ -701,6 +702,7 @@ class ParserEngine(Parser):
reasoning_parts: list[str] = []
seen_tool_event = False
suppress = self._suppress_tool_calls
for event in events:
match event.type:
case EventType.TEXT_CHUNK:
@@ -713,17 +715,21 @@ class ParserEngine(Parser):
case EventType.REASONING_END:
self._reasoning_ended = True
case EventType.TOOL_CALL_START:
seen_tool_event = True
self._ensure_slot(event.tool_index)
if not suppress:
seen_tool_event = True
self._ensure_slot(event.tool_index)
case EventType.TOOL_NAME:
seen_tool_event = True
self._handle_tool_name(event)
if not suppress:
seen_tool_event = True
self._handle_tool_name(event)
case EventType.ARG_VALUE_CHUNK:
seen_tool_event = True
self._handle_arg_chunk(event, tool_call_deltas)
if not suppress:
seen_tool_event = True
self._handle_arg_chunk(event, tool_call_deltas)
case EventType.TOOL_CALL_END:
seen_tool_event = True
self._handle_tool_end(event, tool_call_deltas)
if not suppress:
seen_tool_event = True
self._handle_tool_end(event, tool_call_deltas)
case EventType.REASONING_START:
pass # no delta-level effect
+3 -12
View File
@@ -24,16 +24,6 @@ from functools import cached_property
from vllm.parser.engine.events import EventType
STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset(
{
"<eos>",
"<bos>",
"<pad>",
"<unk>",
"<mask>",
}
)
class ParserState(Enum):
CONTENT = auto()
@@ -90,6 +80,9 @@ class ParserEngineConfig:
arg_structural_chars: frozenset[str] | None = None
# Special tokens exempt from auto-drop but not state-machine terminals.
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
# Prevents trailing-whitespace accumulation across multi-turn conversations.
strip_trailing_reasoning_whitespace: bool = True
@@ -102,8 +95,6 @@ class ParserEngineConfig:
# Reject tool calls whose names are absent from the request tools.
validate_tool_names: bool = False
drop_tokens: frozenset[str] = field(default_factory=frozenset)
@cached_property
def terminal_defs(self):
from vllm.parser.engine.incremental_lexer import terminals_from_literals
+82 -17
View File
@@ -6,15 +6,17 @@ incremental lexing, and state-machine-driven semantic event emission."""
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
from vllm.parser.engine.events import EventType, SemanticEvent
from vllm.parser.engine.incremental_lexer import (
CONTENT_TERMINAL,
IncrementalLexer,
LexerShape,
LexToken,
TerminalDef,
)
from vllm.parser.engine.parser_engine_config import (
STRUCTURAL_DROP_TOKENS,
ParserEngineConfig,
ParserState,
Transition,
@@ -26,6 +28,64 @@ from vllm.parser.engine.token_id_scanner import (
TokenIDScanner,
)
DROP_TERMINAL = "__DROP__"
@dataclass(slots=True)
class _DropInfo:
lexer_shape: LexerShape
extra_token_ids: dict[int, str]
def _build_drop_info(
config: ParserEngineConfig,
tokenizer,
) -> _DropInfo | None:
try:
special_tokens: list[str] = list(tokenizer.all_special_tokens)
special_ids: list[int] = list(tokenizer.all_special_ids)
except (AttributeError, NotImplementedError):
return None
if not special_tokens:
return None
configured_texts = (
set(config.token_id_terminals.values())
| set(config.terminals.values())
| config.preserve_tokens
)
extra_token_ids: dict[int, str] = {}
drop_texts: set[str] = set()
for text, tid in zip(special_tokens, special_ids):
if text not in configured_texts:
extra_token_ids[tid] = DROP_TERMINAL
drop_texts.add(text)
if not drop_texts:
return None
import regex as re
drop_terminal_defs = [
TerminalDef(
name=DROP_TERMINAL,
pattern=re.compile(re.escape(text)),
is_literal=True,
literal=text,
)
for text in drop_texts
]
all_terminal_defs = list(config.terminal_defs) + drop_terminal_defs
lexer_shape = LexerShape(all_terminal_defs)
return _DropInfo(
lexer_shape=lexer_shape,
extra_token_ids=extra_token_ids,
)
class StreamingParserEngine:
"""Consumes ``(delta_text, delta_token_ids)`` pairs and produces a
@@ -60,7 +120,6 @@ class StreamingParserEngine:
self.config = config
resolved_token_ids: dict[int, str] = {}
drop_token_ids: set[int] = set()
if tokenizer is not None:
if vocab is None:
vocab = tokenizer.get_vocab()
@@ -69,32 +128,29 @@ class StreamingParserEngine:
tid = vocab.get(token_text)
if tid is not None:
resolved_token_ids[tid] = terminal_name
all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS
for token_text in all_drop:
tid = vocab.get(token_text)
if tid is not None:
drop_token_ids.add(tid)
for attr in ("eos_token_id", "bos_token_id", "pad_token_id"):
tid = getattr(tokenizer, attr, None)
if tid is not None:
drop_token_ids.add(tid)
drop_info: _DropInfo | None = None
if tokenizer is not None:
drop_info = _build_drop_info(config, tokenizer)
lexer_shape = config.lexer_shape
if drop_info is not None:
resolved_token_ids.update(drop_info.extra_token_ids)
lexer_shape = drop_info.lexer_shape
self._resolved_token_ids = resolved_token_ids
self._drop_token_ids = drop_token_ids
self._has_drops = drop_info is not None
self._scanner = TokenIDScanner(
resolved_token_ids,
tokenizer,
drop_token_ids,
)
self._token_id_terminal_names: frozenset[str] = frozenset(
resolved_token_ids.values()
)
self._lexer = IncrementalLexer(
config.lexer_shape, content_terminal=CONTENT_TERMINAL
)
self._lexer = IncrementalLexer(lexer_shape, content_terminal=CONTENT_TERMINAL)
self._tool_terminals: frozenset[str] = frozenset(
terminal
@@ -150,7 +206,7 @@ class StreamingParserEngine:
):
has_special = False
for tid in delta_token_ids:
if tid in self._resolved_token_ids or tid in self._drop_token_ids:
if tid in self._resolved_token_ids:
has_special = True
break
if not has_special:
@@ -247,6 +303,15 @@ class StreamingParserEngine:
transition = self.config.transitions.get(key)
if transition is None:
if (
self._has_drops
and terminal == DROP_TERMINAL
# Preserve drop tokens when skip_tool_parsing is active so
# the reasoning pass doesn't silently remove tokens that a
# later tool-call pass might need to see.
and not self.skip_tool_parsing
):
return []
return self._emit_for_state(value)
if self.skip_tool_parsing and terminal in self._tool_terminals:
+4 -27
View File
@@ -42,12 +42,10 @@ class TokenIDScanner:
self,
token_id_to_terminal: dict[int, str],
tokenizer,
drop_token_ids: set[int] | None = None,
) -> None:
self.token_id_to_terminal = token_id_to_terminal
self.tokenizer = tokenizer
self._token_text_cache: dict[int, str] = {}
self._drop_token_ids = drop_token_ids or set()
self._deferred_terminals: list[PreLexedTerminal] = []
self._deferred_post_text: str = ""
@@ -74,22 +72,19 @@ class TokenIDScanner:
if self._deferred_terminals:
prefix_items, effective_text = self._resolve_deferred(delta_text)
if not self.token_id_to_terminal and not self._drop_token_ids:
if not self.token_id_to_terminal:
if effective_text:
prefix_items.append(TextChunk(effective_text))
return prefix_items
has_special = False
has_drop = False
token_id_to_terminal = self.token_id_to_terminal
drop_token_ids = self._drop_token_ids
for tid in delta_token_ids:
if tid in token_id_to_terminal:
has_special = True
if tid in drop_token_ids:
has_drop = True
break
if not has_special and not has_drop:
if not has_special:
if effective_text:
if not prefix_items:
return [TextChunk(effective_text)]
@@ -102,8 +97,6 @@ class TokenIDScanner:
text_accum: list[str] = []
for idx, tid in enumerate(delta_token_ids):
if tid in self._drop_token_ids:
continue
terminal = self.token_id_to_terminal.get(tid)
if terminal is not None:
if text_accum:
@@ -121,23 +114,7 @@ class TokenIDScanner:
results.append(TextChunk(joined))
if effective_text:
if has_drop:
clean_delta = effective_text
for idx, tid in enumerate(delta_token_ids):
if tid in self._drop_token_ids:
dropped = token_texts[idx]
pos = clean_delta.find(dropped)
if pos >= 0:
clean_delta = (
clean_delta[:pos] + clean_delta[pos + len(dropped) :]
)
if clean_delta:
if results:
results = self._recover_holdback_text(clean_delta, results)
else:
results = [TextChunk(clean_delta)]
else:
results = self._recover_holdback_text(effective_text, results)
results = self._recover_holdback_text(effective_text, results)
else:
# No detokenizer text to validate against — individually-decoded
# TextChunks are unreliable (context-dependent decoding).
+1 -58
View File
@@ -35,34 +35,6 @@ if TYPE_CHECKING:
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool
# Tokens the model generates that must not leak into response content.
_GEMMA4_MODEL_DROP_TOKENS: set[str] = {
# Turn boundaries
"<|turn>",
"<turn|>",
# Channel / reasoning
"<|channel>",
"<channel|>",
# Tool protocol tokens
"<|tool>",
"<tool|>",
"<|tool_call>",
"<tool_call|>",
"<|tool_response>",
"<tool_response|>",
'<|"|>',
# Thinking
"<|think|>",
# Multi-modal (defensive — not expected during text completion)
"<|image>",
"<|image|>",
"<image|>",
"<|audio>",
"<|audio|>",
"<audio|>",
"<|video|>",
}
CHANNEL_START = "<|channel>"
CHANNEL_END = "<channel|>"
TOOL_CALL_START = "<|tool_call>"
@@ -322,14 +294,6 @@ def _gemma4_arg_converter(raw_args: str, partial: bool) -> str:
@functools.cache
def gemma4_config() -> ParserEngineConfig:
used_tokens = {
CHANNEL_START,
CHANNEL_END,
TOOL_CALL_START,
TOOL_CALL_END,
'<|"|>',
}
return ParserEngineConfig(
name="gemma4",
initial_state=ParserState.CONTENT,
@@ -412,7 +376,7 @@ def gemma4_config() -> ParserEngineConfig:
arg_converter=_gemma4_arg_converter,
tool_args_json=False,
arg_structural_chars=frozenset(",:{}[]<"),
drop_tokens=frozenset(_GEMMA4_MODEL_DROP_TOKENS - used_tokens),
preserve_tokens=frozenset({STRING_DELIM}),
)
@@ -451,27 +415,6 @@ class Gemma4Parser(ParserEngine):
self._prefix_stripped: bool = False
self._is_first_feed: bool = True
def adjust_request(
self,
request: ChatCompletionRequest | ResponsesRequest,
) -> ChatCompletionRequest | ResponsesRequest:
"""Keep special tokens when thinking or tool calls need them.
``skip_special_tokens`` must stay ``False`` when there is something to
preserve: reasoning channel tokens (thinking enabled) or tool-call
delimiters (tools active). Otherwise keep the default so stray
delimiters do not leak into content (e.g. ``tool_choice="none"`` with
thinking disabled).
"""
request = super().adjust_request(request)
chat_template_kwargs = getattr(request, "chat_template_kwargs", None) or {}
enable_thinking = chat_template_kwargs.get("enable_thinking", True)
has_tools = bool(getattr(request, "tools", None))
tools_active = has_tools and request.tool_choice != "none"
if not enable_thinking and not tools_active:
request.skip_special_tokens = True
return request
def _reset(self, initial_state=None) -> None:
super()._reset(initial_state=initial_state)
self._reasoning_text = ""