[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
27d5f78b63
commit
25671cb520
@@ -116,6 +116,7 @@ class ParserEngine(Parser):
|
||||
self._deferred_content: str = ""
|
||||
self._deferred_reasoning: str = ""
|
||||
self._content_has_nonws: bool = False
|
||||
self._suppress_tool_calls: bool = False
|
||||
|
||||
self._arg_converter = parser_engine_config.arg_converter
|
||||
self._arg_structural_chars = parser_engine_config.arg_structural_chars
|
||||
@@ -398,10 +399,10 @@ class ParserEngine(Parser):
|
||||
tools = getattr(request, "tools", None)
|
||||
if tools:
|
||||
self._tools = tools
|
||||
if not self.skip_tool_parsing:
|
||||
if not self.skip_tool_parsing and not self._suppress_tool_calls:
|
||||
tool_choice = getattr(request, "tool_choice", None)
|
||||
if tool_choice == "none" and tools:
|
||||
self.skip_tool_parsing = True
|
||||
self._suppress_tool_calls = True
|
||||
|
||||
def _strip_content_whitespace(
|
||||
self,
|
||||
@@ -642,7 +643,7 @@ class ParserEngine(Parser):
|
||||
events = self._feed(text, token_ids)
|
||||
events.extend(self._engine.finish())
|
||||
|
||||
delta = self._events_to_delta(events)
|
||||
delta = self._events_to_delta(events, finished=True)
|
||||
tool_call_info = self._build_extracted_result()
|
||||
|
||||
reasoning = delta.reasoning if delta else None
|
||||
@@ -701,6 +702,7 @@ class ParserEngine(Parser):
|
||||
reasoning_parts: list[str] = []
|
||||
|
||||
seen_tool_event = False
|
||||
suppress = self._suppress_tool_calls
|
||||
for event in events:
|
||||
match event.type:
|
||||
case EventType.TEXT_CHUNK:
|
||||
@@ -713,17 +715,21 @@ class ParserEngine(Parser):
|
||||
case EventType.REASONING_END:
|
||||
self._reasoning_ended = True
|
||||
case EventType.TOOL_CALL_START:
|
||||
seen_tool_event = True
|
||||
self._ensure_slot(event.tool_index)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._ensure_slot(event.tool_index)
|
||||
case EventType.TOOL_NAME:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_name(event)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_name(event)
|
||||
case EventType.ARG_VALUE_CHUNK:
|
||||
seen_tool_event = True
|
||||
self._handle_arg_chunk(event, tool_call_deltas)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._handle_arg_chunk(event, tool_call_deltas)
|
||||
case EventType.TOOL_CALL_END:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_end(event, tool_call_deltas)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_end(event, tool_call_deltas)
|
||||
case EventType.REASONING_START:
|
||||
pass # no delta-level effect
|
||||
|
||||
|
||||
@@ -24,16 +24,6 @@ from functools import cached_property
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
|
||||
STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset(
|
||||
{
|
||||
"<eos>",
|
||||
"<bos>",
|
||||
"<pad>",
|
||||
"<unk>",
|
||||
"<mask>",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ParserState(Enum):
|
||||
CONTENT = auto()
|
||||
@@ -90,6 +80,9 @@ class ParserEngineConfig:
|
||||
|
||||
arg_structural_chars: frozenset[str] | None = None
|
||||
|
||||
# Special tokens exempt from auto-drop but not state-machine terminals.
|
||||
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
# Prevents trailing-whitespace accumulation across multi-turn conversations.
|
||||
strip_trailing_reasoning_whitespace: bool = True
|
||||
|
||||
@@ -102,8 +95,6 @@ class ParserEngineConfig:
|
||||
# Reject tool calls whose names are absent from the request tools.
|
||||
validate_tool_names: bool = False
|
||||
|
||||
drop_tokens: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
@cached_property
|
||||
def terminal_defs(self):
|
||||
from vllm.parser.engine.incremental_lexer import terminals_from_literals
|
||||
|
||||
@@ -6,15 +6,17 @@ incremental lexing, and state-machine-driven semantic event emission."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.incremental_lexer import (
|
||||
CONTENT_TERMINAL,
|
||||
IncrementalLexer,
|
||||
LexerShape,
|
||||
LexToken,
|
||||
TerminalDef,
|
||||
)
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
STRUCTURAL_DROP_TOKENS,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
@@ -26,6 +28,64 @@ from vllm.parser.engine.token_id_scanner import (
|
||||
TokenIDScanner,
|
||||
)
|
||||
|
||||
DROP_TERMINAL = "__DROP__"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _DropInfo:
|
||||
lexer_shape: LexerShape
|
||||
extra_token_ids: dict[int, str]
|
||||
|
||||
|
||||
def _build_drop_info(
|
||||
config: ParserEngineConfig,
|
||||
tokenizer,
|
||||
) -> _DropInfo | None:
|
||||
try:
|
||||
special_tokens: list[str] = list(tokenizer.all_special_tokens)
|
||||
special_ids: list[int] = list(tokenizer.all_special_ids)
|
||||
except (AttributeError, NotImplementedError):
|
||||
return None
|
||||
|
||||
if not special_tokens:
|
||||
return None
|
||||
|
||||
configured_texts = (
|
||||
set(config.token_id_terminals.values())
|
||||
| set(config.terminals.values())
|
||||
| config.preserve_tokens
|
||||
)
|
||||
|
||||
extra_token_ids: dict[int, str] = {}
|
||||
drop_texts: set[str] = set()
|
||||
for text, tid in zip(special_tokens, special_ids):
|
||||
if text not in configured_texts:
|
||||
extra_token_ids[tid] = DROP_TERMINAL
|
||||
drop_texts.add(text)
|
||||
|
||||
if not drop_texts:
|
||||
return None
|
||||
|
||||
import regex as re
|
||||
|
||||
drop_terminal_defs = [
|
||||
TerminalDef(
|
||||
name=DROP_TERMINAL,
|
||||
pattern=re.compile(re.escape(text)),
|
||||
is_literal=True,
|
||||
literal=text,
|
||||
)
|
||||
for text in drop_texts
|
||||
]
|
||||
|
||||
all_terminal_defs = list(config.terminal_defs) + drop_terminal_defs
|
||||
lexer_shape = LexerShape(all_terminal_defs)
|
||||
|
||||
return _DropInfo(
|
||||
lexer_shape=lexer_shape,
|
||||
extra_token_ids=extra_token_ids,
|
||||
)
|
||||
|
||||
|
||||
class StreamingParserEngine:
|
||||
"""Consumes ``(delta_text, delta_token_ids)`` pairs and produces a
|
||||
@@ -60,7 +120,6 @@ class StreamingParserEngine:
|
||||
self.config = config
|
||||
|
||||
resolved_token_ids: dict[int, str] = {}
|
||||
drop_token_ids: set[int] = set()
|
||||
if tokenizer is not None:
|
||||
if vocab is None:
|
||||
vocab = tokenizer.get_vocab()
|
||||
@@ -69,32 +128,29 @@ class StreamingParserEngine:
|
||||
tid = vocab.get(token_text)
|
||||
if tid is not None:
|
||||
resolved_token_ids[tid] = terminal_name
|
||||
all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS
|
||||
for token_text in all_drop:
|
||||
tid = vocab.get(token_text)
|
||||
if tid is not None:
|
||||
drop_token_ids.add(tid)
|
||||
for attr in ("eos_token_id", "bos_token_id", "pad_token_id"):
|
||||
tid = getattr(tokenizer, attr, None)
|
||||
if tid is not None:
|
||||
drop_token_ids.add(tid)
|
||||
|
||||
drop_info: _DropInfo | None = None
|
||||
if tokenizer is not None:
|
||||
drop_info = _build_drop_info(config, tokenizer)
|
||||
|
||||
lexer_shape = config.lexer_shape
|
||||
if drop_info is not None:
|
||||
resolved_token_ids.update(drop_info.extra_token_ids)
|
||||
lexer_shape = drop_info.lexer_shape
|
||||
|
||||
self._resolved_token_ids = resolved_token_ids
|
||||
self._drop_token_ids = drop_token_ids
|
||||
self._has_drops = drop_info is not None
|
||||
|
||||
self._scanner = TokenIDScanner(
|
||||
resolved_token_ids,
|
||||
tokenizer,
|
||||
drop_token_ids,
|
||||
)
|
||||
|
||||
self._token_id_terminal_names: frozenset[str] = frozenset(
|
||||
resolved_token_ids.values()
|
||||
)
|
||||
|
||||
self._lexer = IncrementalLexer(
|
||||
config.lexer_shape, content_terminal=CONTENT_TERMINAL
|
||||
)
|
||||
self._lexer = IncrementalLexer(lexer_shape, content_terminal=CONTENT_TERMINAL)
|
||||
|
||||
self._tool_terminals: frozenset[str] = frozenset(
|
||||
terminal
|
||||
@@ -150,7 +206,7 @@ class StreamingParserEngine:
|
||||
):
|
||||
has_special = False
|
||||
for tid in delta_token_ids:
|
||||
if tid in self._resolved_token_ids or tid in self._drop_token_ids:
|
||||
if tid in self._resolved_token_ids:
|
||||
has_special = True
|
||||
break
|
||||
if not has_special:
|
||||
@@ -247,6 +303,15 @@ class StreamingParserEngine:
|
||||
transition = self.config.transitions.get(key)
|
||||
|
||||
if transition is None:
|
||||
if (
|
||||
self._has_drops
|
||||
and terminal == DROP_TERMINAL
|
||||
# Preserve drop tokens when skip_tool_parsing is active so
|
||||
# the reasoning pass doesn't silently remove tokens that a
|
||||
# later tool-call pass might need to see.
|
||||
and not self.skip_tool_parsing
|
||||
):
|
||||
return []
|
||||
return self._emit_for_state(value)
|
||||
|
||||
if self.skip_tool_parsing and terminal in self._tool_terminals:
|
||||
|
||||
@@ -42,12 +42,10 @@ class TokenIDScanner:
|
||||
self,
|
||||
token_id_to_terminal: dict[int, str],
|
||||
tokenizer,
|
||||
drop_token_ids: set[int] | None = None,
|
||||
) -> None:
|
||||
self.token_id_to_terminal = token_id_to_terminal
|
||||
self.tokenizer = tokenizer
|
||||
self._token_text_cache: dict[int, str] = {}
|
||||
self._drop_token_ids = drop_token_ids or set()
|
||||
self._deferred_terminals: list[PreLexedTerminal] = []
|
||||
self._deferred_post_text: str = ""
|
||||
|
||||
@@ -74,22 +72,19 @@ class TokenIDScanner:
|
||||
if self._deferred_terminals:
|
||||
prefix_items, effective_text = self._resolve_deferred(delta_text)
|
||||
|
||||
if not self.token_id_to_terminal and not self._drop_token_ids:
|
||||
if not self.token_id_to_terminal:
|
||||
if effective_text:
|
||||
prefix_items.append(TextChunk(effective_text))
|
||||
return prefix_items
|
||||
|
||||
has_special = False
|
||||
has_drop = False
|
||||
token_id_to_terminal = self.token_id_to_terminal
|
||||
drop_token_ids = self._drop_token_ids
|
||||
for tid in delta_token_ids:
|
||||
if tid in token_id_to_terminal:
|
||||
has_special = True
|
||||
if tid in drop_token_ids:
|
||||
has_drop = True
|
||||
break
|
||||
|
||||
if not has_special and not has_drop:
|
||||
if not has_special:
|
||||
if effective_text:
|
||||
if not prefix_items:
|
||||
return [TextChunk(effective_text)]
|
||||
@@ -102,8 +97,6 @@ class TokenIDScanner:
|
||||
text_accum: list[str] = []
|
||||
|
||||
for idx, tid in enumerate(delta_token_ids):
|
||||
if tid in self._drop_token_ids:
|
||||
continue
|
||||
terminal = self.token_id_to_terminal.get(tid)
|
||||
if terminal is not None:
|
||||
if text_accum:
|
||||
@@ -121,23 +114,7 @@ class TokenIDScanner:
|
||||
results.append(TextChunk(joined))
|
||||
|
||||
if effective_text:
|
||||
if has_drop:
|
||||
clean_delta = effective_text
|
||||
for idx, tid in enumerate(delta_token_ids):
|
||||
if tid in self._drop_token_ids:
|
||||
dropped = token_texts[idx]
|
||||
pos = clean_delta.find(dropped)
|
||||
if pos >= 0:
|
||||
clean_delta = (
|
||||
clean_delta[:pos] + clean_delta[pos + len(dropped) :]
|
||||
)
|
||||
if clean_delta:
|
||||
if results:
|
||||
results = self._recover_holdback_text(clean_delta, results)
|
||||
else:
|
||||
results = [TextChunk(clean_delta)]
|
||||
else:
|
||||
results = self._recover_holdback_text(effective_text, results)
|
||||
results = self._recover_holdback_text(effective_text, results)
|
||||
else:
|
||||
# No detokenizer text to validate against — individually-decoded
|
||||
# TextChunks are unreliable (context-dependent decoding).
|
||||
|
||||
Reference in New Issue
Block a user