[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
27d5f78b63
commit
25671cb520
@@ -424,6 +424,12 @@ class DelegatingParser(Parser):
|
||||
if tool_parser is None:
|
||||
return [], content
|
||||
|
||||
if request.tool_choice == "none":
|
||||
if self._engine_based:
|
||||
result = self.extract_tool_calls(content or "", request=request)
|
||||
return [], result.content
|
||||
return [], content
|
||||
|
||||
supports_required_and_named = tool_parser.supports_required_and_named
|
||||
is_named_tool_choice = request.tool_choice and isinstance(
|
||||
request.tool_choice,
|
||||
|
||||
@@ -116,6 +116,7 @@ class ParserEngine(Parser):
|
||||
self._deferred_content: str = ""
|
||||
self._deferred_reasoning: str = ""
|
||||
self._content_has_nonws: bool = False
|
||||
self._suppress_tool_calls: bool = False
|
||||
|
||||
self._arg_converter = parser_engine_config.arg_converter
|
||||
self._arg_structural_chars = parser_engine_config.arg_structural_chars
|
||||
@@ -398,10 +399,10 @@ class ParserEngine(Parser):
|
||||
tools = getattr(request, "tools", None)
|
||||
if tools:
|
||||
self._tools = tools
|
||||
if not self.skip_tool_parsing:
|
||||
if not self.skip_tool_parsing and not self._suppress_tool_calls:
|
||||
tool_choice = getattr(request, "tool_choice", None)
|
||||
if tool_choice == "none" and tools:
|
||||
self.skip_tool_parsing = True
|
||||
self._suppress_tool_calls = True
|
||||
|
||||
def _strip_content_whitespace(
|
||||
self,
|
||||
@@ -642,7 +643,7 @@ class ParserEngine(Parser):
|
||||
events = self._feed(text, token_ids)
|
||||
events.extend(self._engine.finish())
|
||||
|
||||
delta = self._events_to_delta(events)
|
||||
delta = self._events_to_delta(events, finished=True)
|
||||
tool_call_info = self._build_extracted_result()
|
||||
|
||||
reasoning = delta.reasoning if delta else None
|
||||
@@ -701,6 +702,7 @@ class ParserEngine(Parser):
|
||||
reasoning_parts: list[str] = []
|
||||
|
||||
seen_tool_event = False
|
||||
suppress = self._suppress_tool_calls
|
||||
for event in events:
|
||||
match event.type:
|
||||
case EventType.TEXT_CHUNK:
|
||||
@@ -713,17 +715,21 @@ class ParserEngine(Parser):
|
||||
case EventType.REASONING_END:
|
||||
self._reasoning_ended = True
|
||||
case EventType.TOOL_CALL_START:
|
||||
seen_tool_event = True
|
||||
self._ensure_slot(event.tool_index)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._ensure_slot(event.tool_index)
|
||||
case EventType.TOOL_NAME:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_name(event)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_name(event)
|
||||
case EventType.ARG_VALUE_CHUNK:
|
||||
seen_tool_event = True
|
||||
self._handle_arg_chunk(event, tool_call_deltas)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._handle_arg_chunk(event, tool_call_deltas)
|
||||
case EventType.TOOL_CALL_END:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_end(event, tool_call_deltas)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_end(event, tool_call_deltas)
|
||||
case EventType.REASONING_START:
|
||||
pass # no delta-level effect
|
||||
|
||||
|
||||
@@ -24,16 +24,6 @@ from functools import cached_property
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
|
||||
STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset(
|
||||
{
|
||||
"<eos>",
|
||||
"<bos>",
|
||||
"<pad>",
|
||||
"<unk>",
|
||||
"<mask>",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ParserState(Enum):
|
||||
CONTENT = auto()
|
||||
@@ -90,6 +80,9 @@ class ParserEngineConfig:
|
||||
|
||||
arg_structural_chars: frozenset[str] | None = None
|
||||
|
||||
# Special tokens exempt from auto-drop but not state-machine terminals.
|
||||
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
# Prevents trailing-whitespace accumulation across multi-turn conversations.
|
||||
strip_trailing_reasoning_whitespace: bool = True
|
||||
|
||||
@@ -102,8 +95,6 @@ class ParserEngineConfig:
|
||||
# Reject tool calls whose names are absent from the request tools.
|
||||
validate_tool_names: bool = False
|
||||
|
||||
drop_tokens: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
@cached_property
|
||||
def terminal_defs(self):
|
||||
from vllm.parser.engine.incremental_lexer import terminals_from_literals
|
||||
|
||||
@@ -6,15 +6,17 @@ incremental lexing, and state-machine-driven semantic event emission."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.incremental_lexer import (
|
||||
CONTENT_TERMINAL,
|
||||
IncrementalLexer,
|
||||
LexerShape,
|
||||
LexToken,
|
||||
TerminalDef,
|
||||
)
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
STRUCTURAL_DROP_TOKENS,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
@@ -26,6 +28,64 @@ from vllm.parser.engine.token_id_scanner import (
|
||||
TokenIDScanner,
|
||||
)
|
||||
|
||||
DROP_TERMINAL = "__DROP__"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _DropInfo:
|
||||
lexer_shape: LexerShape
|
||||
extra_token_ids: dict[int, str]
|
||||
|
||||
|
||||
def _build_drop_info(
|
||||
config: ParserEngineConfig,
|
||||
tokenizer,
|
||||
) -> _DropInfo | None:
|
||||
try:
|
||||
special_tokens: list[str] = list(tokenizer.all_special_tokens)
|
||||
special_ids: list[int] = list(tokenizer.all_special_ids)
|
||||
except (AttributeError, NotImplementedError):
|
||||
return None
|
||||
|
||||
if not special_tokens:
|
||||
return None
|
||||
|
||||
configured_texts = (
|
||||
set(config.token_id_terminals.values())
|
||||
| set(config.terminals.values())
|
||||
| config.preserve_tokens
|
||||
)
|
||||
|
||||
extra_token_ids: dict[int, str] = {}
|
||||
drop_texts: set[str] = set()
|
||||
for text, tid in zip(special_tokens, special_ids):
|
||||
if text not in configured_texts:
|
||||
extra_token_ids[tid] = DROP_TERMINAL
|
||||
drop_texts.add(text)
|
||||
|
||||
if not drop_texts:
|
||||
return None
|
||||
|
||||
import regex as re
|
||||
|
||||
drop_terminal_defs = [
|
||||
TerminalDef(
|
||||
name=DROP_TERMINAL,
|
||||
pattern=re.compile(re.escape(text)),
|
||||
is_literal=True,
|
||||
literal=text,
|
||||
)
|
||||
for text in drop_texts
|
||||
]
|
||||
|
||||
all_terminal_defs = list(config.terminal_defs) + drop_terminal_defs
|
||||
lexer_shape = LexerShape(all_terminal_defs)
|
||||
|
||||
return _DropInfo(
|
||||
lexer_shape=lexer_shape,
|
||||
extra_token_ids=extra_token_ids,
|
||||
)
|
||||
|
||||
|
||||
class StreamingParserEngine:
|
||||
"""Consumes ``(delta_text, delta_token_ids)`` pairs and produces a
|
||||
@@ -60,7 +120,6 @@ class StreamingParserEngine:
|
||||
self.config = config
|
||||
|
||||
resolved_token_ids: dict[int, str] = {}
|
||||
drop_token_ids: set[int] = set()
|
||||
if tokenizer is not None:
|
||||
if vocab is None:
|
||||
vocab = tokenizer.get_vocab()
|
||||
@@ -69,32 +128,29 @@ class StreamingParserEngine:
|
||||
tid = vocab.get(token_text)
|
||||
if tid is not None:
|
||||
resolved_token_ids[tid] = terminal_name
|
||||
all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS
|
||||
for token_text in all_drop:
|
||||
tid = vocab.get(token_text)
|
||||
if tid is not None:
|
||||
drop_token_ids.add(tid)
|
||||
for attr in ("eos_token_id", "bos_token_id", "pad_token_id"):
|
||||
tid = getattr(tokenizer, attr, None)
|
||||
if tid is not None:
|
||||
drop_token_ids.add(tid)
|
||||
|
||||
drop_info: _DropInfo | None = None
|
||||
if tokenizer is not None:
|
||||
drop_info = _build_drop_info(config, tokenizer)
|
||||
|
||||
lexer_shape = config.lexer_shape
|
||||
if drop_info is not None:
|
||||
resolved_token_ids.update(drop_info.extra_token_ids)
|
||||
lexer_shape = drop_info.lexer_shape
|
||||
|
||||
self._resolved_token_ids = resolved_token_ids
|
||||
self._drop_token_ids = drop_token_ids
|
||||
self._has_drops = drop_info is not None
|
||||
|
||||
self._scanner = TokenIDScanner(
|
||||
resolved_token_ids,
|
||||
tokenizer,
|
||||
drop_token_ids,
|
||||
)
|
||||
|
||||
self._token_id_terminal_names: frozenset[str] = frozenset(
|
||||
resolved_token_ids.values()
|
||||
)
|
||||
|
||||
self._lexer = IncrementalLexer(
|
||||
config.lexer_shape, content_terminal=CONTENT_TERMINAL
|
||||
)
|
||||
self._lexer = IncrementalLexer(lexer_shape, content_terminal=CONTENT_TERMINAL)
|
||||
|
||||
self._tool_terminals: frozenset[str] = frozenset(
|
||||
terminal
|
||||
@@ -150,7 +206,7 @@ class StreamingParserEngine:
|
||||
):
|
||||
has_special = False
|
||||
for tid in delta_token_ids:
|
||||
if tid in self._resolved_token_ids or tid in self._drop_token_ids:
|
||||
if tid in self._resolved_token_ids:
|
||||
has_special = True
|
||||
break
|
||||
if not has_special:
|
||||
@@ -247,6 +303,15 @@ class StreamingParserEngine:
|
||||
transition = self.config.transitions.get(key)
|
||||
|
||||
if transition is None:
|
||||
if (
|
||||
self._has_drops
|
||||
and terminal == DROP_TERMINAL
|
||||
# Preserve drop tokens when skip_tool_parsing is active so
|
||||
# the reasoning pass doesn't silently remove tokens that a
|
||||
# later tool-call pass might need to see.
|
||||
and not self.skip_tool_parsing
|
||||
):
|
||||
return []
|
||||
return self._emit_for_state(value)
|
||||
|
||||
if self.skip_tool_parsing and terminal in self._tool_terminals:
|
||||
|
||||
@@ -42,12 +42,10 @@ class TokenIDScanner:
|
||||
self,
|
||||
token_id_to_terminal: dict[int, str],
|
||||
tokenizer,
|
||||
drop_token_ids: set[int] | None = None,
|
||||
) -> None:
|
||||
self.token_id_to_terminal = token_id_to_terminal
|
||||
self.tokenizer = tokenizer
|
||||
self._token_text_cache: dict[int, str] = {}
|
||||
self._drop_token_ids = drop_token_ids or set()
|
||||
self._deferred_terminals: list[PreLexedTerminal] = []
|
||||
self._deferred_post_text: str = ""
|
||||
|
||||
@@ -74,22 +72,19 @@ class TokenIDScanner:
|
||||
if self._deferred_terminals:
|
||||
prefix_items, effective_text = self._resolve_deferred(delta_text)
|
||||
|
||||
if not self.token_id_to_terminal and not self._drop_token_ids:
|
||||
if not self.token_id_to_terminal:
|
||||
if effective_text:
|
||||
prefix_items.append(TextChunk(effective_text))
|
||||
return prefix_items
|
||||
|
||||
has_special = False
|
||||
has_drop = False
|
||||
token_id_to_terminal = self.token_id_to_terminal
|
||||
drop_token_ids = self._drop_token_ids
|
||||
for tid in delta_token_ids:
|
||||
if tid in token_id_to_terminal:
|
||||
has_special = True
|
||||
if tid in drop_token_ids:
|
||||
has_drop = True
|
||||
break
|
||||
|
||||
if not has_special and not has_drop:
|
||||
if not has_special:
|
||||
if effective_text:
|
||||
if not prefix_items:
|
||||
return [TextChunk(effective_text)]
|
||||
@@ -102,8 +97,6 @@ class TokenIDScanner:
|
||||
text_accum: list[str] = []
|
||||
|
||||
for idx, tid in enumerate(delta_token_ids):
|
||||
if tid in self._drop_token_ids:
|
||||
continue
|
||||
terminal = self.token_id_to_terminal.get(tid)
|
||||
if terminal is not None:
|
||||
if text_accum:
|
||||
@@ -121,23 +114,7 @@ class TokenIDScanner:
|
||||
results.append(TextChunk(joined))
|
||||
|
||||
if effective_text:
|
||||
if has_drop:
|
||||
clean_delta = effective_text
|
||||
for idx, tid in enumerate(delta_token_ids):
|
||||
if tid in self._drop_token_ids:
|
||||
dropped = token_texts[idx]
|
||||
pos = clean_delta.find(dropped)
|
||||
if pos >= 0:
|
||||
clean_delta = (
|
||||
clean_delta[:pos] + clean_delta[pos + len(dropped) :]
|
||||
)
|
||||
if clean_delta:
|
||||
if results:
|
||||
results = self._recover_holdback_text(clean_delta, results)
|
||||
else:
|
||||
results = [TextChunk(clean_delta)]
|
||||
else:
|
||||
results = self._recover_holdback_text(effective_text, results)
|
||||
results = self._recover_holdback_text(effective_text, results)
|
||||
else:
|
||||
# No detokenizer text to validate against — individually-decoded
|
||||
# TextChunks are unreliable (context-dependent decoding).
|
||||
|
||||
+1
-58
@@ -35,34 +35,6 @@ if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
# Tokens the model generates that must not leak into response content.
|
||||
_GEMMA4_MODEL_DROP_TOKENS: set[str] = {
|
||||
# Turn boundaries
|
||||
"<|turn>",
|
||||
"<turn|>",
|
||||
# Channel / reasoning
|
||||
"<|channel>",
|
||||
"<channel|>",
|
||||
# Tool protocol tokens
|
||||
"<|tool>",
|
||||
"<tool|>",
|
||||
"<|tool_call>",
|
||||
"<tool_call|>",
|
||||
"<|tool_response>",
|
||||
"<tool_response|>",
|
||||
'<|"|>',
|
||||
# Thinking
|
||||
"<|think|>",
|
||||
# Multi-modal (defensive — not expected during text completion)
|
||||
"<|image>",
|
||||
"<|image|>",
|
||||
"<image|>",
|
||||
"<|audio>",
|
||||
"<|audio|>",
|
||||
"<audio|>",
|
||||
"<|video|>",
|
||||
}
|
||||
|
||||
CHANNEL_START = "<|channel>"
|
||||
CHANNEL_END = "<channel|>"
|
||||
TOOL_CALL_START = "<|tool_call>"
|
||||
@@ -322,14 +294,6 @@ def _gemma4_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
|
||||
@functools.cache
|
||||
def gemma4_config() -> ParserEngineConfig:
|
||||
used_tokens = {
|
||||
CHANNEL_START,
|
||||
CHANNEL_END,
|
||||
TOOL_CALL_START,
|
||||
TOOL_CALL_END,
|
||||
'<|"|>',
|
||||
}
|
||||
|
||||
return ParserEngineConfig(
|
||||
name="gemma4",
|
||||
initial_state=ParserState.CONTENT,
|
||||
@@ -412,7 +376,7 @@ def gemma4_config() -> ParserEngineConfig:
|
||||
arg_converter=_gemma4_arg_converter,
|
||||
tool_args_json=False,
|
||||
arg_structural_chars=frozenset(",:{}[]<"),
|
||||
drop_tokens=frozenset(_GEMMA4_MODEL_DROP_TOKENS - used_tokens),
|
||||
preserve_tokens=frozenset({STRING_DELIM}),
|
||||
)
|
||||
|
||||
|
||||
@@ -451,27 +415,6 @@ class Gemma4Parser(ParserEngine):
|
||||
self._prefix_stripped: bool = False
|
||||
self._is_first_feed: bool = True
|
||||
|
||||
def adjust_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
"""Keep special tokens when thinking or tool calls need them.
|
||||
|
||||
``skip_special_tokens`` must stay ``False`` when there is something to
|
||||
preserve: reasoning channel tokens (thinking enabled) or tool-call
|
||||
delimiters (tools active). Otherwise keep the default so stray
|
||||
delimiters do not leak into content (e.g. ``tool_choice="none"`` with
|
||||
thinking disabled).
|
||||
"""
|
||||
request = super().adjust_request(request)
|
||||
chat_template_kwargs = getattr(request, "chat_template_kwargs", None) or {}
|
||||
enable_thinking = chat_template_kwargs.get("enable_thinking", True)
|
||||
has_tools = bool(getattr(request, "tools", None))
|
||||
tools_active = has_tools and request.tool_choice != "none"
|
||||
if not enable_thinking and not tools_active:
|
||||
request.skip_special_tokens = True
|
||||
return request
|
||||
|
||||
def _reset(self, initial_state=None) -> None:
|
||||
super()._reset(initial_state=initial_state)
|
||||
self._reasoning_text = ""
|
||||
|
||||
Reference in New Issue
Block a user