[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
27d5f78b63
commit
25671cb520
@@ -15,8 +15,17 @@ def should_do_global_cleanup_after_test() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
|
||||
"""Create a mock tokenizer with the given special-token vocabulary.
|
||||
def make_mock_tokenizer(
|
||||
vocab: dict[str, int],
|
||||
special_tokens: list[str] | None = None,
|
||||
) -> MagicMock:
|
||||
"""Create a mock tokenizer with the given vocabulary.
|
||||
|
||||
Args:
|
||||
vocab: Mapping of token text to token ID.
|
||||
special_tokens: Which tokens to mark as special. When ``None``
|
||||
(the default), every key in *vocab* is treated as special —
|
||||
convenient when the vocab only contains delimiter tokens.
|
||||
|
||||
The returned mock supports get_vocab(), encode(), and decode().
|
||||
decode() maps known token IDs back to their text and falls back to
|
||||
@@ -29,6 +38,9 @@ def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
|
||||
tokenizer.decode.side_effect = lambda ids: "".join(
|
||||
id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
|
||||
)
|
||||
st = special_tokens if special_tokens is not None else list(vocab.keys())
|
||||
tokenizer.all_special_tokens = st
|
||||
tokenizer.all_special_ids = [vocab[t] for t in st if t in vocab]
|
||||
return tokenizer
|
||||
|
||||
|
||||
|
||||
@@ -62,6 +62,7 @@ class MockTokenizer:
|
||||
"eos_token_id",
|
||||
"bos_token_id",
|
||||
"pad_token_id",
|
||||
"_all_special_tokens",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -73,6 +74,7 @@ class MockTokenizer:
|
||||
self._token_ids = [tid for tid, _ in tokens]
|
||||
self._token_decode_map = {tid: text for tid, text in tokens}
|
||||
self._special_ids = set(vocab.values())
|
||||
self._all_special_tokens = list(vocab.keys())
|
||||
self.eos_token_id = None
|
||||
self.bos_token_id = None
|
||||
self.pad_token_id = None
|
||||
@@ -83,6 +85,14 @@ class MockTokenizer:
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return self._vocab
|
||||
|
||||
@property
|
||||
def all_special_tokens(self) -> list[str]:
|
||||
return self._all_special_tokens
|
||||
|
||||
@property
|
||||
def all_special_ids(self) -> list[int]:
|
||||
return [self._vocab[t] for t in self._all_special_tokens if t in self._vocab]
|
||||
|
||||
def encode(self, text: str, **kwargs) -> list[int]:
|
||||
return self._token_ids
|
||||
|
||||
@@ -117,6 +127,37 @@ def _test_request(
|
||||
)
|
||||
|
||||
|
||||
DUMMY_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "stub", "parameters": {"type": "object"}},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def parse_non_streaming(
|
||||
parser,
|
||||
sample: Sample,
|
||||
request: ChatCompletionRequest,
|
||||
) -> ParseOutput:
|
||||
"""Run ``parser.parse()`` and return a :class:`ParseOutput`."""
|
||||
full_text = "".join(text for _, text in sample.tokens)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
full_text,
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
tc_list: list[dict] = []
|
||||
if tool_calls:
|
||||
for tc in tool_calls:
|
||||
tc_list.append({"name": tc.name, "arguments": tc.arguments})
|
||||
return ParseOutput(
|
||||
reasoning=reasoning or "",
|
||||
content=content or "",
|
||||
tool_calls=tc_list,
|
||||
)
|
||||
|
||||
|
||||
def replay_streaming(
|
||||
parser,
|
||||
tokens: list[tuple[int, str]],
|
||||
|
||||
@@ -20,10 +20,14 @@ from pydantic import TypeAdapter
|
||||
|
||||
from tests.parser.engine.replay_harness import (
|
||||
CHUNK_SIZES,
|
||||
DUMMY_TOOLS,
|
||||
MockTokenizer,
|
||||
_test_request,
|
||||
assert_no_terminal_leakage,
|
||||
assert_parse_output,
|
||||
collect_output,
|
||||
make_mock_tokenizer,
|
||||
parse_non_streaming,
|
||||
replay_streaming,
|
||||
)
|
||||
from tests.parser.engine.trace_builder import _BUILDERS, build_samples
|
||||
@@ -142,3 +146,51 @@ def test_delegating_replay(parser_cls, sample, chunk_size):
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
assert_parse_output(output, sample)
|
||||
|
||||
|
||||
_TOOL_CALL_SAMPLES = [
|
||||
(p.parser_cls, p.name, s)
|
||||
for p in _PAIRINGS
|
||||
for s in p.samples
|
||||
if s.expected_tool_calls
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,parser_name,sample",
|
||||
_TOOL_CALL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else "",
|
||||
)
|
||||
def test_delegating_parse_tool_choice_none(parser_cls, parser_name, sample):
|
||||
"""Non-streaming parse() with tool_choice='none' via DelegatingParser
|
||||
must not leak special tokens into content."""
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
validated_tools = (
|
||||
_TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
|
||||
)
|
||||
parser = parser_cls(
|
||||
tokenizer,
|
||||
validated_tools,
|
||||
chat_template_kwargs=sample.chat_template_kwargs,
|
||||
)
|
||||
|
||||
request = _test_request(tools=DUMMY_TOOLS)
|
||||
request.tool_choice = "none"
|
||||
|
||||
output = parse_non_streaming(parser, sample, request)
|
||||
|
||||
assert output.tool_calls == [], (
|
||||
f"Expected no tool calls but got {output.tool_calls}"
|
||||
)
|
||||
|
||||
cfg = parser._tool_parser._parser_engine.parser_engine_config
|
||||
terminals = sorted(
|
||||
v
|
||||
for v in set(cfg.terminals.values()) | set(cfg.token_id_terminals.values())
|
||||
if len(v) > 1
|
||||
)
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
terminals,
|
||||
context=f"parser={parser_name}",
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import (
|
||||
collect_content,
|
||||
collect_function_name,
|
||||
@@ -57,6 +58,8 @@ def _make_tokenizer(sequence: list[tuple[int, str]]) -> MagicMock:
|
||||
return "".join(parts)
|
||||
|
||||
tokenizer.decode.side_effect = decode
|
||||
tokenizer.all_special_tokens = list(SPECIAL_TOKEN_MAP.values())
|
||||
tokenizer.all_special_ids = list(SPECIAL_TOKEN_MAP.keys())
|
||||
return tokenizer
|
||||
|
||||
|
||||
@@ -657,19 +660,15 @@ class TestGemma4ReasoningTruncationWithHoldback:
|
||||
@pytest.fixture
|
||||
def tool_call_tokenizer():
|
||||
"""Mock tokenizer with Gemma4 special token vocab."""
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.encode.return_value = [1, 2, 3]
|
||||
tokenizer.get_vocab.return_value = {
|
||||
"<|tool_call>": TOOL_CALL_START_ID,
|
||||
"<tool_call|>": TOOL_CALL_END_ID,
|
||||
"<|channel>": CHANNEL_START_ID,
|
||||
"<channel|>": CHANNEL_END_ID,
|
||||
'<|"|>': QUOTED_ID,
|
||||
}
|
||||
tokenizer.decode.side_effect = lambda ids: "".join(
|
||||
SPECIAL_TOKEN_MAP.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
|
||||
return make_mock_tokenizer(
|
||||
vocab={
|
||||
"<|tool_call>": TOOL_CALL_START_ID,
|
||||
"<tool_call|>": TOOL_CALL_END_ID,
|
||||
"<|channel>": CHANNEL_START_ID,
|
||||
"<channel|>": CHANNEL_END_ID,
|
||||
'<|"|>': QUOTED_ID,
|
||||
},
|
||||
)
|
||||
return tokenizer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -1407,3 +1406,133 @@ class TestBareThoughtWithoutChannelOpener:
|
||||
assert reasoning == ""
|
||||
assert content == ""
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
# ── Regression: commas inside <|"|>-delimited string values ─────────
|
||||
#
|
||||
# _make_tokenizer sets all_special_tokens, which activates the auto-drop
|
||||
# mechanism in _build_drop_info. If <|"|> is not in configured_texts,
|
||||
# it gets silently dropped and commas inside string values become field
|
||||
# separators, e.g. "San Francisco, CA" → {"location": "San Francisco"}.
|
||||
|
||||
|
||||
COMMA_TOKEN_SEQUENCE: list[tuple[int, str]] = [
|
||||
(TOOL_CALL_START_ID, "<|tool_call>"),
|
||||
(4000, "call"),
|
||||
(4001, ":"),
|
||||
(4002, "get_weather"),
|
||||
(4003, "{"),
|
||||
(4004, "location"),
|
||||
(4005, ":"),
|
||||
(QUOTED_ID, '<|"|>'),
|
||||
(4006, "San Francisco"),
|
||||
(4007, ", CA"),
|
||||
(QUOTED_ID, '<|"|>'),
|
||||
(4008, ","),
|
||||
(4009, "unit"),
|
||||
(4010, ":"),
|
||||
(QUOTED_ID, '<|"|>'),
|
||||
(4011, "celsius"),
|
||||
(QUOTED_ID, '<|"|>'),
|
||||
(4012, "}"),
|
||||
(TOOL_CALL_END_ID, "<tool_call|>"),
|
||||
]
|
||||
|
||||
MULTI_COMMA_TOKEN_SEQUENCE: list[tuple[int, str]] = [
|
||||
(TOOL_CALL_START_ID, "<|tool_call>"),
|
||||
(4000, "call"),
|
||||
(4001, ":"),
|
||||
(4020, "send_message"),
|
||||
(4003, "{"),
|
||||
(4021, "destination"),
|
||||
(4005, ":"),
|
||||
(QUOTED_ID, '<|"|>'),
|
||||
(4022, "456 Oakwood Avenue"),
|
||||
(4023, ", Rivermist"),
|
||||
(4024, ", 83214"),
|
||||
(QUOTED_ID, '<|"|>'),
|
||||
(4012, "}"),
|
||||
(TOOL_CALL_END_ID, "<tool_call|>"),
|
||||
]
|
||||
|
||||
|
||||
class TestCommaInStringValueRegression:
|
||||
"""Regression: <|"|> delimiters must not be auto-dropped.
|
||||
|
||||
When _build_drop_info discovers <|"|> as a special token and it is
|
||||
not in configured_texts, the delimiter is silently removed. Without
|
||||
it, _parse_gemma4_args treats commas inside string values as field
|
||||
separators.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def comma_tokenizer(self):
|
||||
return _make_tokenizer(COMMA_TOKEN_SEQUENCE)
|
||||
|
||||
@pytest.fixture
|
||||
def comma_parser(self, comma_tokenizer):
|
||||
return Gemma4Parser(comma_tokenizer)
|
||||
|
||||
@pytest.fixture
|
||||
def multi_comma_tokenizer(self):
|
||||
return _make_tokenizer(MULTI_COMMA_TOKEN_SEQUENCE)
|
||||
|
||||
@pytest.fixture
|
||||
def multi_comma_parser(self, multi_comma_tokenizer):
|
||||
return Gemma4Parser(multi_comma_tokenizer)
|
||||
|
||||
def test_batched_streaming_comma_in_value(
|
||||
self, comma_parser, comma_tokenizer, request_obj
|
||||
):
|
||||
results = _stream_tokens_batched(
|
||||
comma_parser,
|
||||
comma_tokenizer,
|
||||
request_obj,
|
||||
batch_size=1,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
_, _, tool_calls = _collect_fields(results)
|
||||
assert len(tool_calls) > 0
|
||||
args_text = "".join(
|
||||
tc.function.arguments
|
||||
for tc in tool_calls
|
||||
if tc.function and tc.function.arguments
|
||||
)
|
||||
parsed = json.loads(args_text)
|
||||
assert parsed["location"] == "San Francisco, CA"
|
||||
assert parsed["unit"] == "celsius"
|
||||
|
||||
def test_batched_streaming_multiple_commas(
|
||||
self, multi_comma_parser, multi_comma_tokenizer, request_obj
|
||||
):
|
||||
results = _stream_tokens_batched(
|
||||
multi_comma_parser,
|
||||
multi_comma_tokenizer,
|
||||
request_obj,
|
||||
batch_size=1,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
_, _, tool_calls = _collect_fields(results)
|
||||
assert len(tool_calls) > 0
|
||||
args_text = "".join(
|
||||
tc.function.arguments
|
||||
for tc in tool_calls
|
||||
if tc.function and tc.function.arguments
|
||||
)
|
||||
parsed = json.loads(args_text)
|
||||
assert parsed["destination"] == "456 Oakwood Avenue, Rivermist, 83214"
|
||||
|
||||
def test_non_streaming_comma_in_value(self, comma_parser, request_obj):
|
||||
text = "".join(text for _, text in COMMA_TOKEN_SEQUENCE)
|
||||
result = comma_parser.extract_tool_calls(text, request_obj)
|
||||
assert result.tools_called is True
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args["location"] == "San Francisco, CA"
|
||||
assert args["unit"] == "celsius"
|
||||
|
||||
def test_non_streaming_multiple_commas(self, multi_comma_parser, request_obj):
|
||||
text = "".join(text for _, text in MULTI_COMMA_TOKEN_SEQUENCE)
|
||||
result = multi_comma_parser.extract_tool_calls(text, request_obj)
|
||||
assert result.tools_called is True
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args["destination"] == "456 Oakwood Avenue, Rivermist, 83214"
|
||||
|
||||
@@ -119,8 +119,10 @@ def _hermes_config() -> ParserEngineConfig:
|
||||
def _make_engine(
|
||||
config: ParserEngineConfig | None = None,
|
||||
tools: list | None = None,
|
||||
vocab: dict[str, int] | None = None,
|
||||
special_tokens: list[str] | None = None,
|
||||
) -> ParserEngine:
|
||||
tokenizer = make_mock_tokenizer(_VOCAB)
|
||||
tokenizer = make_mock_tokenizer(vocab or _VOCAB, special_tokens=special_tokens)
|
||||
cfg = config or _combined_config()
|
||||
return ParserEngine(
|
||||
tokenizer,
|
||||
@@ -843,6 +845,7 @@ class TestParseTokenIdPassthrough:
|
||||
|
||||
assert content is not None
|
||||
assert "<tool_call>" in content
|
||||
assert content == "Use <tool_call> to call tools."
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "f"
|
||||
@@ -1490,3 +1493,181 @@ class TestCoercionInstabilityRegression:
|
||||
assert parsed["val"] == "4e"
|
||||
assert isinstance(parsed["val"], str)
|
||||
assert parsed["extra"] == "ok"
|
||||
|
||||
|
||||
_DROP_VOCAB: dict[str, int] = {
|
||||
**_VOCAB,
|
||||
"<bos>": 204,
|
||||
"<eos>": 205,
|
||||
}
|
||||
|
||||
|
||||
class TestDropSpecialTokens:
|
||||
"""Special token dropping via the __DROP__ terminal mechanism."""
|
||||
|
||||
def test_drops_special_token_by_id_from_content(self):
|
||||
"""A special token (not a configured terminal) is dropped when
|
||||
it arrives as its actual token ID."""
|
||||
config = ParserEngineConfig(
|
||||
name="drop_content_test",
|
||||
terminals={},
|
||||
token_id_terminals={},
|
||||
transitions={},
|
||||
initial_state=ParserState.CONTENT,
|
||||
content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
|
||||
)
|
||||
engine = _make_engine(
|
||||
config=config,
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
)
|
||||
engine._engine.reset()
|
||||
events = engine._engine.feed("hello<bos>world", [72, 204, 73])
|
||||
delta = engine._events_to_delta(events)
|
||||
assert delta is not None
|
||||
assert "<bos>" not in delta.content
|
||||
assert delta.content == "helloworld"
|
||||
|
||||
def test_drops_special_token_by_id_from_reasoning(self):
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
)
|
||||
engine._engine.reset()
|
||||
events = engine._engine.feed("thinking<eos>more", [72, 205, 73])
|
||||
delta = engine._events_to_delta(events)
|
||||
assert delta is not None
|
||||
assert "<eos>" not in delta.reasoning
|
||||
assert delta.reasoning == "thinkingmore"
|
||||
|
||||
def test_drops_via_text_fallback_when_no_token_ids(self):
|
||||
"""When no token IDs are provided, text-based lexer catches
|
||||
drop tokens as a fallback."""
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
)
|
||||
engine._engine.reset()
|
||||
events = engine._engine.feed("hello<bos>world", [])
|
||||
delta = engine._events_to_delta(events)
|
||||
assert delta is not None
|
||||
assert "<bos>" not in delta.reasoning
|
||||
|
||||
def test_regular_tokens_spelling_special_survive(self):
|
||||
"""Regular tokens that spell out a drop-token string survive
|
||||
when token IDs prove they are not the special token."""
|
||||
vocab = {**_DROP_VOCAB, "h": 72, "<": 73, "bos": 74, ">": 75, "w": 76}
|
||||
engine = _make_engine(
|
||||
vocab=vocab,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
)
|
||||
engine._engine.reset()
|
||||
# Feed regular token IDs (73, 74, 75) that spell <bos>,
|
||||
# not the special token ID 204.
|
||||
events = engine._engine.feed("h<bos>w", [72, 73, 74, 75, 76])
|
||||
delta = engine._events_to_delta(events)
|
||||
assert delta is not None
|
||||
assert "<bos>" in delta.reasoning
|
||||
|
||||
def test_configured_terminal_not_treated_as_drop(self):
|
||||
"""Tokens already in config.terminals (like <think>) are handled
|
||||
by the state machine, not the drop mechanism."""
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
)
|
||||
# Start from CONTENT so <think> triggers REASONING_START
|
||||
engine._engine.reset(initial_state=ParserState.CONTENT)
|
||||
events = engine._engine.feed("<think>", [200])
|
||||
has_reasoning_start = any(e.type == EventType.REASONING_START for e in events)
|
||||
assert has_reasoning_start
|
||||
|
||||
def test_tool_name_not_affected_by_drops(self):
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
)
|
||||
engine._engine.reset()
|
||||
events = [
|
||||
SemanticEvent(EventType.TOOL_CALL_START, tool_index=0),
|
||||
SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0),
|
||||
SemanticEvent(EventType.ARG_VALUE_CHUNK, '{"city": "NYC"}', tool_index=0),
|
||||
SemanticEvent(EventType.TOOL_CALL_END, tool_index=0),
|
||||
]
|
||||
delta = engine._events_to_delta(events)
|
||||
assert delta is not None
|
||||
assert delta.tool_calls is not None
|
||||
names = [
|
||||
tc.function.name
|
||||
for tc in delta.tool_calls
|
||||
if tc.function and tc.function.name
|
||||
]
|
||||
assert "get_weather" in names
|
||||
|
||||
def test_adjacent_drop_tokens_by_id(self):
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
)
|
||||
engine._engine.reset()
|
||||
events = engine._engine.feed("<bos><eos>", [204, 205])
|
||||
delta = engine._events_to_delta(events)
|
||||
assert delta is None
|
||||
|
||||
def test_no_special_tokens_means_no_drops(self):
|
||||
"""Engine with no special tokens passes all text through."""
|
||||
engine = _make_engine(special_tokens=[])
|
||||
engine._engine.reset()
|
||||
events = engine._engine.feed("hello<bos>world", [])
|
||||
delta = engine._events_to_delta(events)
|
||||
assert delta is not None
|
||||
assert "<bos>" in delta.reasoning
|
||||
|
||||
def test_drops_suppressed_with_skip_tool_parsing(self):
|
||||
"""When skip_tool_parsing is active, drop tokens are preserved
|
||||
as content so a later tool-call pass can see them."""
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
)
|
||||
engine._engine.skip_tool_parsing = True
|
||||
engine._engine.reset()
|
||||
events = engine._engine.feed("hello<bos>world", [72, 204, 73])
|
||||
delta = engine._events_to_delta(events)
|
||||
assert delta is not None
|
||||
assert "<bos>" in delta.reasoning
|
||||
|
||||
def test_drops_in_tool_args_state(self):
|
||||
"""Drop tokens in TOOL_ARGS state are silently discarded."""
|
||||
vocab = {**_DROP_VOCAB, '{"a":1}': 300}
|
||||
engine = _make_engine(
|
||||
vocab=vocab,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
)
|
||||
engine._engine.reset(initial_state=ParserState.CONTENT)
|
||||
events = engine._engine.feed(
|
||||
'<tool_call>{"a":1}<bos></tool_call>',
|
||||
[202, 300, 204, 203],
|
||||
)
|
||||
arg_chunks = [e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK]
|
||||
combined = "".join(arg_chunks)
|
||||
assert "<bos>" not in combined
|
||||
assert '{"a":1}' in combined
|
||||
|
||||
def test_mixed_configured_and_drop_terminals(self):
|
||||
"""Configured terminals trigger transitions while drop terminals
|
||||
are silently removed in the same stream."""
|
||||
engine = _make_engine(
|
||||
vocab=_DROP_VOCAB,
|
||||
special_tokens=list(_DROP_VOCAB.keys()),
|
||||
)
|
||||
engine._engine.reset()
|
||||
events = engine._engine.feed("thought<bos></think>answer", [72, 204, 201, 73])
|
||||
types = [e.type for e in events]
|
||||
assert EventType.REASONING_CHUNK in types
|
||||
assert EventType.REASONING_END in types
|
||||
assert EventType.TEXT_CHUNK in types
|
||||
reasoning_text = "".join(
|
||||
e.value for e in events if e.type == EventType.REASONING_CHUNK
|
||||
)
|
||||
assert "<bos>" not in reasoning_text
|
||||
|
||||
@@ -18,18 +18,21 @@ from typing import NamedTuple
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.replay_harness import (
|
||||
DUMMY_TOOLS,
|
||||
MockTokenizer,
|
||||
_test_request,
|
||||
assert_no_terminal_leakage,
|
||||
assert_parse_output,
|
||||
collect_output,
|
||||
make_mock_tokenizer,
|
||||
parse_non_streaming,
|
||||
replay_streaming,
|
||||
replay_with_text_holdback,
|
||||
)
|
||||
from tests.parser.engine.trace_builder import _BUILDERS, build_samples
|
||||
from vllm.parser.engine import registered_adapters as _adapters_mod
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import ParserState
|
||||
|
||||
# ── Parser discovery ─────────────────────────────────────────────────
|
||||
|
||||
@@ -78,7 +81,11 @@ def _discover_parsers() -> list[_ParserInfo]:
|
||||
terminals=sorted(v for v in all_vals if len(v) > 1),
|
||||
tool_end=tool_end,
|
||||
think_end=cfg.terminals.get("THINK_END", ""),
|
||||
tool_start=cfg.terminals.get("TOOL_START", ""),
|
||||
tool_start=(
|
||||
cfg.terminals["TOOL_SECTION_START"]
|
||||
if (ParserState.CONTENT, "TOOL_SECTION_START") in cfg.transitions
|
||||
else cfg.terminals.get("TOOL_START", "")
|
||||
),
|
||||
)
|
||||
)
|
||||
if missing_builders:
|
||||
@@ -287,16 +294,18 @@ _TOOL_CALL_SAMPLES = [
|
||||
]
|
||||
|
||||
|
||||
def _suppressed_expectations(
|
||||
sample, think_end: str, tool_start: str
|
||||
def _tool_suppression_expectations(
|
||||
sample, think_end: str, tool_start: str, *, include_tool_block: bool
|
||||
) -> tuple[str, str]:
|
||||
"""Compute expected (reasoning, content) when tools are suppressed.
|
||||
"""Expected (reasoning, content) when tool calls are not extracted.
|
||||
|
||||
When an explicit reasoning-end delimiter is present, reasoning ends
|
||||
there and the tool call block becomes content. When reasoning ends
|
||||
implicitly (the tool-start token triggers both REASONING_END and
|
||||
TOOL_CALL_START), reasoning still ends at the tool start and the raw
|
||||
tool call block becomes content text.
|
||||
With ``include_tool_block=True`` (skip_tool_parsing / reasoning
|
||||
adapter first pass), tool terminal text is preserved as content so
|
||||
a second-pass parser can see it.
|
||||
|
||||
With ``include_tool_block=False`` (_suppress_tool_calls /
|
||||
tool_choice='none'), the state machine consumes tool blocks and
|
||||
only non-tool content survives.
|
||||
"""
|
||||
full_text = "".join(text for _, text in sample.tokens)
|
||||
reasoning = sample.expected_reasoning
|
||||
@@ -307,36 +316,42 @@ def _suppressed_expectations(
|
||||
if think_end:
|
||||
pos = after_reasoning.find(think_end)
|
||||
if pos >= 0:
|
||||
return (reasoning, after_reasoning[pos + len(think_end) :])
|
||||
if include_tool_block:
|
||||
return (reasoning, after_reasoning[pos + len(think_end) :])
|
||||
after_reasoning = after_reasoning[pos + len(think_end) :]
|
||||
if tool_start:
|
||||
pos = after_reasoning.find(tool_start)
|
||||
if pos >= 0:
|
||||
return (reasoning, after_reasoning[pos:])
|
||||
return (full_text, "")
|
||||
|
||||
|
||||
_DUMMY_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "stub", "parameters": {"type": "object"}},
|
||||
}
|
||||
]
|
||||
if include_tool_block:
|
||||
return (reasoning, after_reasoning[pos:])
|
||||
return (reasoning, after_reasoning[:pos])
|
||||
if include_tool_block:
|
||||
return (full_text, "")
|
||||
return (reasoning, after_reasoning)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")
|
||||
@pytest.mark.parametrize(
|
||||
"mode",
|
||||
["skip_tool_parsing", "suppress_tool_calls"],
|
||||
ids=["skip_tool_parsing", "suppress_tool_calls"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample,think_end,tool_start",
|
||||
_TOOL_CALL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
|
||||
)
|
||||
class TestSkipToolParsingReplay:
|
||||
"""Replay with skip_tool_parsing=True (tool_choice='none').
|
||||
class TestToolCallFilteringReplay:
|
||||
"""Replay with tool calls not extracted, in both filtering modes.
|
||||
|
||||
Verifies that reasoning is extracted normally and the raw tool call
|
||||
block appears as content text with no tool calls parsed.
|
||||
``skip_tool_parsing`` (reasoning adapter first pass): tool terminal
|
||||
text is preserved as content for a second-pass tool parser.
|
||||
|
||||
``suppress_tool_calls`` (tool_choice='none'): tool call blocks are
|
||||
consumed by the state machine and do not leak into content.
|
||||
"""
|
||||
|
||||
def test_replay(self, parser_cls, sample, think_end, tool_start, chunk_size):
|
||||
def test_replay(self, parser_cls, sample, think_end, tool_start, mode, chunk_size):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
kwargs = {}
|
||||
if sample.chat_template_kwargs:
|
||||
@@ -344,8 +359,11 @@ class TestSkipToolParsingReplay:
|
||||
parser = parser_cls(tokenizer, **kwargs)
|
||||
|
||||
request = _test_request()
|
||||
request.tool_choice = "none"
|
||||
request.tools = _DUMMY_TOOLS
|
||||
request.tools = DUMMY_TOOLS
|
||||
if mode == "skip_tool_parsing":
|
||||
parser.skip_tool_parsing = True
|
||||
else:
|
||||
request.tool_choice = "none"
|
||||
|
||||
all_ids = [tid for tid, _ in sample.tokens]
|
||||
all_texts = [text for _, text in sample.tokens]
|
||||
@@ -370,10 +388,51 @@ class TestSkipToolParsingReplay:
|
||||
|
||||
output = collect_output(results)
|
||||
|
||||
expected_reasoning, expected_content = _suppressed_expectations(
|
||||
sample, think_end, tool_start
|
||||
include_block = mode == "skip_tool_parsing"
|
||||
expected_reasoning, expected_content = _tool_suppression_expectations(
|
||||
sample, think_end, tool_start, include_tool_block=include_block
|
||||
)
|
||||
|
||||
assert output.reasoning == expected_reasoning, (
|
||||
f"Reasoning mismatch (mode={mode}):\n"
|
||||
f" expected: {expected_reasoning!r}\n"
|
||||
f" actual: {output.reasoning!r}"
|
||||
)
|
||||
assert output.tool_calls == [], (
|
||||
f"Expected no tool calls (mode={mode}) but got {output.tool_calls}"
|
||||
)
|
||||
assert output.content == expected_content, (
|
||||
f"Content mismatch (mode={mode}):\n"
|
||||
f" expected: {expected_content!r}\n"
|
||||
f" actual: {output.content!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample,think_end,tool_start",
|
||||
_TOOL_CALL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
|
||||
)
|
||||
class TestToolCallFilteringNonStreaming:
|
||||
"""Non-streaming parse() with tool_choice='none' must suppress tool
|
||||
calls and not leak special tokens into content."""
|
||||
|
||||
def test_parse(self, parser_cls, sample, think_end, tool_start):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
kwargs = {}
|
||||
if sample.chat_template_kwargs:
|
||||
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
|
||||
parser = parser_cls(tokenizer, **kwargs)
|
||||
|
||||
request = _test_request()
|
||||
request.tools = DUMMY_TOOLS
|
||||
request.tool_choice = "none"
|
||||
|
||||
output = parse_non_streaming(parser, sample, request)
|
||||
|
||||
expected_reasoning, expected_content = _tool_suppression_expectations(
|
||||
sample, think_end, tool_start, include_tool_block=False
|
||||
)
|
||||
assert output.reasoning == expected_reasoning, (
|
||||
f"Reasoning mismatch:\n"
|
||||
f" expected: {expected_reasoning!r}\n"
|
||||
@@ -389,6 +448,135 @@ class TestSkipToolParsingReplay:
|
||||
)
|
||||
|
||||
|
||||
_WS_TOOL_SAMPLES = [(t[0], t[1]) for t in _TOOL_CALL_SAMPLES if "whitespace" in t[1].id]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample",
|
||||
_WS_TOOL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else getattr(v, "__name__", ""),
|
||||
)
|
||||
class TestToolChoiceNoneStreamingParity:
|
||||
"""Streaming and non-streaming must return the same content
|
||||
when tool_choice='none' suppresses tool calls."""
|
||||
|
||||
def test_content_matches(self, parser_cls, sample):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
kwargs = {}
|
||||
if sample.chat_template_kwargs:
|
||||
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
|
||||
request = _test_request()
|
||||
request.tools = DUMMY_TOOLS
|
||||
request.tool_choice = "none"
|
||||
|
||||
ns_output = parse_non_streaming(
|
||||
parser_cls(tokenizer, **kwargs),
|
||||
sample,
|
||||
request,
|
||||
)
|
||||
|
||||
s_parser = parser_cls(tokenizer, **kwargs)
|
||||
results = []
|
||||
for i, (tid, text) in enumerate(sample.tokens):
|
||||
is_last = i == len(sample.tokens) - 1
|
||||
results.append(
|
||||
s_parser.parse_delta(
|
||||
text,
|
||||
[tid],
|
||||
request,
|
||||
prompt_token_ids=(sample.prompt_token_ids or [])
|
||||
if i == 0
|
||||
else None,
|
||||
finished=is_last,
|
||||
)
|
||||
)
|
||||
s_output = collect_output(results)
|
||||
|
||||
assert ns_output.content == s_output.content, (
|
||||
f"Streaming/non-streaming content mismatch:\n"
|
||||
f" streaming: {s_output.content!r}\n"
|
||||
f" non-streaming: {ns_output.content!r}"
|
||||
)
|
||||
|
||||
|
||||
_DROP_TOKENS = {"<bos>": 99990, "<eos>": 99991}
|
||||
|
||||
|
||||
def _inject_drop_tokens(sample):
|
||||
"""Insert <bos> at stream start and <eos> between the first two tokens."""
|
||||
new_vocab = {**sample.vocab, **_DROP_TOKENS}
|
||||
tokens = list(sample.tokens)
|
||||
tokens.insert(0, (99990, "<bos>"))
|
||||
if len(tokens) >= 3:
|
||||
tokens.insert(2, (99991, "<eos>"))
|
||||
else:
|
||||
tokens.append((99991, "<eos>"))
|
||||
return dataclasses.replace(sample, vocab=new_vocab, tokens=tokens)
|
||||
|
||||
|
||||
class TestDropTokenReplay:
|
||||
"""Verify unconfigured special tokens are silently dropped across
|
||||
all parsers and chunk sizes."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_info",
|
||||
_PARSERS,
|
||||
ids=[p.name for p in _PARSERS],
|
||||
)
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, None])
|
||||
def test_drop_tokens_removed_from_output(self, parser_info, chunk_size):
|
||||
for sample in parser_info.samples:
|
||||
injected = _inject_drop_tokens(sample)
|
||||
tokenizer = make_mock_tokenizer(injected)
|
||||
parser = parser_info.parser_cls(
|
||||
tokenizer,
|
||||
tools=sample.tools,
|
||||
)
|
||||
|
||||
results = replay_streaming(
|
||||
parser,
|
||||
injected.tokens,
|
||||
chunk_size=chunk_size,
|
||||
tools=sample.tools,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(results)
|
||||
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
list(_DROP_TOKENS.keys()),
|
||||
context=f"parser={parser_info.name}, chunk={chunk_size}",
|
||||
)
|
||||
assert_parse_output(output, sample)
|
||||
|
||||
|
||||
class TestDropTokenNonStreaming:
|
||||
"""Non-streaming parse() must also strip unconfigured special tokens."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_info",
|
||||
_PARSERS,
|
||||
ids=[p.name for p in _PARSERS],
|
||||
)
|
||||
def test_drop_tokens_removed_from_output(self, parser_info):
|
||||
for sample in parser_info.samples:
|
||||
injected = _inject_drop_tokens(sample)
|
||||
tokenizer = make_mock_tokenizer(injected)
|
||||
parser = parser_info.parser_cls(
|
||||
tokenizer,
|
||||
tools=sample.tools,
|
||||
)
|
||||
|
||||
request = _test_request(tools=sample.tools)
|
||||
output = parse_non_streaming(parser, injected, request)
|
||||
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
list(_DROP_TOKENS.keys()),
|
||||
context=f"parser={parser_info.name}",
|
||||
)
|
||||
|
||||
|
||||
class TestAdapterReferences:
|
||||
"""Verify make_adapters sets reasoning/tool parser class refs on parser engine
|
||||
parser classes so the serving layer finds them and calls adjust_request."""
|
||||
|
||||
@@ -57,7 +57,7 @@ class TestJoinDecodedTextReturnsStr:
|
||||
|
||||
@pytest.fixture
|
||||
def bare_scanner(self):
|
||||
return TokenIDScanner({}, tokenizer=None, drop_token_ids=set())
|
||||
return TokenIDScanner({}, tokenizer=None)
|
||||
|
||||
def test_mixed_items(self, bare_scanner):
|
||||
items = [
|
||||
@@ -265,43 +265,6 @@ class TestHoldbackTextRecovery:
|
||||
assert "reasoning end." in combined
|
||||
|
||||
|
||||
class TestDropTokens:
|
||||
def test_drop_token_with_holdback(self, tokenizer):
|
||||
"""Drop tokens stripped; hold-back text preserved."""
|
||||
drop_id = 300
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
drop_id: "<eos>",
|
||||
}.get(ids[0], "?")
|
||||
|
||||
scanner = TokenIDScanner(
|
||||
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
|
||||
tokenizer=tokenizer,
|
||||
drop_token_ids={drop_id},
|
||||
)
|
||||
|
||||
result = scanner.scan(
|
||||
delta_text="holdback<eos>",
|
||||
delta_token_ids=[drop_id, CHANNEL_END_ID],
|
||||
)
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
result2 = scanner.scan(
|
||||
delta_text="<channel|>content",
|
||||
delta_token_ids=[20],
|
||||
)
|
||||
pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
|
||||
assert len(pre_lexed) == 1
|
||||
assert pre_lexed[0].terminal == "THINK_END"
|
||||
texts = [r.text for r in result2 if isinstance(r, TextChunk)]
|
||||
combined = "".join(texts)
|
||||
assert "holdback" in combined
|
||||
assert "<eos>" not in combined
|
||||
|
||||
assert len(scanner.flush_pending()) == 0
|
||||
|
||||
|
||||
class TestEndToEndReasoningHoldback:
|
||||
"""End-to-end engine tests with detokenizer hold-back."""
|
||||
|
||||
|
||||
@@ -100,14 +100,24 @@ def _build_chat_request(
|
||||
class _StubTokenizer:
|
||||
"""Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``."""
|
||||
|
||||
_VOCAB: dict[str, int] = {
|
||||
"<|tool_call>": 256_000,
|
||||
"<tool_call|>": 256_001,
|
||||
'<|"|>': 52,
|
||||
"<|channel>": 256_002,
|
||||
"<channel|>": 256_003,
|
||||
}
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {
|
||||
"<|tool_call>": 256_000,
|
||||
"<tool_call|>": 256_001,
|
||||
'<|"|>': 52,
|
||||
"<|channel>": 256_002,
|
||||
"<channel|>": 256_003,
|
||||
}
|
||||
return dict(self._VOCAB)
|
||||
|
||||
@property
|
||||
def all_special_tokens(self) -> list[str]:
|
||||
return list(self._VOCAB.keys())
|
||||
|
||||
@property
|
||||
def all_special_ids(self) -> list[int]:
|
||||
return list(self._VOCAB.values())
|
||||
|
||||
|
||||
def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None:
|
||||
@@ -235,10 +245,10 @@ def test_gemma4_keeps_special_tokens_with_tools_thinking_disabled() -> None:
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:
|
||||
"""No active tools + thinking disabled: keep the default
|
||||
(``skip_special_tokens=True``) so stray delimiters do not leak into
|
||||
content.
|
||||
def test_gemma4_keeps_skip_special_tokens_false_when_nothing_to_preserve() -> None:
|
||||
"""No active tools + thinking disabled: ``skip_special_tokens`` stays
|
||||
``False`` because the parser engine's ``__DROP__`` terminal mechanism
|
||||
strips unconfigured special tokens automatically.
|
||||
"""
|
||||
parser = Gemma4ToolParser(_StubTokenizer())
|
||||
request = _build_chat_request(
|
||||
@@ -247,4 +257,4 @@ def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:
|
||||
|
||||
parser.adjust_request(request)
|
||||
|
||||
assert request.skip_special_tokens is True
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
@@ -424,6 +424,12 @@ class DelegatingParser(Parser):
|
||||
if tool_parser is None:
|
||||
return [], content
|
||||
|
||||
if request.tool_choice == "none":
|
||||
if self._engine_based:
|
||||
result = self.extract_tool_calls(content or "", request=request)
|
||||
return [], result.content
|
||||
return [], content
|
||||
|
||||
supports_required_and_named = tool_parser.supports_required_and_named
|
||||
is_named_tool_choice = request.tool_choice and isinstance(
|
||||
request.tool_choice,
|
||||
|
||||
@@ -116,6 +116,7 @@ class ParserEngine(Parser):
|
||||
self._deferred_content: str = ""
|
||||
self._deferred_reasoning: str = ""
|
||||
self._content_has_nonws: bool = False
|
||||
self._suppress_tool_calls: bool = False
|
||||
|
||||
self._arg_converter = parser_engine_config.arg_converter
|
||||
self._arg_structural_chars = parser_engine_config.arg_structural_chars
|
||||
@@ -398,10 +399,10 @@ class ParserEngine(Parser):
|
||||
tools = getattr(request, "tools", None)
|
||||
if tools:
|
||||
self._tools = tools
|
||||
if not self.skip_tool_parsing:
|
||||
if not self.skip_tool_parsing and not self._suppress_tool_calls:
|
||||
tool_choice = getattr(request, "tool_choice", None)
|
||||
if tool_choice == "none" and tools:
|
||||
self.skip_tool_parsing = True
|
||||
self._suppress_tool_calls = True
|
||||
|
||||
def _strip_content_whitespace(
|
||||
self,
|
||||
@@ -642,7 +643,7 @@ class ParserEngine(Parser):
|
||||
events = self._feed(text, token_ids)
|
||||
events.extend(self._engine.finish())
|
||||
|
||||
delta = self._events_to_delta(events)
|
||||
delta = self._events_to_delta(events, finished=True)
|
||||
tool_call_info = self._build_extracted_result()
|
||||
|
||||
reasoning = delta.reasoning if delta else None
|
||||
@@ -701,6 +702,7 @@ class ParserEngine(Parser):
|
||||
reasoning_parts: list[str] = []
|
||||
|
||||
seen_tool_event = False
|
||||
suppress = self._suppress_tool_calls
|
||||
for event in events:
|
||||
match event.type:
|
||||
case EventType.TEXT_CHUNK:
|
||||
@@ -713,17 +715,21 @@ class ParserEngine(Parser):
|
||||
case EventType.REASONING_END:
|
||||
self._reasoning_ended = True
|
||||
case EventType.TOOL_CALL_START:
|
||||
seen_tool_event = True
|
||||
self._ensure_slot(event.tool_index)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._ensure_slot(event.tool_index)
|
||||
case EventType.TOOL_NAME:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_name(event)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_name(event)
|
||||
case EventType.ARG_VALUE_CHUNK:
|
||||
seen_tool_event = True
|
||||
self._handle_arg_chunk(event, tool_call_deltas)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._handle_arg_chunk(event, tool_call_deltas)
|
||||
case EventType.TOOL_CALL_END:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_end(event, tool_call_deltas)
|
||||
if not suppress:
|
||||
seen_tool_event = True
|
||||
self._handle_tool_end(event, tool_call_deltas)
|
||||
case EventType.REASONING_START:
|
||||
pass # no delta-level effect
|
||||
|
||||
|
||||
@@ -24,16 +24,6 @@ from functools import cached_property
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
|
||||
STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset(
|
||||
{
|
||||
"<eos>",
|
||||
"<bos>",
|
||||
"<pad>",
|
||||
"<unk>",
|
||||
"<mask>",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ParserState(Enum):
|
||||
CONTENT = auto()
|
||||
@@ -90,6 +80,9 @@ class ParserEngineConfig:
|
||||
|
||||
arg_structural_chars: frozenset[str] | None = None
|
||||
|
||||
# Special tokens exempt from auto-drop but not state-machine terminals.
|
||||
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
# Prevents trailing-whitespace accumulation across multi-turn conversations.
|
||||
strip_trailing_reasoning_whitespace: bool = True
|
||||
|
||||
@@ -102,8 +95,6 @@ class ParserEngineConfig:
|
||||
# Reject tool calls whose names are absent from the request tools.
|
||||
validate_tool_names: bool = False
|
||||
|
||||
drop_tokens: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
@cached_property
|
||||
def terminal_defs(self):
|
||||
from vllm.parser.engine.incremental_lexer import terminals_from_literals
|
||||
|
||||
@@ -6,15 +6,17 @@ incremental lexing, and state-machine-driven semantic event emission."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.incremental_lexer import (
|
||||
CONTENT_TERMINAL,
|
||||
IncrementalLexer,
|
||||
LexerShape,
|
||||
LexToken,
|
||||
TerminalDef,
|
||||
)
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
STRUCTURAL_DROP_TOKENS,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
@@ -26,6 +28,64 @@ from vllm.parser.engine.token_id_scanner import (
|
||||
TokenIDScanner,
|
||||
)
|
||||
|
||||
DROP_TERMINAL = "__DROP__"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _DropInfo:
|
||||
lexer_shape: LexerShape
|
||||
extra_token_ids: dict[int, str]
|
||||
|
||||
|
||||
def _build_drop_info(
|
||||
config: ParserEngineConfig,
|
||||
tokenizer,
|
||||
) -> _DropInfo | None:
|
||||
try:
|
||||
special_tokens: list[str] = list(tokenizer.all_special_tokens)
|
||||
special_ids: list[int] = list(tokenizer.all_special_ids)
|
||||
except (AttributeError, NotImplementedError):
|
||||
return None
|
||||
|
||||
if not special_tokens:
|
||||
return None
|
||||
|
||||
configured_texts = (
|
||||
set(config.token_id_terminals.values())
|
||||
| set(config.terminals.values())
|
||||
| config.preserve_tokens
|
||||
)
|
||||
|
||||
extra_token_ids: dict[int, str] = {}
|
||||
drop_texts: set[str] = set()
|
||||
for text, tid in zip(special_tokens, special_ids):
|
||||
if text not in configured_texts:
|
||||
extra_token_ids[tid] = DROP_TERMINAL
|
||||
drop_texts.add(text)
|
||||
|
||||
if not drop_texts:
|
||||
return None
|
||||
|
||||
import regex as re
|
||||
|
||||
drop_terminal_defs = [
|
||||
TerminalDef(
|
||||
name=DROP_TERMINAL,
|
||||
pattern=re.compile(re.escape(text)),
|
||||
is_literal=True,
|
||||
literal=text,
|
||||
)
|
||||
for text in drop_texts
|
||||
]
|
||||
|
||||
all_terminal_defs = list(config.terminal_defs) + drop_terminal_defs
|
||||
lexer_shape = LexerShape(all_terminal_defs)
|
||||
|
||||
return _DropInfo(
|
||||
lexer_shape=lexer_shape,
|
||||
extra_token_ids=extra_token_ids,
|
||||
)
|
||||
|
||||
|
||||
class StreamingParserEngine:
|
||||
"""Consumes ``(delta_text, delta_token_ids)`` pairs and produces a
|
||||
@@ -60,7 +120,6 @@ class StreamingParserEngine:
|
||||
self.config = config
|
||||
|
||||
resolved_token_ids: dict[int, str] = {}
|
||||
drop_token_ids: set[int] = set()
|
||||
if tokenizer is not None:
|
||||
if vocab is None:
|
||||
vocab = tokenizer.get_vocab()
|
||||
@@ -69,32 +128,29 @@ class StreamingParserEngine:
|
||||
tid = vocab.get(token_text)
|
||||
if tid is not None:
|
||||
resolved_token_ids[tid] = terminal_name
|
||||
all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS
|
||||
for token_text in all_drop:
|
||||
tid = vocab.get(token_text)
|
||||
if tid is not None:
|
||||
drop_token_ids.add(tid)
|
||||
for attr in ("eos_token_id", "bos_token_id", "pad_token_id"):
|
||||
tid = getattr(tokenizer, attr, None)
|
||||
if tid is not None:
|
||||
drop_token_ids.add(tid)
|
||||
|
||||
drop_info: _DropInfo | None = None
|
||||
if tokenizer is not None:
|
||||
drop_info = _build_drop_info(config, tokenizer)
|
||||
|
||||
lexer_shape = config.lexer_shape
|
||||
if drop_info is not None:
|
||||
resolved_token_ids.update(drop_info.extra_token_ids)
|
||||
lexer_shape = drop_info.lexer_shape
|
||||
|
||||
self._resolved_token_ids = resolved_token_ids
|
||||
self._drop_token_ids = drop_token_ids
|
||||
self._has_drops = drop_info is not None
|
||||
|
||||
self._scanner = TokenIDScanner(
|
||||
resolved_token_ids,
|
||||
tokenizer,
|
||||
drop_token_ids,
|
||||
)
|
||||
|
||||
self._token_id_terminal_names: frozenset[str] = frozenset(
|
||||
resolved_token_ids.values()
|
||||
)
|
||||
|
||||
self._lexer = IncrementalLexer(
|
||||
config.lexer_shape, content_terminal=CONTENT_TERMINAL
|
||||
)
|
||||
self._lexer = IncrementalLexer(lexer_shape, content_terminal=CONTENT_TERMINAL)
|
||||
|
||||
self._tool_terminals: frozenset[str] = frozenset(
|
||||
terminal
|
||||
@@ -150,7 +206,7 @@ class StreamingParserEngine:
|
||||
):
|
||||
has_special = False
|
||||
for tid in delta_token_ids:
|
||||
if tid in self._resolved_token_ids or tid in self._drop_token_ids:
|
||||
if tid in self._resolved_token_ids:
|
||||
has_special = True
|
||||
break
|
||||
if not has_special:
|
||||
@@ -247,6 +303,15 @@ class StreamingParserEngine:
|
||||
transition = self.config.transitions.get(key)
|
||||
|
||||
if transition is None:
|
||||
if (
|
||||
self._has_drops
|
||||
and terminal == DROP_TERMINAL
|
||||
# Preserve drop tokens when skip_tool_parsing is active so
|
||||
# the reasoning pass doesn't silently remove tokens that a
|
||||
# later tool-call pass might need to see.
|
||||
and not self.skip_tool_parsing
|
||||
):
|
||||
return []
|
||||
return self._emit_for_state(value)
|
||||
|
||||
if self.skip_tool_parsing and terminal in self._tool_terminals:
|
||||
|
||||
@@ -42,12 +42,10 @@ class TokenIDScanner:
|
||||
self,
|
||||
token_id_to_terminal: dict[int, str],
|
||||
tokenizer,
|
||||
drop_token_ids: set[int] | None = None,
|
||||
) -> None:
|
||||
self.token_id_to_terminal = token_id_to_terminal
|
||||
self.tokenizer = tokenizer
|
||||
self._token_text_cache: dict[int, str] = {}
|
||||
self._drop_token_ids = drop_token_ids or set()
|
||||
self._deferred_terminals: list[PreLexedTerminal] = []
|
||||
self._deferred_post_text: str = ""
|
||||
|
||||
@@ -74,22 +72,19 @@ class TokenIDScanner:
|
||||
if self._deferred_terminals:
|
||||
prefix_items, effective_text = self._resolve_deferred(delta_text)
|
||||
|
||||
if not self.token_id_to_terminal and not self._drop_token_ids:
|
||||
if not self.token_id_to_terminal:
|
||||
if effective_text:
|
||||
prefix_items.append(TextChunk(effective_text))
|
||||
return prefix_items
|
||||
|
||||
has_special = False
|
||||
has_drop = False
|
||||
token_id_to_terminal = self.token_id_to_terminal
|
||||
drop_token_ids = self._drop_token_ids
|
||||
for tid in delta_token_ids:
|
||||
if tid in token_id_to_terminal:
|
||||
has_special = True
|
||||
if tid in drop_token_ids:
|
||||
has_drop = True
|
||||
break
|
||||
|
||||
if not has_special and not has_drop:
|
||||
if not has_special:
|
||||
if effective_text:
|
||||
if not prefix_items:
|
||||
return [TextChunk(effective_text)]
|
||||
@@ -102,8 +97,6 @@ class TokenIDScanner:
|
||||
text_accum: list[str] = []
|
||||
|
||||
for idx, tid in enumerate(delta_token_ids):
|
||||
if tid in self._drop_token_ids:
|
||||
continue
|
||||
terminal = self.token_id_to_terminal.get(tid)
|
||||
if terminal is not None:
|
||||
if text_accum:
|
||||
@@ -121,23 +114,7 @@ class TokenIDScanner:
|
||||
results.append(TextChunk(joined))
|
||||
|
||||
if effective_text:
|
||||
if has_drop:
|
||||
clean_delta = effective_text
|
||||
for idx, tid in enumerate(delta_token_ids):
|
||||
if tid in self._drop_token_ids:
|
||||
dropped = token_texts[idx]
|
||||
pos = clean_delta.find(dropped)
|
||||
if pos >= 0:
|
||||
clean_delta = (
|
||||
clean_delta[:pos] + clean_delta[pos + len(dropped) :]
|
||||
)
|
||||
if clean_delta:
|
||||
if results:
|
||||
results = self._recover_holdback_text(clean_delta, results)
|
||||
else:
|
||||
results = [TextChunk(clean_delta)]
|
||||
else:
|
||||
results = self._recover_holdback_text(effective_text, results)
|
||||
results = self._recover_holdback_text(effective_text, results)
|
||||
else:
|
||||
# No detokenizer text to validate against — individually-decoded
|
||||
# TextChunks are unreliable (context-dependent decoding).
|
||||
|
||||
+1
-58
@@ -35,34 +35,6 @@ if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
# Tokens the model generates that must not leak into response content.
|
||||
_GEMMA4_MODEL_DROP_TOKENS: set[str] = {
|
||||
# Turn boundaries
|
||||
"<|turn>",
|
||||
"<turn|>",
|
||||
# Channel / reasoning
|
||||
"<|channel>",
|
||||
"<channel|>",
|
||||
# Tool protocol tokens
|
||||
"<|tool>",
|
||||
"<tool|>",
|
||||
"<|tool_call>",
|
||||
"<tool_call|>",
|
||||
"<|tool_response>",
|
||||
"<tool_response|>",
|
||||
'<|"|>',
|
||||
# Thinking
|
||||
"<|think|>",
|
||||
# Multi-modal (defensive — not expected during text completion)
|
||||
"<|image>",
|
||||
"<|image|>",
|
||||
"<image|>",
|
||||
"<|audio>",
|
||||
"<|audio|>",
|
||||
"<audio|>",
|
||||
"<|video|>",
|
||||
}
|
||||
|
||||
CHANNEL_START = "<|channel>"
|
||||
CHANNEL_END = "<channel|>"
|
||||
TOOL_CALL_START = "<|tool_call>"
|
||||
@@ -322,14 +294,6 @@ def _gemma4_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
|
||||
@functools.cache
|
||||
def gemma4_config() -> ParserEngineConfig:
|
||||
used_tokens = {
|
||||
CHANNEL_START,
|
||||
CHANNEL_END,
|
||||
TOOL_CALL_START,
|
||||
TOOL_CALL_END,
|
||||
'<|"|>',
|
||||
}
|
||||
|
||||
return ParserEngineConfig(
|
||||
name="gemma4",
|
||||
initial_state=ParserState.CONTENT,
|
||||
@@ -412,7 +376,7 @@ def gemma4_config() -> ParserEngineConfig:
|
||||
arg_converter=_gemma4_arg_converter,
|
||||
tool_args_json=False,
|
||||
arg_structural_chars=frozenset(",:{}[]<"),
|
||||
drop_tokens=frozenset(_GEMMA4_MODEL_DROP_TOKENS - used_tokens),
|
||||
preserve_tokens=frozenset({STRING_DELIM}),
|
||||
)
|
||||
|
||||
|
||||
@@ -451,27 +415,6 @@ class Gemma4Parser(ParserEngine):
|
||||
self._prefix_stripped: bool = False
|
||||
self._is_first_feed: bool = True
|
||||
|
||||
def adjust_request(
|
||||
self,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
"""Keep special tokens when thinking or tool calls need them.
|
||||
|
||||
``skip_special_tokens`` must stay ``False`` when there is something to
|
||||
preserve: reasoning channel tokens (thinking enabled) or tool-call
|
||||
delimiters (tools active). Otherwise keep the default so stray
|
||||
delimiters do not leak into content (e.g. ``tool_choice="none"`` with
|
||||
thinking disabled).
|
||||
"""
|
||||
request = super().adjust_request(request)
|
||||
chat_template_kwargs = getattr(request, "chat_template_kwargs", None) or {}
|
||||
enable_thinking = chat_template_kwargs.get("enable_thinking", True)
|
||||
has_tools = bool(getattr(request, "tools", None))
|
||||
tools_active = has_tools and request.tool_choice != "none"
|
||||
if not enable_thinking and not tools_active:
|
||||
request.skip_special_tokens = True
|
||||
return request
|
||||
|
||||
def _reset(self, initial_state=None) -> None:
|
||||
super()._reset(initial_state=initial_state)
|
||||
self._reasoning_text = ""
|
||||
|
||||
Reference in New Issue
Block a user