[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ben Browning
2026-06-30 13:46:53 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 27d5f78b63
commit 25671cb520
14 changed files with 783 additions and 219 deletions
+41
View File
@@ -62,6 +62,7 @@ class MockTokenizer:
"eos_token_id",
"bos_token_id",
"pad_token_id",
"_all_special_tokens",
)
def __init__(
@@ -73,6 +74,7 @@ class MockTokenizer:
self._token_ids = [tid for tid, _ in tokens]
self._token_decode_map = {tid: text for tid, text in tokens}
self._special_ids = set(vocab.values())
self._all_special_tokens = list(vocab.keys())
self.eos_token_id = None
self.bos_token_id = None
self.pad_token_id = None
@@ -83,6 +85,14 @@ class MockTokenizer:
def get_vocab(self) -> dict[str, int]:
return self._vocab
@property
def all_special_tokens(self) -> list[str]:
return self._all_special_tokens
@property
def all_special_ids(self) -> list[int]:
return [self._vocab[t] for t in self._all_special_tokens if t in self._vocab]
def encode(self, text: str, **kwargs) -> list[int]:
return self._token_ids
@@ -117,6 +127,37 @@ def _test_request(
)
DUMMY_TOOLS = [
{
"type": "function",
"function": {"name": "stub", "parameters": {"type": "object"}},
},
]
def parse_non_streaming(
parser,
sample: Sample,
request: ChatCompletionRequest,
) -> ParseOutput:
"""Run ``parser.parse()`` and return a :class:`ParseOutput`."""
full_text = "".join(text for _, text in sample.tokens)
reasoning, content, tool_calls = parser.parse(
full_text,
request,
enable_auto_tools=True,
)
tc_list: list[dict] = []
if tool_calls:
for tc in tool_calls:
tc_list.append({"name": tc.name, "arguments": tc.arguments})
return ParseOutput(
reasoning=reasoning or "",
content=content or "",
tool_calls=tc_list,
)
def replay_streaming(
parser,
tokens: list[tuple[int, str]],