[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
27d5f78b63
commit
25671cb520
@@ -62,6 +62,7 @@ class MockTokenizer:
|
||||
"eos_token_id",
|
||||
"bos_token_id",
|
||||
"pad_token_id",
|
||||
"_all_special_tokens",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -73,6 +74,7 @@ class MockTokenizer:
|
||||
self._token_ids = [tid for tid, _ in tokens]
|
||||
self._token_decode_map = {tid: text for tid, text in tokens}
|
||||
self._special_ids = set(vocab.values())
|
||||
self._all_special_tokens = list(vocab.keys())
|
||||
self.eos_token_id = None
|
||||
self.bos_token_id = None
|
||||
self.pad_token_id = None
|
||||
@@ -83,6 +85,14 @@ class MockTokenizer:
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return self._vocab
|
||||
|
||||
@property
|
||||
def all_special_tokens(self) -> list[str]:
|
||||
return self._all_special_tokens
|
||||
|
||||
@property
|
||||
def all_special_ids(self) -> list[int]:
|
||||
return [self._vocab[t] for t in self._all_special_tokens if t in self._vocab]
|
||||
|
||||
def encode(self, text: str, **kwargs) -> list[int]:
|
||||
return self._token_ids
|
||||
|
||||
@@ -117,6 +127,37 @@ def _test_request(
|
||||
)
|
||||
|
||||
|
||||
DUMMY_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "stub", "parameters": {"type": "object"}},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def parse_non_streaming(
|
||||
parser,
|
||||
sample: Sample,
|
||||
request: ChatCompletionRequest,
|
||||
) -> ParseOutput:
|
||||
"""Run ``parser.parse()`` and return a :class:`ParseOutput`."""
|
||||
full_text = "".join(text for _, text in sample.tokens)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
full_text,
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
tc_list: list[dict] = []
|
||||
if tool_calls:
|
||||
for tc in tool_calls:
|
||||
tc_list.append({"name": tc.name, "arguments": tc.arguments})
|
||||
return ParseOutput(
|
||||
reasoning=reasoning or "",
|
||||
content=content or "",
|
||||
tool_calls=tc_list,
|
||||
)
|
||||
|
||||
|
||||
def replay_streaming(
|
||||
parser,
|
||||
tokens: list[tuple[int, str]],
|
||||
|
||||
Reference in New Issue
Block a user