[BUG]: fix HF tokenizer concurrent borrow in tool parsers (#40059)

Signed-off-by: Yifan <yzong@redhat.com>
Co-authored-by: timon0305 <timon0305@outlook.com>
Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
yzong-rh
2026-04-23 18:20:30 -07:00
committed by GitHub
co-authored by timon0305 sfeng33
parent 3041344287
commit c9bf77df92
3 changed files with 37 additions and 38 deletions
+15 -27
View File
@@ -34,6 +34,21 @@ class FunctionGemmaToolParser(ToolParser):
<start_function_call>call:func_name{param:<escape>value<escape>}<end_function_call>
"""
# FunctionGemma tokens
tool_call_start_token: str = "<start_function_call>"
tool_call_end_token: str = "<end_function_call>"
# Regex patterns
tool_call_regex: re.Pattern = re.compile(
r"<start_function_call>call:(\w+)\{(.*?)\}<end_function_call>"
r"|<start_function_call>call:(\w+)\{(.*)",
re.DOTALL,
)
arg_regex: re.Pattern = re.compile(
r"(\w+):<escape>(.*?)<escape>",
re.DOTALL,
)
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
super().__init__(tokenizer, tools)
@@ -42,33 +57,6 @@ class FunctionGemmaToolParser(ToolParser):
self.prev_tool_call_arr: list[dict] = []
self.current_tool_id: int = -1
self.streamed_args_for_tool: list[str] = []
# FunctionGemma tokens
self.tool_call_start_token: str = "<start_function_call>"
self.tool_call_end_token: str = "<end_function_call>"
# Regex patterns
self.tool_call_regex = re.compile(
r"<start_function_call>call:(\w+)\{(.*?)\}<end_function_call>"
r"|<start_function_call>call:(\w+)\{(.*)",
re.DOTALL,
)
self.arg_regex = re.compile(
r"(\w+):<escape>(.*?)<escape>",
re.DOTALL,
)
if self.model_tokenizer:
self.tool_call_start_token_ids = self.model_tokenizer.encode(
self.tool_call_start_token, add_special_tokens=False
)
self.tool_call_end_token_ids = self.model_tokenizer.encode(
self.tool_call_end_token, add_special_tokens=False
)
else:
self.tool_call_start_token_ids = []
self.tool_call_end_token_ids = []
self.buffered_delta_text = ""
def _parse_arguments(self, args_str: str) -> dict: