[BUG]: fix HF tokenizer concurrent borrow in tool parsers (#40059)

Signed-off-by: Yifan <yzong@redhat.com>
Co-authored-by: timon0305 <timon0305@outlook.com>
Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
yzong-rh
2026-04-23 18:20:30 -07:00
committed by GitHub
co-authored by timon0305 sfeng33
parent 3041344287
commit c9bf77df92
3 changed files with 37 additions and 38 deletions
@@ -4,15 +4,22 @@
from unittest.mock import MagicMock, patch
import pytest
from transformers import AutoTokenizer
from vllm.entrypoints.openai.engine.protocol import ExtractedToolCallInformation
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
LLAMA_MODEL = "meta-llama/Llama-3.2-1B-Instruct"
@pytest.fixture(scope="module")
def llama_tokenizer():
return AutoTokenizer.from_pretrained(LLAMA_MODEL)
@pytest.fixture
def parser(default_tokenizer: TokenizerLike):
return Llama3JsonToolParser(default_tokenizer)
def parser(llama_tokenizer):
return Llama3JsonToolParser(llama_tokenizer)
def test_extract_tool_calls_simple(parser):
+15 -27
View File
@@ -34,6 +34,21 @@ class FunctionGemmaToolParser(ToolParser):
<start_function_call>call:func_name{param:<escape>value<escape>}<end_function_call>
"""
# FunctionGemma tokens
tool_call_start_token: str = "<start_function_call>"
tool_call_end_token: str = "<end_function_call>"
# Regex patterns
tool_call_regex: re.Pattern = re.compile(
r"<start_function_call>call:(\w+)\{(.*?)\}<end_function_call>"
r"|<start_function_call>call:(\w+)\{(.*)",
re.DOTALL,
)
arg_regex: re.Pattern = re.compile(
r"(\w+):<escape>(.*?)<escape>",
re.DOTALL,
)
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
super().__init__(tokenizer, tools)
@@ -42,33 +57,6 @@ class FunctionGemmaToolParser(ToolParser):
self.prev_tool_call_arr: list[dict] = []
self.current_tool_id: int = -1
self.streamed_args_for_tool: list[str] = []
# FunctionGemma tokens
self.tool_call_start_token: str = "<start_function_call>"
self.tool_call_end_token: str = "<end_function_call>"
# Regex patterns
self.tool_call_regex = re.compile(
r"<start_function_call>call:(\w+)\{(.*?)\}<end_function_call>"
r"|<start_function_call>call:(\w+)\{(.*)",
re.DOTALL,
)
self.arg_regex = re.compile(
r"(\w+):<escape>(.*?)<escape>",
re.DOTALL,
)
if self.model_tokenizer:
self.tool_call_start_token_ids = self.model_tokenizer.encode(
self.tool_call_start_token, add_special_tokens=False
)
self.tool_call_end_token_ids = self.model_tokenizer.encode(
self.tool_call_end_token, add_special_tokens=False
)
else:
self.tool_call_start_token_ids = []
self.tool_call_end_token_ids = []
self.buffered_delta_text = ""
def _parse_arguments(self, args_str: str) -> dict:
+12 -8
View File
@@ -45,6 +45,12 @@ class Llama3JsonToolParser(ToolParser):
llama4_json are set.
"""
bot_token: str = "<|python_tag|>"
# Simple regex to find opening braces - we'll use JSON decoder for parsing
# This handles arbitrary nesting depth correctly
tool_call_start_regex: re.Pattern = re.compile(r"\{")
json_decoder: json.JSONDecoder = json.JSONDecoder()
def __init__(
self,
tokenizer: PreTrainedTokenizerBase,
@@ -60,14 +66,12 @@ class Llama3JsonToolParser(ToolParser):
self.streamed_args_for_tool: list[
str
] = [] # map what has been streamed for each tool so far to a list
self.bot_token = "<|python_tag|>"
self.bot_token_id = tokenizer.encode(self.bot_token, add_special_tokens=False)[
0
]
# Simple regex to find opening braces - we'll use JSON decoder for parsing
# This handles arbitrary nesting depth correctly
self.tool_call_start_regex = re.compile(r"\{")
self.json_decoder = json.JSONDecoder()
self.bot_token_id = self.vocab.get(self.bot_token)
if self.bot_token_id is None:
raise RuntimeError(
"Llama3JsonToolParser could not locate the bot token "
f"'{self.bot_token}' in the tokenizer."
)
def extract_tool_calls(
self, model_output: str, request: ChatCompletionRequest