[Bugfix] Fix RuntimeError: Already borrowed by adding thread-safe Hugging Face fast-tokenizer wrappers (#41181)

Signed-off-by: Yifan Zong <yzong@redhat.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
yzong-rh
2026-05-05 14:04:01 +00:00
committed by GitHub
co-authored by wang.yuqi
parent 6fca518157
commit 20dcd984f9
5 changed files with 107 additions and 35 deletions
-22
View File
@@ -268,28 +268,6 @@ class InputProcessingContext:
try:
output = hf_processor(**data, **allowed_kwargs)
except Exception as exc:
# See https://github.com/huggingface/tokenizers/issues/537
if (
isinstance(exc, RuntimeError)
and exc
and exc.args[0] == "Already borrowed"
and num_tries < max_tries
):
logger.warning(
"Failed to acquire tokenizer in current thread. "
"Retrying (%d/%d)...",
num_tries,
max_tries,
)
time.sleep(0.5)
return self.call_hf_processor(
hf_processor,
data,
kwargs,
num_tries=num_tries + 1,
max_tries=max_tries,
)
msg = (
f"Failed to apply {type(hf_processor).__name__} "
f"on data={data} with kwargs={allowed_kwargs}"