[Bugfix] Fix RuntimeError: Already borrowed by adding thread-safe Hugging Face fast-tokenizer wrappers (#41181)
Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -268,28 +268,6 @@ class InputProcessingContext:
|
||||
try:
|
||||
output = hf_processor(**data, **allowed_kwargs)
|
||||
except Exception as exc:
|
||||
# See https://github.com/huggingface/tokenizers/issues/537
|
||||
if (
|
||||
isinstance(exc, RuntimeError)
|
||||
and exc
|
||||
and exc.args[0] == "Already borrowed"
|
||||
and num_tries < max_tries
|
||||
):
|
||||
logger.warning(
|
||||
"Failed to acquire tokenizer in current thread. "
|
||||
"Retrying (%d/%d)...",
|
||||
num_tries,
|
||||
max_tries,
|
||||
)
|
||||
time.sleep(0.5)
|
||||
return self.call_hf_processor(
|
||||
hf_processor,
|
||||
data,
|
||||
kwargs,
|
||||
num_tries=num_tries + 1,
|
||||
max_tries=max_tries,
|
||||
)
|
||||
|
||||
msg = (
|
||||
f"Failed to apply {type(hf_processor).__name__} "
|
||||
f"on data={data} with kwargs={allowed_kwargs}"
|
||||
|
||||
Reference in New Issue
Block a user