[Bugfix] Fix RuntimeError: Already borrowed by adding thread-safe Hugging Face fast-tokenizer wrappers (#41181)
Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
+2
-11
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import copy
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping, Sequence
|
||||
@@ -108,17 +107,10 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
if mm_registry.supports_multimodal_inputs(config.model_config):
|
||||
mm_processor_cache = mm_registry.processor_cache_from_config(config)
|
||||
|
||||
# Deep-copy the tokenizer so the multimodal processor gets its
|
||||
# own Rust tokenizer backend. Without this, concurrent access
|
||||
# from AsyncMicrobatchTokenizer and call_hf_processor causes
|
||||
# "RuntimeError: Already borrowed" from the Rust RefCell.
|
||||
# See: https://github.com/huggingface/tokenizers/issues/537
|
||||
mm_tokenizer = copy.deepcopy(tokenizer)
|
||||
|
||||
with set_default_torch_num_threads():
|
||||
self.mm_processor = mm_registry.create_processor(
|
||||
config.model_config,
|
||||
tokenizer=mm_tokenizer,
|
||||
tokenizer=self.tokenizer,
|
||||
cache=mm_processor_cache,
|
||||
)
|
||||
|
||||
@@ -130,11 +122,10 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
# requests don't pollute the sender cache.
|
||||
ro_cache = mm_registry.processor_only_cache_from_config(config)
|
||||
if ro_cache is not None:
|
||||
ro_tokenizer = copy.deepcopy(tokenizer)
|
||||
with set_default_torch_num_threads():
|
||||
self._readonly_mm_processor = mm_registry.create_processor(
|
||||
config.model_config,
|
||||
tokenizer=ro_tokenizer,
|
||||
tokenizer=self.tokenizer,
|
||||
cache=ro_cache,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user