[Perf] Isolate MM preprocessing on its own executor (#49524)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
This commit is contained in:
+19
-8
@@ -1558,20 +1558,31 @@ def test_fault_tolerance_requires_single_api_server():
|
|||||||
|
|
||||||
|
|
||||||
def test_renderer_num_workers_with_mm_cache():
|
def test_renderer_num_workers_with_mm_cache():
|
||||||
"""Disallow renderer_num_workers > 1 when mm processor cache is enabled,
|
"""Disallow renderer_num_workers > 1 with the mm processor cache only for
|
||||||
since neither cache type is thread-safe."""
|
pooling models, whose preprocessing runs on the renderer workers."""
|
||||||
mm_model = "Qwen/Qwen2-VL-2B-Instruct"
|
mm_model = "Qwen/Qwen2-VL-2B-Instruct"
|
||||||
|
|
||||||
# Should raise: multi-worker + cache enabled (default cache_gb=4)
|
# Should raise: pooling + multi-worker + cache enabled (default cache_gb=4)
|
||||||
with pytest.raises(ValueError, match="renderer-num-workers"):
|
with pytest.raises(ValueError, match="renderer-num-workers"):
|
||||||
ModelConfig(mm_model, renderer_num_workers=4)
|
ModelConfig(mm_model, runner="pooling", renderer_num_workers=4)
|
||||||
|
|
||||||
# Should raise: multi-worker + explicit cache size
|
# Should raise: pooling + multi-worker + explicit cache size
|
||||||
with pytest.raises(ValueError, match="renderer-num-workers"):
|
with pytest.raises(ValueError, match="renderer-num-workers"):
|
||||||
ModelConfig(mm_model, renderer_num_workers=2, mm_processor_cache_gb=1.0)
|
ModelConfig(
|
||||||
|
mm_model,
|
||||||
|
runner="pooling",
|
||||||
|
renderer_num_workers=2,
|
||||||
|
mm_processor_cache_gb=1.0,
|
||||||
|
)
|
||||||
|
|
||||||
# Should pass: multi-worker + cache disabled
|
# Should pass: pooling + multi-worker + cache disabled
|
||||||
config = ModelConfig(mm_model, renderer_num_workers=4, mm_processor_cache_gb=0)
|
config = ModelConfig(
|
||||||
|
mm_model, runner="pooling", renderer_num_workers=4, mm_processor_cache_gb=0
|
||||||
|
)
|
||||||
|
assert config.renderer_num_workers == 4
|
||||||
|
|
||||||
|
# Should pass: generate models preprocess on the dedicated mm executor
|
||||||
|
config = ModelConfig(mm_model, renderer_num_workers=4)
|
||||||
assert config.renderer_num_workers == 4
|
assert config.renderer_num_workers == 4
|
||||||
|
|
||||||
# Should pass: single worker + cache enabled (default)
|
# Should pass: single worker + cache enabled (default)
|
||||||
|
|||||||
@@ -748,12 +748,13 @@ class ModelConfig:
|
|||||||
if (
|
if (
|
||||||
self.renderer_num_workers > 1
|
self.renderer_num_workers > 1
|
||||||
and self.multimodal_config.mm_processor_cache_gb > 0
|
and self.multimodal_config.mm_processor_cache_gb > 0
|
||||||
|
and self.runner_type == "pooling"
|
||||||
):
|
):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Cannot use --renderer-num-workers > 1 with the "
|
"Cannot use --renderer-num-workers > 1 with the "
|
||||||
"multimodal processor cache enabled. The cache is "
|
"multimodal processor cache enabled for pooling models. "
|
||||||
"not thread-safe and does not support concurrent "
|
"Pooling preprocessing runs on the renderer workers, and "
|
||||||
"renderer workers. Please set "
|
"the cache is not thread-safe. Please set "
|
||||||
"--renderer-num-workers 1 (the default), or "
|
"--renderer-num-workers 1 (the default), or "
|
||||||
"disable the cache with --mm-processor-cache-gb 0."
|
"disable the cache with --mm-processor-cache-gb 0."
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -79,16 +79,15 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
|
|
||||||
self.tokenizer = tokenizer
|
self.tokenizer = tokenizer
|
||||||
|
|
||||||
# Shared thread pool executor for blocking tokenizer and
|
# Thread pool executor for blocking tokenizer operations. The
|
||||||
# multimodal preprocessing operations. The multimodal processor
|
# multimodal processor receives a deep-copied tokenizer (see #36557)
|
||||||
# receives a deep-copied tokenizer (see #36557) so it is safe to
|
# so it is safe to run tokenization and MM preprocessing concurrently.
|
||||||
# run tokenization and MM preprocessing concurrently.
|
|
||||||
pool_workers = config.model_config.renderer_num_workers
|
pool_workers = config.model_config.renderer_num_workers
|
||||||
self._executor = ThreadPoolExecutor(max_workers=pool_workers)
|
self._executor = ThreadPoolExecutor(max_workers=pool_workers)
|
||||||
|
|
||||||
# Multimodal preprocessing is always offloaded to the thread pool
|
# Separate single-worker executor so tokenization never queues behind
|
||||||
# to keep the asyncio event loop responsive under concurrent load.
|
# MM preprocessing; must stay single-worker per #38418 (P0/P1 order).
|
||||||
self._mm_executor: Executor = self._executor
|
self._mm_executor: Executor = ThreadPoolExecutor(max_workers=1)
|
||||||
|
|
||||||
# Offload tokenization to the thread pool. The sync
|
# Offload tokenization to the thread pool. The sync
|
||||||
# ``_tokenize_prompt`` already encapsulates the unified ``__call__``
|
# ``_tokenize_prompt`` already encapsulates the unified ``__call__``
|
||||||
@@ -103,7 +102,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
self._readonly_mm_processor: BaseMultiModalProcessor | None = None
|
self._readonly_mm_processor: BaseMultiModalProcessor | None = None
|
||||||
self._mm_cache_stats: MultiModalCacheStats | None = None
|
self._mm_cache_stats: MultiModalCacheStats | None = None
|
||||||
self._clear_mm_cache_async = make_async(
|
self._clear_mm_cache_async = make_async(
|
||||||
self.clear_mm_cache, executor=self._executor
|
self.clear_mm_cache, executor=self._mm_executor
|
||||||
)
|
)
|
||||||
self._process_multimodal_async = make_async(
|
self._process_multimodal_async = make_async(
|
||||||
self._process_multimodal, executor=self._mm_executor
|
self._process_multimodal, executor=self._mm_executor
|
||||||
@@ -282,7 +281,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
|||||||
self._clear_processor_cache(self._readonly_mm_processor)
|
self._clear_processor_cache(self._readonly_mm_processor)
|
||||||
|
|
||||||
async def clear_mm_cache_async(self) -> None:
|
async def clear_mm_cache_async(self) -> None:
|
||||||
"""Serialize clear_mm_cache through the shared executor to avoid
|
"""Serialize clear_mm_cache through the multimodal executor to avoid
|
||||||
races with concurrent process_inputs on the mm_processor_cache."""
|
races with concurrent process_inputs on the mm_processor_cache."""
|
||||||
await self._clear_mm_cache_async()
|
await self._clear_mm_cache_async()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user