[Perf] Isolate MM preprocessing on its own executor (#49524)

Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
This commit is contained in:
Guan-Ming Chiu
2026-07-26 08:04:17 +00:00
committed by GitHub
parent 0164022c90
commit 8d28b48d01
3 changed files with 31 additions and 20 deletions
+19 -8
View File
@@ -1558,20 +1558,31 @@ def test_fault_tolerance_requires_single_api_server():
def test_renderer_num_workers_with_mm_cache():
"""Disallow renderer_num_workers > 1 when mm processor cache is enabled,
since neither cache type is thread-safe."""
"""Disallow renderer_num_workers > 1 with the mm processor cache only for
pooling models, whose preprocessing runs on the renderer workers."""
mm_model = "Qwen/Qwen2-VL-2B-Instruct"
# Should raise: multi-worker + cache enabled (default cache_gb=4)
# Should raise: pooling + multi-worker + cache enabled (default cache_gb=4)
with pytest.raises(ValueError, match="renderer-num-workers"):
ModelConfig(mm_model, renderer_num_workers=4)
ModelConfig(mm_model, runner="pooling", renderer_num_workers=4)
# Should raise: multi-worker + explicit cache size
# Should raise: pooling + multi-worker + explicit cache size
with pytest.raises(ValueError, match="renderer-num-workers"):
ModelConfig(mm_model, renderer_num_workers=2, mm_processor_cache_gb=1.0)
ModelConfig(
mm_model,
runner="pooling",
renderer_num_workers=2,
mm_processor_cache_gb=1.0,
)
# Should pass: multi-worker + cache disabled
config = ModelConfig(mm_model, renderer_num_workers=4, mm_processor_cache_gb=0)
# Should pass: pooling + multi-worker + cache disabled
config = ModelConfig(
mm_model, runner="pooling", renderer_num_workers=4, mm_processor_cache_gb=0
)
assert config.renderer_num_workers == 4
# Should pass: generate models preprocess on the dedicated mm executor
config = ModelConfig(mm_model, renderer_num_workers=4)
assert config.renderer_num_workers == 4
# Should pass: single worker + cache enabled (default)