[Refactor] Use dynamic MM cache in processor (#57674)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-19 15:38:00 +00:00
committed by GitHub
parent 41c4a3ed4e
commit 751f6807d9
23 changed files with 343 additions and 401 deletions
+42 -62
View File
@@ -127,8 +127,7 @@ class BaseRenderer(ABC, Generic[_T]):
self._async_tokenizer_decode = make_async(self._decode, executor=self._executor)
self.mm_processor: BaseMultiModalProcessor | None = None
self._readonly_mm_processor: BaseMultiModalProcessor | None = None
self._mm_cache_stats: MultiModalCacheStats | None = None
self._clear_mm_cache_async = make_async(
self.clear_mm_cache, executor=self._mm_executor
)
@@ -138,6 +137,16 @@ class BaseRenderer(ABC, Generic[_T]):
self._safe_load_prompt_embeds_async = make_async(
safe_load_prompt_embeds, executor=self._executor
)
self._mm_processor_cache: BaseMultiModalProcessorCache | None = None
# A second processor with its own processor-only cache.
# Used by the tokenize endpoint so that tokenize-only
# requests don't pollute the sender cache.
self._mm_processor_only_cache: BaseMultiModalProcessorCache | None = None
self._mm_cache_stats: MultiModalCacheStats | None = None
if mm_registry.supports_multimodal_inputs(config.model_config):
# Install the process-global GPU memory pool used to gate
# frontend GPU-side multimodal decoding (no-op when the budget
@@ -149,30 +158,16 @@ class BaseRenderer(ABC, Generic[_T]):
config.parallel_config._api_process_count,
)
mm_processor_cache = mm_registry.processor_cache_from_config(config)
with set_default_torch_num_threads():
self.mm_processor = mm_registry.create_processor(
config.model_config,
tokenizer=self.tokenizer,
cache=mm_processor_cache,
)
if mm_processor_cache:
self._mm_cache_stats = MultiModalCacheStats()
self._resources.callback(mm_processor_cache.close)
# A second processor with its own processor-only cache.
# Used by the tokenize endpoint so that tokenize-only
# requests don't pollute the sender cache.
ro_cache = mm_registry.processor_only_cache_from_config(config)
if ro_cache is not None:
with set_default_torch_num_threads():
self._readonly_mm_processor = mm_registry.create_processor(
config.model_config,
tokenizer=self.tokenizer,
cache=ro_cache,
)
self._mm_processor_cache = mm_registry.processor_cache_from_config(config)
self._mm_processor_only_cache = (
mm_registry.processor_only_cache_from_config(config)
)
# This is used to generate internal request ID for MM processing
# It has no relation to the request ID for engine core
@@ -181,6 +176,12 @@ class BaseRenderer(ABC, Generic[_T]):
config.observability_config
)
if self._mm_processor_cache:
self._mm_cache_stats = MultiModalCacheStats()
self._resources.callback(self._mm_processor_cache.close)
if self._mm_processor_only_cache:
self._resources.callback(self._mm_processor_only_cache.close)
def get_tokenizer(self) -> _T:
tokenizer = self.tokenizer
if tokenizer is None:
@@ -198,11 +199,8 @@ class BaseRenderer(ABC, Generic[_T]):
return self.mm_processor
@property
def mm_processor_cache(self) -> "BaseMultiModalProcessorCache | None":
if self.mm_processor is None:
return None
return self.mm_processor.cache
def mm_processor_cache(self) -> BaseMultiModalProcessorCache | None:
return self._mm_processor_cache
def stat_mm_cache(self) -> MultiModalCacheStats | None:
mm_cache_stats = self._mm_cache_stats
@@ -214,7 +212,7 @@ class BaseRenderer(ABC, Generic[_T]):
return mm_cache_stats
def update_mm_cache_stats(self) -> None:
mm_processor_cache = self.mm_processor_cache
mm_processor_cache = self._mm_processor_cache
mm_cache_stats = self._mm_cache_stats
if mm_processor_cache and mm_cache_stats:
@@ -222,38 +220,26 @@ class BaseRenderer(ABC, Generic[_T]):
mm_cache_stats.record(delta.total, delta.hits)
def clear_mm_cache(self) -> None:
mm_processor_cache = self.mm_processor_cache
mm_processor_cache = self._mm_processor_cache
if mm_processor_cache is not None:
mm_processor_cache.clear_cache()
if self._mm_cache_stats is not None:
self._mm_cache_stats.reset = True
@staticmethod
def _clear_processor_cache(
processor: "BaseMultiModalProcessor | None",
) -> None:
if processor is None:
return
processor_cache = processor.cache
if processor_cache is not None:
processor_cache.clear_cache()
def _warmup_mm_processor(
self,
processor: "BaseMultiModalProcessor",
processor: BaseMultiModalProcessor,
cache: BaseMultiModalProcessorCache | None,
*,
log_prefix: str,
) -> None:
model_config = self.model_config
mm_limits = {k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0}
start_time = time.perf_counter()
_ = mm_registry.get_dummy_mm_inputs(
model_config,
mm_counts=dict.fromkeys(mm_limits, 1),
processor=processor,
_ = processor.get_dummy_mm_inputs(
dict.fromkeys(mm_limits, 1),
cache=cache,
scheduler_config=self.config.scheduler_config,
)
@@ -274,28 +260,20 @@ class BaseRenderer(ABC, Generic[_T]):
# prevent MM processor hangs
with set_default_torch_num_threads(1):
if self.mm_processor:
mm_processor_only_cache = self._mm_processor_only_cache
try:
logger.debug("Warming up multi-modal processing...")
self._warmup_mm_processor(
self.mm_processor,
mm_processor_only_cache,
log_prefix="Multi-modal",
)
except Exception:
logger.warning("Multi-modal warmup failed")
finally:
self.clear_mm_cache()
if self._readonly_mm_processor is not None:
try:
logger.debug("Warming up readonly multi-modal processing...")
self._warmup_mm_processor(
self._readonly_mm_processor,
log_prefix="Readonly multi-modal",
)
except Exception:
logger.warning("Readonly multi-modal warmup failed")
finally:
self._clear_processor_cache(self._readonly_mm_processor)
if mm_processor_only_cache:
mm_processor_only_cache.clear_cache()
finally:
self._mm_warmup_done = True
@@ -323,7 +301,7 @@ class BaseRenderer(ABC, Generic[_T]):
# which is not designed for concurrent access (its get_and_update must
# stay ordered with the engine-core cache) — is never touched
# concurrently by the warmup and the serving path.
if self.mm_processor is None and self._readonly_mm_processor is None:
if self.mm_processor is None:
return # text-only model: nothing to warm up
if self._mm_warmup_future is not None:
return # already launched
@@ -853,10 +831,7 @@ class BaseRenderer(ABC, Generic[_T]):
*,
skip_mm_cache: bool = False,
) -> "MultiModalInput":
if skip_mm_cache and self._readonly_mm_processor is not None:
mm_processor = self._readonly_mm_processor
else:
mm_processor = self.get_mm_processor()
mm_processor = self.get_mm_processor()
mm_req_id = f"renderer{self.api_process_rank}-mm-{self._mm_req_counter.inc(1)}"
@@ -873,6 +848,11 @@ class BaseRenderer(ABC, Generic[_T]):
mm_uuid_items,
hf_processor_mm_kwargs=mm_processor_kwargs or {},
media_io_kwargs=media_io_kwargs or {},
cache=(
self._mm_processor_only_cache
if skip_mm_cache
else self._mm_processor_cache
),
)
mm_timing_ctx = self._mm_timing_registry.get(mm_req_id)