[Refactor] Use dynamic MM cache in processor (#57674)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+42
-62
@@ -127,8 +127,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
self._async_tokenizer_decode = make_async(self._decode, executor=self._executor)
|
||||
|
||||
self.mm_processor: BaseMultiModalProcessor | None = None
|
||||
self._readonly_mm_processor: BaseMultiModalProcessor | None = None
|
||||
self._mm_cache_stats: MultiModalCacheStats | None = None
|
||||
|
||||
self._clear_mm_cache_async = make_async(
|
||||
self.clear_mm_cache, executor=self._mm_executor
|
||||
)
|
||||
@@ -138,6 +137,16 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
self._safe_load_prompt_embeds_async = make_async(
|
||||
safe_load_prompt_embeds, executor=self._executor
|
||||
)
|
||||
|
||||
self._mm_processor_cache: BaseMultiModalProcessorCache | None = None
|
||||
|
||||
# A second processor with its own processor-only cache.
|
||||
# Used by the tokenize endpoint so that tokenize-only
|
||||
# requests don't pollute the sender cache.
|
||||
self._mm_processor_only_cache: BaseMultiModalProcessorCache | None = None
|
||||
|
||||
self._mm_cache_stats: MultiModalCacheStats | None = None
|
||||
|
||||
if mm_registry.supports_multimodal_inputs(config.model_config):
|
||||
# Install the process-global GPU memory pool used to gate
|
||||
# frontend GPU-side multimodal decoding (no-op when the budget
|
||||
@@ -149,30 +158,16 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
config.parallel_config._api_process_count,
|
||||
)
|
||||
|
||||
mm_processor_cache = mm_registry.processor_cache_from_config(config)
|
||||
|
||||
with set_default_torch_num_threads():
|
||||
self.mm_processor = mm_registry.create_processor(
|
||||
config.model_config,
|
||||
tokenizer=self.tokenizer,
|
||||
cache=mm_processor_cache,
|
||||
)
|
||||
|
||||
if mm_processor_cache:
|
||||
self._mm_cache_stats = MultiModalCacheStats()
|
||||
self._resources.callback(mm_processor_cache.close)
|
||||
|
||||
# A second processor with its own processor-only cache.
|
||||
# Used by the tokenize endpoint so that tokenize-only
|
||||
# requests don't pollute the sender cache.
|
||||
ro_cache = mm_registry.processor_only_cache_from_config(config)
|
||||
if ro_cache is not None:
|
||||
with set_default_torch_num_threads():
|
||||
self._readonly_mm_processor = mm_registry.create_processor(
|
||||
config.model_config,
|
||||
tokenizer=self.tokenizer,
|
||||
cache=ro_cache,
|
||||
)
|
||||
self._mm_processor_cache = mm_registry.processor_cache_from_config(config)
|
||||
self._mm_processor_only_cache = (
|
||||
mm_registry.processor_only_cache_from_config(config)
|
||||
)
|
||||
|
||||
# This is used to generate internal request ID for MM processing
|
||||
# It has no relation to the request ID for engine core
|
||||
@@ -181,6 +176,12 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
config.observability_config
|
||||
)
|
||||
|
||||
if self._mm_processor_cache:
|
||||
self._mm_cache_stats = MultiModalCacheStats()
|
||||
self._resources.callback(self._mm_processor_cache.close)
|
||||
if self._mm_processor_only_cache:
|
||||
self._resources.callback(self._mm_processor_only_cache.close)
|
||||
|
||||
def get_tokenizer(self) -> _T:
|
||||
tokenizer = self.tokenizer
|
||||
if tokenizer is None:
|
||||
@@ -198,11 +199,8 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
return self.mm_processor
|
||||
|
||||
@property
|
||||
def mm_processor_cache(self) -> "BaseMultiModalProcessorCache | None":
|
||||
if self.mm_processor is None:
|
||||
return None
|
||||
|
||||
return self.mm_processor.cache
|
||||
def mm_processor_cache(self) -> BaseMultiModalProcessorCache | None:
|
||||
return self._mm_processor_cache
|
||||
|
||||
def stat_mm_cache(self) -> MultiModalCacheStats | None:
|
||||
mm_cache_stats = self._mm_cache_stats
|
||||
@@ -214,7 +212,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
return mm_cache_stats
|
||||
|
||||
def update_mm_cache_stats(self) -> None:
|
||||
mm_processor_cache = self.mm_processor_cache
|
||||
mm_processor_cache = self._mm_processor_cache
|
||||
mm_cache_stats = self._mm_cache_stats
|
||||
|
||||
if mm_processor_cache and mm_cache_stats:
|
||||
@@ -222,38 +220,26 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
mm_cache_stats.record(delta.total, delta.hits)
|
||||
|
||||
def clear_mm_cache(self) -> None:
|
||||
mm_processor_cache = self.mm_processor_cache
|
||||
mm_processor_cache = self._mm_processor_cache
|
||||
if mm_processor_cache is not None:
|
||||
mm_processor_cache.clear_cache()
|
||||
|
||||
if self._mm_cache_stats is not None:
|
||||
self._mm_cache_stats.reset = True
|
||||
|
||||
@staticmethod
|
||||
def _clear_processor_cache(
|
||||
processor: "BaseMultiModalProcessor | None",
|
||||
) -> None:
|
||||
if processor is None:
|
||||
return
|
||||
|
||||
processor_cache = processor.cache
|
||||
if processor_cache is not None:
|
||||
processor_cache.clear_cache()
|
||||
|
||||
def _warmup_mm_processor(
|
||||
self,
|
||||
processor: "BaseMultiModalProcessor",
|
||||
processor: BaseMultiModalProcessor,
|
||||
cache: BaseMultiModalProcessorCache | None,
|
||||
*,
|
||||
log_prefix: str,
|
||||
) -> None:
|
||||
model_config = self.model_config
|
||||
mm_limits = {k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0}
|
||||
|
||||
start_time = time.perf_counter()
|
||||
_ = mm_registry.get_dummy_mm_inputs(
|
||||
model_config,
|
||||
mm_counts=dict.fromkeys(mm_limits, 1),
|
||||
processor=processor,
|
||||
_ = processor.get_dummy_mm_inputs(
|
||||
dict.fromkeys(mm_limits, 1),
|
||||
cache=cache,
|
||||
scheduler_config=self.config.scheduler_config,
|
||||
)
|
||||
|
||||
@@ -274,28 +260,20 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
# prevent MM processor hangs
|
||||
with set_default_torch_num_threads(1):
|
||||
if self.mm_processor:
|
||||
mm_processor_only_cache = self._mm_processor_only_cache
|
||||
|
||||
try:
|
||||
logger.debug("Warming up multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self.mm_processor,
|
||||
mm_processor_only_cache,
|
||||
log_prefix="Multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Multi-modal warmup failed")
|
||||
finally:
|
||||
self.clear_mm_cache()
|
||||
|
||||
if self._readonly_mm_processor is not None:
|
||||
try:
|
||||
logger.debug("Warming up readonly multi-modal processing...")
|
||||
self._warmup_mm_processor(
|
||||
self._readonly_mm_processor,
|
||||
log_prefix="Readonly multi-modal",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Readonly multi-modal warmup failed")
|
||||
finally:
|
||||
self._clear_processor_cache(self._readonly_mm_processor)
|
||||
if mm_processor_only_cache:
|
||||
mm_processor_only_cache.clear_cache()
|
||||
finally:
|
||||
self._mm_warmup_done = True
|
||||
|
||||
@@ -323,7 +301,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
# which is not designed for concurrent access (its get_and_update must
|
||||
# stay ordered with the engine-core cache) — is never touched
|
||||
# concurrently by the warmup and the serving path.
|
||||
if self.mm_processor is None and self._readonly_mm_processor is None:
|
||||
if self.mm_processor is None:
|
||||
return # text-only model: nothing to warm up
|
||||
if self._mm_warmup_future is not None:
|
||||
return # already launched
|
||||
@@ -853,10 +831,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> "MultiModalInput":
|
||||
if skip_mm_cache and self._readonly_mm_processor is not None:
|
||||
mm_processor = self._readonly_mm_processor
|
||||
else:
|
||||
mm_processor = self.get_mm_processor()
|
||||
mm_processor = self.get_mm_processor()
|
||||
|
||||
mm_req_id = f"renderer{self.api_process_rank}-mm-{self._mm_req_counter.inc(1)}"
|
||||
|
||||
@@ -873,6 +848,11 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
mm_uuid_items,
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs or {},
|
||||
media_io_kwargs=media_io_kwargs or {},
|
||||
cache=(
|
||||
self._mm_processor_only_cache
|
||||
if skip_mm_cache
|
||||
else self._mm_processor_cache
|
||||
),
|
||||
)
|
||||
mm_timing_ctx = self._mm_timing_registry.get(mm_req_id)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user