[warmup] overlap renderer warmup and engine core initialization (#54557)
Signed-off-by: Andy Xie <andy.xning@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -102,12 +102,17 @@ class LLMEngine:
|
||||
)
|
||||
|
||||
# EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
|
||||
# Hand the renderer to the client. In multiprocess mode the client
|
||||
# starts the MM warmup only after engine-core fork (the why is in
|
||||
# BaseRenderer.start_mm_warmup_in_background); InprocClient takes no
|
||||
# renderer, so MM warmup stays inside renderer.warmup() there.
|
||||
self.engine_core = EngineCoreClient.make_client(
|
||||
multiprocess_mode=multiprocess_mode,
|
||||
asyncio_mode=False,
|
||||
vllm_config=vllm_config,
|
||||
executor_class=executor_class,
|
||||
log_stats=self.log_stats,
|
||||
renderer=renderer,
|
||||
)
|
||||
|
||||
self.logger_manager: StatLoggerManager | None = None
|
||||
@@ -343,6 +348,9 @@ class LLMEngine:
|
||||
self.engine_core.profile(False)
|
||||
|
||||
def reset_mm_cache(self):
|
||||
# Join the background MM warmup first: the mm_processor_cache is not
|
||||
# safe for concurrent access with its apply/clear.
|
||||
self.renderer._join_mm_warmup()
|
||||
self.renderer.clear_mm_cache()
|
||||
self.engine_core.reset_mm_cache()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user