[Core] Cleanup engine pause/sleep logic (#34528)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-02-24 19:33:34 -08:00
committed by GitHub
parent 3bbb2046ff
commit dbf0da817a
10 changed files with 321 additions and 217 deletions
+9 -4
View File
@@ -753,6 +753,13 @@ class AsyncLLM(EngineClient):
)
mode = "wait"
await self.engine_core.pause_scheduler_async(mode=mode, clear_cache=clear_cache)
# Small sleep to help ensure that final outputs from any in-flight requests are
# returned prior to this method returning. These outputs come out of the engine
# prior to the wait-for-idle completion event, but involve additional async
# tasks in output processing.
# Note that this is not required for correctness, just more intuitive ordering
# of events from caller's pov.
await asyncio.sleep(0.02)
async def resume_generation(self) -> None:
"""Resume generation after :meth:`pause_generation`."""
@@ -890,10 +897,8 @@ class AsyncLLM(EngineClient):
async def reset_encoder_cache(self) -> None:
await self.engine_core.reset_encoder_cache_async()
async def sleep(self, level: int = 1) -> None:
if level > 0:
await self.reset_prefix_cache()
await self.engine_core.sleep_async(level)
async def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
await self.engine_core.sleep_async(level, mode)
if self.logger_manager is not None:
self.logger_manager.record_sleep_state(1, level)