[Core] Cleanup engine pause/sleep logic (#34528)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -753,6 +753,13 @@ class AsyncLLM(EngineClient):
|
||||
)
|
||||
mode = "wait"
|
||||
await self.engine_core.pause_scheduler_async(mode=mode, clear_cache=clear_cache)
|
||||
# Small sleep to help ensure that final outputs from any in-flight requests are
|
||||
# returned prior to this method returning. These outputs come out of the engine
|
||||
# prior to the wait-for-idle completion event, but involve additional async
|
||||
# tasks in output processing.
|
||||
# Note that this is not required for correctness, just more intuitive ordering
|
||||
# of events from caller's pov.
|
||||
await asyncio.sleep(0.02)
|
||||
|
||||
async def resume_generation(self) -> None:
|
||||
"""Resume generation after :meth:`pause_generation`."""
|
||||
@@ -890,10 +897,8 @@ class AsyncLLM(EngineClient):
|
||||
async def reset_encoder_cache(self) -> None:
|
||||
await self.engine_core.reset_encoder_cache_async()
|
||||
|
||||
async def sleep(self, level: int = 1) -> None:
|
||||
if level > 0:
|
||||
await self.reset_prefix_cache()
|
||||
await self.engine_core.sleep_async(level)
|
||||
async def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None:
|
||||
await self.engine_core.sleep_async(level, mode)
|
||||
|
||||
if self.logger_manager is not None:
|
||||
self.logger_manager.record_sleep_state(1, level)
|
||||
|
||||
Reference in New Issue
Block a user