[Core] Cleanup engine pause/sleep logic (#34528)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
+108
-62
@@ -9,6 +9,7 @@ from collections import defaultdict, deque
|
||||
from collections.abc import Callable, Generator
|
||||
from concurrent.futures import Future
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from functools import partial
|
||||
from inspect import isclass, signature
|
||||
from logging import DEBUG
|
||||
from typing import Any, TypeVar, cast
|
||||
@@ -211,7 +212,7 @@ class EngineCore:
|
||||
|
||||
self.aborts_queue = queue.Queue[list[str]]()
|
||||
|
||||
self.per_step_hooks: set[Callable] = set()
|
||||
self._idle_state_callbacks: list[Callable] = []
|
||||
|
||||
# Mark the startup heap as static so that it's ignored by GC.
|
||||
# Reduces pause times of oldest generation collections.
|
||||
@@ -592,21 +593,51 @@ class EngineCore:
|
||||
# Reset the GPU model runner's encoder cache (physical storage)
|
||||
self.model_executor.reset_encoder_cache()
|
||||
|
||||
def _reset_caches(self, reset_running_requests=True) -> None:
|
||||
self.reset_prefix_cache(reset_running_requests=reset_running_requests)
|
||||
self.reset_mm_cache()
|
||||
self.reset_encoder_cache()
|
||||
|
||||
def pause_scheduler(
|
||||
self, mode: PauseMode = "abort", clear_cache: bool = True
|
||||
) -> Future[Any] | None:
|
||||
"""Pause scheduling. No-op in base EngineCore; overridden in EngineCoreProc."""
|
||||
) -> Future | None:
|
||||
"""Pause generation; behavior depends on mode.
|
||||
|
||||
All pause modes queue new adds -- "abort" and "keep" skip step();
|
||||
"wait" allows step() so in-flight requests can drain.
|
||||
|
||||
- ``abort``: Set PAUSED_NEW, abort all requests, wait for abort
|
||||
outputs to be sent (when running with output_queue), optionally
|
||||
clear caches, then complete the returned Future.
|
||||
- ``wait``: Set PAUSED_NEW (queue adds, keep stepping); when drained,
|
||||
optionally clear caches, then complete the returned Future.
|
||||
- ``keep``: Set PAUSED_ALL; return a Future that completes when the
|
||||
output queue is empty.
|
||||
"""
|
||||
if mode not in ("keep", "abort", "wait"):
|
||||
raise ValueError(f"Invalid pause mode: {mode}")
|
||||
if mode == "wait":
|
||||
raise ValueError("'wait' mode can't be used in inproc-engine mode")
|
||||
|
||||
if mode == "abort":
|
||||
self.scheduler.finish_requests(None, RequestStatus.FINISHED_ABORTED)
|
||||
|
||||
pause_state = PauseState.PAUSED_ALL if mode == "keep" else PauseState.PAUSED_NEW
|
||||
self.scheduler.set_pause_state(pause_state)
|
||||
if clear_cache:
|
||||
self._reset_caches()
|
||||
|
||||
return None
|
||||
|
||||
def resume_scheduler(self) -> None:
|
||||
"""Resume scheduling. No-op in base EngineCore; overridden in EngineCoreProc."""
|
||||
"""Resume the scheduler and flush any requests queued while paused."""
|
||||
self.scheduler.set_pause_state(PauseState.UNPAUSED)
|
||||
|
||||
def is_scheduler_paused(self) -> bool:
|
||||
"""Return whether the scheduler is in any pause state. False in base EngineCore
|
||||
and overridden in EngineCoreProc."""
|
||||
return False
|
||||
"""Return whether the scheduler is in any pause state."""
|
||||
return self.scheduler.pause_state != PauseState.UNPAUSED
|
||||
|
||||
def sleep(self, level: int = 1):
|
||||
def sleep(self, level: int = 1, mode: PauseMode = "abort") -> None | Future:
|
||||
"""Put the engine to sleep at the specified level.
|
||||
|
||||
Args:
|
||||
@@ -615,13 +646,34 @@ class EngineCore:
|
||||
but not processed. No GPU memory changes.
|
||||
- Level 1: Offload model weights to CPU, discard KV cache.
|
||||
- Level 2: Discard all GPU memory.
|
||||
mode: Pause mode - how to deal with any existing requests, see
|
||||
documentation of pause_scheduler method.
|
||||
"""
|
||||
if level == 0:
|
||||
# Level 0: Just pause scheduling, don't touch GPU
|
||||
self.pause_scheduler()
|
||||
else:
|
||||
# Level 1+: Delegate to executor for GPU memory management
|
||||
self.model_executor.sleep(level)
|
||||
|
||||
# Pause scheduler before sleeping.
|
||||
clear_prefix_cache = level >= 1
|
||||
pause_future = self.pause_scheduler(mode=mode, clear_cache=clear_prefix_cache)
|
||||
if level < 1:
|
||||
return pause_future
|
||||
|
||||
# Level 1+: Delegate to executor for GPU memory management
|
||||
model_executor = self.model_executor
|
||||
if pause_future is None:
|
||||
model_executor.sleep(level)
|
||||
return None
|
||||
|
||||
future = Future[Any]()
|
||||
|
||||
def pause_complete(f: Future):
|
||||
try:
|
||||
f.result() # propagate any exception
|
||||
future.set_result(model_executor.sleep(level))
|
||||
except Exception as e:
|
||||
future.set_exception(e)
|
||||
|
||||
logger.info("Waiting for in-flight requests to complete before sleeping...")
|
||||
pause_future.add_done_callback(pause_complete)
|
||||
return future
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
"""Wake up the engine from sleep.
|
||||
@@ -630,17 +682,15 @@ class EngineCore:
|
||||
tags: Tags to wake up. Use ["scheduling"] for level 0 wake up.
|
||||
"""
|
||||
if tags is not None and "scheduling" in tags:
|
||||
# Level 0 wake up: Resume scheduling
|
||||
self.resume_scheduler()
|
||||
# Remove "scheduling" from tags if there are other tags to process
|
||||
remaining_tags = [t for t in tags if t != "scheduling"]
|
||||
if remaining_tags:
|
||||
self.model_executor.wake_up(remaining_tags)
|
||||
else:
|
||||
# Full wake up
|
||||
self.resume_scheduler()
|
||||
# Remove "scheduling" from tags if there are other tags to process.
|
||||
tags = [t for t in tags if t != "scheduling"]
|
||||
|
||||
if tags is None or tags:
|
||||
self.model_executor.wake_up(tags)
|
||||
|
||||
# Resume scheduling (applies to all levels)
|
||||
self.resume_scheduler()
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
"""Check if engine is sleeping at any level."""
|
||||
return self.is_scheduler_paused() or self.model_executor.is_sleeping
|
||||
@@ -1038,6 +1088,14 @@ class EngineCoreProc(EngineCore):
|
||||
def _init_data_parallel(self, vllm_config: VllmConfig):
|
||||
pass
|
||||
|
||||
def has_work(self) -> bool:
|
||||
"""Returns true if the engine should be stepped."""
|
||||
return (
|
||||
self.engines_running
|
||||
or self.scheduler.has_requests()
|
||||
or bool(self.batch_queue)
|
||||
)
|
||||
|
||||
def run_busy_loop(self):
|
||||
"""Core busy loop of the EngineCore."""
|
||||
|
||||
@@ -1047,19 +1105,14 @@ class EngineCoreProc(EngineCore):
|
||||
self._process_input_queue()
|
||||
# 2) Step the engine core and return the outputs.
|
||||
self._process_engine_step()
|
||||
# 3) Run any per-step hooks.
|
||||
self._process_per_step_hooks()
|
||||
|
||||
def _process_input_queue(self):
|
||||
"""Exits when an engine step needs to be performed."""
|
||||
|
||||
waited = False
|
||||
while (
|
||||
not self.engines_running
|
||||
and not self.scheduler.has_requests()
|
||||
and not self.batch_queue
|
||||
and not self.per_step_hooks
|
||||
):
|
||||
while not self.has_work():
|
||||
# Notify callbacks waiting for engine to become idle.
|
||||
self._notify_idle_state_callbacks()
|
||||
if self.input_queue.empty():
|
||||
# Drain aborts queue; all aborts are also processed via input_queue.
|
||||
with self.aborts_queue.mutex:
|
||||
@@ -1098,12 +1151,10 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
return model_executed
|
||||
|
||||
def _process_per_step_hooks(self) -> None:
|
||||
if self.per_step_hooks:
|
||||
for hook in list(self.per_step_hooks):
|
||||
finished = hook(self)
|
||||
if finished:
|
||||
self.per_step_hooks.discard(hook)
|
||||
def _notify_idle_state_callbacks(self) -> None:
|
||||
while self._idle_state_callbacks:
|
||||
callback = self._idle_state_callbacks.pop()
|
||||
callback(self)
|
||||
|
||||
def _handle_client_request(
|
||||
self, request_type: EngineCoreRequestType, request: Any
|
||||
@@ -1377,19 +1428,10 @@ class EngineCoreProc(EngineCore):
|
||||
if mode not in ("keep", "abort", "wait"):
|
||||
raise ValueError(f"Invalid pause mode: {mode}")
|
||||
|
||||
future: Future[Any] = Future()
|
||||
|
||||
def wait_until_idle(engine: "EngineCoreProc") -> bool:
|
||||
scheduler = engine.scheduler
|
||||
out_queue = engine.output_queue
|
||||
if scheduler.has_requests() or engine.batch_queue or not out_queue.empty():
|
||||
return False
|
||||
def engine_idle_callback(engine: "EngineCoreProc", future: Future[Any]) -> None:
|
||||
if clear_cache:
|
||||
engine.reset_prefix_cache(reset_running_requests=True)
|
||||
engine.reset_mm_cache()
|
||||
engine.reset_encoder_cache()
|
||||
engine._reset_caches()
|
||||
future.set_result(None)
|
||||
return True
|
||||
|
||||
if mode == "abort":
|
||||
aborted_reqs = self.scheduler.finish_requests(
|
||||
@@ -1399,12 +1441,17 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
pause_state = PauseState.PAUSED_ALL if mode == "keep" else PauseState.PAUSED_NEW
|
||||
self.scheduler.set_pause_state(pause_state)
|
||||
if not wait_until_idle(self):
|
||||
self.per_step_hooks.add(wait_until_idle)
|
||||
return future
|
||||
return None
|
||||
if not self.has_work():
|
||||
if clear_cache:
|
||||
self._reset_caches()
|
||||
return None
|
||||
|
||||
future = Future[Any]()
|
||||
self._idle_state_callbacks.append(partial(engine_idle_callback, future=future))
|
||||
return future
|
||||
|
||||
def _send_abort_outputs(self, aborted_reqs: list[tuple[str, int]]) -> None:
|
||||
# TODO(nick) this will be moved inside the scheduler
|
||||
if aborted_reqs:
|
||||
# Map client_index to list of request_ids that belong to that client.
|
||||
by_client = defaultdict[int, set[str]](set)
|
||||
@@ -1418,14 +1465,6 @@ class EngineCoreProc(EngineCore):
|
||||
eco = EngineCoreOutputs(finished_requests=req_ids, outputs=outputs)
|
||||
self.output_queue.put_nowait((client_index, eco))
|
||||
|
||||
def resume_scheduler(self) -> None:
|
||||
"""Resume the scheduler and flush any requests queued while paused."""
|
||||
self.scheduler.set_pause_state(PauseState.UNPAUSED)
|
||||
|
||||
def is_scheduler_paused(self) -> bool:
|
||||
"""Return whether the scheduler is in any pause state."""
|
||||
return self.scheduler.pause_state != PauseState.UNPAUSED
|
||||
|
||||
|
||||
class DPEngineCoreProc(EngineCoreProc):
|
||||
"""ZMQ-wrapper for running EngineCore in background process
|
||||
@@ -1481,6 +1520,7 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
stateless_destroy_torch_distributed_process_group(dp_group)
|
||||
|
||||
def add_request(self, request: Request, request_wave: int = 0):
|
||||
super().add_request(request, request_wave)
|
||||
if self.has_coordinator and request_wave != self.current_wave:
|
||||
if request_wave > self.current_wave:
|
||||
self.current_wave = request_wave
|
||||
@@ -1491,7 +1531,13 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
(-1, EngineCoreOutputs(start_wave=self.current_wave))
|
||||
)
|
||||
|
||||
super().add_request(request, request_wave)
|
||||
def resume_scheduler(self):
|
||||
super().resume_scheduler()
|
||||
if not self.engines_running and self.scheduler.has_unfinished_requests():
|
||||
# Wake up other DP engines.
|
||||
self.output_queue.put_nowait(
|
||||
(-1, EngineCoreOutputs(start_wave=self.current_wave))
|
||||
)
|
||||
|
||||
def _handle_client_request(
|
||||
self, request_type: EngineCoreRequestType, request: Any
|
||||
@@ -1532,8 +1578,8 @@ class DPEngineCoreProc(EngineCoreProc):
|
||||
# 2) Step the engine core.
|
||||
executed = self._process_engine_step()
|
||||
self._maybe_publish_request_counts()
|
||||
local_unfinished_reqs = self.scheduler.has_unfinished_requests()
|
||||
|
||||
local_unfinished_reqs = self.scheduler.has_unfinished_requests()
|
||||
if not executed:
|
||||
if not local_unfinished_reqs and not self.engines_running:
|
||||
# All engines are idle.
|
||||
|
||||
Reference in New Issue
Block a user