[Feat][RL] Pause and Resume with keep requests for single engine (#32351)
Signed-off-by: ahao-anyscale <ahao@anyscale.com> Signed-off-by: Aaron Hao <ahao@anyscale.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -209,6 +209,10 @@ class EngineCore:
|
||||
self.async_scheduling = vllm_config.scheduler_config.async_scheduling
|
||||
|
||||
self.aborts_queue = queue.Queue[list[str]]()
|
||||
|
||||
# Pause state for "keep" mode - freezes requests in queue.
|
||||
self._scheduler_paused = False
|
||||
|
||||
# Mark the startup heap as static so that it's ignored by GC.
|
||||
# Reduces pause times of oldest generation collections.
|
||||
freeze_gc_heap()
|
||||
@@ -322,6 +326,20 @@ class EngineCore:
|
||||
# (i.e. client-aborted vs stop criteria met).
|
||||
self.scheduler.finish_requests(request_ids, RequestStatus.FINISHED_ABORTED)
|
||||
|
||||
def pause_scheduler(self) -> None:
|
||||
"""Pause the scheduler, keeping requests frozen in queue.
|
||||
|
||||
Requests are kept frozen in queue and can be resumed later.
|
||||
"""
|
||||
self._scheduler_paused = True
|
||||
|
||||
def resume_scheduler(self) -> None:
|
||||
"""Resume the scheduler after a pause.
|
||||
|
||||
Resumes processing of frozen requests in the queue.
|
||||
"""
|
||||
self._scheduler_paused = False
|
||||
|
||||
@contextmanager
|
||||
def log_error_detail(self, scheduler_output: SchedulerOutput):
|
||||
"""Execute the model and log detailed info on failure."""
|
||||
@@ -375,6 +393,10 @@ class EngineCore:
|
||||
was executed.
|
||||
"""
|
||||
|
||||
# If paused, don't schedule any work.
|
||||
if self._scheduler_paused:
|
||||
return {}, False
|
||||
|
||||
# Check for any requests remaining in the scheduler - unfinished,
|
||||
# or finished and not yet removed from the batch.
|
||||
if not self.scheduler.has_requests():
|
||||
@@ -425,6 +447,10 @@ class EngineCore:
|
||||
batch in the job queue is finished.
|
||||
3. Update the scheduler from the output.
|
||||
"""
|
||||
# If paused, don't schedule any work.
|
||||
if self._scheduler_paused:
|
||||
return {}, False
|
||||
|
||||
batch_queue = self.batch_queue
|
||||
assert batch_queue is not None
|
||||
|
||||
@@ -1007,6 +1033,7 @@ class EngineCoreProc(EngineCore):
|
||||
not self.engines_running
|
||||
and not self.scheduler.has_requests()
|
||||
and not self.batch_queue
|
||||
and not self._scheduler_paused
|
||||
):
|
||||
if self.input_queue.empty():
|
||||
# Drain aborts queue; all aborts are also processed via input_queue.
|
||||
|
||||
Reference in New Issue
Block a user