[RL] Add Pause and Resume Generation for Asynchronous RL Training (#28037)
Signed-off-by: SamitHuang <285365963@qq.com> Signed-off-by: Samit <285365963@qq.com> Signed-off-by: samithuang <285365963@qq.com> Co-authored-by: 22quinn <33176974+22quinn@users.noreply.github.com>
This commit is contained in:
@@ -350,6 +350,8 @@ class OutputProcessor:
|
||||
self.parent_requests: dict[str, ParentRequest] = {}
|
||||
self.lora_states = LoRARequestStates(log_stats)
|
||||
self.tracer: Tracer | None = None
|
||||
self._requests_drained = asyncio.Event()
|
||||
self._requests_drained.set()
|
||||
|
||||
def get_num_unfinished_requests(self):
|
||||
return len(self.request_states)
|
||||
@@ -357,6 +359,11 @@ class OutputProcessor:
|
||||
def has_unfinished_requests(self) -> bool:
|
||||
return len(self.request_states) > 0
|
||||
|
||||
async def wait_for_requests_to_drain(self) -> None:
|
||||
if not self.request_states:
|
||||
return
|
||||
await self._requests_drained.wait()
|
||||
|
||||
def propagate_error(self, e: Exception):
|
||||
"""Propagate error to all generate() tasks."""
|
||||
|
||||
@@ -396,6 +403,8 @@ class OutputProcessor:
|
||||
child_reqs = self.abort_requests(child_reqs)
|
||||
request_ids_to_abort.extend(child_reqs)
|
||||
self.parent_requests.pop(request_id, None)
|
||||
if not self.request_states:
|
||||
self._requests_drained.set()
|
||||
return request_ids_to_abort
|
||||
|
||||
def add_request(
|
||||
@@ -420,6 +429,8 @@ class OutputProcessor:
|
||||
log_stats=self.log_stats,
|
||||
stream_interval=self.stream_interval,
|
||||
)
|
||||
if self._requests_drained.is_set():
|
||||
self._requests_drained.clear()
|
||||
self.request_states[request_id] = req_state
|
||||
if parent_req:
|
||||
self.parent_requests[parent_req.request_id] = parent_req
|
||||
@@ -511,6 +522,8 @@ class OutputProcessor:
|
||||
parent_req = req_state.parent_req
|
||||
if parent_req and not parent_req.child_requests:
|
||||
self.parent_requests.pop(parent_req.request_id, None)
|
||||
if not self.request_states:
|
||||
self._requests_drained.set()
|
||||
if not engine_core_output.finished:
|
||||
# If req not finished in EngineCore, but Detokenizer
|
||||
# detected stop string, abort needed in EngineCore.
|
||||
|
||||
Reference in New Issue
Block a user