[PLE] Carry the host-mapped pull transport onto v0.30.0
Restores the fork's PLE offload package (not upstream) and its pull transport: the offload process holds no CUDA context; the captured op waits on a device-mapped host flag and copies host->GPU on the model stream; publication is fenced; releases are synchronous across dummy/capture work; failures close the rank. Includes the test-only fault-injection cuts for the transport. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -352,9 +352,10 @@ class MultiprocExecutor(Executor):
|
||||
proc = next(h.proc for h in workers if h.proc.sentinel == died[0])
|
||||
logger.error(
|
||||
"Worker proc %s died unexpectedly (exit code: %s), "
|
||||
"shutting down executor.",
|
||||
"shutting down executor. executor_shutdown_monotonic=%.9f",
|
||||
proc.name,
|
||||
proc.exitcode,
|
||||
time.monotonic(),
|
||||
)
|
||||
_self.shutdown()
|
||||
callback = _self.failure_callback
|
||||
@@ -1039,6 +1040,28 @@ class WorkerProc:
|
||||
SUCCESS = auto()
|
||||
FAILURE = auto()
|
||||
|
||||
def _fail_ple_rpc_if_pending(
|
||||
self, where: str, error: BaseException
|
||||
) -> None:
|
||||
"""Fail closed if RPC recovery would abandon queued PLE GPU work."""
|
||||
owners = (self.worker, getattr(self.worker, "worker", None))
|
||||
for owner in owners:
|
||||
if owner is None:
|
||||
continue
|
||||
runner = getattr(owner, "model_runner", None)
|
||||
connector = getattr(runner, "_ple_offload_connector", None)
|
||||
if connector is None:
|
||||
continue
|
||||
try:
|
||||
pending = connector.has_pending_completions
|
||||
except BaseException as state_error:
|
||||
connector._fail_closed(
|
||||
f"{where}:pending-state", state_error
|
||||
)
|
||||
if pending:
|
||||
connector._fail_closed(where, error)
|
||||
return
|
||||
|
||||
def enqueue_output(self, output: Any):
|
||||
"""Prepares output from the worker and enqueues it to the
|
||||
worker_response_mq. If the output is an Exception, it is
|
||||
@@ -1049,6 +1072,9 @@ class WorkerProc:
|
||||
output = output.get_output()
|
||||
except Exception as e:
|
||||
logger.exception("Error getting async model runner output")
|
||||
self._fail_ple_rpc_if_pending(
|
||||
"async-output-copy", e
|
||||
)
|
||||
output = e
|
||||
|
||||
if isinstance(output, Exception):
|
||||
@@ -1108,6 +1134,9 @@ class WorkerProc:
|
||||
if output_rank is None or self.rank == output_rank:
|
||||
self.handle_output(output)
|
||||
except Exception as e:
|
||||
self._fail_ple_rpc_if_pending(
|
||||
f"worker-rpc:{method}", e
|
||||
)
|
||||
# Notes have been introduced in python 3.11
|
||||
if hasattr(e, "add_note"):
|
||||
e.add_note(traceback.format_exc())
|
||||
|
||||
Reference in New Issue
Block a user