[PLE] Carry the host-mapped pull transport onto v0.30.0

Restores the fork's PLE offload package (not upstream) and its pull
transport: the offload process holds no CUDA context; the captured op waits on
a device-mapped host flag and copies host->GPU on the model stream; publication
is fenced; releases are synchronous across dummy/capture work; failures close
the rank. Includes the test-only fault-injection cuts for the transport.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 19:49:16 +07:00
co-authored by Hinata
parent bc0a1cee82
commit 025fd08603
13 changed files with 3755 additions and 5 deletions
+30 -1
View File
@@ -352,9 +352,10 @@ class MultiprocExecutor(Executor):
proc = next(h.proc for h in workers if h.proc.sentinel == died[0])
logger.error(
"Worker proc %s died unexpectedly (exit code: %s), "
"shutting down executor.",
"shutting down executor. executor_shutdown_monotonic=%.9f",
proc.name,
proc.exitcode,
time.monotonic(),
)
_self.shutdown()
callback = _self.failure_callback
@@ -1039,6 +1040,28 @@ class WorkerProc:
SUCCESS = auto()
FAILURE = auto()
def _fail_ple_rpc_if_pending(
self, where: str, error: BaseException
) -> None:
"""Fail closed if RPC recovery would abandon queued PLE GPU work."""
owners = (self.worker, getattr(self.worker, "worker", None))
for owner in owners:
if owner is None:
continue
runner = getattr(owner, "model_runner", None)
connector = getattr(runner, "_ple_offload_connector", None)
if connector is None:
continue
try:
pending = connector.has_pending_completions
except BaseException as state_error:
connector._fail_closed(
f"{where}:pending-state", state_error
)
if pending:
connector._fail_closed(where, error)
return
def enqueue_output(self, output: Any):
"""Prepares output from the worker and enqueues it to the
worker_response_mq. If the output is an Exception, it is
@@ -1049,6 +1072,9 @@ class WorkerProc:
output = output.get_output()
except Exception as e:
logger.exception("Error getting async model runner output")
self._fail_ple_rpc_if_pending(
"async-output-copy", e
)
output = e
if isinstance(output, Exception):
@@ -1108,6 +1134,9 @@ class WorkerProc:
if output_rank is None or self.rank == output_rank:
self.handle_output(output)
except Exception as e:
self._fail_ple_rpc_if_pending(
f"worker-rpc:{method}", e
)
# Notes have been introduced in python 3.11
if hasattr(e, "add_note"):
e.add_note(traceback.format_exc())