[Bugfix] Make metadata send non-blocking in GroupCoordinator.isend_tensor_dict (#49274)
Signed-off-by: z00897177 <zhouxinyi6@huawei.com>
This commit is contained in:
@@ -27,7 +27,7 @@ import contextlib
|
||||
import gc
|
||||
import pickle
|
||||
import weakref
|
||||
from collections import namedtuple
|
||||
from collections import deque, namedtuple
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
@@ -78,6 +78,37 @@ class Handle(Protocol):
|
||||
def wait(self) -> None: ...
|
||||
|
||||
|
||||
class _RetainedHandle:
|
||||
"""Handle that retains source tensors until all posted work completes.
|
||||
|
||||
Used by ``isend_object`` to keep the serialized CPU tensors alive while
|
||||
gloo copies them in the background; ``wait`` drops the refs once drained.
|
||||
|
||||
``wait`` must be idempotent: gloo ``Work.wait()`` is single-shot for
|
||||
point-to-point operations — calling it a second time on an already
|
||||
completed send blocks forever (verified empirically). Both the worker's
|
||||
top-of-step drain of ``_pp_send_work`` and ``_reap_completed_isends``
|
||||
may wait the same entry's metadata handle, so the second caller must
|
||||
not re-wait the underlying works.
|
||||
"""
|
||||
|
||||
def __init__(self, works: list[Any], retained: tuple[torch.Tensor, ...]) -> None:
|
||||
self._works = works
|
||||
self._retained = retained
|
||||
self._waited = False
|
||||
|
||||
def is_completed(self) -> bool:
|
||||
return all(w.is_completed() for w in self._works)
|
||||
|
||||
def wait(self) -> None:
|
||||
if self._waited:
|
||||
return
|
||||
for w in self._works:
|
||||
w.wait()
|
||||
self._waited = True
|
||||
self._retained = ()
|
||||
|
||||
|
||||
def _split_tensor_dict(
|
||||
tensor_dict: dict[str, torch.Tensor | Any],
|
||||
) -> tuple[list[tuple[str, Any]], list[torch.Tensor]]:
|
||||
@@ -532,6 +563,15 @@ class GroupCoordinator:
|
||||
and getattr(self.device_communicator, "supports_tensor_dict", False)
|
||||
)
|
||||
|
||||
# FIFO of in-flight ``isend_tensor_dict`` calls: one entry per call,
|
||||
# (handles, sent source tensors). Self-retention keeps the send
|
||||
# buffers alive for fire-and-forget callers that drop the returned
|
||||
# handles; entries are reaped lazily on the next ``isend_tensor_dict``
|
||||
# (see ``_reap_completed_isends``). Lazily created instead of plain
|
||||
# attribute init because some subclasses replicate ``__init__``
|
||||
# without calling ``super().__init__``.
|
||||
self._pending_isends: deque[tuple[list[Handle], list[torch.Tensor]]] = deque()
|
||||
|
||||
def make_sibling_device_group(self, group_desc: str | None = None) -> ProcessGroup:
|
||||
"""Create a new device-side ProcessGroup with the same per-rank membership
|
||||
as this coordinator's `device_group`, but backed by a distinct communicator.
|
||||
@@ -882,6 +922,33 @@ class GroupCoordinator:
|
||||
|
||||
return obj
|
||||
|
||||
def isend_object(self, obj: Any, dst: int) -> Handle:
|
||||
"""Non-blocking ``send_object``: isend size + pickled object on the
|
||||
CPU group. Returns a handle that retains the serialized source
|
||||
tensors until ``wait`` drains both sends.
|
||||
"""
|
||||
assert dst < self.world_size, f"Invalid dst rank ({dst})"
|
||||
assert dst != self.rank_in_group, (
|
||||
"Invalid destination rank. Destination rank is the same "
|
||||
"as the current rank."
|
||||
)
|
||||
|
||||
object_tensor = torch.frombuffer(pickle.dumps(obj), dtype=torch.uint8)
|
||||
size_tensor = torch.tensor(
|
||||
[object_tensor.numel()], dtype=torch.long, device="cpu"
|
||||
)
|
||||
|
||||
retained = (size_tensor, object_tensor)
|
||||
works: list[Any] = []
|
||||
for tensor in retained:
|
||||
work = torch.distributed.isend(
|
||||
tensor, dst=self.ranks[dst], group=self.cpu_group
|
||||
)
|
||||
if work is not None:
|
||||
works.append(work)
|
||||
|
||||
return _RetainedHandle(works, retained)
|
||||
|
||||
def broadcast_tensor_dict(
|
||||
self,
|
||||
tensor_dict: dict[str, torch.Tensor | Any] | None = None,
|
||||
@@ -1014,8 +1081,51 @@ class GroupCoordinator:
|
||||
)
|
||||
for handle in handles:
|
||||
handle.wait()
|
||||
# The sync path waited every handle, so the self-retention entry
|
||||
# appended by ``isend_tensor_dict`` is already drained; drop it
|
||||
# instead of leaking the tensor refs until the next async send.
|
||||
pending = getattr(self, "_pending_isends", None)
|
||||
if pending and pending[-1][0] is handles:
|
||||
pending.pop()
|
||||
return None
|
||||
|
||||
def _reap_completed_isends(self) -> None:
|
||||
"""Lazily drop self-retained ``isend_tensor_dict`` entries that have
|
||||
completed, oldest first (FIFO).
|
||||
|
||||
gloo ``Work.is_completed()`` is unreliable (it can report completion
|
||||
before the background copy of the source buffer finishes), so the
|
||||
metadata handle (``handles[0]``, a gloo-backed ``_RetainedHandle``) is
|
||||
never used as the completion gate. Instead we gate on the tensor-send
|
||||
handles (``handles[1:]``), which run on the device backend where
|
||||
``is_completed()`` is reliable. Once all tensor sends are done the
|
||||
peer must already have posted the matching tensor recvs — and since
|
||||
the receiver ingests metadata before tensors, the gloo metadata send
|
||||
is then guaranteed complete, so ``wait()`` on it is ~instant and only
|
||||
serves to drop the retained refs.
|
||||
|
||||
Metadata-only entries (``handles[1:]`` empty, e.g. an empty
|
||||
``IntermediateTensors``) have no reliable completion signal; drop them
|
||||
best-effort without blocking on the gloo metadata wait.
|
||||
"""
|
||||
pending = getattr(self, "_pending_isends", None)
|
||||
if pending is None:
|
||||
# Subclasses that replicate __init__ without calling
|
||||
# super().__init__ may not have the attribute yet.
|
||||
self._pending_isends = pending = deque()
|
||||
while pending:
|
||||
handles, _ = pending[0]
|
||||
tensor_handles = handles[1:]
|
||||
if not tensor_handles:
|
||||
pending.popleft()
|
||||
continue
|
||||
if not all(h.is_completed() for h in tensor_handles):
|
||||
# The oldest send is still in flight; FIFO ordering means the
|
||||
# rest are no further along, so stop here.
|
||||
break
|
||||
handles[0].wait()
|
||||
pending.popleft()
|
||||
|
||||
def isend_tensor_dict(
|
||||
self,
|
||||
tensor_dict: dict[str, torch.Tensor | Any],
|
||||
@@ -1023,6 +1133,16 @@ class GroupCoordinator:
|
||||
all_gather_group: "GroupCoordinator | None" = None,
|
||||
all_gather_tensors: dict[str, bool] | None = None,
|
||||
) -> list[Handle]:
|
||||
"""Send the input tensor dictionary asynchronously.
|
||||
|
||||
The returned handles are additionally self-retained in
|
||||
``self._pending_isends`` (together with the sent source tensors) and
|
||||
reaped lazily on subsequent calls, so fire-and-forget callers that
|
||||
drop the handles cannot free or overwrite the send buffers while the
|
||||
sends are still in flight. Callers that do keep the handles may
|
||||
``wait()`` them as before; the retention entry is dropped on reap
|
||||
either way.
|
||||
"""
|
||||
if self.world_size <= 1:
|
||||
return []
|
||||
|
||||
@@ -1048,12 +1168,16 @@ class GroupCoordinator:
|
||||
metadata_group = self.cpu_group
|
||||
|
||||
metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
|
||||
self.send_object(metadata_list, dst=dst)
|
||||
|
||||
tensor_keys = [k for k, v in tensor_dict.items() if isinstance(v, torch.Tensor)]
|
||||
assert len(tensor_keys) == len(tensor_list)
|
||||
|
||||
handles: list[Handle] = []
|
||||
# Reap completed sends before posting new ones, bounding the
|
||||
# self-retention FIFO.
|
||||
self._reap_completed_isends()
|
||||
handles: list[Handle] = [self.isend_object(metadata_list, dst=dst)]
|
||||
|
||||
sent_tensors: list[torch.Tensor] = []
|
||||
for key, tensor in zip(tensor_keys, tensor_list):
|
||||
if tensor.numel() == 0:
|
||||
continue
|
||||
@@ -1070,6 +1194,19 @@ class GroupCoordinator:
|
||||
if tensor.is_cuda:
|
||||
tensor.record_stream(torch.cuda.current_stream(tensor.device))
|
||||
handles.append(handle)
|
||||
sent_tensors.append(tensor)
|
||||
|
||||
# Self-retain for fire-and-forget callers: keep the handles and
|
||||
# the source tensors alive until the sends complete (reaped
|
||||
# lazily by ``_reap_completed_isends``). Without this, dropping
|
||||
# the returned handles would GC the ``_RetainedHandle``'s
|
||||
# serialized metadata buffers mid-send, and non-CUDA devices
|
||||
# have no ``record_stream`` equivalent protecting the tensor
|
||||
# sources.
|
||||
pending = getattr(self, "_pending_isends", None)
|
||||
if pending is None:
|
||||
self._pending_isends = pending = deque()
|
||||
pending.append((handles, sent_tensors))
|
||||
|
||||
return handles
|
||||
|
||||
|
||||
Reference in New Issue
Block a user