[Bugfix] Make metadata send non-blocking in GroupCoordinator.isend_tensor_dict (#49274)

Signed-off-by: z00897177 <zhouxinyi6@huawei.com>
This commit is contained in:
AI-Infra Rookie
2026-08-30 23:51:25 +00:00
committed by GitHub
parent 7ab2923489
commit c92b29a1d4
3 changed files with 282 additions and 13 deletions
+136
View File
@@ -5,6 +5,7 @@
Run `pytest tests/distributed/test_comm_ops.py`.
"""
from collections import deque
from collections.abc import Callable
from typing import Any
from unittest.mock import Mock
@@ -208,9 +209,14 @@ def send_recv_tensor_dict_test_worker(
class _DummyWork:
def __init__(self) -> None:
self.wait_calls = 0
self.completed = False
def wait(self) -> None:
self.wait_calls += 1
self.completed = True
def is_completed(self) -> bool:
return self.completed
class _DummyAllGatherGroup:
@@ -235,6 +241,7 @@ def _make_group_for_unit_test(
g.use_cpu_custom_send_recv = False
g.device_group = None
g.cpu_group = None
g._pending_isends = deque()
return g
@@ -402,6 +409,135 @@ def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) ->
nccl_selector.assert_not_called()
def test_isend_object_posts_size_then_object_and_releases_on_wait(
monkeypatch: pytest.MonkeyPatch,
) -> None:
posted: list[tuple[torch.Tensor, _DummyWork]] = []
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
w = _DummyWork()
posted.append((t, w))
return w
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
handle = g.isend_object({"k": [1, 2, 3]}, dst=1)
# two sends, in size-then-object order (preserves gloo FIFO).
assert len(posted) == 2
assert posted[0][0].dtype == torch.long
assert posted[0][0].shape == torch.Size([1])
assert posted[0][0].item() == posted[1][0].numel()
assert posted[1][0].dtype == torch.uint8
# retain holds both source tensors until wait.
assert handle._retained == (posted[0][0], posted[1][0])
handle.wait()
# both underlying works drained, retain dropped.
assert all(w.wait_calls == 1 for _, w in posted)
assert handle._retained == ()
# wait is idempotent: gloo Work.wait() is single-shot for p2p sends (a
# second wait blocks forever), and both the lazy FIFO reap in
# ``_reap_completed_isends`` and any explicit isend caller may wait the
# same metadata handle.
handle.wait()
assert all(w.wait_calls == 1 for _, w in posted)
assert handle._retained == ()
def test_isend_tensor_dict_includes_metadata_handle(
monkeypatch: pytest.MonkeyPatch,
) -> None:
posted: list[torch.Tensor] = []
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
posted.append(t)
return _DummyWork()
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")}
handles = g.isend_tensor_dict(td, dst=1)
# size + object (metadata) + one tensor send.
assert len(posted) == 3
assert posted[0].dtype == torch.long
assert posted[1].dtype == torch.uint8
assert posted[2].dtype == torch.float32
# composite metadata handle + one tensor handle.
assert len(handles) == 2
for handle in handles:
handle.wait()
assert handles[0]._retained == ()
def test_isend_tensor_dict_self_retains_for_fire_and_forget(
monkeypatch: pytest.MonkeyPatch,
) -> None:
works: list[_DummyWork] = []
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
w = _DummyWork()
works.append(w)
return w
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")}
# fire-and-forget: the returned handles are dropped by the caller, so the
# group must self-retain the handles and the source tensor.
g.isend_tensor_dict(td, dst=1)
assert len(g._pending_isends) == 1
handles0, tensors0 = g._pending_isends[0]
assert len(handles0) == 2 # metadata composite + one tensor send
assert tensors0 == [td["a"]]
# the first send is still in flight, so the next call's reap keeps it.
g.isend_tensor_dict(td, dst=1)
assert len(g._pending_isends) == 2
# metadata-only send (no tensors) is dropped best-effort on the next
# reap, since it has no reliable completion signal.
g.isend_tensor_dict({"meta": "no-tensors"}, dst=1)
assert len(g._pending_isends) == 3
# complete every posted work; the next call reaps all three old entries
# (the metadata-only one unconditionally) and keeps only its own.
for w in works:
w.completed = True
g.isend_tensor_dict(td, dst=1)
assert len(g._pending_isends) == 1
# reaped entries had their metadata handles waited as a backstop.
assert handles0[0]._retained == ()
def test_send_tensor_dict_sync_path_does_not_self_retain(
monkeypatch: pytest.MonkeyPatch,
) -> None:
def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork:
return _DummyWork()
monkeypatch.setattr(torch.distributed, "isend", fake_isend)
monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True)
monkeypatch.setattr(GroupCoordinator, "send_object", lambda self, obj, dst: None)
g = _make_group_for_unit_test(rank_in_group=0, world_size=2)
td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")}
g.send_tensor_dict(td, dst=1)
# the sync path waited every handle, so no retention entry may leak.
assert len(g._pending_isends) == 0
def test_async_intermediate_tensors_lazy_wait() -> None:
work = _DummyWork()
post_calls = {"n": 0}
+140 -3
View File
@@ -27,7 +27,7 @@ import contextlib
import gc
import pickle
import weakref
from collections import namedtuple
from collections import deque, namedtuple
from collections.abc import Callable
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
@@ -78,6 +78,37 @@ class Handle(Protocol):
def wait(self) -> None: ...
class _RetainedHandle:
"""Handle that retains source tensors until all posted work completes.
Used by ``isend_object`` to keep the serialized CPU tensors alive while
gloo copies them in the background; ``wait`` drops the refs once drained.
``wait`` must be idempotent: gloo ``Work.wait()`` is single-shot for
point-to-point operations — calling it a second time on an already
completed send blocks forever (verified empirically). Both the worker's
top-of-step drain of ``_pp_send_work`` and ``_reap_completed_isends``
may wait the same entry's metadata handle, so the second caller must
not re-wait the underlying works.
"""
def __init__(self, works: list[Any], retained: tuple[torch.Tensor, ...]) -> None:
self._works = works
self._retained = retained
self._waited = False
def is_completed(self) -> bool:
return all(w.is_completed() for w in self._works)
def wait(self) -> None:
if self._waited:
return
for w in self._works:
w.wait()
self._waited = True
self._retained = ()
def _split_tensor_dict(
tensor_dict: dict[str, torch.Tensor | Any],
) -> tuple[list[tuple[str, Any]], list[torch.Tensor]]:
@@ -532,6 +563,15 @@ class GroupCoordinator:
and getattr(self.device_communicator, "supports_tensor_dict", False)
)
# FIFO of in-flight ``isend_tensor_dict`` calls: one entry per call,
# (handles, sent source tensors). Self-retention keeps the send
# buffers alive for fire-and-forget callers that drop the returned
# handles; entries are reaped lazily on the next ``isend_tensor_dict``
# (see ``_reap_completed_isends``). Lazily created instead of plain
# attribute init because some subclasses replicate ``__init__``
# without calling ``super().__init__``.
self._pending_isends: deque[tuple[list[Handle], list[torch.Tensor]]] = deque()
def make_sibling_device_group(self, group_desc: str | None = None) -> ProcessGroup:
"""Create a new device-side ProcessGroup with the same per-rank membership
as this coordinator's `device_group`, but backed by a distinct communicator.
@@ -882,6 +922,33 @@ class GroupCoordinator:
return obj
def isend_object(self, obj: Any, dst: int) -> Handle:
"""Non-blocking ``send_object``: isend size + pickled object on the
CPU group. Returns a handle that retains the serialized source
tensors until ``wait`` drains both sends.
"""
assert dst < self.world_size, f"Invalid dst rank ({dst})"
assert dst != self.rank_in_group, (
"Invalid destination rank. Destination rank is the same "
"as the current rank."
)
object_tensor = torch.frombuffer(pickle.dumps(obj), dtype=torch.uint8)
size_tensor = torch.tensor(
[object_tensor.numel()], dtype=torch.long, device="cpu"
)
retained = (size_tensor, object_tensor)
works: list[Any] = []
for tensor in retained:
work = torch.distributed.isend(
tensor, dst=self.ranks[dst], group=self.cpu_group
)
if work is not None:
works.append(work)
return _RetainedHandle(works, retained)
def broadcast_tensor_dict(
self,
tensor_dict: dict[str, torch.Tensor | Any] | None = None,
@@ -1014,8 +1081,51 @@ class GroupCoordinator:
)
for handle in handles:
handle.wait()
# The sync path waited every handle, so the self-retention entry
# appended by ``isend_tensor_dict`` is already drained; drop it
# instead of leaking the tensor refs until the next async send.
pending = getattr(self, "_pending_isends", None)
if pending and pending[-1][0] is handles:
pending.pop()
return None
def _reap_completed_isends(self) -> None:
"""Lazily drop self-retained ``isend_tensor_dict`` entries that have
completed, oldest first (FIFO).
gloo ``Work.is_completed()`` is unreliable (it can report completion
before the background copy of the source buffer finishes), so the
metadata handle (``handles[0]``, a gloo-backed ``_RetainedHandle``) is
never used as the completion gate. Instead we gate on the tensor-send
handles (``handles[1:]``), which run on the device backend where
``is_completed()`` is reliable. Once all tensor sends are done the
peer must already have posted the matching tensor recvs — and since
the receiver ingests metadata before tensors, the gloo metadata send
is then guaranteed complete, so ``wait()`` on it is ~instant and only
serves to drop the retained refs.
Metadata-only entries (``handles[1:]`` empty, e.g. an empty
``IntermediateTensors``) have no reliable completion signal; drop them
best-effort without blocking on the gloo metadata wait.
"""
pending = getattr(self, "_pending_isends", None)
if pending is None:
# Subclasses that replicate __init__ without calling
# super().__init__ may not have the attribute yet.
self._pending_isends = pending = deque()
while pending:
handles, _ = pending[0]
tensor_handles = handles[1:]
if not tensor_handles:
pending.popleft()
continue
if not all(h.is_completed() for h in tensor_handles):
# The oldest send is still in flight; FIFO ordering means the
# rest are no further along, so stop here.
break
handles[0].wait()
pending.popleft()
def isend_tensor_dict(
self,
tensor_dict: dict[str, torch.Tensor | Any],
@@ -1023,6 +1133,16 @@ class GroupCoordinator:
all_gather_group: "GroupCoordinator | None" = None,
all_gather_tensors: dict[str, bool] | None = None,
) -> list[Handle]:
"""Send the input tensor dictionary asynchronously.
The returned handles are additionally self-retained in
``self._pending_isends`` (together with the sent source tensors) and
reaped lazily on subsequent calls, so fire-and-forget callers that
drop the handles cannot free or overwrite the send buffers while the
sends are still in flight. Callers that do keep the handles may
``wait()`` them as before; the retention entry is dropped on reap
either way.
"""
if self.world_size <= 1:
return []
@@ -1048,12 +1168,16 @@ class GroupCoordinator:
metadata_group = self.cpu_group
metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
self.send_object(metadata_list, dst=dst)
tensor_keys = [k for k, v in tensor_dict.items() if isinstance(v, torch.Tensor)]
assert len(tensor_keys) == len(tensor_list)
handles: list[Handle] = []
# Reap completed sends before posting new ones, bounding the
# self-retention FIFO.
self._reap_completed_isends()
handles: list[Handle] = [self.isend_object(metadata_list, dst=dst)]
sent_tensors: list[torch.Tensor] = []
for key, tensor in zip(tensor_keys, tensor_list):
if tensor.numel() == 0:
continue
@@ -1070,6 +1194,19 @@ class GroupCoordinator:
if tensor.is_cuda:
tensor.record_stream(torch.cuda.current_stream(tensor.device))
handles.append(handle)
sent_tensors.append(tensor)
# Self-retain for fire-and-forget callers: keep the handles and
# the source tensors alive until the sends complete (reaped
# lazily by ``_reap_completed_isends``). Without this, dropping
# the returned handles would GC the ``_RetainedHandle``'s
# serialized metadata buffers mid-send, and non-CUDA devices
# have no ``record_stream`` equivalent protecting the tensor
# sources.
pending = getattr(self, "_pending_isends", None)
if pending is None:
self._pending_isends = pending = deque()
pending.append((handles, sent_tensors))
return handles
+6 -10
View File
@@ -216,8 +216,6 @@ class Worker(WorkerBase):
self.profiler_config = vllm_config.profiler_config
self.use_v2_model_runner = vllm_config.use_v2_model_runner
# pending non-blocking PP send work from the previous iteration
self._pp_send_work: list[Handle] = []
# Resolved lazily on first sleep/wake; persists worker-process state.
self._sleep_mode_backend: SleepModeBackend | None = None
@@ -1108,12 +1106,6 @@ class Worker(WorkerBase):
def execute_model(
self, scheduler_output: "SchedulerOutput"
) -> ModelRunnerOutput | AsyncModelRunnerOutput | None:
# ensure any previous non-blocking PP sends are complete
if self._pp_send_work:
for handle in self._pp_send_work:
handle.wait()
self._pp_send_work = []
intermediate_tensors = None
forward_pass = scheduler_output.total_num_scheduled_tokens > 0
num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
@@ -1186,8 +1178,12 @@ class Worker(WorkerBase):
and not get_pp_group().is_last_rank
)
# launch non-blocking send of intermediate tensors
self._pp_send_work = get_pp_group().isend_tensor_dict(
# launch non-blocking send of intermediate tensors; the
# GroupCoordinator self-retains the source tensors and the
# metadata handle in ``_pending_isends`` and reaps them lazily on
# the next ``isend_tensor_dict(is_async=True)`` call, so this
# step returns without waiting for the receiver to drain.
get_pp_group().isend_tensor_dict(
output.tensors,
all_gather_group=get_tp_group(),
all_gather_tensors=all_gather_tensors,