From c92b29a1d40644da710209f862b1be0ebd5c2e74 Mon Sep 17 00:00:00 2001 From: AI-Infra Rookie Date: Mon, 31 Aug 2026 07:51:25 +0800 Subject: [PATCH] [Bugfix] Make metadata send non-blocking in GroupCoordinator.isend_tensor_dict (#49274) Signed-off-by: z00897177 --- tests/distributed/test_comm_ops.py | 136 +++++++++++++++++++++++++++ vllm/distributed/parallel_state.py | 143 ++++++++++++++++++++++++++++- vllm/v1/worker/gpu_worker.py | 16 ++-- 3 files changed, 282 insertions(+), 13 deletions(-) diff --git a/tests/distributed/test_comm_ops.py b/tests/distributed/test_comm_ops.py index 5144a1a4cb..a7d59ff2f8 100644 --- a/tests/distributed/test_comm_ops.py +++ b/tests/distributed/test_comm_ops.py @@ -5,6 +5,7 @@ Run `pytest tests/distributed/test_comm_ops.py`. """ +from collections import deque from collections.abc import Callable from typing import Any from unittest.mock import Mock @@ -208,9 +209,14 @@ def send_recv_tensor_dict_test_worker( class _DummyWork: def __init__(self) -> None: self.wait_calls = 0 + self.completed = False def wait(self) -> None: self.wait_calls += 1 + self.completed = True + + def is_completed(self) -> bool: + return self.completed class _DummyAllGatherGroup: @@ -235,6 +241,7 @@ def _make_group_for_unit_test( g.use_cpu_custom_send_recv = False g.device_group = None g.cpu_group = None + g._pending_isends = deque() return g @@ -402,6 +409,135 @@ def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) -> nccl_selector.assert_not_called() +def test_isend_object_posts_size_then_object_and_releases_on_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + posted: list[tuple[torch.Tensor, _DummyWork]] = [] + + def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork: + w = _DummyWork() + posted.append((t, w)) + return w + + monkeypatch.setattr(torch.distributed, "isend", fake_isend) + + g = _make_group_for_unit_test(rank_in_group=0, world_size=2) + handle = g.isend_object({"k": [1, 2, 3]}, dst=1) + + # two sends, in size-then-object order (preserves gloo FIFO). + assert len(posted) == 2 + assert posted[0][0].dtype == torch.long + assert posted[0][0].shape == torch.Size([1]) + assert posted[0][0].item() == posted[1][0].numel() + assert posted[1][0].dtype == torch.uint8 + + # retain holds both source tensors until wait. + assert handle._retained == (posted[0][0], posted[1][0]) + + handle.wait() + + # both underlying works drained, retain dropped. + assert all(w.wait_calls == 1 for _, w in posted) + assert handle._retained == () + + # wait is idempotent: gloo Work.wait() is single-shot for p2p sends (a + # second wait blocks forever), and both the lazy FIFO reap in + # ``_reap_completed_isends`` and any explicit isend caller may wait the + # same metadata handle. + handle.wait() + assert all(w.wait_calls == 1 for _, w in posted) + assert handle._retained == () + + +def test_isend_tensor_dict_includes_metadata_handle( + monkeypatch: pytest.MonkeyPatch, +) -> None: + posted: list[torch.Tensor] = [] + + def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork: + posted.append(t) + return _DummyWork() + + monkeypatch.setattr(torch.distributed, "isend", fake_isend) + + g = _make_group_for_unit_test(rank_in_group=0, world_size=2) + td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")} + handles = g.isend_tensor_dict(td, dst=1) + + # size + object (metadata) + one tensor send. + assert len(posted) == 3 + assert posted[0].dtype == torch.long + assert posted[1].dtype == torch.uint8 + assert posted[2].dtype == torch.float32 + + # composite metadata handle + one tensor handle. + assert len(handles) == 2 + + for handle in handles: + handle.wait() + assert handles[0]._retained == () + + +def test_isend_tensor_dict_self_retains_for_fire_and_forget( + monkeypatch: pytest.MonkeyPatch, +) -> None: + works: list[_DummyWork] = [] + + def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork: + w = _DummyWork() + works.append(w) + return w + + monkeypatch.setattr(torch.distributed, "isend", fake_isend) + + g = _make_group_for_unit_test(rank_in_group=0, world_size=2) + td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")} + + # fire-and-forget: the returned handles are dropped by the caller, so the + # group must self-retain the handles and the source tensor. + g.isend_tensor_dict(td, dst=1) + assert len(g._pending_isends) == 1 + handles0, tensors0 = g._pending_isends[0] + assert len(handles0) == 2 # metadata composite + one tensor send + assert tensors0 == [td["a"]] + + # the first send is still in flight, so the next call's reap keeps it. + g.isend_tensor_dict(td, dst=1) + assert len(g._pending_isends) == 2 + + # metadata-only send (no tensors) is dropped best-effort on the next + # reap, since it has no reliable completion signal. + g.isend_tensor_dict({"meta": "no-tensors"}, dst=1) + assert len(g._pending_isends) == 3 + + # complete every posted work; the next call reaps all three old entries + # (the metadata-only one unconditionally) and keeps only its own. + for w in works: + w.completed = True + g.isend_tensor_dict(td, dst=1) + assert len(g._pending_isends) == 1 + # reaped entries had their metadata handles waited as a backstop. + assert handles0[0]._retained == () + + +def test_send_tensor_dict_sync_path_does_not_self_retain( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_isend(t: torch.Tensor, *args: Any, **kwargs: Any) -> _DummyWork: + return _DummyWork() + + monkeypatch.setattr(torch.distributed, "isend", fake_isend) + monkeypatch.setattr(torch.distributed, "is_initialized", lambda: True) + monkeypatch.setattr(GroupCoordinator, "send_object", lambda self, obj, dst: None) + + g = _make_group_for_unit_test(rank_in_group=0, world_size=2) + td = {"a": torch.arange(4, dtype=torch.float32, device="cpu")} + g.send_tensor_dict(td, dst=1) + + # the sync path waited every handle, so no retention entry may leak. + assert len(g._pending_isends) == 0 + + def test_async_intermediate_tensors_lazy_wait() -> None: work = _DummyWork() post_calls = {"n": 0} diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index 96a95a69fd..0291a3d0ae 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -27,7 +27,7 @@ import contextlib import gc import pickle import weakref -from collections import namedtuple +from collections import deque, namedtuple from collections.abc import Callable from contextlib import contextmanager, nullcontext from dataclasses import dataclass @@ -78,6 +78,37 @@ class Handle(Protocol): def wait(self) -> None: ... +class _RetainedHandle: + """Handle that retains source tensors until all posted work completes. + + Used by ``isend_object`` to keep the serialized CPU tensors alive while + gloo copies them in the background; ``wait`` drops the refs once drained. + + ``wait`` must be idempotent: gloo ``Work.wait()`` is single-shot for + point-to-point operations — calling it a second time on an already + completed send blocks forever (verified empirically). Both the worker's + top-of-step drain of ``_pp_send_work`` and ``_reap_completed_isends`` + may wait the same entry's metadata handle, so the second caller must + not re-wait the underlying works. + """ + + def __init__(self, works: list[Any], retained: tuple[torch.Tensor, ...]) -> None: + self._works = works + self._retained = retained + self._waited = False + + def is_completed(self) -> bool: + return all(w.is_completed() for w in self._works) + + def wait(self) -> None: + if self._waited: + return + for w in self._works: + w.wait() + self._waited = True + self._retained = () + + def _split_tensor_dict( tensor_dict: dict[str, torch.Tensor | Any], ) -> tuple[list[tuple[str, Any]], list[torch.Tensor]]: @@ -532,6 +563,15 @@ class GroupCoordinator: and getattr(self.device_communicator, "supports_tensor_dict", False) ) + # FIFO of in-flight ``isend_tensor_dict`` calls: one entry per call, + # (handles, sent source tensors). Self-retention keeps the send + # buffers alive for fire-and-forget callers that drop the returned + # handles; entries are reaped lazily on the next ``isend_tensor_dict`` + # (see ``_reap_completed_isends``). Lazily created instead of plain + # attribute init because some subclasses replicate ``__init__`` + # without calling ``super().__init__``. + self._pending_isends: deque[tuple[list[Handle], list[torch.Tensor]]] = deque() + def make_sibling_device_group(self, group_desc: str | None = None) -> ProcessGroup: """Create a new device-side ProcessGroup with the same per-rank membership as this coordinator's `device_group`, but backed by a distinct communicator. @@ -882,6 +922,33 @@ class GroupCoordinator: return obj + def isend_object(self, obj: Any, dst: int) -> Handle: + """Non-blocking ``send_object``: isend size + pickled object on the + CPU group. Returns a handle that retains the serialized source + tensors until ``wait`` drains both sends. + """ + assert dst < self.world_size, f"Invalid dst rank ({dst})" + assert dst != self.rank_in_group, ( + "Invalid destination rank. Destination rank is the same " + "as the current rank." + ) + + object_tensor = torch.frombuffer(pickle.dumps(obj), dtype=torch.uint8) + size_tensor = torch.tensor( + [object_tensor.numel()], dtype=torch.long, device="cpu" + ) + + retained = (size_tensor, object_tensor) + works: list[Any] = [] + for tensor in retained: + work = torch.distributed.isend( + tensor, dst=self.ranks[dst], group=self.cpu_group + ) + if work is not None: + works.append(work) + + return _RetainedHandle(works, retained) + def broadcast_tensor_dict( self, tensor_dict: dict[str, torch.Tensor | Any] | None = None, @@ -1014,8 +1081,51 @@ class GroupCoordinator: ) for handle in handles: handle.wait() + # The sync path waited every handle, so the self-retention entry + # appended by ``isend_tensor_dict`` is already drained; drop it + # instead of leaking the tensor refs until the next async send. + pending = getattr(self, "_pending_isends", None) + if pending and pending[-1][0] is handles: + pending.pop() return None + def _reap_completed_isends(self) -> None: + """Lazily drop self-retained ``isend_tensor_dict`` entries that have + completed, oldest first (FIFO). + + gloo ``Work.is_completed()`` is unreliable (it can report completion + before the background copy of the source buffer finishes), so the + metadata handle (``handles[0]``, a gloo-backed ``_RetainedHandle``) is + never used as the completion gate. Instead we gate on the tensor-send + handles (``handles[1:]``), which run on the device backend where + ``is_completed()`` is reliable. Once all tensor sends are done the + peer must already have posted the matching tensor recvs — and since + the receiver ingests metadata before tensors, the gloo metadata send + is then guaranteed complete, so ``wait()`` on it is ~instant and only + serves to drop the retained refs. + + Metadata-only entries (``handles[1:]`` empty, e.g. an empty + ``IntermediateTensors``) have no reliable completion signal; drop them + best-effort without blocking on the gloo metadata wait. + """ + pending = getattr(self, "_pending_isends", None) + if pending is None: + # Subclasses that replicate __init__ without calling + # super().__init__ may not have the attribute yet. + self._pending_isends = pending = deque() + while pending: + handles, _ = pending[0] + tensor_handles = handles[1:] + if not tensor_handles: + pending.popleft() + continue + if not all(h.is_completed() for h in tensor_handles): + # The oldest send is still in flight; FIFO ordering means the + # rest are no further along, so stop here. + break + handles[0].wait() + pending.popleft() + def isend_tensor_dict( self, tensor_dict: dict[str, torch.Tensor | Any], @@ -1023,6 +1133,16 @@ class GroupCoordinator: all_gather_group: "GroupCoordinator | None" = None, all_gather_tensors: dict[str, bool] | None = None, ) -> list[Handle]: + """Send the input tensor dictionary asynchronously. + + The returned handles are additionally self-retained in + ``self._pending_isends`` (together with the sent source tensors) and + reaped lazily on subsequent calls, so fire-and-forget callers that + drop the handles cannot free or overwrite the send buffers while the + sends are still in flight. Callers that do keep the handles may + ``wait()`` them as before; the retention entry is dropped on reap + either way. + """ if self.world_size <= 1: return [] @@ -1048,12 +1168,16 @@ class GroupCoordinator: metadata_group = self.cpu_group metadata_list, tensor_list = _split_tensor_dict(tensor_dict) - self.send_object(metadata_list, dst=dst) tensor_keys = [k for k, v in tensor_dict.items() if isinstance(v, torch.Tensor)] assert len(tensor_keys) == len(tensor_list) - handles: list[Handle] = [] + # Reap completed sends before posting new ones, bounding the + # self-retention FIFO. + self._reap_completed_isends() + handles: list[Handle] = [self.isend_object(metadata_list, dst=dst)] + + sent_tensors: list[torch.Tensor] = [] for key, tensor in zip(tensor_keys, tensor_list): if tensor.numel() == 0: continue @@ -1070,6 +1194,19 @@ class GroupCoordinator: if tensor.is_cuda: tensor.record_stream(torch.cuda.current_stream(tensor.device)) handles.append(handle) + sent_tensors.append(tensor) + + # Self-retain for fire-and-forget callers: keep the handles and + # the source tensors alive until the sends complete (reaped + # lazily by ``_reap_completed_isends``). Without this, dropping + # the returned handles would GC the ``_RetainedHandle``'s + # serialized metadata buffers mid-send, and non-CUDA devices + # have no ``record_stream`` equivalent protecting the tensor + # sources. + pending = getattr(self, "_pending_isends", None) + if pending is None: + self._pending_isends = pending = deque() + pending.append((handles, sent_tensors)) return handles diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 15f194ffe5..6b8fdb94ef 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -216,8 +216,6 @@ class Worker(WorkerBase): self.profiler_config = vllm_config.profiler_config self.use_v2_model_runner = vllm_config.use_v2_model_runner - # pending non-blocking PP send work from the previous iteration - self._pp_send_work: list[Handle] = [] # Resolved lazily on first sleep/wake; persists worker-process state. self._sleep_mode_backend: SleepModeBackend | None = None @@ -1108,12 +1106,6 @@ class Worker(WorkerBase): def execute_model( self, scheduler_output: "SchedulerOutput" ) -> ModelRunnerOutput | AsyncModelRunnerOutput | None: - # ensure any previous non-blocking PP sends are complete - if self._pp_send_work: - for handle in self._pp_send_work: - handle.wait() - self._pp_send_work = [] - intermediate_tensors = None forward_pass = scheduler_output.total_num_scheduled_tokens > 0 num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens @@ -1186,8 +1178,12 @@ class Worker(WorkerBase): and not get_pp_group().is_last_rank ) - # launch non-blocking send of intermediate tensors - self._pp_send_work = get_pp_group().isend_tensor_dict( + # launch non-blocking send of intermediate tensors; the + # GroupCoordinator self-retains the source tensors and the + # metadata handle in ``_pending_isends`` and reaps them lazily on + # the next ``isend_tensor_dict(is_async=True)`` call, so this + # step returns without waiting for the receiver to drain. + get_pp_group().isend_tensor_dict( output.tensors, all_gather_group=get_tp_group(), all_gather_tensors=all_gather_tensors,