[Core] Release NCCL communicator memory in sleep mode (#51485)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-01 14:31:16 +08:00
committed by GitHub
co-authored by aoshen02 Claude Fable 5
parent ec32f669bb
commit 4ac452ad98
12 changed files with 182 additions and 6 deletions
@@ -226,6 +226,12 @@ class DeviceCommunicatorBase:
def checkpoint_restore(self) -> None:
"""Restore communicator state after checkpoint (default: no-op)."""
def suspend(self) -> None:
"""Release reclaimable communicator memory (default: no-op)."""
def resume(self) -> None:
"""Restore memory released by ``suspend`` (default: no-op)."""
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
if dim < 0:
# Convert negative dim to positive.
@@ -631,6 +631,14 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.all2all_manager.destroy()
self.all2all_manager = None # type: ignore[assignment]
def suspend(self) -> None:
if self.pynccl_comm is not None:
self.pynccl_comm.suspend()
def resume(self) -> None:
if self.pynccl_comm is not None:
self.pynccl_comm.resume()
def checkpoint_prepare(self) -> None:
# Only FlashInfer all-reduce and FlashInfer all2all are supported for now.
from .flashinfer_all_reduce import checkpoint_prepare_fi_ar_workspaces
@@ -27,6 +27,8 @@ logger = init_logger(__name__)
_NCCL_SYMM_OPS_REGISTERED = False
_NCCL_SUSPEND_MEM = 0x01
def register_nccl_symmetric_ops(pynccl_comm):
from vllm.distributed.device_communicators.pynccl_allocator import (
@@ -105,6 +107,7 @@ class PyNcclCommunicator:
self.available = True
self.disabled = False
self._suspended = False
self.nccl_version = self.nccl.ncclGetRawVersion()
if self.rank == 0:
@@ -479,6 +482,27 @@ class PyNcclCommunicator:
def deregister_comm_window(self, window):
return self.nccl.ncclCommWindowDeregister(self.comm, window)
def suspend(self):
"""Release comm GPU memory (collective, idempotent); keeps topology."""
if self.disabled or self._suspended:
return
if not self.nccl.has_symbol("ncclCommSuspend"):
logger.warning_once(
"ncclCommSuspend is not available in the loaded NCCL/RCCL "
"library (requires NCCL >= 2.29.7); skipping communicator "
"memory suspension."
)
return
self.nccl.ncclCommSuspend(self.comm, _NCCL_SUSPEND_MEM)
self._suspended = True
def resume(self):
"""Restore a suspended comm (collective); no-op unless suspended."""
if self.disabled or not self._suspended:
return
self.nccl.ncclCommResume(self.comm)
self._suspended = False
def batch_isend_irecv(self, p2p_ops: list, stream=None):
if self.disabled:
return
@@ -329,6 +329,10 @@ class NCCLLibrary:
# shutdown when peer ranks may already be gone.
# ncclResult_t ncclCommAbort(ncclComm_t comm);
Function("ncclCommAbort", ncclResult_t, [ncclComm_t]),
# ncclResult_t ncclCommSuspend(ncclComm_t comm, int flags);
Function("ncclCommSuspend", ncclResult_t, [ncclComm_t, ctypes.c_int]),
# ncclResult_t ncclCommResume(ncclComm_t comm);
Function("ncclCommResume", ncclResult_t, [ncclComm_t]),
# ncclResult_t ncclGroupStart();
Function("ncclGroupStart", ncclResult_t, []),
# ncclResult_t ncclGroupEnd();
@@ -417,10 +421,21 @@ class NCCLLibrary:
elif func.name == "ncclCommQueryProperties":
# Optional on NCCL versions older than 2.29.
continue
elif func.name in ("ncclCommSuspend", "ncclCommResume"):
# RCCL doesn't export these; NCCL >= 2.29.7 does, and
# vLLM's CUDA path already requires that version.
# PyNcclCommunicator checks has_symbol() before
# calling either.
if current_platform.is_rocm():
continue
raise
NCCLLibrary.path_to_dict_mapping[so_file] = _funcs
self._funcs = NCCLLibrary.path_to_dict_mapping[so_file]
def has_symbol(self, name: str) -> bool:
"""Whether the loaded NCCL/RCCL library exports the given symbol."""
return name in self._funcs
def ncclGetErrorString(self, result: ncclResult_t) -> str:
return self._funcs["ncclGetErrorString"](result).decode("utf-8")
@@ -599,6 +614,12 @@ class NCCLLibrary:
def ncclCommAbort(self, comm: ncclComm_t) -> None:
self.NCCL_CHECK(self._funcs["ncclCommAbort"](comm))
def ncclCommSuspend(self, comm: ncclComm_t, flags: int) -> None:
self.NCCL_CHECK(self._funcs["ncclCommSuspend"](comm, flags))
def ncclCommResume(self, comm: ncclComm_t) -> None:
self.NCCL_CHECK(self._funcs["ncclCommResume"](comm))
def ncclGroupStart(self) -> None:
self.NCCL_CHECK(self._funcs["ncclGroupStart"]())
+10
View File
@@ -180,6 +180,16 @@ def _apply_to_device_comms(
action(dc)
def suspend_device_comms() -> None:
"""Release device communicator memory (collective; comms must be idle)."""
_apply_to_device_comms(lambda comm: comm.suspend())
def resume_device_comms() -> None:
"""Restore suspended device communicators (collective)."""
_apply_to_device_comms(lambda comm: comm.resume())
def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
assert group_name in _groups, f"Group {group_name} is not found."
group = _groups[group_name]()