[Core] Release NCCL communicator memory in sleep mode (#51485)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Fable 5
parent
ec32f669bb
commit
4ac452ad98
@@ -226,6 +226,12 @@ class DeviceCommunicatorBase:
|
||||
def checkpoint_restore(self) -> None:
|
||||
"""Restore communicator state after checkpoint (default: no-op)."""
|
||||
|
||||
def suspend(self) -> None:
|
||||
"""Release reclaimable communicator memory (default: no-op)."""
|
||||
|
||||
def resume(self) -> None:
|
||||
"""Restore memory released by ``suspend`` (default: no-op)."""
|
||||
|
||||
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
|
||||
@@ -631,6 +631,14 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.all2all_manager.destroy()
|
||||
self.all2all_manager = None # type: ignore[assignment]
|
||||
|
||||
def suspend(self) -> None:
|
||||
if self.pynccl_comm is not None:
|
||||
self.pynccl_comm.suspend()
|
||||
|
||||
def resume(self) -> None:
|
||||
if self.pynccl_comm is not None:
|
||||
self.pynccl_comm.resume()
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
# Only FlashInfer all-reduce and FlashInfer all2all are supported for now.
|
||||
from .flashinfer_all_reduce import checkpoint_prepare_fi_ar_workspaces
|
||||
|
||||
@@ -27,6 +27,8 @@ logger = init_logger(__name__)
|
||||
|
||||
_NCCL_SYMM_OPS_REGISTERED = False
|
||||
|
||||
_NCCL_SUSPEND_MEM = 0x01
|
||||
|
||||
|
||||
def register_nccl_symmetric_ops(pynccl_comm):
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import (
|
||||
@@ -105,6 +107,7 @@ class PyNcclCommunicator:
|
||||
|
||||
self.available = True
|
||||
self.disabled = False
|
||||
self._suspended = False
|
||||
|
||||
self.nccl_version = self.nccl.ncclGetRawVersion()
|
||||
if self.rank == 0:
|
||||
@@ -479,6 +482,27 @@ class PyNcclCommunicator:
|
||||
def deregister_comm_window(self, window):
|
||||
return self.nccl.ncclCommWindowDeregister(self.comm, window)
|
||||
|
||||
def suspend(self):
|
||||
"""Release comm GPU memory (collective, idempotent); keeps topology."""
|
||||
if self.disabled or self._suspended:
|
||||
return
|
||||
if not self.nccl.has_symbol("ncclCommSuspend"):
|
||||
logger.warning_once(
|
||||
"ncclCommSuspend is not available in the loaded NCCL/RCCL "
|
||||
"library (requires NCCL >= 2.29.7); skipping communicator "
|
||||
"memory suspension."
|
||||
)
|
||||
return
|
||||
self.nccl.ncclCommSuspend(self.comm, _NCCL_SUSPEND_MEM)
|
||||
self._suspended = True
|
||||
|
||||
def resume(self):
|
||||
"""Restore a suspended comm (collective); no-op unless suspended."""
|
||||
if self.disabled or not self._suspended:
|
||||
return
|
||||
self.nccl.ncclCommResume(self.comm)
|
||||
self._suspended = False
|
||||
|
||||
def batch_isend_irecv(self, p2p_ops: list, stream=None):
|
||||
if self.disabled:
|
||||
return
|
||||
|
||||
@@ -329,6 +329,10 @@ class NCCLLibrary:
|
||||
# shutdown when peer ranks may already be gone.
|
||||
# ncclResult_t ncclCommAbort(ncclComm_t comm);
|
||||
Function("ncclCommAbort", ncclResult_t, [ncclComm_t]),
|
||||
# ncclResult_t ncclCommSuspend(ncclComm_t comm, int flags);
|
||||
Function("ncclCommSuspend", ncclResult_t, [ncclComm_t, ctypes.c_int]),
|
||||
# ncclResult_t ncclCommResume(ncclComm_t comm);
|
||||
Function("ncclCommResume", ncclResult_t, [ncclComm_t]),
|
||||
# ncclResult_t ncclGroupStart();
|
||||
Function("ncclGroupStart", ncclResult_t, []),
|
||||
# ncclResult_t ncclGroupEnd();
|
||||
@@ -417,10 +421,21 @@ class NCCLLibrary:
|
||||
elif func.name == "ncclCommQueryProperties":
|
||||
# Optional on NCCL versions older than 2.29.
|
||||
continue
|
||||
elif func.name in ("ncclCommSuspend", "ncclCommResume"):
|
||||
# RCCL doesn't export these; NCCL >= 2.29.7 does, and
|
||||
# vLLM's CUDA path already requires that version.
|
||||
# PyNcclCommunicator checks has_symbol() before
|
||||
# calling either.
|
||||
if current_platform.is_rocm():
|
||||
continue
|
||||
raise
|
||||
NCCLLibrary.path_to_dict_mapping[so_file] = _funcs
|
||||
self._funcs = NCCLLibrary.path_to_dict_mapping[so_file]
|
||||
|
||||
def has_symbol(self, name: str) -> bool:
|
||||
"""Whether the loaded NCCL/RCCL library exports the given symbol."""
|
||||
return name in self._funcs
|
||||
|
||||
def ncclGetErrorString(self, result: ncclResult_t) -> str:
|
||||
return self._funcs["ncclGetErrorString"](result).decode("utf-8")
|
||||
|
||||
@@ -599,6 +614,12 @@ class NCCLLibrary:
|
||||
def ncclCommAbort(self, comm: ncclComm_t) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclCommAbort"](comm))
|
||||
|
||||
def ncclCommSuspend(self, comm: ncclComm_t, flags: int) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclCommSuspend"](comm, flags))
|
||||
|
||||
def ncclCommResume(self, comm: ncclComm_t) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclCommResume"](comm))
|
||||
|
||||
def ncclGroupStart(self) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclGroupStart"]())
|
||||
|
||||
|
||||
@@ -180,6 +180,16 @@ def _apply_to_device_comms(
|
||||
action(dc)
|
||||
|
||||
|
||||
def suspend_device_comms() -> None:
|
||||
"""Release device communicator memory (collective; comms must be idle)."""
|
||||
_apply_to_device_comms(lambda comm: comm.suspend())
|
||||
|
||||
|
||||
def resume_device_comms() -> None:
|
||||
"""Restore suspended device communicators (collective)."""
|
||||
_apply_to_device_comms(lambda comm: comm.resume())
|
||||
|
||||
|
||||
def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
|
||||
assert group_name in _groups, f"Group {group_name} is not found."
|
||||
group = _groups[group_name]()
|
||||
|
||||
Reference in New Issue
Block a user