[Core] Release NCCL communicator memory in sleep mode (#51485)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-01 14:31:16 +08:00
committed by GitHub
co-authored by aoshen02 Claude Fable 5
parent ec32f669bb
commit 4ac452ad98
12 changed files with 182 additions and 6 deletions
+39
View File
@@ -472,3 +472,42 @@ def test_ncclGetUniqueId():
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
# as long as the function doesn't raise an exception, we're good
assert unique_id is not None
def test_pynccl_suspend_resume_idempotent():
"""Repeated suspend/resume (e.g. staged wake-ups) reach NCCL once each."""
from unittest.mock import Mock
comm = object.__new__(PyNcclCommunicator)
comm.disabled = False
comm._suspended = False
comm.comm = object()
comm.nccl = Mock()
comm.suspend()
comm.suspend()
comm.resume()
comm.resume()
assert comm.nccl.ncclCommSuspend.call_count == 1
assert comm.nccl.ncclCommResume.call_count == 1
def test_pynccl_suspend_noop_without_symbol():
"""RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not
crash, and resume() must then no-op too since nothing was suspended."""
from unittest.mock import Mock
comm = object.__new__(PyNcclCommunicator)
comm.disabled = False
comm._suspended = False
comm.comm = object()
comm.nccl = Mock()
comm.nccl.has_symbol.return_value = False
comm.suspend()
comm.resume()
comm.nccl.ncclCommSuspend.assert_not_called()
comm.nccl.ncclCommResume.assert_not_called()
assert comm._suspended is False
@@ -113,11 +113,14 @@ def test_kv_wake_does_not_run_model_runner_recovery() -> None:
worker = cast(
Worker,
SimpleNamespace(
_get_sleep_mode_backend=lambda: SimpleNamespace(resume=lambda tags: None),
sleep_mode_backend=SimpleNamespace(resume=lambda tags: None),
_sleep_saved_buffers={},
_sleep_saved_draft_buffers={},
model_runner=runner,
synchronize_device=lambda: None,
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(enable_nccl_comm_suspend=False)
),
),
)
layout_tensors = runner.layout_tensors
@@ -38,6 +38,55 @@ def test_new_backend_starts_in_running_state():
assert CuMemBackend().state() == "RUNNING"
@pytest.mark.parametrize("enable_nccl_comm_suspend", [True, False])
def test_worker_drives_communicator_suspension(monkeypatch, enable_nccl_comm_suspend):
"""Comm walkers run around sleep/wake only when explicitly enabled."""
from types import SimpleNamespace
from vllm.v1.worker.gpu_worker import Worker
calls: list[tuple[str, object]] = []
class Backend:
def suspend(self, level: int = 1) -> None:
calls.append(("backend.suspend", level))
def resume(self, tags: list[str] | None = None) -> None:
calls.append(("backend.resume", tuple(tags) if tags else None))
worker = object.__new__(Worker)
worker._sleep_mode_backend = Backend()
worker._sleep_saved_buffers = {}
worker._sleep_saved_draft_buffers = {}
worker.vllm_config = SimpleNamespace(
model_config=SimpleNamespace(enable_nccl_comm_suspend=enable_nccl_comm_suspend)
)
monkeypatch.setattr("torch.accelerator.synchronize", lambda: None)
monkeypatch.setattr("torch.accelerator.get_memory_info", lambda: (0, 0))
monkeypatch.setattr(
"vllm.v1.worker.gpu_worker.suspend_device_comms",
lambda: calls.append(("comms.suspend", None)),
)
monkeypatch.setattr(
"vllm.v1.worker.gpu_worker.resume_device_comms",
lambda: calls.append(("comms.resume", None)),
)
worker.sleep(level=1)
worker.wake_up(tags=["weights"])
expected = [
("backend.suspend", 1),
("comms.suspend", None),
("backend.resume", ("weights",)),
("comms.resume", None),
]
if not enable_nccl_comm_suspend:
expected = [c for c in expected if not c[0].startswith("comms.")]
assert calls == expected
def test_unknown_backend_raises():
with pytest.raises(ValueError, match="Unsupported sleep-mode backend"):
SleepModeBackendFactory.get_backend_class("does-not-exist")
+4
View File
@@ -343,6 +343,10 @@ class ModelConfig:
(default) uses the built-in ``CuMemAllocator`` and is behavior-compatible
with prior releases. Additional backends (CUDA checkpoint, CRIU, durable
snapshot) may be registered in-tree or by plugins (RFC #34303)."""
enable_nccl_comm_suspend: bool = False
"""Enable releasing NCCL communicator memory during sleep mode
(``ncclCommSuspend``/``ncclCommResume``). Experimental; when disabled
(the default) sleep still releases weights/KV-cache memory as before."""
enable_cumem_allocator: bool = False
"""Enable the custom cumem allocator to leverage advanced GPU memory
allocation features such as multi-node NVLink support.
+1 -2
View File
@@ -134,8 +134,7 @@ class CuMemBackend(SleepModeBackend):
@classmethod
def preserves_communicators(cls) -> bool:
# Communicator buffers (e.g. NCCL) live outside CuMemAllocator's pool, so
# an allocator-level sleep leaves them intact (no reinit needed on resume).
# Comm identity/topology survive memory suspension; no reinit needed.
return True
@@ -226,6 +226,12 @@ class DeviceCommunicatorBase:
def checkpoint_restore(self) -> None:
"""Restore communicator state after checkpoint (default: no-op)."""
def suspend(self) -> None:
"""Release reclaimable communicator memory (default: no-op)."""
def resume(self) -> None:
"""Restore memory released by ``suspend`` (default: no-op)."""
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
if dim < 0:
# Convert negative dim to positive.
@@ -631,6 +631,14 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.all2all_manager.destroy()
self.all2all_manager = None # type: ignore[assignment]
def suspend(self) -> None:
if self.pynccl_comm is not None:
self.pynccl_comm.suspend()
def resume(self) -> None:
if self.pynccl_comm is not None:
self.pynccl_comm.resume()
def checkpoint_prepare(self) -> None:
# Only FlashInfer all-reduce and FlashInfer all2all are supported for now.
from .flashinfer_all_reduce import checkpoint_prepare_fi_ar_workspaces
@@ -27,6 +27,8 @@ logger = init_logger(__name__)
_NCCL_SYMM_OPS_REGISTERED = False
_NCCL_SUSPEND_MEM = 0x01
def register_nccl_symmetric_ops(pynccl_comm):
from vllm.distributed.device_communicators.pynccl_allocator import (
@@ -105,6 +107,7 @@ class PyNcclCommunicator:
self.available = True
self.disabled = False
self._suspended = False
self.nccl_version = self.nccl.ncclGetRawVersion()
if self.rank == 0:
@@ -479,6 +482,27 @@ class PyNcclCommunicator:
def deregister_comm_window(self, window):
return self.nccl.ncclCommWindowDeregister(self.comm, window)
def suspend(self):
"""Release comm GPU memory (collective, idempotent); keeps topology."""
if self.disabled or self._suspended:
return
if not self.nccl.has_symbol("ncclCommSuspend"):
logger.warning_once(
"ncclCommSuspend is not available in the loaded NCCL/RCCL "
"library (requires NCCL >= 2.29.7); skipping communicator "
"memory suspension."
)
return
self.nccl.ncclCommSuspend(self.comm, _NCCL_SUSPEND_MEM)
self._suspended = True
def resume(self):
"""Restore a suspended comm (collective); no-op unless suspended."""
if self.disabled or not self._suspended:
return
self.nccl.ncclCommResume(self.comm)
self._suspended = False
def batch_isend_irecv(self, p2p_ops: list, stream=None):
if self.disabled:
return
@@ -329,6 +329,10 @@ class NCCLLibrary:
# shutdown when peer ranks may already be gone.
# ncclResult_t ncclCommAbort(ncclComm_t comm);
Function("ncclCommAbort", ncclResult_t, [ncclComm_t]),
# ncclResult_t ncclCommSuspend(ncclComm_t comm, int flags);
Function("ncclCommSuspend", ncclResult_t, [ncclComm_t, ctypes.c_int]),
# ncclResult_t ncclCommResume(ncclComm_t comm);
Function("ncclCommResume", ncclResult_t, [ncclComm_t]),
# ncclResult_t ncclGroupStart();
Function("ncclGroupStart", ncclResult_t, []),
# ncclResult_t ncclGroupEnd();
@@ -417,10 +421,21 @@ class NCCLLibrary:
elif func.name == "ncclCommQueryProperties":
# Optional on NCCL versions older than 2.29.
continue
elif func.name in ("ncclCommSuspend", "ncclCommResume"):
# RCCL doesn't export these; NCCL >= 2.29.7 does, and
# vLLM's CUDA path already requires that version.
# PyNcclCommunicator checks has_symbol() before
# calling either.
if current_platform.is_rocm():
continue
raise
NCCLLibrary.path_to_dict_mapping[so_file] = _funcs
self._funcs = NCCLLibrary.path_to_dict_mapping[so_file]
def has_symbol(self, name: str) -> bool:
"""Whether the loaded NCCL/RCCL library exports the given symbol."""
return name in self._funcs
def ncclGetErrorString(self, result: ncclResult_t) -> str:
return self._funcs["ncclGetErrorString"](result).decode("utf-8")
@@ -599,6 +614,12 @@ class NCCLLibrary:
def ncclCommAbort(self, comm: ncclComm_t) -> None:
self.NCCL_CHECK(self._funcs["ncclCommAbort"](comm))
def ncclCommSuspend(self, comm: ncclComm_t, flags: int) -> None:
self.NCCL_CHECK(self._funcs["ncclCommSuspend"](comm, flags))
def ncclCommResume(self, comm: ncclComm_t) -> None:
self.NCCL_CHECK(self._funcs["ncclCommResume"](comm))
def ncclGroupStart(self) -> None:
self.NCCL_CHECK(self._funcs["ncclGroupStart"]())
+10
View File
@@ -180,6 +180,16 @@ def _apply_to_device_comms(
action(dc)
def suspend_device_comms() -> None:
"""Release device communicator memory (collective; comms must be idle)."""
_apply_to_device_comms(lambda comm: comm.suspend())
def resume_device_comms() -> None:
"""Restore suspended device communicators (collective)."""
_apply_to_device_comms(lambda comm: comm.resume())
def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
assert group_name in _groups, f"Group {group_name} is not found."
group = _groups[group_name]()
+6
View File
@@ -710,6 +710,7 @@ class EngineArgs:
generation_config: str = ModelConfig.generation_config
enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
enable_cumem_allocator: bool = ModelConfig.enable_cumem_allocator
enable_nccl_comm_suspend: bool = ModelConfig.enable_nccl_comm_suspend
override_generation_config: dict[str, Any] = get_field(
ModelConfig, "override_generation_config"
)
@@ -932,6 +933,10 @@ class EngineArgs:
model_group.add_argument(
"--enable-cumem-allocator", **model_kwargs["enable_cumem_allocator"]
)
model_group.add_argument(
"--enable-nccl-comm-suspend",
**model_kwargs["enable_nccl_comm_suspend"],
)
model_group.add_argument("--model-impl", **model_kwargs["model_impl"])
model_group.add_argument(
"--logits-processors", **model_kwargs["logits_processors"]
@@ -1823,6 +1828,7 @@ class EngineArgs:
override_generation_config=self.override_generation_config,
enable_sleep_mode=self.enable_sleep_mode,
enable_cumem_allocator=self.enable_cumem_allocator,
enable_nccl_comm_suspend=self.enable_nccl_comm_suspend,
model_impl=self.model_impl,
logits_processors=self.logits_processors,
video_pruning_rate=self.video_pruning_rate,
+10 -3
View File
@@ -45,6 +45,8 @@ from vllm.distributed.parallel_state import (
checkpoint_restore_distributed_state,
get_pp_group,
get_tp_group,
resume_device_comms,
suspend_device_comms,
)
from vllm.distributed.weight_transfer import (
WeightTransferEngine,
@@ -220,7 +222,8 @@ class Worker(WorkerBase):
# Resolved lazily on first sleep/wake; persists worker-process state.
self._sleep_mode_backend: SleepModeBackend | None = None
def _get_sleep_mode_backend(self) -> "SleepModeBackend":
@property
def sleep_mode_backend(self) -> "SleepModeBackend":
if self._sleep_mode_backend is None:
from vllm.device_allocator.sleep_mode_backend import (
SleepModeBackendFactory,
@@ -247,7 +250,9 @@ class Worker(WorkerBase):
name: buffer.cpu().clone() for name, buffer in draft.named_buffers()
}
self._get_sleep_mode_backend().suspend(level)
self.sleep_mode_backend.suspend(level)
if self.vllm_config.model_config.enable_nccl_comm_suspend:
suspend_device_comms()
torch.accelerator.synchronize()
deadline = time.monotonic() + (5.0 if current_platform.is_rocm() else 0)
@@ -267,7 +272,9 @@ class Worker(WorkerBase):
)
def wake_up(self, tags: list[str] | None = None) -> None:
self._get_sleep_mode_backend().resume(tags)
self.sleep_mode_backend.resume(tags)
if self.vllm_config.model_config.enable_nccl_comm_suspend:
resume_device_comms()
# Restore the buffers after level 2 sleep
wake_weights = tags is None or "weights" in tags