[Core] Release NCCL communicator memory in sleep mode (#51485)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Fable 5
parent
ec32f669bb
commit
4ac452ad98
@@ -472,3 +472,42 @@ def test_ncclGetUniqueId():
|
||||
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
|
||||
# as long as the function doesn't raise an exception, we're good
|
||||
assert unique_id is not None
|
||||
|
||||
|
||||
def test_pynccl_suspend_resume_idempotent():
|
||||
"""Repeated suspend/resume (e.g. staged wake-ups) reach NCCL once each."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
comm = object.__new__(PyNcclCommunicator)
|
||||
comm.disabled = False
|
||||
comm._suspended = False
|
||||
comm.comm = object()
|
||||
comm.nccl = Mock()
|
||||
|
||||
comm.suspend()
|
||||
comm.suspend()
|
||||
comm.resume()
|
||||
comm.resume()
|
||||
|
||||
assert comm.nccl.ncclCommSuspend.call_count == 1
|
||||
assert comm.nccl.ncclCommResume.call_count == 1
|
||||
|
||||
|
||||
def test_pynccl_suspend_noop_without_symbol():
|
||||
"""RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not
|
||||
crash, and resume() must then no-op too since nothing was suspended."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
comm = object.__new__(PyNcclCommunicator)
|
||||
comm.disabled = False
|
||||
comm._suspended = False
|
||||
comm.comm = object()
|
||||
comm.nccl = Mock()
|
||||
comm.nccl.has_symbol.return_value = False
|
||||
|
||||
comm.suspend()
|
||||
comm.resume()
|
||||
|
||||
comm.nccl.ncclCommSuspend.assert_not_called()
|
||||
comm.nccl.ncclCommResume.assert_not_called()
|
||||
assert comm._suspended is False
|
||||
|
||||
@@ -113,11 +113,14 @@ def test_kv_wake_does_not_run_model_runner_recovery() -> None:
|
||||
worker = cast(
|
||||
Worker,
|
||||
SimpleNamespace(
|
||||
_get_sleep_mode_backend=lambda: SimpleNamespace(resume=lambda tags: None),
|
||||
sleep_mode_backend=SimpleNamespace(resume=lambda tags: None),
|
||||
_sleep_saved_buffers={},
|
||||
_sleep_saved_draft_buffers={},
|
||||
model_runner=runner,
|
||||
synchronize_device=lambda: None,
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(enable_nccl_comm_suspend=False)
|
||||
),
|
||||
),
|
||||
)
|
||||
layout_tensors = runner.layout_tensors
|
||||
|
||||
@@ -38,6 +38,55 @@ def test_new_backend_starts_in_running_state():
|
||||
assert CuMemBackend().state() == "RUNNING"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_nccl_comm_suspend", [True, False])
|
||||
def test_worker_drives_communicator_suspension(monkeypatch, enable_nccl_comm_suspend):
|
||||
"""Comm walkers run around sleep/wake only when explicitly enabled."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
calls: list[tuple[str, object]] = []
|
||||
|
||||
class Backend:
|
||||
def suspend(self, level: int = 1) -> None:
|
||||
calls.append(("backend.suspend", level))
|
||||
|
||||
def resume(self, tags: list[str] | None = None) -> None:
|
||||
calls.append(("backend.resume", tuple(tags) if tags else None))
|
||||
|
||||
worker = object.__new__(Worker)
|
||||
worker._sleep_mode_backend = Backend()
|
||||
worker._sleep_saved_buffers = {}
|
||||
worker._sleep_saved_draft_buffers = {}
|
||||
worker.vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(enable_nccl_comm_suspend=enable_nccl_comm_suspend)
|
||||
)
|
||||
|
||||
monkeypatch.setattr("torch.accelerator.synchronize", lambda: None)
|
||||
monkeypatch.setattr("torch.accelerator.get_memory_info", lambda: (0, 0))
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu_worker.suspend_device_comms",
|
||||
lambda: calls.append(("comms.suspend", None)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu_worker.resume_device_comms",
|
||||
lambda: calls.append(("comms.resume", None)),
|
||||
)
|
||||
|
||||
worker.sleep(level=1)
|
||||
worker.wake_up(tags=["weights"])
|
||||
|
||||
expected = [
|
||||
("backend.suspend", 1),
|
||||
("comms.suspend", None),
|
||||
("backend.resume", ("weights",)),
|
||||
("comms.resume", None),
|
||||
]
|
||||
if not enable_nccl_comm_suspend:
|
||||
expected = [c for c in expected if not c[0].startswith("comms.")]
|
||||
assert calls == expected
|
||||
|
||||
|
||||
def test_unknown_backend_raises():
|
||||
with pytest.raises(ValueError, match="Unsupported sleep-mode backend"):
|
||||
SleepModeBackendFactory.get_backend_class("does-not-exist")
|
||||
|
||||
@@ -343,6 +343,10 @@ class ModelConfig:
|
||||
(default) uses the built-in ``CuMemAllocator`` and is behavior-compatible
|
||||
with prior releases. Additional backends (CUDA checkpoint, CRIU, durable
|
||||
snapshot) may be registered in-tree or by plugins (RFC #34303)."""
|
||||
enable_nccl_comm_suspend: bool = False
|
||||
"""Enable releasing NCCL communicator memory during sleep mode
|
||||
(``ncclCommSuspend``/``ncclCommResume``). Experimental; when disabled
|
||||
(the default) sleep still releases weights/KV-cache memory as before."""
|
||||
enable_cumem_allocator: bool = False
|
||||
"""Enable the custom cumem allocator to leverage advanced GPU memory
|
||||
allocation features such as multi-node NVLink support.
|
||||
|
||||
@@ -134,8 +134,7 @@ class CuMemBackend(SleepModeBackend):
|
||||
|
||||
@classmethod
|
||||
def preserves_communicators(cls) -> bool:
|
||||
# Communicator buffers (e.g. NCCL) live outside CuMemAllocator's pool, so
|
||||
# an allocator-level sleep leaves them intact (no reinit needed on resume).
|
||||
# Comm identity/topology survive memory suspension; no reinit needed.
|
||||
return True
|
||||
|
||||
|
||||
|
||||
@@ -226,6 +226,12 @@ class DeviceCommunicatorBase:
|
||||
def checkpoint_restore(self) -> None:
|
||||
"""Restore communicator state after checkpoint (default: no-op)."""
|
||||
|
||||
def suspend(self) -> None:
|
||||
"""Release reclaimable communicator memory (default: no-op)."""
|
||||
|
||||
def resume(self) -> None:
|
||||
"""Restore memory released by ``suspend`` (default: no-op)."""
|
||||
|
||||
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
|
||||
@@ -631,6 +631,14 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.all2all_manager.destroy()
|
||||
self.all2all_manager = None # type: ignore[assignment]
|
||||
|
||||
def suspend(self) -> None:
|
||||
if self.pynccl_comm is not None:
|
||||
self.pynccl_comm.suspend()
|
||||
|
||||
def resume(self) -> None:
|
||||
if self.pynccl_comm is not None:
|
||||
self.pynccl_comm.resume()
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
# Only FlashInfer all-reduce and FlashInfer all2all are supported for now.
|
||||
from .flashinfer_all_reduce import checkpoint_prepare_fi_ar_workspaces
|
||||
|
||||
@@ -27,6 +27,8 @@ logger = init_logger(__name__)
|
||||
|
||||
_NCCL_SYMM_OPS_REGISTERED = False
|
||||
|
||||
_NCCL_SUSPEND_MEM = 0x01
|
||||
|
||||
|
||||
def register_nccl_symmetric_ops(pynccl_comm):
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import (
|
||||
@@ -105,6 +107,7 @@ class PyNcclCommunicator:
|
||||
|
||||
self.available = True
|
||||
self.disabled = False
|
||||
self._suspended = False
|
||||
|
||||
self.nccl_version = self.nccl.ncclGetRawVersion()
|
||||
if self.rank == 0:
|
||||
@@ -479,6 +482,27 @@ class PyNcclCommunicator:
|
||||
def deregister_comm_window(self, window):
|
||||
return self.nccl.ncclCommWindowDeregister(self.comm, window)
|
||||
|
||||
def suspend(self):
|
||||
"""Release comm GPU memory (collective, idempotent); keeps topology."""
|
||||
if self.disabled or self._suspended:
|
||||
return
|
||||
if not self.nccl.has_symbol("ncclCommSuspend"):
|
||||
logger.warning_once(
|
||||
"ncclCommSuspend is not available in the loaded NCCL/RCCL "
|
||||
"library (requires NCCL >= 2.29.7); skipping communicator "
|
||||
"memory suspension."
|
||||
)
|
||||
return
|
||||
self.nccl.ncclCommSuspend(self.comm, _NCCL_SUSPEND_MEM)
|
||||
self._suspended = True
|
||||
|
||||
def resume(self):
|
||||
"""Restore a suspended comm (collective); no-op unless suspended."""
|
||||
if self.disabled or not self._suspended:
|
||||
return
|
||||
self.nccl.ncclCommResume(self.comm)
|
||||
self._suspended = False
|
||||
|
||||
def batch_isend_irecv(self, p2p_ops: list, stream=None):
|
||||
if self.disabled:
|
||||
return
|
||||
|
||||
@@ -329,6 +329,10 @@ class NCCLLibrary:
|
||||
# shutdown when peer ranks may already be gone.
|
||||
# ncclResult_t ncclCommAbort(ncclComm_t comm);
|
||||
Function("ncclCommAbort", ncclResult_t, [ncclComm_t]),
|
||||
# ncclResult_t ncclCommSuspend(ncclComm_t comm, int flags);
|
||||
Function("ncclCommSuspend", ncclResult_t, [ncclComm_t, ctypes.c_int]),
|
||||
# ncclResult_t ncclCommResume(ncclComm_t comm);
|
||||
Function("ncclCommResume", ncclResult_t, [ncclComm_t]),
|
||||
# ncclResult_t ncclGroupStart();
|
||||
Function("ncclGroupStart", ncclResult_t, []),
|
||||
# ncclResult_t ncclGroupEnd();
|
||||
@@ -417,10 +421,21 @@ class NCCLLibrary:
|
||||
elif func.name == "ncclCommQueryProperties":
|
||||
# Optional on NCCL versions older than 2.29.
|
||||
continue
|
||||
elif func.name in ("ncclCommSuspend", "ncclCommResume"):
|
||||
# RCCL doesn't export these; NCCL >= 2.29.7 does, and
|
||||
# vLLM's CUDA path already requires that version.
|
||||
# PyNcclCommunicator checks has_symbol() before
|
||||
# calling either.
|
||||
if current_platform.is_rocm():
|
||||
continue
|
||||
raise
|
||||
NCCLLibrary.path_to_dict_mapping[so_file] = _funcs
|
||||
self._funcs = NCCLLibrary.path_to_dict_mapping[so_file]
|
||||
|
||||
def has_symbol(self, name: str) -> bool:
|
||||
"""Whether the loaded NCCL/RCCL library exports the given symbol."""
|
||||
return name in self._funcs
|
||||
|
||||
def ncclGetErrorString(self, result: ncclResult_t) -> str:
|
||||
return self._funcs["ncclGetErrorString"](result).decode("utf-8")
|
||||
|
||||
@@ -599,6 +614,12 @@ class NCCLLibrary:
|
||||
def ncclCommAbort(self, comm: ncclComm_t) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclCommAbort"](comm))
|
||||
|
||||
def ncclCommSuspend(self, comm: ncclComm_t, flags: int) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclCommSuspend"](comm, flags))
|
||||
|
||||
def ncclCommResume(self, comm: ncclComm_t) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclCommResume"](comm))
|
||||
|
||||
def ncclGroupStart(self) -> None:
|
||||
self.NCCL_CHECK(self._funcs["ncclGroupStart"]())
|
||||
|
||||
|
||||
@@ -180,6 +180,16 @@ def _apply_to_device_comms(
|
||||
action(dc)
|
||||
|
||||
|
||||
def suspend_device_comms() -> None:
|
||||
"""Release device communicator memory (collective; comms must be idle)."""
|
||||
_apply_to_device_comms(lambda comm: comm.suspend())
|
||||
|
||||
|
||||
def resume_device_comms() -> None:
|
||||
"""Restore suspended device communicators (collective)."""
|
||||
_apply_to_device_comms(lambda comm: comm.resume())
|
||||
|
||||
|
||||
def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
|
||||
assert group_name in _groups, f"Group {group_name} is not found."
|
||||
group = _groups[group_name]()
|
||||
|
||||
@@ -710,6 +710,7 @@ class EngineArgs:
|
||||
generation_config: str = ModelConfig.generation_config
|
||||
enable_sleep_mode: bool = ModelConfig.enable_sleep_mode
|
||||
enable_cumem_allocator: bool = ModelConfig.enable_cumem_allocator
|
||||
enable_nccl_comm_suspend: bool = ModelConfig.enable_nccl_comm_suspend
|
||||
override_generation_config: dict[str, Any] = get_field(
|
||||
ModelConfig, "override_generation_config"
|
||||
)
|
||||
@@ -932,6 +933,10 @@ class EngineArgs:
|
||||
model_group.add_argument(
|
||||
"--enable-cumem-allocator", **model_kwargs["enable_cumem_allocator"]
|
||||
)
|
||||
model_group.add_argument(
|
||||
"--enable-nccl-comm-suspend",
|
||||
**model_kwargs["enable_nccl_comm_suspend"],
|
||||
)
|
||||
model_group.add_argument("--model-impl", **model_kwargs["model_impl"])
|
||||
model_group.add_argument(
|
||||
"--logits-processors", **model_kwargs["logits_processors"]
|
||||
@@ -1823,6 +1828,7 @@ class EngineArgs:
|
||||
override_generation_config=self.override_generation_config,
|
||||
enable_sleep_mode=self.enable_sleep_mode,
|
||||
enable_cumem_allocator=self.enable_cumem_allocator,
|
||||
enable_nccl_comm_suspend=self.enable_nccl_comm_suspend,
|
||||
model_impl=self.model_impl,
|
||||
logits_processors=self.logits_processors,
|
||||
video_pruning_rate=self.video_pruning_rate,
|
||||
|
||||
@@ -45,6 +45,8 @@ from vllm.distributed.parallel_state import (
|
||||
checkpoint_restore_distributed_state,
|
||||
get_pp_group,
|
||||
get_tp_group,
|
||||
resume_device_comms,
|
||||
suspend_device_comms,
|
||||
)
|
||||
from vllm.distributed.weight_transfer import (
|
||||
WeightTransferEngine,
|
||||
@@ -220,7 +222,8 @@ class Worker(WorkerBase):
|
||||
# Resolved lazily on first sleep/wake; persists worker-process state.
|
||||
self._sleep_mode_backend: SleepModeBackend | None = None
|
||||
|
||||
def _get_sleep_mode_backend(self) -> "SleepModeBackend":
|
||||
@property
|
||||
def sleep_mode_backend(self) -> "SleepModeBackend":
|
||||
if self._sleep_mode_backend is None:
|
||||
from vllm.device_allocator.sleep_mode_backend import (
|
||||
SleepModeBackendFactory,
|
||||
@@ -247,7 +250,9 @@ class Worker(WorkerBase):
|
||||
name: buffer.cpu().clone() for name, buffer in draft.named_buffers()
|
||||
}
|
||||
|
||||
self._get_sleep_mode_backend().suspend(level)
|
||||
self.sleep_mode_backend.suspend(level)
|
||||
if self.vllm_config.model_config.enable_nccl_comm_suspend:
|
||||
suspend_device_comms()
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
deadline = time.monotonic() + (5.0 if current_platform.is_rocm() else 0)
|
||||
@@ -267,7 +272,9 @@ class Worker(WorkerBase):
|
||||
)
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
self._get_sleep_mode_backend().resume(tags)
|
||||
self.sleep_mode_backend.resume(tags)
|
||||
if self.vllm_config.model_config.enable_nccl_comm_suspend:
|
||||
resume_device_comms()
|
||||
|
||||
# Restore the buffers after level 2 sleep
|
||||
wake_weights = tags is None or "weights" in tags
|
||||
|
||||
Reference in New Issue
Block a user