diff --git a/tests/distributed/test_pynccl.py b/tests/distributed/test_pynccl.py index 62eba4843a..91dccd5798 100644 --- a/tests/distributed/test_pynccl.py +++ b/tests/distributed/test_pynccl.py @@ -472,3 +472,42 @@ def test_ncclGetUniqueId(): # 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] # as long as the function doesn't raise an exception, we're good assert unique_id is not None + + +def test_pynccl_suspend_resume_idempotent(): + """Repeated suspend/resume (e.g. staged wake-ups) reach NCCL once each.""" + from unittest.mock import Mock + + comm = object.__new__(PyNcclCommunicator) + comm.disabled = False + comm._suspended = False + comm.comm = object() + comm.nccl = Mock() + + comm.suspend() + comm.suspend() + comm.resume() + comm.resume() + + assert comm.nccl.ncclCommSuspend.call_count == 1 + assert comm.nccl.ncclCommResume.call_count == 1 + + +def test_pynccl_suspend_noop_without_symbol(): + """RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not + crash, and resume() must then no-op too since nothing was suspended.""" + from unittest.mock import Mock + + comm = object.__new__(PyNcclCommunicator) + comm.disabled = False + comm._suspended = False + comm.comm = object() + comm.nccl = Mock() + comm.nccl.has_symbol.return_value = False + + comm.suspend() + comm.resume() + + comm.nccl.ncclCommSuspend.assert_not_called() + comm.nccl.ncclCommResume.assert_not_called() + assert comm._suspended is False diff --git a/tests/v1/worker/test_kv_cache_allocation_scope.py b/tests/v1/worker/test_kv_cache_allocation_scope.py index 81afe44275..1818f173d9 100644 --- a/tests/v1/worker/test_kv_cache_allocation_scope.py +++ b/tests/v1/worker/test_kv_cache_allocation_scope.py @@ -113,11 +113,14 @@ def test_kv_wake_does_not_run_model_runner_recovery() -> None: worker = cast( Worker, SimpleNamespace( - _get_sleep_mode_backend=lambda: SimpleNamespace(resume=lambda tags: None), + sleep_mode_backend=SimpleNamespace(resume=lambda tags: None), _sleep_saved_buffers={}, _sleep_saved_draft_buffers={}, model_runner=runner, synchronize_device=lambda: None, + vllm_config=SimpleNamespace( + model_config=SimpleNamespace(enable_nccl_comm_suspend=False) + ), ), ) layout_tensors = runner.layout_tensors diff --git a/tests/v1/worker/test_sleep_mode_backend.py b/tests/v1/worker/test_sleep_mode_backend.py index 4ec9f0bf12..e2514f84e1 100644 --- a/tests/v1/worker/test_sleep_mode_backend.py +++ b/tests/v1/worker/test_sleep_mode_backend.py @@ -38,6 +38,55 @@ def test_new_backend_starts_in_running_state(): assert CuMemBackend().state() == "RUNNING" +@pytest.mark.parametrize("enable_nccl_comm_suspend", [True, False]) +def test_worker_drives_communicator_suspension(monkeypatch, enable_nccl_comm_suspend): + """Comm walkers run around sleep/wake only when explicitly enabled.""" + from types import SimpleNamespace + + from vllm.v1.worker.gpu_worker import Worker + + calls: list[tuple[str, object]] = [] + + class Backend: + def suspend(self, level: int = 1) -> None: + calls.append(("backend.suspend", level)) + + def resume(self, tags: list[str] | None = None) -> None: + calls.append(("backend.resume", tuple(tags) if tags else None)) + + worker = object.__new__(Worker) + worker._sleep_mode_backend = Backend() + worker._sleep_saved_buffers = {} + worker._sleep_saved_draft_buffers = {} + worker.vllm_config = SimpleNamespace( + model_config=SimpleNamespace(enable_nccl_comm_suspend=enable_nccl_comm_suspend) + ) + + monkeypatch.setattr("torch.accelerator.synchronize", lambda: None) + monkeypatch.setattr("torch.accelerator.get_memory_info", lambda: (0, 0)) + monkeypatch.setattr( + "vllm.v1.worker.gpu_worker.suspend_device_comms", + lambda: calls.append(("comms.suspend", None)), + ) + monkeypatch.setattr( + "vllm.v1.worker.gpu_worker.resume_device_comms", + lambda: calls.append(("comms.resume", None)), + ) + + worker.sleep(level=1) + worker.wake_up(tags=["weights"]) + + expected = [ + ("backend.suspend", 1), + ("comms.suspend", None), + ("backend.resume", ("weights",)), + ("comms.resume", None), + ] + if not enable_nccl_comm_suspend: + expected = [c for c in expected if not c[0].startswith("comms.")] + assert calls == expected + + def test_unknown_backend_raises(): with pytest.raises(ValueError, match="Unsupported sleep-mode backend"): SleepModeBackendFactory.get_backend_class("does-not-exist") diff --git a/vllm/config/model.py b/vllm/config/model.py index 2b69d7ce4c..cfb2ce3e63 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -343,6 +343,10 @@ class ModelConfig: (default) uses the built-in ``CuMemAllocator`` and is behavior-compatible with prior releases. Additional backends (CUDA checkpoint, CRIU, durable snapshot) may be registered in-tree or by plugins (RFC #34303).""" + enable_nccl_comm_suspend: bool = False + """Enable releasing NCCL communicator memory during sleep mode + (``ncclCommSuspend``/``ncclCommResume``). Experimental; when disabled + (the default) sleep still releases weights/KV-cache memory as before.""" enable_cumem_allocator: bool = False """Enable the custom cumem allocator to leverage advanced GPU memory allocation features such as multi-node NVLink support. diff --git a/vllm/device_allocator/sleep_mode_backend.py b/vllm/device_allocator/sleep_mode_backend.py index 0f1e243344..45a635d613 100644 --- a/vllm/device_allocator/sleep_mode_backend.py +++ b/vllm/device_allocator/sleep_mode_backend.py @@ -134,8 +134,7 @@ class CuMemBackend(SleepModeBackend): @classmethod def preserves_communicators(cls) -> bool: - # Communicator buffers (e.g. NCCL) live outside CuMemAllocator's pool, so - # an allocator-level sleep leaves them intact (no reinit needed on resume). + # Comm identity/topology survive memory suspension; no reinit needed. return True diff --git a/vllm/distributed/device_communicators/base_device_communicator.py b/vllm/distributed/device_communicators/base_device_communicator.py index fc4def37f7..83ae25ff44 100644 --- a/vllm/distributed/device_communicators/base_device_communicator.py +++ b/vllm/distributed/device_communicators/base_device_communicator.py @@ -226,6 +226,12 @@ class DeviceCommunicatorBase: def checkpoint_restore(self) -> None: """Restore communicator state after checkpoint (default: no-op).""" + def suspend(self) -> None: + """Release reclaimable communicator memory (default: no-op).""" + + def resume(self) -> None: + """Restore memory released by ``suspend`` (default: no-op).""" + def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor: if dim < 0: # Convert negative dim to positive. diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index 5cef8cba3f..25b500b8fa 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -631,6 +631,14 @@ class CudaCommunicator(DeviceCommunicatorBase): self.all2all_manager.destroy() self.all2all_manager = None # type: ignore[assignment] + def suspend(self) -> None: + if self.pynccl_comm is not None: + self.pynccl_comm.suspend() + + def resume(self) -> None: + if self.pynccl_comm is not None: + self.pynccl_comm.resume() + def checkpoint_prepare(self) -> None: # Only FlashInfer all-reduce and FlashInfer all2all are supported for now. from .flashinfer_all_reduce import checkpoint_prepare_fi_ar_workspaces diff --git a/vllm/distributed/device_communicators/pynccl.py b/vllm/distributed/device_communicators/pynccl.py index 9415d215e1..9c8a3f6005 100644 --- a/vllm/distributed/device_communicators/pynccl.py +++ b/vllm/distributed/device_communicators/pynccl.py @@ -27,6 +27,8 @@ logger = init_logger(__name__) _NCCL_SYMM_OPS_REGISTERED = False +_NCCL_SUSPEND_MEM = 0x01 + def register_nccl_symmetric_ops(pynccl_comm): from vllm.distributed.device_communicators.pynccl_allocator import ( @@ -105,6 +107,7 @@ class PyNcclCommunicator: self.available = True self.disabled = False + self._suspended = False self.nccl_version = self.nccl.ncclGetRawVersion() if self.rank == 0: @@ -479,6 +482,27 @@ class PyNcclCommunicator: def deregister_comm_window(self, window): return self.nccl.ncclCommWindowDeregister(self.comm, window) + def suspend(self): + """Release comm GPU memory (collective, idempotent); keeps topology.""" + if self.disabled or self._suspended: + return + if not self.nccl.has_symbol("ncclCommSuspend"): + logger.warning_once( + "ncclCommSuspend is not available in the loaded NCCL/RCCL " + "library (requires NCCL >= 2.29.7); skipping communicator " + "memory suspension." + ) + return + self.nccl.ncclCommSuspend(self.comm, _NCCL_SUSPEND_MEM) + self._suspended = True + + def resume(self): + """Restore a suspended comm (collective); no-op unless suspended.""" + if self.disabled or not self._suspended: + return + self.nccl.ncclCommResume(self.comm) + self._suspended = False + def batch_isend_irecv(self, p2p_ops: list, stream=None): if self.disabled: return diff --git a/vllm/distributed/device_communicators/pynccl_wrapper.py b/vllm/distributed/device_communicators/pynccl_wrapper.py index 7056bc04ec..03ac549dd8 100644 --- a/vllm/distributed/device_communicators/pynccl_wrapper.py +++ b/vllm/distributed/device_communicators/pynccl_wrapper.py @@ -329,6 +329,10 @@ class NCCLLibrary: # shutdown when peer ranks may already be gone. # ncclResult_t ncclCommAbort(ncclComm_t comm); Function("ncclCommAbort", ncclResult_t, [ncclComm_t]), + # ncclResult_t ncclCommSuspend(ncclComm_t comm, int flags); + Function("ncclCommSuspend", ncclResult_t, [ncclComm_t, ctypes.c_int]), + # ncclResult_t ncclCommResume(ncclComm_t comm); + Function("ncclCommResume", ncclResult_t, [ncclComm_t]), # ncclResult_t ncclGroupStart(); Function("ncclGroupStart", ncclResult_t, []), # ncclResult_t ncclGroupEnd(); @@ -417,10 +421,21 @@ class NCCLLibrary: elif func.name == "ncclCommQueryProperties": # Optional on NCCL versions older than 2.29. continue + elif func.name in ("ncclCommSuspend", "ncclCommResume"): + # RCCL doesn't export these; NCCL >= 2.29.7 does, and + # vLLM's CUDA path already requires that version. + # PyNcclCommunicator checks has_symbol() before + # calling either. + if current_platform.is_rocm(): + continue raise NCCLLibrary.path_to_dict_mapping[so_file] = _funcs self._funcs = NCCLLibrary.path_to_dict_mapping[so_file] + def has_symbol(self, name: str) -> bool: + """Whether the loaded NCCL/RCCL library exports the given symbol.""" + return name in self._funcs + def ncclGetErrorString(self, result: ncclResult_t) -> str: return self._funcs["ncclGetErrorString"](result).decode("utf-8") @@ -599,6 +614,12 @@ class NCCLLibrary: def ncclCommAbort(self, comm: ncclComm_t) -> None: self.NCCL_CHECK(self._funcs["ncclCommAbort"](comm)) + def ncclCommSuspend(self, comm: ncclComm_t, flags: int) -> None: + self.NCCL_CHECK(self._funcs["ncclCommSuspend"](comm, flags)) + + def ncclCommResume(self, comm: ncclComm_t) -> None: + self.NCCL_CHECK(self._funcs["ncclCommResume"](comm)) + def ncclGroupStart(self) -> None: self.NCCL_CHECK(self._funcs["ncclGroupStart"]()) diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index c3d8e70358..9a0f79fcc2 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -180,6 +180,16 @@ def _apply_to_device_comms( action(dc) +def suspend_device_comms() -> None: + """Release device communicator memory (collective; comms must be idle).""" + _apply_to_device_comms(lambda comm: comm.suspend()) + + +def resume_device_comms() -> None: + """Restore suspended device communicators (collective).""" + _apply_to_device_comms(lambda comm: comm.resume()) + + def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor: assert group_name in _groups, f"Group {group_name} is not found." group = _groups[group_name]() diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 08ec2962f7..66373df6fc 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -710,6 +710,7 @@ class EngineArgs: generation_config: str = ModelConfig.generation_config enable_sleep_mode: bool = ModelConfig.enable_sleep_mode enable_cumem_allocator: bool = ModelConfig.enable_cumem_allocator + enable_nccl_comm_suspend: bool = ModelConfig.enable_nccl_comm_suspend override_generation_config: dict[str, Any] = get_field( ModelConfig, "override_generation_config" ) @@ -932,6 +933,10 @@ class EngineArgs: model_group.add_argument( "--enable-cumem-allocator", **model_kwargs["enable_cumem_allocator"] ) + model_group.add_argument( + "--enable-nccl-comm-suspend", + **model_kwargs["enable_nccl_comm_suspend"], + ) model_group.add_argument("--model-impl", **model_kwargs["model_impl"]) model_group.add_argument( "--logits-processors", **model_kwargs["logits_processors"] @@ -1823,6 +1828,7 @@ class EngineArgs: override_generation_config=self.override_generation_config, enable_sleep_mode=self.enable_sleep_mode, enable_cumem_allocator=self.enable_cumem_allocator, + enable_nccl_comm_suspend=self.enable_nccl_comm_suspend, model_impl=self.model_impl, logits_processors=self.logits_processors, video_pruning_rate=self.video_pruning_rate, diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 6b8fdb94ef..9ee217936c 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -45,6 +45,8 @@ from vllm.distributed.parallel_state import ( checkpoint_restore_distributed_state, get_pp_group, get_tp_group, + resume_device_comms, + suspend_device_comms, ) from vllm.distributed.weight_transfer import ( WeightTransferEngine, @@ -220,7 +222,8 @@ class Worker(WorkerBase): # Resolved lazily on first sleep/wake; persists worker-process state. self._sleep_mode_backend: SleepModeBackend | None = None - def _get_sleep_mode_backend(self) -> "SleepModeBackend": + @property + def sleep_mode_backend(self) -> "SleepModeBackend": if self._sleep_mode_backend is None: from vllm.device_allocator.sleep_mode_backend import ( SleepModeBackendFactory, @@ -247,7 +250,9 @@ class Worker(WorkerBase): name: buffer.cpu().clone() for name, buffer in draft.named_buffers() } - self._get_sleep_mode_backend().suspend(level) + self.sleep_mode_backend.suspend(level) + if self.vllm_config.model_config.enable_nccl_comm_suspend: + suspend_device_comms() torch.accelerator.synchronize() deadline = time.monotonic() + (5.0 if current_platform.is_rocm() else 0) @@ -267,7 +272,9 @@ class Worker(WorkerBase): ) def wake_up(self, tags: list[str] | None = None) -> None: - self._get_sleep_mode_backend().resume(tags) + self.sleep_mode_backend.resume(tags) + if self.vllm_config.model_config.enable_nccl_comm_suspend: + resume_device_comms() # Restore the buffers after level 2 sleep wake_weights = tags is None or "weights" in tags