[Core] Release NCCL communicator memory in sleep mode (#51485)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-01 14:31:16 +08:00
committed by GitHub
co-authored by aoshen02 Claude Fable 5
parent ec32f669bb
commit 4ac452ad98
12 changed files with 182 additions and 6 deletions
+39
View File
@@ -472,3 +472,42 @@ def test_ncclGetUniqueId():
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
# as long as the function doesn't raise an exception, we're good
assert unique_id is not None
def test_pynccl_suspend_resume_idempotent():
"""Repeated suspend/resume (e.g. staged wake-ups) reach NCCL once each."""
from unittest.mock import Mock
comm = object.__new__(PyNcclCommunicator)
comm.disabled = False
comm._suspended = False
comm.comm = object()
comm.nccl = Mock()
comm.suspend()
comm.suspend()
comm.resume()
comm.resume()
assert comm.nccl.ncclCommSuspend.call_count == 1
assert comm.nccl.ncclCommResume.call_count == 1
def test_pynccl_suspend_noop_without_symbol():
"""RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not
crash, and resume() must then no-op too since nothing was suspended."""
from unittest.mock import Mock
comm = object.__new__(PyNcclCommunicator)
comm.disabled = False
comm._suspended = False
comm.comm = object()
comm.nccl = Mock()
comm.nccl.has_symbol.return_value = False
comm.suspend()
comm.resume()
comm.nccl.ncclCommSuspend.assert_not_called()
comm.nccl.ncclCommResume.assert_not_called()
assert comm._suspended is False
@@ -113,11 +113,14 @@ def test_kv_wake_does_not_run_model_runner_recovery() -> None:
worker = cast(
Worker,
SimpleNamespace(
_get_sleep_mode_backend=lambda: SimpleNamespace(resume=lambda tags: None),
sleep_mode_backend=SimpleNamespace(resume=lambda tags: None),
_sleep_saved_buffers={},
_sleep_saved_draft_buffers={},
model_runner=runner,
synchronize_device=lambda: None,
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(enable_nccl_comm_suspend=False)
),
),
)
layout_tensors = runner.layout_tensors
@@ -38,6 +38,55 @@ def test_new_backend_starts_in_running_state():
assert CuMemBackend().state() == "RUNNING"
@pytest.mark.parametrize("enable_nccl_comm_suspend", [True, False])
def test_worker_drives_communicator_suspension(monkeypatch, enable_nccl_comm_suspend):
"""Comm walkers run around sleep/wake only when explicitly enabled."""
from types import SimpleNamespace
from vllm.v1.worker.gpu_worker import Worker
calls: list[tuple[str, object]] = []
class Backend:
def suspend(self, level: int = 1) -> None:
calls.append(("backend.suspend", level))
def resume(self, tags: list[str] | None = None) -> None:
calls.append(("backend.resume", tuple(tags) if tags else None))
worker = object.__new__(Worker)
worker._sleep_mode_backend = Backend()
worker._sleep_saved_buffers = {}
worker._sleep_saved_draft_buffers = {}
worker.vllm_config = SimpleNamespace(
model_config=SimpleNamespace(enable_nccl_comm_suspend=enable_nccl_comm_suspend)
)
monkeypatch.setattr("torch.accelerator.synchronize", lambda: None)
monkeypatch.setattr("torch.accelerator.get_memory_info", lambda: (0, 0))
monkeypatch.setattr(
"vllm.v1.worker.gpu_worker.suspend_device_comms",
lambda: calls.append(("comms.suspend", None)),
)
monkeypatch.setattr(
"vllm.v1.worker.gpu_worker.resume_device_comms",
lambda: calls.append(("comms.resume", None)),
)
worker.sleep(level=1)
worker.wake_up(tags=["weights"])
expected = [
("backend.suspend", 1),
("comms.suspend", None),
("backend.resume", ("weights",)),
("comms.resume", None),
]
if not enable_nccl_comm_suspend:
expected = [c for c in expected if not c[0].startswith("comms.")]
assert calls == expected
def test_unknown_backend_raises():
with pytest.raises(ValueError, match="Unsupported sleep-mode backend"):
SleepModeBackendFactory.get_backend_class("does-not-exist")