[Core] Release NCCL communicator memory in sleep mode (#51485)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Fable 5
parent
ec32f669bb
commit
4ac452ad98
@@ -472,3 +472,42 @@ def test_ncclGetUniqueId():
|
||||
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
|
||||
# as long as the function doesn't raise an exception, we're good
|
||||
assert unique_id is not None
|
||||
|
||||
|
||||
def test_pynccl_suspend_resume_idempotent():
|
||||
"""Repeated suspend/resume (e.g. staged wake-ups) reach NCCL once each."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
comm = object.__new__(PyNcclCommunicator)
|
||||
comm.disabled = False
|
||||
comm._suspended = False
|
||||
comm.comm = object()
|
||||
comm.nccl = Mock()
|
||||
|
||||
comm.suspend()
|
||||
comm.suspend()
|
||||
comm.resume()
|
||||
comm.resume()
|
||||
|
||||
assert comm.nccl.ncclCommSuspend.call_count == 1
|
||||
assert comm.nccl.ncclCommResume.call_count == 1
|
||||
|
||||
|
||||
def test_pynccl_suspend_noop_without_symbol():
|
||||
"""RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not
|
||||
crash, and resume() must then no-op too since nothing was suspended."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
comm = object.__new__(PyNcclCommunicator)
|
||||
comm.disabled = False
|
||||
comm._suspended = False
|
||||
comm.comm = object()
|
||||
comm.nccl = Mock()
|
||||
comm.nccl.has_symbol.return_value = False
|
||||
|
||||
comm.suspend()
|
||||
comm.resume()
|
||||
|
||||
comm.nccl.ncclCommSuspend.assert_not_called()
|
||||
comm.nccl.ncclCommResume.assert_not_called()
|
||||
assert comm._suspended is False
|
||||
|
||||
@@ -113,11 +113,14 @@ def test_kv_wake_does_not_run_model_runner_recovery() -> None:
|
||||
worker = cast(
|
||||
Worker,
|
||||
SimpleNamespace(
|
||||
_get_sleep_mode_backend=lambda: SimpleNamespace(resume=lambda tags: None),
|
||||
sleep_mode_backend=SimpleNamespace(resume=lambda tags: None),
|
||||
_sleep_saved_buffers={},
|
||||
_sleep_saved_draft_buffers={},
|
||||
model_runner=runner,
|
||||
synchronize_device=lambda: None,
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(enable_nccl_comm_suspend=False)
|
||||
),
|
||||
),
|
||||
)
|
||||
layout_tensors = runner.layout_tensors
|
||||
|
||||
@@ -38,6 +38,55 @@ def test_new_backend_starts_in_running_state():
|
||||
assert CuMemBackend().state() == "RUNNING"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_nccl_comm_suspend", [True, False])
|
||||
def test_worker_drives_communicator_suspension(monkeypatch, enable_nccl_comm_suspend):
|
||||
"""Comm walkers run around sleep/wake only when explicitly enabled."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
calls: list[tuple[str, object]] = []
|
||||
|
||||
class Backend:
|
||||
def suspend(self, level: int = 1) -> None:
|
||||
calls.append(("backend.suspend", level))
|
||||
|
||||
def resume(self, tags: list[str] | None = None) -> None:
|
||||
calls.append(("backend.resume", tuple(tags) if tags else None))
|
||||
|
||||
worker = object.__new__(Worker)
|
||||
worker._sleep_mode_backend = Backend()
|
||||
worker._sleep_saved_buffers = {}
|
||||
worker._sleep_saved_draft_buffers = {}
|
||||
worker.vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(enable_nccl_comm_suspend=enable_nccl_comm_suspend)
|
||||
)
|
||||
|
||||
monkeypatch.setattr("torch.accelerator.synchronize", lambda: None)
|
||||
monkeypatch.setattr("torch.accelerator.get_memory_info", lambda: (0, 0))
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu_worker.suspend_device_comms",
|
||||
lambda: calls.append(("comms.suspend", None)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu_worker.resume_device_comms",
|
||||
lambda: calls.append(("comms.resume", None)),
|
||||
)
|
||||
|
||||
worker.sleep(level=1)
|
||||
worker.wake_up(tags=["weights"])
|
||||
|
||||
expected = [
|
||||
("backend.suspend", 1),
|
||||
("comms.suspend", None),
|
||||
("backend.resume", ("weights",)),
|
||||
("comms.resume", None),
|
||||
]
|
||||
if not enable_nccl_comm_suspend:
|
||||
expected = [c for c in expected if not c[0].startswith("comms.")]
|
||||
assert calls == expected
|
||||
|
||||
|
||||
def test_unknown_backend_raises():
|
||||
with pytest.raises(ValueError, match="Unsupported sleep-mode backend"):
|
||||
SleepModeBackendFactory.get_backend_class("does-not-exist")
|
||||
|
||||
Reference in New Issue
Block a user