[Core] Release NCCL communicator memory in sleep mode (#51485)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Fable 5
parent
ec32f669bb
commit
4ac452ad98
@@ -472,3 +472,42 @@ def test_ncclGetUniqueId():
|
||||
# 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
|
||||
# as long as the function doesn't raise an exception, we're good
|
||||
assert unique_id is not None
|
||||
|
||||
|
||||
def test_pynccl_suspend_resume_idempotent():
|
||||
"""Repeated suspend/resume (e.g. staged wake-ups) reach NCCL once each."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
comm = object.__new__(PyNcclCommunicator)
|
||||
comm.disabled = False
|
||||
comm._suspended = False
|
||||
comm.comm = object()
|
||||
comm.nccl = Mock()
|
||||
|
||||
comm.suspend()
|
||||
comm.suspend()
|
||||
comm.resume()
|
||||
comm.resume()
|
||||
|
||||
assert comm.nccl.ncclCommSuspend.call_count == 1
|
||||
assert comm.nccl.ncclCommResume.call_count == 1
|
||||
|
||||
|
||||
def test_pynccl_suspend_noop_without_symbol():
|
||||
"""RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not
|
||||
crash, and resume() must then no-op too since nothing was suspended."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
comm = object.__new__(PyNcclCommunicator)
|
||||
comm.disabled = False
|
||||
comm._suspended = False
|
||||
comm.comm = object()
|
||||
comm.nccl = Mock()
|
||||
comm.nccl.has_symbol.return_value = False
|
||||
|
||||
comm.suspend()
|
||||
comm.resume()
|
||||
|
||||
comm.nccl.ncclCommSuspend.assert_not_called()
|
||||
comm.nccl.ncclCommResume.assert_not_called()
|
||||
assert comm._suspended is False
|
||||
|
||||
Reference in New Issue
Block a user