[Core] Release NCCL communicator memory in sleep mode (#51485)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Fable 5
parent
ec32f669bb
commit
4ac452ad98
@@ -180,6 +180,16 @@ def _apply_to_device_comms(
|
||||
action(dc)
|
||||
|
||||
|
||||
def suspend_device_comms() -> None:
|
||||
"""Release device communicator memory (collective; comms must be idle)."""
|
||||
_apply_to_device_comms(lambda comm: comm.suspend())
|
||||
|
||||
|
||||
def resume_device_comms() -> None:
|
||||
"""Restore suspended device communicators (collective)."""
|
||||
_apply_to_device_comms(lambda comm: comm.resume())
|
||||
|
||||
|
||||
def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
|
||||
assert group_name in _groups, f"Group {group_name} is not found."
|
||||
group = _groups[group_name]()
|
||||
|
||||
Reference in New Issue
Block a user