[Core] Release FlashInfer allreduce workspace with --enable-nccl-comm-suspend (#59360)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
193922d6ff
commit
1dfe9fbdae
@@ -325,6 +325,77 @@ def test_cuda_communicator_checkpoints_flashinfer_workspaces(
|
||||
workspace.checkpoint_restore.assert_called_once_with(group)
|
||||
|
||||
|
||||
def _fi_ar_communicator(
|
||||
monkeypatch: pytest.MonkeyPatch, group: object, normal: Mock, quant: Mock
|
||||
) -> CudaCommunicator:
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", normal)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", quant)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"_fi_ar_workspace_groups",
|
||||
{id(normal): group, id(quant): group},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce, "TorchDistBackend", lambda group: group, raising=False
|
||||
)
|
||||
communicator = CudaCommunicator.__new__(CudaCommunicator)
|
||||
communicator.cpu_group = group
|
||||
communicator.pynccl_comm = Mock()
|
||||
communicator.all2all_manager = None
|
||||
return communicator
|
||||
|
||||
|
||||
def _symm_mem_workspace() -> Mock:
|
||||
workspace = Mock()
|
||||
workspace.checkpoint_prepare.side_effect = NotImplementedError
|
||||
workspace.checkpoint_restore.side_effect = NotImplementedError
|
||||
return workspace
|
||||
|
||||
|
||||
@pytest.mark.parametrize("symm_mem", [False, True], ids=["mnnvl", "symm-mem+mnnvl"])
|
||||
def test_cuda_communicator_suspend_detaches_flashinfer_workspaces(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
symm_mem: bool,
|
||||
) -> None:
|
||||
group = object()
|
||||
workspaces = (_symm_mem_workspace() if symm_mem else Mock(), Mock())
|
||||
communicator = _fi_ar_communicator(monkeypatch, group, *workspaces)
|
||||
communicator.suspend()
|
||||
communicator.resume()
|
||||
|
||||
for workspace in workspaces:
|
||||
workspace.checkpoint_prepare.assert_called_once_with()
|
||||
workspace.checkpoint_restore.assert_called_once_with(group)
|
||||
communicator.pynccl_comm.suspend.assert_called_once_with()
|
||||
communicator.pynccl_comm.resume.assert_called_once_with()
|
||||
|
||||
|
||||
def test_cuda_communicator_checkpoint_rejects_symm_mem_workspace(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
communicator = _fi_ar_communicator(
|
||||
monkeypatch, object(), _symm_mem_workspace(), Mock()
|
||||
)
|
||||
with pytest.raises(NotImplementedError):
|
||||
communicator.checkpoint_prepare()
|
||||
with pytest.raises(NotImplementedError):
|
||||
communicator.checkpoint_restore()
|
||||
|
||||
|
||||
def test_cuda_communicator_resume_retries_failed_restore(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
workspace = Mock()
|
||||
workspace.checkpoint_restore.side_effect = [RuntimeError("cuMemCreate"), None]
|
||||
communicator = _fi_ar_communicator(monkeypatch, object(), workspace, Mock())
|
||||
communicator.suspend()
|
||||
with pytest.raises(RuntimeError):
|
||||
communicator.resume()
|
||||
communicator.resume()
|
||||
|
||||
assert workspace.checkpoint_restore.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "capability", "world_size", "nodes", "expected"),
|
||||
[
|
||||
|
||||
@@ -814,12 +814,19 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
return bd is not None and bd.uniform
|
||||
|
||||
def suspend(self) -> None:
|
||||
from .flashinfer_all_reduce import checkpoint_prepare_fi_ar_workspaces
|
||||
|
||||
# FlashInfer AR syncs over the gloo cpu_group, so order vs. NCCL is free.
|
||||
checkpoint_prepare_fi_ar_workspaces(self.cpu_group, skip_unsupported=True)
|
||||
if self.pynccl_comm is not None:
|
||||
self.pynccl_comm.suspend()
|
||||
|
||||
def resume(self) -> None:
|
||||
from .flashinfer_all_reduce import checkpoint_restore_fi_ar_workspaces
|
||||
|
||||
if self.pynccl_comm is not None:
|
||||
self.pynccl_comm.resume()
|
||||
checkpoint_restore_fi_ar_workspaces(self.cpu_group, skip_unsupported=True)
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
# Only FlashInfer all-reduce and FlashInfer all2all are supported for now.
|
||||
|
||||
@@ -360,14 +360,30 @@ def _fi_ar_workspaces_for_group(group: ProcessGroup) -> list[Any]:
|
||||
return group_workspaces
|
||||
|
||||
|
||||
def checkpoint_prepare_fi_ar_workspaces(group: ProcessGroup) -> None:
|
||||
def checkpoint_prepare_fi_ar_workspaces(
|
||||
group: ProcessGroup, skip_unsupported: bool = False
|
||||
) -> None:
|
||||
for workspace in _fi_ar_workspaces_for_group(group):
|
||||
workspace.checkpoint_prepare()
|
||||
try:
|
||||
workspace.checkpoint_prepare()
|
||||
except NotImplementedError:
|
||||
if not skip_unsupported:
|
||||
raise
|
||||
logger.info_once(
|
||||
"FlashInfer all-reduce workspace backed by torch symmetric memory "
|
||||
"stays resident during suspend."
|
||||
)
|
||||
|
||||
|
||||
def checkpoint_restore_fi_ar_workspaces(group: ProcessGroup) -> None:
|
||||
def checkpoint_restore_fi_ar_workspaces(
|
||||
group: ProcessGroup, skip_unsupported: bool = False
|
||||
) -> None:
|
||||
for workspace in _fi_ar_workspaces_for_group(group):
|
||||
workspace.checkpoint_restore(TorchDistBackend(group=group))
|
||||
try:
|
||||
workspace.checkpoint_restore(TorchDistBackend(group=group))
|
||||
except NotImplementedError:
|
||||
if not skip_unsupported:
|
||||
raise
|
||||
|
||||
|
||||
atexit.register(destroy_fi_ar_workspace)
|
||||
|
||||
Reference in New Issue
Block a user