diff --git a/tests/distributed/test_comm_ops.py b/tests/distributed/test_comm_ops.py index 253ec8a188..78aee1efd1 100644 --- a/tests/distributed/test_comm_ops.py +++ b/tests/distributed/test_comm_ops.py @@ -325,6 +325,77 @@ def test_cuda_communicator_checkpoints_flashinfer_workspaces( workspace.checkpoint_restore.assert_called_once_with(group) +def _fi_ar_communicator( + monkeypatch: pytest.MonkeyPatch, group: object, normal: Mock, quant: Mock +) -> CudaCommunicator: + monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", normal) + monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", quant) + monkeypatch.setattr( + flashinfer_all_reduce, + "_fi_ar_workspace_groups", + {id(normal): group, id(quant): group}, + ) + monkeypatch.setattr( + flashinfer_all_reduce, "TorchDistBackend", lambda group: group, raising=False + ) + communicator = CudaCommunicator.__new__(CudaCommunicator) + communicator.cpu_group = group + communicator.pynccl_comm = Mock() + communicator.all2all_manager = None + return communicator + + +def _symm_mem_workspace() -> Mock: + workspace = Mock() + workspace.checkpoint_prepare.side_effect = NotImplementedError + workspace.checkpoint_restore.side_effect = NotImplementedError + return workspace + + +@pytest.mark.parametrize("symm_mem", [False, True], ids=["mnnvl", "symm-mem+mnnvl"]) +def test_cuda_communicator_suspend_detaches_flashinfer_workspaces( + monkeypatch: pytest.MonkeyPatch, + symm_mem: bool, +) -> None: + group = object() + workspaces = (_symm_mem_workspace() if symm_mem else Mock(), Mock()) + communicator = _fi_ar_communicator(monkeypatch, group, *workspaces) + communicator.suspend() + communicator.resume() + + for workspace in workspaces: + workspace.checkpoint_prepare.assert_called_once_with() + workspace.checkpoint_restore.assert_called_once_with(group) + communicator.pynccl_comm.suspend.assert_called_once_with() + communicator.pynccl_comm.resume.assert_called_once_with() + + +def test_cuda_communicator_checkpoint_rejects_symm_mem_workspace( + monkeypatch: pytest.MonkeyPatch, +) -> None: + communicator = _fi_ar_communicator( + monkeypatch, object(), _symm_mem_workspace(), Mock() + ) + with pytest.raises(NotImplementedError): + communicator.checkpoint_prepare() + with pytest.raises(NotImplementedError): + communicator.checkpoint_restore() + + +def test_cuda_communicator_resume_retries_failed_restore( + monkeypatch: pytest.MonkeyPatch, +) -> None: + workspace = Mock() + workspace.checkpoint_restore.side_effect = [RuntimeError("cuMemCreate"), None] + communicator = _fi_ar_communicator(monkeypatch, object(), workspace, Mock()) + communicator.suspend() + with pytest.raises(RuntimeError): + communicator.resume() + communicator.resume() + + assert workspace.checkpoint_restore.call_count == 2 + + @pytest.mark.parametrize( ("backend", "capability", "world_size", "nodes", "expected"), [ diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index 992d07e15d..29d5bae89d 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -814,12 +814,19 @@ class CudaCommunicator(DeviceCommunicatorBase): return bd is not None and bd.uniform def suspend(self) -> None: + from .flashinfer_all_reduce import checkpoint_prepare_fi_ar_workspaces + + # FlashInfer AR syncs over the gloo cpu_group, so order vs. NCCL is free. + checkpoint_prepare_fi_ar_workspaces(self.cpu_group, skip_unsupported=True) if self.pynccl_comm is not None: self.pynccl_comm.suspend() def resume(self) -> None: + from .flashinfer_all_reduce import checkpoint_restore_fi_ar_workspaces + if self.pynccl_comm is not None: self.pynccl_comm.resume() + checkpoint_restore_fi_ar_workspaces(self.cpu_group, skip_unsupported=True) def checkpoint_prepare(self) -> None: # Only FlashInfer all-reduce and FlashInfer all2all are supported for now. diff --git a/vllm/distributed/device_communicators/flashinfer_all_reduce.py b/vllm/distributed/device_communicators/flashinfer_all_reduce.py index d51e9e0c0e..22bcbb775a 100644 --- a/vllm/distributed/device_communicators/flashinfer_all_reduce.py +++ b/vllm/distributed/device_communicators/flashinfer_all_reduce.py @@ -360,14 +360,30 @@ def _fi_ar_workspaces_for_group(group: ProcessGroup) -> list[Any]: return group_workspaces -def checkpoint_prepare_fi_ar_workspaces(group: ProcessGroup) -> None: +def checkpoint_prepare_fi_ar_workspaces( + group: ProcessGroup, skip_unsupported: bool = False +) -> None: for workspace in _fi_ar_workspaces_for_group(group): - workspace.checkpoint_prepare() + try: + workspace.checkpoint_prepare() + except NotImplementedError: + if not skip_unsupported: + raise + logger.info_once( + "FlashInfer all-reduce workspace backed by torch symmetric memory " + "stays resident during suspend." + ) -def checkpoint_restore_fi_ar_workspaces(group: ProcessGroup) -> None: +def checkpoint_restore_fi_ar_workspaces( + group: ProcessGroup, skip_unsupported: bool = False +) -> None: for workspace in _fi_ar_workspaces_for_group(group): - workspace.checkpoint_restore(TorchDistBackend(group=group)) + try: + workspace.checkpoint_restore(TorchDistBackend(group=group)) + except NotImplementedError: + if not skip_unsupported: + raise atexit.register(destroy_fi_ar_workspace)