[Core][Distributed] Add process-checkpoint lifecycle hooks for communicators (starting with Flashinfer) (#46877)

Signed-off-by: Schwinn Saereesitthipitak <schwinns@nvidia.com>
This commit is contained in:
Schwinn Saereesitthipitak
2026-07-26 14:47:50 -04:00
committed by GitHub
parent b68d7ef262
commit b5b61c622c
10 changed files with 185 additions and 4 deletions
+40
View File
@@ -7,6 +7,7 @@ Run `pytest tests/distributed/test_comm_ops.py`.
from collections.abc import Callable
from typing import Any
from unittest.mock import Mock
import pytest
import ray
@@ -19,6 +20,8 @@ from vllm.distributed import (
tensor_model_parallel_all_reduce,
tensor_model_parallel_reduce_scatter,
)
from vllm.distributed.device_communicators import flashinfer_all_reduce
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
from vllm.distributed.parallel_state import GroupCoordinator, TensorMetadata
from vllm.v1.worker.gpu_worker import AsyncIntermediateTensors
@@ -278,6 +281,43 @@ def test_irecv_tensor_dict_send_allgather_postprocess_binds_keys(
torch.testing.assert_close(td["b"], torch.ones(4, dtype=torch.int32))
@pytest.mark.parametrize("aliased", [False, True])
def test_cuda_communicator_checkpoints_flashinfer_workspaces(
monkeypatch: pytest.MonkeyPatch,
aliased: bool,
) -> None:
group = object()
normal_workspace = Mock()
quant_workspace = normal_workspace if aliased else Mock()
unique_workspaces = (
[normal_workspace] if aliased else [normal_workspace, quant_workspace]
)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", normal_workspace)
monkeypatch.setattr(
flashinfer_all_reduce, "_fi_ar_quant_workspace", quant_workspace
)
monkeypatch.setattr(
flashinfer_all_reduce,
"_fi_ar_workspace_groups",
{id(workspace): group for workspace in unique_workspaces},
)
monkeypatch.setattr(
flashinfer_all_reduce, "TorchDistBackend", lambda group: group, raising=False
)
communicator = CudaCommunicator.__new__(CudaCommunicator)
communicator.cpu_group = group
communicator.fi_ar_comm = None
communicator.all2all_manager = None
communicator.checkpoint_prepare()
communicator.checkpoint_restore()
for workspace in unique_workspaces:
workspace.checkpoint_prepare.assert_called_once_with()
workspace.checkpoint_restore.assert_called_once_with(group)
def test_async_intermediate_tensors_lazy_wait() -> None:
work = _DummyWork()
post_calls = {"n": 0}