[Core][Distributed] Add process-checkpoint lifecycle hooks for communicators (starting with Flashinfer) (#46877)

Signed-off-by: Schwinn Saereesitthipitak <schwinns@nvidia.com>
This commit is contained in:
Schwinn Saereesitthipitak
2026-07-26 14:47:50 -04:00
committed by GitHub
parent b68d7ef262
commit b5b61c622c
10 changed files with 185 additions and 4 deletions
+40
View File
@@ -7,6 +7,7 @@ Run `pytest tests/distributed/test_comm_ops.py`.
from collections.abc import Callable
from typing import Any
from unittest.mock import Mock
import pytest
import ray
@@ -19,6 +20,8 @@ from vllm.distributed import (
tensor_model_parallel_all_reduce,
tensor_model_parallel_reduce_scatter,
)
from vllm.distributed.device_communicators import flashinfer_all_reduce
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
from vllm.distributed.parallel_state import GroupCoordinator, TensorMetadata
from vllm.v1.worker.gpu_worker import AsyncIntermediateTensors
@@ -278,6 +281,43 @@ def test_irecv_tensor_dict_send_allgather_postprocess_binds_keys(
torch.testing.assert_close(td["b"], torch.ones(4, dtype=torch.int32))
@pytest.mark.parametrize("aliased", [False, True])
def test_cuda_communicator_checkpoints_flashinfer_workspaces(
monkeypatch: pytest.MonkeyPatch,
aliased: bool,
) -> None:
group = object()
normal_workspace = Mock()
quant_workspace = normal_workspace if aliased else Mock()
unique_workspaces = (
[normal_workspace] if aliased else [normal_workspace, quant_workspace]
)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", normal_workspace)
monkeypatch.setattr(
flashinfer_all_reduce, "_fi_ar_quant_workspace", quant_workspace
)
monkeypatch.setattr(
flashinfer_all_reduce,
"_fi_ar_workspace_groups",
{id(workspace): group for workspace in unique_workspaces},
)
monkeypatch.setattr(
flashinfer_all_reduce, "TorchDistBackend", lambda group: group, raising=False
)
communicator = CudaCommunicator.__new__(CudaCommunicator)
communicator.cpu_group = group
communicator.fi_ar_comm = None
communicator.all2all_manager = None
communicator.checkpoint_prepare()
communicator.checkpoint_restore()
for workspace in unique_workspaces:
workspace.checkpoint_prepare.assert_called_once_with()
workspace.checkpoint_restore.assert_called_once_with(group)
def test_async_intermediate_tensors_lazy_wait() -> None:
work = _DummyWork()
post_calls = {"n": 0}
@@ -225,7 +225,7 @@ if flashinfer_comm is not None:
max_token_num=max_token_num,
hidden_dim=hidden_size,
dtype=allreduce_in.dtype,
group=get_tp_group().device_group,
group=get_tp_group().cpu_group,
)
assert workspace is not None, (
"Flashinfer allreduce workspace must be initialized when using flashinfer"
@@ -996,7 +996,7 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
)
return
self.hidden_dim = config.model_config.get_hidden_size()
self.group = get_tp_group().device_group
self.group = get_tp_group().cpu_group
rank = get_tensor_model_parallel_rank()
if flashinfer_comm is None:
logger.warning(
@@ -885,6 +885,20 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
self.mapping = None
self.initialized = False
def checkpoint_prepare(self) -> None:
if self.initialized:
assert self.moe_alltoall is not None
self.moe_alltoall.checkpoint_prepare()
def checkpoint_restore(self) -> None:
if self.initialized:
assert self.moe_alltoall is not None
from vllm.distributed.device_communicators.mnnvl_compat import (
CustomCommunicator,
)
self.moe_alltoall.checkpoint_restore(CustomCommunicator(self.cpu_group))
class MoriAll2AllManager(All2AllManagerBase):
def __init__(self, cpu_group, all2all_backend: str):
@@ -7,8 +7,11 @@ import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup
from vllm.logger import init_logger
from vllm.utils import is_moe_layer
logger = init_logger(__name__)
class Cache:
def __init__(self):
@@ -137,6 +140,18 @@ class All2AllManagerBase:
def max_sms_used(self) -> int | None:
return None # None means it could use the whole GPU
def checkpoint_prepare(self) -> None:
logger.warning_once(
"%s.checkpoint_prepare is not implemented; skipping.",
type(self).__name__,
)
def checkpoint_restore(self) -> None:
logger.warning_once(
"%s.checkpoint_restore is not implemented; skipping.",
type(self).__name__,
)
def combine(self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False):
raise NotImplementedError
@@ -216,6 +231,12 @@ class DeviceCommunicatorBase:
dist.all_reduce(input_, group=self.device_group)
return input_
def checkpoint_prepare(self) -> None:
"""Prepare reclaimable communicator state for checkpoint (default: no-op)."""
def checkpoint_restore(self) -> None:
"""Restore communicator state after checkpoint (default: no-op)."""
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
if dim < 0:
# Convert negative dim to positive.
@@ -571,6 +571,22 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.all2all_manager.destroy()
self.all2all_manager = None # type: ignore[assignment]
def checkpoint_prepare(self) -> None:
# Only FlashInfer all-reduce and FlashInfer all2all are supported for now.
from .flashinfer_all_reduce import checkpoint_prepare_fi_ar_workspaces
checkpoint_prepare_fi_ar_workspaces(self.cpu_group)
if self.all2all_manager is not None:
self.all2all_manager.checkpoint_prepare()
def checkpoint_restore(self) -> None:
# Only FlashInfer all-reduce and FlashInfer all2all are supported for now.
from .flashinfer_all_reduce import checkpoint_restore_fi_ar_workspaces
checkpoint_restore_fi_ar_workspaces(self.cpu_group)
if self.all2all_manager is not None:
self.all2all_manager.checkpoint_restore()
def all_gatherv(
self,
input_: torch.Tensor | list[torch.Tensor],
@@ -6,6 +6,7 @@ import atexit
import os
import random
import threading
from typing import Any
import torch
import torch.distributed as dist
@@ -39,6 +40,7 @@ _fi_ar_workspace = None
# allreduce backend or a fallback backend when the primary workspace is not
# available on the current topology.
_fi_ar_quant_workspace = None
_fi_ar_workspace_groups: dict[int, ProcessGroup] = {}
def _create_workspace(
@@ -81,6 +83,14 @@ def _create_workspace(
return None
finally:
random.setstate(rng_state)
workspace_id = id(workspace)
workspace_group = _fi_ar_workspace_groups.get(workspace_id)
if workspace_group is not None and workspace_group is not group:
raise RuntimeError(
"FlashInfer returned an all-reduce workspace already associated "
"with a different process group"
)
_fi_ar_workspace_groups[workspace_id] = group
logger.debug(
"Initialized FlashInfer All Reduce workspace: backend=%s, "
"world_size=%d, rank=%d, max_token_num=%d, hidden_dim=%d, dtype=%s",
@@ -111,7 +121,7 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]:
# Default to mnnvl for both single- and multi-node setups. The mnnvl
# cudagraph hang that previously forced single-node to trtllm
# (https://github.com/vllm-project/vllm/issues/35772) was fixed upstream in
# FlashInfer (>= 0.6.12, vLLM pins 0.6.13), so mnnvl is safe here. trtllm
# FlashInfer (>= 0.6.12, vLLM pins 0.6.15), so mnnvl is safe here. trtllm
# does not support multi-node allreduce, so mnnvl is required there anyway.
# mnnvl needs NVSwitch multicast; on single-node topologies without it,
# fall back to trtllm so fused allreduce stays enabled.
@@ -268,6 +278,36 @@ def destroy_fi_ar_workspace():
_fi_ar_quant_workspace.destroy()
_fi_ar_workspace = _fi_ar_quant_workspace = None
_fi_ar_workspace_groups.clear()
def _fi_ar_workspaces_for_group(group: ProcessGroup) -> list[Any]:
workspaces = [_fi_ar_workspace]
if _fi_ar_quant_workspace is not _fi_ar_workspace:
workspaces.append(_fi_ar_quant_workspace)
group_workspaces = []
for workspace in workspaces:
if workspace is None:
continue
workspace_group = _fi_ar_workspace_groups.get(id(workspace))
if workspace_group is None:
raise RuntimeError(
"FlashInfer all-reduce workspace process group was not retained"
)
if workspace_group is group:
group_workspaces.append(workspace)
return group_workspaces
def checkpoint_prepare_fi_ar_workspaces(group: ProcessGroup) -> None:
for workspace in _fi_ar_workspaces_for_group(group):
workspace.checkpoint_prepare()
def checkpoint_restore_fi_ar_workspaces(group: ProcessGroup) -> None:
for workspace in _fi_ar_workspaces_for_group(group):
workspace.checkpoint_restore(TorchDistBackend(group=group))
atexit.register(destroy_fi_ar_workspace)
+36
View File
@@ -127,6 +127,28 @@ def _register_group(group: "GroupCoordinator") -> None:
_groups[group.unique_name] = weakref.ref(group)
def _apply_to_device_comms(
action: Callable[[DeviceCommunicatorBase], None],
) -> None:
"""Apply ``action`` to every group's device communicator.
Walks the registered parallel groups and skips those without a device
communicator (absent at ``world_size == 1``).
"""
comms = []
for group_ref in _groups.values():
group = group_ref()
if group is None:
continue
dc = group.device_communicator
if dc is None:
continue
comms.append(dc)
for dc in comms:
action(dc)
def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
assert group_name in _groups, f"Group {group_name} is not found."
group = _groups[group_name]()
@@ -2028,6 +2050,20 @@ def prepare_communication_buffer_for_model(model: torch.nn.Module):
_EPLB.prepare_communication_buffer_for_model(model)
def checkpoint_prepare_distributed_state() -> None:
"""Prepare every device communicator for a process checkpoint."""
torch.accelerator.synchronize()
_apply_to_device_comms(lambda comm: comm.checkpoint_prepare())
torch.accelerator.synchronize()
def checkpoint_restore_distributed_state() -> None:
"""Restore every device communicator after a process checkpoint."""
torch.accelerator.synchronize()
_apply_to_device_comms(lambda comm: comm.checkpoint_restore())
torch.accelerator.synchronize()
def model_parallel_is_initialized():
"""Check if tensor and pipeline parallel groups are initialized."""
return _TP is not None and _PP is not None
@@ -93,7 +93,7 @@ def _can_use_flashinfer(hidden_states: torch.Tensor, tp_size: int) -> tuple[bool
max_token_num=max_token_num,
hidden_dim=hidden_size,
dtype=hidden_states.dtype,
group=get_tp_group().device_group,
group=get_tp_group().cpu_group,
)
if workspace is None:
return False, 0
+6
View File
@@ -943,6 +943,12 @@ class AsyncLLM(EngineClient):
if self.logger_manager is not None:
self.logger_manager.record_sleep_state(0, 0)
async def checkpoint_prepare(self) -> None:
await self.collective_rpc("checkpoint_prepare")
async def checkpoint_restore(self) -> None:
await self.collective_rpc("checkpoint_restore")
async def is_sleeping(self) -> bool:
return await self.engine_core.is_sleeping_async()
+8
View File
@@ -41,6 +41,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
)
from vllm.distributed.parallel_state import (
Handle,
checkpoint_prepare_distributed_state,
checkpoint_restore_distributed_state,
get_pp_group,
get_tp_group,
)
@@ -244,6 +246,12 @@ class Worker(WorkerBase):
if tags is None or "kv_cache" in tags:
self.model_runner.post_kv_cache_wake_up()
def checkpoint_prepare(self) -> None:
checkpoint_prepare_distributed_state()
def checkpoint_restore(self) -> None:
checkpoint_restore_distributed_state()
def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager:
if (
current_platform.is_cuda_alike()