From b5b61c622c941426cf78a83672e878b5b829fe90 Mon Sep 17 00:00:00 2001 From: Schwinn Saereesitthipitak <17022745+galletas1712@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:47:50 -0700 Subject: [PATCH] [Core][Distributed] Add process-checkpoint lifecycle hooks for communicators (starting with Flashinfer) (#46877) Signed-off-by: Schwinn Saereesitthipitak --- tests/distributed/test_comm_ops.py | 40 ++++++++++++++++++ .../passes/fusion/allreduce_rms_fusion.py | 4 +- .../device_communicators/all2all.py | 14 +++++++ .../base_device_communicator.py | 21 ++++++++++ .../device_communicators/cuda_communicator.py | 16 +++++++ .../flashinfer_all_reduce.py | 42 ++++++++++++++++++- vllm/distributed/parallel_state.py | 36 ++++++++++++++++ .../layers/fused_allreduce_gemma_rms_norm.py | 2 +- vllm/v1/engine/async_llm.py | 6 +++ vllm/v1/worker/gpu_worker.py | 8 ++++ 10 files changed, 185 insertions(+), 4 deletions(-) diff --git a/tests/distributed/test_comm_ops.py b/tests/distributed/test_comm_ops.py index 48b007da66..19a095c7c6 100644 --- a/tests/distributed/test_comm_ops.py +++ b/tests/distributed/test_comm_ops.py @@ -7,6 +7,7 @@ Run `pytest tests/distributed/test_comm_ops.py`. from collections.abc import Callable from typing import Any +from unittest.mock import Mock import pytest import ray @@ -19,6 +20,8 @@ from vllm.distributed import ( tensor_model_parallel_all_reduce, tensor_model_parallel_reduce_scatter, ) +from vllm.distributed.device_communicators import flashinfer_all_reduce +from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator from vllm.distributed.parallel_state import GroupCoordinator, TensorMetadata from vllm.v1.worker.gpu_worker import AsyncIntermediateTensors @@ -278,6 +281,43 @@ def test_irecv_tensor_dict_send_allgather_postprocess_binds_keys( torch.testing.assert_close(td["b"], torch.ones(4, dtype=torch.int32)) +@pytest.mark.parametrize("aliased", [False, True]) +def test_cuda_communicator_checkpoints_flashinfer_workspaces( + monkeypatch: pytest.MonkeyPatch, + aliased: bool, +) -> None: + group = object() + normal_workspace = Mock() + quant_workspace = normal_workspace if aliased else Mock() + unique_workspaces = ( + [normal_workspace] if aliased else [normal_workspace, quant_workspace] + ) + + monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", normal_workspace) + monkeypatch.setattr( + flashinfer_all_reduce, "_fi_ar_quant_workspace", quant_workspace + ) + monkeypatch.setattr( + flashinfer_all_reduce, + "_fi_ar_workspace_groups", + {id(workspace): group for workspace in unique_workspaces}, + ) + monkeypatch.setattr( + flashinfer_all_reduce, "TorchDistBackend", lambda group: group, raising=False + ) + + communicator = CudaCommunicator.__new__(CudaCommunicator) + communicator.cpu_group = group + communicator.fi_ar_comm = None + communicator.all2all_manager = None + communicator.checkpoint_prepare() + communicator.checkpoint_restore() + + for workspace in unique_workspaces: + workspace.checkpoint_prepare.assert_called_once_with() + workspace.checkpoint_restore.assert_called_once_with(group) + + def test_async_intermediate_tensors_lazy_wait() -> None: work = _DummyWork() post_calls = {"n": 0} diff --git a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py index 4716e7012f..f7ff7df66c 100644 --- a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py +++ b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py @@ -225,7 +225,7 @@ if flashinfer_comm is not None: max_token_num=max_token_num, hidden_dim=hidden_size, dtype=allreduce_in.dtype, - group=get_tp_group().device_group, + group=get_tp_group().cpu_group, ) assert workspace is not None, ( "Flashinfer allreduce workspace must be initialized when using flashinfer" @@ -996,7 +996,7 @@ class AllReduceFusionPass(VllmPatternMatcherPass): ) return self.hidden_dim = config.model_config.get_hidden_size() - self.group = get_tp_group().device_group + self.group = get_tp_group().cpu_group rank = get_tensor_model_parallel_rank() if flashinfer_comm is None: logger.warning( diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index ee404f688a..5abe7568a2 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -885,6 +885,20 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase): self.mapping = None self.initialized = False + def checkpoint_prepare(self) -> None: + if self.initialized: + assert self.moe_alltoall is not None + self.moe_alltoall.checkpoint_prepare() + + def checkpoint_restore(self) -> None: + if self.initialized: + assert self.moe_alltoall is not None + from vllm.distributed.device_communicators.mnnvl_compat import ( + CustomCommunicator, + ) + + self.moe_alltoall.checkpoint_restore(CustomCommunicator(self.cpu_group)) + class MoriAll2AllManager(All2AllManagerBase): def __init__(self, cpu_group, all2all_backend: str): diff --git a/vllm/distributed/device_communicators/base_device_communicator.py b/vllm/distributed/device_communicators/base_device_communicator.py index dc26714338..45438a5469 100644 --- a/vllm/distributed/device_communicators/base_device_communicator.py +++ b/vllm/distributed/device_communicators/base_device_communicator.py @@ -7,8 +7,11 @@ import torch import torch.distributed as dist from torch.distributed import ProcessGroup +from vllm.logger import init_logger from vllm.utils import is_moe_layer +logger = init_logger(__name__) + class Cache: def __init__(self): @@ -137,6 +140,18 @@ class All2AllManagerBase: def max_sms_used(self) -> int | None: return None # None means it could use the whole GPU + def checkpoint_prepare(self) -> None: + logger.warning_once( + "%s.checkpoint_prepare is not implemented; skipping.", + type(self).__name__, + ) + + def checkpoint_restore(self) -> None: + logger.warning_once( + "%s.checkpoint_restore is not implemented; skipping.", + type(self).__name__, + ) + def combine(self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False): raise NotImplementedError @@ -216,6 +231,12 @@ class DeviceCommunicatorBase: dist.all_reduce(input_, group=self.device_group) return input_ + def checkpoint_prepare(self) -> None: + """Prepare reclaimable communicator state for checkpoint (default: no-op).""" + + def checkpoint_restore(self) -> None: + """Restore communicator state after checkpoint (default: no-op).""" + def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor: if dim < 0: # Convert negative dim to positive. diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index b613819477..fccc6ba60c 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -571,6 +571,22 @@ class CudaCommunicator(DeviceCommunicatorBase): self.all2all_manager.destroy() self.all2all_manager = None # type: ignore[assignment] + def checkpoint_prepare(self) -> None: + # Only FlashInfer all-reduce and FlashInfer all2all are supported for now. + from .flashinfer_all_reduce import checkpoint_prepare_fi_ar_workspaces + + checkpoint_prepare_fi_ar_workspaces(self.cpu_group) + if self.all2all_manager is not None: + self.all2all_manager.checkpoint_prepare() + + def checkpoint_restore(self) -> None: + # Only FlashInfer all-reduce and FlashInfer all2all are supported for now. + from .flashinfer_all_reduce import checkpoint_restore_fi_ar_workspaces + + checkpoint_restore_fi_ar_workspaces(self.cpu_group) + if self.all2all_manager is not None: + self.all2all_manager.checkpoint_restore() + def all_gatherv( self, input_: torch.Tensor | list[torch.Tensor], diff --git a/vllm/distributed/device_communicators/flashinfer_all_reduce.py b/vllm/distributed/device_communicators/flashinfer_all_reduce.py index 881b265d24..8d3e817092 100644 --- a/vllm/distributed/device_communicators/flashinfer_all_reduce.py +++ b/vllm/distributed/device_communicators/flashinfer_all_reduce.py @@ -6,6 +6,7 @@ import atexit import os import random import threading +from typing import Any import torch import torch.distributed as dist @@ -39,6 +40,7 @@ _fi_ar_workspace = None # allreduce backend or a fallback backend when the primary workspace is not # available on the current topology. _fi_ar_quant_workspace = None +_fi_ar_workspace_groups: dict[int, ProcessGroup] = {} def _create_workspace( @@ -81,6 +83,14 @@ def _create_workspace( return None finally: random.setstate(rng_state) + workspace_id = id(workspace) + workspace_group = _fi_ar_workspace_groups.get(workspace_id) + if workspace_group is not None and workspace_group is not group: + raise RuntimeError( + "FlashInfer returned an all-reduce workspace already associated " + "with a different process group" + ) + _fi_ar_workspace_groups[workspace_id] = group logger.debug( "Initialized FlashInfer All Reduce workspace: backend=%s, " "world_size=%d, rank=%d, max_token_num=%d, hidden_dim=%d, dtype=%s", @@ -111,7 +121,7 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]: # Default to mnnvl for both single- and multi-node setups. The mnnvl # cudagraph hang that previously forced single-node to trtllm # (https://github.com/vllm-project/vllm/issues/35772) was fixed upstream in - # FlashInfer (>= 0.6.12, vLLM pins 0.6.13), so mnnvl is safe here. trtllm + # FlashInfer (>= 0.6.12, vLLM pins 0.6.15), so mnnvl is safe here. trtllm # does not support multi-node allreduce, so mnnvl is required there anyway. # mnnvl needs NVSwitch multicast; on single-node topologies without it, # fall back to trtllm so fused allreduce stays enabled. @@ -268,6 +278,36 @@ def destroy_fi_ar_workspace(): _fi_ar_quant_workspace.destroy() _fi_ar_workspace = _fi_ar_quant_workspace = None + _fi_ar_workspace_groups.clear() + + +def _fi_ar_workspaces_for_group(group: ProcessGroup) -> list[Any]: + workspaces = [_fi_ar_workspace] + if _fi_ar_quant_workspace is not _fi_ar_workspace: + workspaces.append(_fi_ar_quant_workspace) + + group_workspaces = [] + for workspace in workspaces: + if workspace is None: + continue + workspace_group = _fi_ar_workspace_groups.get(id(workspace)) + if workspace_group is None: + raise RuntimeError( + "FlashInfer all-reduce workspace process group was not retained" + ) + if workspace_group is group: + group_workspaces.append(workspace) + return group_workspaces + + +def checkpoint_prepare_fi_ar_workspaces(group: ProcessGroup) -> None: + for workspace in _fi_ar_workspaces_for_group(group): + workspace.checkpoint_prepare() + + +def checkpoint_restore_fi_ar_workspaces(group: ProcessGroup) -> None: + for workspace in _fi_ar_workspaces_for_group(group): + workspace.checkpoint_restore(TorchDistBackend(group=group)) atexit.register(destroy_fi_ar_workspace) diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index b3545d54b4..4284a609d6 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -127,6 +127,28 @@ def _register_group(group: "GroupCoordinator") -> None: _groups[group.unique_name] = weakref.ref(group) +def _apply_to_device_comms( + action: Callable[[DeviceCommunicatorBase], None], +) -> None: + """Apply ``action`` to every group's device communicator. + + Walks the registered parallel groups and skips those without a device + communicator (absent at ``world_size == 1``). + """ + comms = [] + for group_ref in _groups.values(): + group = group_ref() + if group is None: + continue + dc = group.device_communicator + if dc is None: + continue + comms.append(dc) + + for dc in comms: + action(dc) + + def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor: assert group_name in _groups, f"Group {group_name} is not found." group = _groups[group_name]() @@ -2028,6 +2050,20 @@ def prepare_communication_buffer_for_model(model: torch.nn.Module): _EPLB.prepare_communication_buffer_for_model(model) +def checkpoint_prepare_distributed_state() -> None: + """Prepare every device communicator for a process checkpoint.""" + torch.accelerator.synchronize() + _apply_to_device_comms(lambda comm: comm.checkpoint_prepare()) + torch.accelerator.synchronize() + + +def checkpoint_restore_distributed_state() -> None: + """Restore every device communicator after a process checkpoint.""" + torch.accelerator.synchronize() + _apply_to_device_comms(lambda comm: comm.checkpoint_restore()) + torch.accelerator.synchronize() + + def model_parallel_is_initialized(): """Check if tensor and pipeline parallel groups are initialized.""" return _TP is not None and _PP is not None diff --git a/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py b/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py index e49e135b26..94d783e730 100644 --- a/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py +++ b/vllm/model_executor/layers/fused_allreduce_gemma_rms_norm.py @@ -93,7 +93,7 @@ def _can_use_flashinfer(hidden_states: torch.Tensor, tp_size: int) -> tuple[bool max_token_num=max_token_num, hidden_dim=hidden_size, dtype=hidden_states.dtype, - group=get_tp_group().device_group, + group=get_tp_group().cpu_group, ) if workspace is None: return False, 0 diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index f1e7132339..922a8aa598 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -943,6 +943,12 @@ class AsyncLLM(EngineClient): if self.logger_manager is not None: self.logger_manager.record_sleep_state(0, 0) + async def checkpoint_prepare(self) -> None: + await self.collective_rpc("checkpoint_prepare") + + async def checkpoint_restore(self) -> None: + await self.collective_rpc("checkpoint_restore") + async def is_sleeping(self) -> bool: return await self.engine_core.is_sleeping_async() diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 7c3d033668..556b1e6c7d 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -41,6 +41,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( ) from vllm.distributed.parallel_state import ( Handle, + checkpoint_prepare_distributed_state, + checkpoint_restore_distributed_state, get_pp_group, get_tp_group, ) @@ -244,6 +246,12 @@ class Worker(WorkerBase): if tags is None or "kv_cache" in tags: self.model_runner.post_kv_cache_wake_up() + def checkpoint_prepare(self) -> None: + checkpoint_prepare_distributed_state() + + def checkpoint_restore(self) -> None: + checkpoint_restore_distributed_state() + def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager: if ( current_platform.is_cuda_alike()