[Core][Distributed] Add process-checkpoint lifecycle hooks for communicators (starting with Flashinfer) (#46877)
Signed-off-by: Schwinn Saereesitthipitak <schwinns@nvidia.com>
This commit is contained in:
@@ -7,6 +7,7 @@ Run `pytest tests/distributed/test_comm_ops.py`.
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
@@ -19,6 +20,8 @@ from vllm.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
tensor_model_parallel_reduce_scatter,
|
||||
)
|
||||
from vllm.distributed.device_communicators import flashinfer_all_reduce
|
||||
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
|
||||
from vllm.distributed.parallel_state import GroupCoordinator, TensorMetadata
|
||||
from vllm.v1.worker.gpu_worker import AsyncIntermediateTensors
|
||||
|
||||
@@ -278,6 +281,43 @@ def test_irecv_tensor_dict_send_allgather_postprocess_binds_keys(
|
||||
torch.testing.assert_close(td["b"], torch.ones(4, dtype=torch.int32))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("aliased", [False, True])
|
||||
def test_cuda_communicator_checkpoints_flashinfer_workspaces(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
aliased: bool,
|
||||
) -> None:
|
||||
group = object()
|
||||
normal_workspace = Mock()
|
||||
quant_workspace = normal_workspace if aliased else Mock()
|
||||
unique_workspaces = (
|
||||
[normal_workspace] if aliased else [normal_workspace, quant_workspace]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", normal_workspace)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce, "_fi_ar_quant_workspace", quant_workspace
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"_fi_ar_workspace_groups",
|
||||
{id(workspace): group for workspace in unique_workspaces},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce, "TorchDistBackend", lambda group: group, raising=False
|
||||
)
|
||||
|
||||
communicator = CudaCommunicator.__new__(CudaCommunicator)
|
||||
communicator.cpu_group = group
|
||||
communicator.fi_ar_comm = None
|
||||
communicator.all2all_manager = None
|
||||
communicator.checkpoint_prepare()
|
||||
communicator.checkpoint_restore()
|
||||
|
||||
for workspace in unique_workspaces:
|
||||
workspace.checkpoint_prepare.assert_called_once_with()
|
||||
workspace.checkpoint_restore.assert_called_once_with(group)
|
||||
|
||||
|
||||
def test_async_intermediate_tensors_lazy_wait() -> None:
|
||||
work = _DummyWork()
|
||||
post_calls = {"n": 0}
|
||||
|
||||
@@ -225,7 +225,7 @@ if flashinfer_comm is not None:
|
||||
max_token_num=max_token_num,
|
||||
hidden_dim=hidden_size,
|
||||
dtype=allreduce_in.dtype,
|
||||
group=get_tp_group().device_group,
|
||||
group=get_tp_group().cpu_group,
|
||||
)
|
||||
assert workspace is not None, (
|
||||
"Flashinfer allreduce workspace must be initialized when using flashinfer"
|
||||
@@ -996,7 +996,7 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
|
||||
)
|
||||
return
|
||||
self.hidden_dim = config.model_config.get_hidden_size()
|
||||
self.group = get_tp_group().device_group
|
||||
self.group = get_tp_group().cpu_group
|
||||
rank = get_tensor_model_parallel_rank()
|
||||
if flashinfer_comm is None:
|
||||
logger.warning(
|
||||
|
||||
@@ -885,6 +885,20 @@ class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
|
||||
self.mapping = None
|
||||
self.initialized = False
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
if self.initialized:
|
||||
assert self.moe_alltoall is not None
|
||||
self.moe_alltoall.checkpoint_prepare()
|
||||
|
||||
def checkpoint_restore(self) -> None:
|
||||
if self.initialized:
|
||||
assert self.moe_alltoall is not None
|
||||
from vllm.distributed.device_communicators.mnnvl_compat import (
|
||||
CustomCommunicator,
|
||||
)
|
||||
|
||||
self.moe_alltoall.checkpoint_restore(CustomCommunicator(self.cpu_group))
|
||||
|
||||
|
||||
class MoriAll2AllManager(All2AllManagerBase):
|
||||
def __init__(self, cpu_group, all2all_backend: str):
|
||||
|
||||
@@ -7,8 +7,11 @@ import torch
|
||||
import torch.distributed as dist
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import is_moe_layer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Cache:
|
||||
def __init__(self):
|
||||
@@ -137,6 +140,18 @@ class All2AllManagerBase:
|
||||
def max_sms_used(self) -> int | None:
|
||||
return None # None means it could use the whole GPU
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
logger.warning_once(
|
||||
"%s.checkpoint_prepare is not implemented; skipping.",
|
||||
type(self).__name__,
|
||||
)
|
||||
|
||||
def checkpoint_restore(self) -> None:
|
||||
logger.warning_once(
|
||||
"%s.checkpoint_restore is not implemented; skipping.",
|
||||
type(self).__name__,
|
||||
)
|
||||
|
||||
def combine(self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False):
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -216,6 +231,12 @@ class DeviceCommunicatorBase:
|
||||
dist.all_reduce(input_, group=self.device_group)
|
||||
return input_
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
"""Prepare reclaimable communicator state for checkpoint (default: no-op)."""
|
||||
|
||||
def checkpoint_restore(self) -> None:
|
||||
"""Restore communicator state after checkpoint (default: no-op)."""
|
||||
|
||||
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
|
||||
@@ -571,6 +571,22 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
self.all2all_manager.destroy()
|
||||
self.all2all_manager = None # type: ignore[assignment]
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
# Only FlashInfer all-reduce and FlashInfer all2all are supported for now.
|
||||
from .flashinfer_all_reduce import checkpoint_prepare_fi_ar_workspaces
|
||||
|
||||
checkpoint_prepare_fi_ar_workspaces(self.cpu_group)
|
||||
if self.all2all_manager is not None:
|
||||
self.all2all_manager.checkpoint_prepare()
|
||||
|
||||
def checkpoint_restore(self) -> None:
|
||||
# Only FlashInfer all-reduce and FlashInfer all2all are supported for now.
|
||||
from .flashinfer_all_reduce import checkpoint_restore_fi_ar_workspaces
|
||||
|
||||
checkpoint_restore_fi_ar_workspaces(self.cpu_group)
|
||||
if self.all2all_manager is not None:
|
||||
self.all2all_manager.checkpoint_restore()
|
||||
|
||||
def all_gatherv(
|
||||
self,
|
||||
input_: torch.Tensor | list[torch.Tensor],
|
||||
|
||||
@@ -6,6 +6,7 @@ import atexit
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -39,6 +40,7 @@ _fi_ar_workspace = None
|
||||
# allreduce backend or a fallback backend when the primary workspace is not
|
||||
# available on the current topology.
|
||||
_fi_ar_quant_workspace = None
|
||||
_fi_ar_workspace_groups: dict[int, ProcessGroup] = {}
|
||||
|
||||
|
||||
def _create_workspace(
|
||||
@@ -81,6 +83,14 @@ def _create_workspace(
|
||||
return None
|
||||
finally:
|
||||
random.setstate(rng_state)
|
||||
workspace_id = id(workspace)
|
||||
workspace_group = _fi_ar_workspace_groups.get(workspace_id)
|
||||
if workspace_group is not None and workspace_group is not group:
|
||||
raise RuntimeError(
|
||||
"FlashInfer returned an all-reduce workspace already associated "
|
||||
"with a different process group"
|
||||
)
|
||||
_fi_ar_workspace_groups[workspace_id] = group
|
||||
logger.debug(
|
||||
"Initialized FlashInfer All Reduce workspace: backend=%s, "
|
||||
"world_size=%d, rank=%d, max_token_num=%d, hidden_dim=%d, dtype=%s",
|
||||
@@ -111,7 +121,7 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]:
|
||||
# Default to mnnvl for both single- and multi-node setups. The mnnvl
|
||||
# cudagraph hang that previously forced single-node to trtllm
|
||||
# (https://github.com/vllm-project/vllm/issues/35772) was fixed upstream in
|
||||
# FlashInfer (>= 0.6.12, vLLM pins 0.6.13), so mnnvl is safe here. trtllm
|
||||
# FlashInfer (>= 0.6.12, vLLM pins 0.6.15), so mnnvl is safe here. trtllm
|
||||
# does not support multi-node allreduce, so mnnvl is required there anyway.
|
||||
# mnnvl needs NVSwitch multicast; on single-node topologies without it,
|
||||
# fall back to trtllm so fused allreduce stays enabled.
|
||||
@@ -268,6 +278,36 @@ def destroy_fi_ar_workspace():
|
||||
_fi_ar_quant_workspace.destroy()
|
||||
|
||||
_fi_ar_workspace = _fi_ar_quant_workspace = None
|
||||
_fi_ar_workspace_groups.clear()
|
||||
|
||||
|
||||
def _fi_ar_workspaces_for_group(group: ProcessGroup) -> list[Any]:
|
||||
workspaces = [_fi_ar_workspace]
|
||||
if _fi_ar_quant_workspace is not _fi_ar_workspace:
|
||||
workspaces.append(_fi_ar_quant_workspace)
|
||||
|
||||
group_workspaces = []
|
||||
for workspace in workspaces:
|
||||
if workspace is None:
|
||||
continue
|
||||
workspace_group = _fi_ar_workspace_groups.get(id(workspace))
|
||||
if workspace_group is None:
|
||||
raise RuntimeError(
|
||||
"FlashInfer all-reduce workspace process group was not retained"
|
||||
)
|
||||
if workspace_group is group:
|
||||
group_workspaces.append(workspace)
|
||||
return group_workspaces
|
||||
|
||||
|
||||
def checkpoint_prepare_fi_ar_workspaces(group: ProcessGroup) -> None:
|
||||
for workspace in _fi_ar_workspaces_for_group(group):
|
||||
workspace.checkpoint_prepare()
|
||||
|
||||
|
||||
def checkpoint_restore_fi_ar_workspaces(group: ProcessGroup) -> None:
|
||||
for workspace in _fi_ar_workspaces_for_group(group):
|
||||
workspace.checkpoint_restore(TorchDistBackend(group=group))
|
||||
|
||||
|
||||
atexit.register(destroy_fi_ar_workspace)
|
||||
|
||||
@@ -127,6 +127,28 @@ def _register_group(group: "GroupCoordinator") -> None:
|
||||
_groups[group.unique_name] = weakref.ref(group)
|
||||
|
||||
|
||||
def _apply_to_device_comms(
|
||||
action: Callable[[DeviceCommunicatorBase], None],
|
||||
) -> None:
|
||||
"""Apply ``action`` to every group's device communicator.
|
||||
|
||||
Walks the registered parallel groups and skips those without a device
|
||||
communicator (absent at ``world_size == 1``).
|
||||
"""
|
||||
comms = []
|
||||
for group_ref in _groups.values():
|
||||
group = group_ref()
|
||||
if group is None:
|
||||
continue
|
||||
dc = group.device_communicator
|
||||
if dc is None:
|
||||
continue
|
||||
comms.append(dc)
|
||||
|
||||
for dc in comms:
|
||||
action(dc)
|
||||
|
||||
|
||||
def all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
|
||||
assert group_name in _groups, f"Group {group_name} is not found."
|
||||
group = _groups[group_name]()
|
||||
@@ -2028,6 +2050,20 @@ def prepare_communication_buffer_for_model(model: torch.nn.Module):
|
||||
_EPLB.prepare_communication_buffer_for_model(model)
|
||||
|
||||
|
||||
def checkpoint_prepare_distributed_state() -> None:
|
||||
"""Prepare every device communicator for a process checkpoint."""
|
||||
torch.accelerator.synchronize()
|
||||
_apply_to_device_comms(lambda comm: comm.checkpoint_prepare())
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
|
||||
def checkpoint_restore_distributed_state() -> None:
|
||||
"""Restore every device communicator after a process checkpoint."""
|
||||
torch.accelerator.synchronize()
|
||||
_apply_to_device_comms(lambda comm: comm.checkpoint_restore())
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
|
||||
def model_parallel_is_initialized():
|
||||
"""Check if tensor and pipeline parallel groups are initialized."""
|
||||
return _TP is not None and _PP is not None
|
||||
|
||||
@@ -93,7 +93,7 @@ def _can_use_flashinfer(hidden_states: torch.Tensor, tp_size: int) -> tuple[bool
|
||||
max_token_num=max_token_num,
|
||||
hidden_dim=hidden_size,
|
||||
dtype=hidden_states.dtype,
|
||||
group=get_tp_group().device_group,
|
||||
group=get_tp_group().cpu_group,
|
||||
)
|
||||
if workspace is None:
|
||||
return False, 0
|
||||
|
||||
@@ -943,6 +943,12 @@ class AsyncLLM(EngineClient):
|
||||
if self.logger_manager is not None:
|
||||
self.logger_manager.record_sleep_state(0, 0)
|
||||
|
||||
async def checkpoint_prepare(self) -> None:
|
||||
await self.collective_rpc("checkpoint_prepare")
|
||||
|
||||
async def checkpoint_restore(self) -> None:
|
||||
await self.collective_rpc("checkpoint_restore")
|
||||
|
||||
async def is_sleeping(self) -> bool:
|
||||
return await self.engine_core.is_sleeping_async()
|
||||
|
||||
|
||||
@@ -41,6 +41,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
Handle,
|
||||
checkpoint_prepare_distributed_state,
|
||||
checkpoint_restore_distributed_state,
|
||||
get_pp_group,
|
||||
get_tp_group,
|
||||
)
|
||||
@@ -244,6 +246,12 @@ class Worker(WorkerBase):
|
||||
if tags is None or "kv_cache" in tags:
|
||||
self.model_runner.post_kv_cache_wake_up()
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
checkpoint_prepare_distributed_state()
|
||||
|
||||
def checkpoint_restore(self) -> None:
|
||||
checkpoint_restore_distributed_state()
|
||||
|
||||
def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager:
|
||||
if (
|
||||
current_platform.is_cuda_alike()
|
||||
|
||||
Reference in New Issue
Block a user