[EPLB] Add alternative communication for EPLB weight exchange (#33176)

Signed-off-by: ilmarkov <markovilya197@gmail.com>
Signed-off-by: Markov Ilya <markovilya19@gmail.com>
Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
Ilya Markov
2026-03-31 08:17:12 -04:00
committed by GitHub
co-authored by Markov Ilya
parent 0c63739135
commit abdbb68386
12 changed files with 635 additions and 183 deletions
@@ -23,6 +23,7 @@
# variable in the code.
import ctypes
import functools
import platform
from dataclasses import dataclass
from typing import Any
@@ -75,26 +76,34 @@ class ncclDataTypeEnum:
ncclFloat8e4m3 = 10
ncclNumTypes = 11
@classmethod
@functools.lru_cache(maxsize=1)
def _torch_to_nccl_map(cls) -> dict[torch.dtype, int]:
return {
torch.int8: cls.ncclInt8,
torch.uint8: cls.ncclUint8,
torch.int32: cls.ncclInt32,
torch.int64: cls.ncclInt64,
torch.float16: cls.ncclFloat16,
torch.float32: cls.ncclFloat32,
torch.float64: cls.ncclFloat64,
torch.bfloat16: cls.ncclBfloat16,
current_platform.fp8_dtype(): cls.ncclFloat8e4m3,
}
@classmethod
def supports_torch_dtype(cls, dtype: torch.dtype) -> bool:
return dtype in cls._torch_to_nccl_map()
@classmethod
def try_from_torch(cls, dtype: torch.dtype) -> int | None:
return cls._torch_to_nccl_map().get(dtype)
@classmethod
def from_torch(cls, dtype: torch.dtype) -> int:
if dtype == torch.int8:
return cls.ncclInt8
if dtype == torch.uint8:
return cls.ncclUint8
if dtype == torch.int32:
return cls.ncclInt32
if dtype == torch.int64:
return cls.ncclInt64
if dtype == torch.float16:
return cls.ncclFloat16
if dtype == torch.float32:
return cls.ncclFloat32
if dtype == torch.float64:
return cls.ncclFloat64
if dtype == torch.bfloat16:
return cls.ncclBfloat16
if dtype == current_platform.fp8_dtype():
return cls.ncclFloat8e4m3
nccl_dtype = cls.try_from_torch(dtype)
if nccl_dtype is not None:
return nccl_dtype
raise ValueError(
f"Unsupported dtype {dtype}: should be one of "
f"int8, uint8, int32, int64, float16, float32, float64, bfloat16,"
@@ -29,8 +29,10 @@ from vllm.distributed.elastic_ep.standby_state import (
get_standby_ep_group,
pop_standby_groups,
)
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
from vllm.distributed.parallel_state import (
_replace_active_groups,
get_eplb_group,
prepare_communication_buffer_for_model,
)
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
@@ -411,6 +413,13 @@ class ElasticEPScalingExecutor:
module.quant_method = module.quant_method.old_quant_method
module.runner = module._init_runner()
prepare_communication_buffer_for_model(self.worker.model_runner.model)
eplb_model_state.communicator = create_eplb_communicator(
group_coordinator=get_eplb_group(),
backend=parallel_config.eplb_config.communicator,
expert_weights=model.expert_weights[0],
)
if (
self.worker.vllm_config.compilation_config.mode
== CompilationMode.STOCK_TORCH_COMPILE
+3
View File
@@ -98,6 +98,8 @@ async def transfer_run_periodically(
assert state.is_async
for model_state in state.model_states.values():
# Set the async worker's CUDA stream on the communicator
model_state.communicator.set_stream(cuda_stream)
rebalancing_algorithm_executed = False
physical_to_logical_map_cpu = None
current_num_layers = model_state.model.num_moe_layers
@@ -157,6 +159,7 @@ async def transfer_run_periodically(
expert_weights=model_state.model.expert_weights[layer_idx],
expert_weights_buffer=model_state.expert_buffer,
ep_group=eplb_group,
communicator=model_state.communicator,
is_profile=is_profile,
cuda_stream=cuda_stream,
)
+277
View File
@@ -0,0 +1,277 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
EPLB communicator implementations and factory.
"""
from abc import ABC, abstractmethod
from collections.abc import Sequence
import torch
from torch.distributed import (
P2POp,
ProcessGroup,
batch_isend_irecv,
)
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.device_communicators.pynccl_wrapper import (
ncclDataTypeEnum,
)
from vllm.distributed.parallel_state import GroupCoordinator, is_local_first_rank
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
from vllm.logger import init_logger
logger = init_logger(__name__)
class EplbCommunicator(ABC):
"""Abstract EPLB communicator for expert weight transfers."""
@abstractmethod
def add_send(self, tensor: torch.Tensor, dst_rank: int) -> None:
pass
@abstractmethod
def add_recv(self, tensor: torch.Tensor, src_rank: int) -> None:
pass
@abstractmethod
def execute(self) -> None:
pass
def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None:
self._cuda_stream = cuda_stream
def _log_initialized(self) -> None:
if is_local_first_rank():
logger.info("Initialized EPLB communicator: %s.", self.__class__.__name__)
class TorchDistNcclEplbCommunicator(EplbCommunicator):
"""EPLB communicator backed by torch.distributed isend/irecv."""
def __init__(
self,
ep_group: ProcessGroup,
cuda_stream: torch.cuda.Stream | None = None,
) -> None:
self._ep_group = ep_group
self._cuda_stream = cuda_stream
self._p2p_ops: list[P2POp] = []
self._log_initialized()
def add_send(self, tensor: torch.Tensor, dst_rank: int) -> None:
self._p2p_ops.append(
P2POp(
torch.distributed.isend,
tensor,
dst_rank,
self._ep_group,
)
)
def add_recv(self, tensor: torch.Tensor, src_rank: int) -> None:
self._p2p_ops.append(
P2POp(
torch.distributed.irecv,
tensor,
src_rank,
self._ep_group,
)
)
def execute(self) -> None:
if not self._p2p_ops:
return
try:
with torch.cuda.stream(self._cuda_stream):
reqs = batch_isend_irecv(self._p2p_ops)
for req in reqs:
req.wait()
finally:
self._p2p_ops.clear()
class TorchDistGlooStagedEplbCommunicator(EplbCommunicator):
"""EPLB communicator using gloo P2P with CPU staging."""
def __init__(
self,
cpu_group: ProcessGroup,
cuda_stream: torch.cuda.Stream | None = None,
) -> None:
self._cpu_group = cpu_group
self._cuda_stream = cuda_stream
self._ops: list[tuple[str, torch.Tensor, int]] = []
self._log_initialized()
def add_send(self, tensor: torch.Tensor, dst_rank: int) -> None:
self._ops.append(("send", tensor, dst_rank))
def add_recv(self, tensor: torch.Tensor, src_rank: int) -> None:
self._ops.append(("recv", tensor, src_rank))
def execute(self) -> None:
if not self._ops:
return
p2p_ops: list[P2POp] = []
recv_staging: list[tuple[torch.Tensor, torch.Tensor]] = []
def build_ops() -> None:
for op, tensor, peer_rank in self._ops:
if op == "send":
cpu_tensor = tensor.to(device="cpu", non_blocking=True)
p2p_ops.append(
P2POp(
torch.distributed.isend,
cpu_tensor,
peer_rank,
self._cpu_group,
)
)
continue
cpu_tensor = torch.empty_like(tensor, device="cpu")
p2p_ops.append(
P2POp(
torch.distributed.irecv,
cpu_tensor,
peer_rank,
self._cpu_group,
)
)
recv_staging.append((tensor, cpu_tensor))
try:
with torch.cuda.stream(self._cuda_stream):
build_ops()
finally:
self._ops.clear()
# Wait for all D2H copies to finish
# before issuing gloo batch_isend_irecv operations.
if self._cuda_stream is not None:
self._cuda_stream.synchronize()
else:
torch.cuda.current_stream().synchronize()
reqs = batch_isend_irecv(p2p_ops)
for req in reqs:
req.wait()
if not recv_staging:
return
with torch.cuda.stream(self._cuda_stream):
for dst_tensor, cpu_tensor in recv_staging:
dst_tensor.copy_(cpu_tensor, non_blocking=True)
class PyNcclEplbCommunicator(EplbCommunicator):
"""EPLB communicator backed by PyNcclCommunicator using ncclSend/ncclRecv."""
def __init__(
self,
pynccl_comm: PyNcclCommunicator,
cuda_stream: torch.cuda.Stream | None = None,
) -> None:
self._pynccl_comm = pynccl_comm
self._cuda_stream = cuda_stream
self._group_started = False
self._log_initialized()
def _ensure_group_started(self) -> None:
if not self._group_started:
self._pynccl_comm.group_start()
self._group_started = True
def add_send(self, tensor: torch.Tensor, dst_rank: int) -> None:
self._ensure_group_started()
self._pynccl_comm.send(tensor, dst_rank, stream=self._cuda_stream)
def add_recv(self, tensor: torch.Tensor, src_rank: int) -> None:
self._ensure_group_started()
self._pynccl_comm.recv(tensor, src_rank, stream=self._cuda_stream)
def execute(self) -> None:
if self._group_started:
self._pynccl_comm.group_end()
self._group_started = False
def create_eplb_communicator(
group_coordinator: GroupCoordinator,
backend: str | None,
expert_weights: Sequence[torch.Tensor],
) -> EplbCommunicator:
# Keep a safe default for callers that have not resolved communicator yet.
if backend is None:
backend = "torch_nccl"
tensor_device_type = expert_weights[0].device.type if expert_weights else "cpu"
torch_group = (
group_coordinator.cpu_group
if tensor_device_type == "cpu"
else group_coordinator.device_group
)
def _create_pynccl() -> EplbCommunicator:
if tensor_device_type == "cpu":
raise RuntimeError(
"EPLB communicator 'pynccl' supports only cuda-like devices "
f"(got {tensor_device_type})."
)
unsupported_dtypes = sorted(
{
tensor.dtype
for tensor in expert_weights
if not ncclDataTypeEnum.supports_torch_dtype(tensor.dtype)
},
key=str,
)
if unsupported_dtypes:
raise RuntimeError(
"EPLB communicator 'pynccl' requested but expert weights contain "
"unsupported dtypes: "
f"({', '.join(str(dtype) for dtype in unsupported_dtypes)})."
)
device_comm = group_coordinator.device_communicator
pynccl_comm = (
getattr(device_comm, "pynccl_comm", None)
if device_comm is not None
else None
)
if pynccl_comm is None or pynccl_comm.disabled or not pynccl_comm.available:
raise RuntimeError("EPLB communicator 'pynccl' requested but unavailable.")
try:
return PyNcclEplbCommunicator(pynccl_comm=pynccl_comm)
except Exception as exc:
raise RuntimeError(
f"Failed to initialize PyNcclEplbCommunicator ({exc})."
) from exc
is_stateless = isinstance(group_coordinator, StatelessGroupCoordinator)
if is_stateless:
if backend not in ("torch_nccl", "pynccl"):
raise ValueError(
f"Elastic EP requires 'torch_nccl' or 'pynccl' EPLB communicator "
f"(got '{backend}'). torch_gloo is not supported with stateless groups."
)
if backend == "torch_nccl":
logger.warning(
"Stateless elastic EP requires PyNCCL backend. "
"Forcing EPLB communicator to 'pynccl'."
)
backend = "pynccl"
return _create_pynccl()
if backend == "torch_gloo":
return TorchDistGlooStagedEplbCommunicator(
cpu_group=group_coordinator.cpu_group,
)
elif backend == "torch_nccl":
return TorchDistNcclEplbCommunicator(ep_group=torch_group)
elif backend == "pynccl":
return _create_pynccl()
raise ValueError(f"Unknown EPLB communicator backend: {backend}")
+23 -5
View File
@@ -37,6 +37,7 @@ from torch.distributed import ProcessGroup, all_reduce
from vllm.config import ModelConfig, ParallelConfig
from vllm.distributed.parallel_state import (
get_ep_group,
get_eplb_group,
get_node_count,
in_the_same_node_as,
)
@@ -46,6 +47,7 @@ from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import MixtureOfExperts
from .async_worker import start_async_worker
from .eplb_communicator import EplbCommunicator, create_eplb_communicator
from .policy import EPLB_POLICIES, AbstractEplbPolicy, DefaultEplbPolicy
from .rebalance_execute import (
RecvMetadata,
@@ -225,6 +227,10 @@ class EplbModelState:
"""
CUDA device index for the async EPLB worker thread.
"""
communicator: EplbCommunicator
"""
The communicator for expert weight transfers.
"""
new_physical_to_logical_map: torch.Tensor | None = None
"""
intermediate variable between `move_to_buffer` and `move_to_workspace`.
@@ -472,6 +478,12 @@ class EplbState:
self._init_should_record_tensor(model)
expert_buffer = [torch.empty_like(w) for w in model.expert_weights[0]]
communicator = create_eplb_communicator(
group_coordinator=get_eplb_group(),
backend=self.parallel_config.eplb_config.communicator,
expert_weights=model.expert_weights[0],
)
model_state = EplbModelState(
physical_to_logical_map=physical_to_logical_map,
logical_to_physical_map=logical_to_physical_map,
@@ -498,6 +510,7 @@ class EplbState:
recv_dst_rows=np.array([]),
),
cuda_device_index=self.cuda_device_index,
communicator=communicator,
new_physical_to_logical_map=None,
)
self.model_states[model_config.compute_hash()] = model_state
@@ -800,6 +813,7 @@ class EplbState:
new_physical_to_logical_map,
eplb_model_state.model.expert_weights,
ep_group,
eplb_model_state.communicator,
is_profile,
rank_mapping,
)
@@ -923,11 +937,8 @@ class EplbState:
new_indices=new_indices,
ep_rank=ep_group.rank(),
)
# Record event after consuming buffer to signal async thread
# that it's safe to overwrite the intermediate buffer
consumed_event = torch.cuda.Event()
consumed_event.record()
model_state.buffer_consumed_event = consumed_event
transferred_layer = model_state.layer_to_transfer
transferred_layer = model_state.layer_to_transfer
assert model_state.new_physical_to_logical_map is not None
@@ -936,6 +947,13 @@ class EplbState:
new_physical_to_logical_map=model_state.new_physical_to_logical_map,
layer=transferred_layer,
)
# Record event after consuming buffer to signal async thread
# that it's safe to overwrite the intermediate buffer
consumed_event = torch.cuda.Event()
consumed_event.record()
model_state.buffer_consumed_event = consumed_event
# After the main thread consumes, advance layer_to_transfer
model_state.layer_to_transfer += 1
model_state.ep_buffer_ready = 0
+13 -2
View File
@@ -21,6 +21,10 @@ def override_envs_for_eplb(parallel_config: ParallelConfig) -> None:
is_eplb_enabled = parallel_config.enable_eplb
async_eplb = parallel_config.eplb_config.use_async
is_deepep_ll = parallel_config.all2all_backend == "deepep_low_latency"
is_nccl_based_eplb_communicator = parallel_config.eplb_config.communicator in (
"torch_nccl",
"pynccl",
)
# Override NCCL_MAX_CTAS to avoid hangs when using async EPLB with the
# DeepEP low-latency backend.
@@ -39,7 +43,13 @@ def override_envs_for_eplb(parallel_config: ParallelConfig) -> None:
# Limiting NCCL occupancy via NCCL_MAX_CTAS leaves space for the DeepEP
# cooperative kernel to launch and complete, breaking the deadlock.
# See: https://github.com/deepseek-ai/DeepEP/issues/496
if is_data_parallel and is_eplb_enabled and is_deepep_ll and async_eplb:
if (
is_data_parallel
and is_eplb_enabled
and is_deepep_ll
and async_eplb
and is_nccl_based_eplb_communicator
):
current_value_str = os.getenv("NCCL_MAX_CTAS")
if current_value_str and current_value_str.isdigit():
@@ -49,6 +59,7 @@ def override_envs_for_eplb(parallel_config: ParallelConfig) -> None:
os.environ["NCCL_MAX_CTAS"] = str(override_value)
logger.info_once(
f"EPLB: Setting NCCL_MAX_CTAS={override_value} "
"for expert parallel with EPLB and deepep_low_latency backend",
"for expert parallel with NCCL-based EPLB communicator and "
"deepep_low_latency backend",
scope="global",
)
+24 -84
View File
@@ -11,19 +11,9 @@ from dataclasses import dataclass
import numpy as np
import torch
from torch.distributed import (
P2POp,
ProcessGroup,
all_gather,
batch_isend_irecv,
get_global_rank,
)
from torch.distributed import ProcessGroup, all_gather
from vllm.distributed.parallel_state import get_ep_group
from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator
from vllm.logger import init_logger
logger = init_logger(__name__)
from .eplb_communicator import EplbCommunicator
@dataclass
@@ -158,7 +148,8 @@ def move_to_buffer(
expert_weights: Sequence[torch.Tensor],
expert_weights_buffers: Sequence[torch.Tensor],
cuda_stream: torch.cuda.Stream | None,
ep_group: ProcessGroup,
ep_rank: int,
communicator: EplbCommunicator,
) -> MoveToBufferResult:
"""
Rearranges expert weights during EPLB rebalancing.
@@ -172,7 +163,8 @@ def move_to_buffer(
expert_weights: Original expert weights for the layer.
expert_weights_buffers: Intermediate buffers (one per tensor).
cuda_stream: CUDA stream for async copies (can be None for sync mode).
ep_group: Distributed process group for expert parallel comms.
ep_rank: Rank of this process in expert parallel group.
communicator: EplbCommunicator instance for P2P communication.
Returns:
is_unchanged (np.ndarray): (num_local_experts,), True where an expert row
@@ -182,8 +174,6 @@ def move_to_buffer(
RecvMetadata: Metadata needed for completing remote weight transfers.
"""
assert old_indices.shape == new_indices.shape
ep_rank = ep_group.rank()
recv_primary_mask = np.zeros((num_local_experts,), dtype=np.bool_)
send_expert_ids = np.full((num_local_experts,), -1, dtype=np.int64)
send_src_rows = np.full((num_local_experts,), -1, dtype=np.int32)
@@ -247,22 +237,9 @@ def move_to_buffer(
expert = new_local_expert_ids[dst]
src_local = expert_to_src_map.get(expert, -1)
if src_local != -1:
for w, b in zip(expert_weights, expert_weights_buffers):
b[dst].copy_(w[src_local], non_blocking=True)
p2p_ops: list[P2POp] = []
if isinstance(get_ep_group(), StatelessGroupCoordinator):
ep_group = get_ep_group()
is_stateless = True
else:
is_stateless = False
# Pre-compute global ranks mapping (only needed for non-stateless groups)
ep_size = ep_group.size()
if not is_stateless:
rank_to_global = {
rank: get_global_rank(ep_group, rank) for rank in range(ep_size)
}
with torch.cuda.stream(cuda_stream):
for w, b in zip(expert_weights, expert_weights_buffers):
b[dst].copy_(w[src_local], non_blocking=True)
# 2. Post sends
if send_count > 0:
@@ -294,23 +271,8 @@ def move_to_buffer(
if recver_pos < len(ranks_to_recv):
recv_ranks.append(ranks_to_recv[recver_pos])
for dst in recv_ranks:
if is_stateless:
for w in expert_weights:
op = object.__new__(P2POp)
op.op = torch.distributed.isend
op.tensor = w[src]
op.group_peer = dst
p2p_ops.append(op)
else:
dst_global = rank_to_global[dst]
p2p_ops += [
P2POp(
torch.distributed.isend,
w[src],
dst_global,
)
for w in expert_weights
]
for w in expert_weights:
communicator.add_send(w[src], dst)
# 3. Post recvs
if recv_count > 0:
@@ -339,40 +301,11 @@ def move_to_buffer(
src = ranks_to_send[recver_pos // num_dst_per_sender]
else:
src = ranks_to_send[recver_pos - remainder_start]
if is_stateless:
for b in expert_weights_buffers:
op = object.__new__(P2POp)
op.op = torch.distributed.irecv
op.tensor = b[dst]
op.group_peer = src
p2p_ops.append(op)
else:
src_global = rank_to_global[src]
p2p_ops += [
P2POp(
torch.distributed.irecv,
b[dst],
src_global,
)
for b in expert_weights_buffers
]
for b in expert_weights_buffers:
communicator.add_recv(b[dst], src)
# 4. Execute the P2P operations. The real communication happens here.
if p2p_ops and cuda_stream is not None:
with torch.cuda.stream(cuda_stream):
if is_stateless:
ep_group.device_communicator.batch_isend_irecv(p2p_ops)
else:
reqs = batch_isend_irecv(p2p_ops)
for req in reqs:
req.wait()
elif p2p_ops:
if is_stateless:
ep_group.device_communicator.batch_isend_irecv(p2p_ops)
else:
reqs = batch_isend_irecv(p2p_ops)
for req in reqs:
req.wait()
communicator.execute()
# wait for the communication to finish
return (
is_unchanged,
@@ -471,6 +404,7 @@ async def transfer_layer(
expert_weights: Sequence[torch.Tensor],
expert_weights_buffer: Sequence[torch.Tensor],
ep_group: ProcessGroup,
communicator: EplbCommunicator,
is_profile: bool = False,
cuda_stream: torch.cuda.Stream | None = None,
rank_mapping: dict[int, int] | None = None,
@@ -489,6 +423,7 @@ async def transfer_layer(
For example, a linear layer may have up and down projection.
expert_weights_buffer: Intermediate buffers (one per weight tensor).
ep_group: The device process group for expert parallelism.
communicator: EplbCommunicator instance for P2P communication.
is_profile (bool): If `True`, do not perform any actual weight copy.
This is used during profile run, where we only perform dummy
communications to reserve enough memory for the buffers.
@@ -542,7 +477,8 @@ async def transfer_layer(
expert_weights=expert_weights,
expert_weights_buffers=expert_weights_buffer,
cuda_stream=cuda_stream,
ep_group=ep_group,
ep_rank=ep_group.rank(),
communicator=communicator,
)
return is_unchanged, is_received_locally, recv_metadata
@@ -552,6 +488,7 @@ def rearrange_expert_weights_inplace(
new_global_expert_indices: torch.Tensor,
expert_weights: Sequence[Sequence[torch.Tensor]],
ep_group: ProcessGroup,
communicator: EplbCommunicator,
is_profile: bool = False,
rank_mapping: dict[int, int] | None = None,
) -> None:
@@ -569,6 +506,7 @@ def rearrange_expert_weights_inplace(
For example, a linear layer may have up and down projection,
so weight_count = 2. Each weight's hidden size can be different.
ep_group: The device process group for expert parallelism.
communicator: EplbCommunicator instance for P2P communication.
is_profile (bool): If `True`, do not perform any actual weight copy.
This is used during profile run, where we only perform dummy
communications to reserve enough memory for the buffers.
@@ -599,6 +537,7 @@ def rearrange_expert_weights_inplace(
assert new_global_expert_indices.shape == (num_moe_layers, num_physical_experts)
ep_size = ep_group.size()
ep_rank = ep_group.rank()
assert num_physical_experts == ep_size * num_local_physical_experts
first_layer_weights = list(expert_weights[0])
@@ -635,7 +574,8 @@ def rearrange_expert_weights_inplace(
expert_weights=expert_weights[layer_idx],
expert_weights_buffers=weights_buffer,
cuda_stream=None,
ep_group=ep_group,
ep_rank=ep_rank,
communicator=communicator,
)
move_from_buffer(
@@ -645,7 +585,7 @@ def rearrange_expert_weights_inplace(
is_received_locally=is_received_locally,
recv_metadata=recv_metadata,
new_indices=new_global_expert_indices_cpu[layer_idx],
ep_rank=ep_group.rank(),
ep_rank=ep_rank,
)
+1 -5
View File
@@ -1690,11 +1690,7 @@ def initialize_model_parallel(
# using torch.distributed in execution with torch.distributed in EPLB.
global _EPLB
assert _EPLB is None, "EPLB group is already initialized"
if (
config is not None
and config.parallel_config is not None
and config.parallel_config.enable_eplb
):
if config.parallel_config.enable_eplb:
if enable_elastic_ep:
_EPLB = _init_stateless_group(
group_ranks,