[EPLB] Enable nixl eplb communicator for elastic ep (#45013)

Signed-off-by: Markov Ilya <markovilya197@gmail.com>
Signed-off-by: Markov Ilya <markovilya19@gmail.com>
Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
Ilya Markov
2026-06-22 10:54:08 -07:00
committed by GitHub
co-authored by Markov Ilya
parent f2069b005b
commit ac614587f5
7 changed files with 329 additions and 97 deletions
+34 -35
View File
@@ -59,9 +59,8 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
return accuracy
@multi_gpu_test(num_gpus=4)
def test_elastic_ep_scaling():
vllm_serve_args = [
def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
args = [
"--trust-remote-code",
"--tensor-parallel-size",
"1",
@@ -79,7 +78,11 @@ def test_elastic_ep_scaling():
"--eplb-config.num_redundant_experts",
"0",
"--eplb-config.use_async",
"false",
"true" if use_async_eplb else "false",
"--eplb-config.step_interval",
"10",
"--eplb-config.window_size",
"5",
"--data-parallel-backend",
"ray",
"--data-parallel-size",
@@ -90,7 +93,23 @@ def test_elastic_ep_scaling():
leader_address = os.environ.get("LEADER_ADDRESS")
if leader_address:
vllm_serve_args.extend(["--data-parallel-address", leader_address])
args.extend(["--data-parallel-address", leader_address])
return args
@pytest.mark.parametrize(
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
)
@multi_gpu_test(num_gpus=4)
def test_elastic_ep_scaling(use_async_eplb: bool):
if use_async_eplb:
from vllm.distributed.eplb.eplb_communicator import has_nixl
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
vllm_serve_args = _base_serve_args(use_async_eplb)
with RemoteOpenAIServer(
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
@@ -128,44 +147,24 @@ def test_elastic_ep_scaling():
print(f" Tolerance: {ACCURACY_TOL:.3f}")
@pytest.mark.parametrize(
"use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
)
@multi_gpu_test(num_gpus=4)
def test_elastic_ep_scaling_uneven():
def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
"""Test scale up with uneven worker distribution.
This tests the case where num_new_workers % old_dp_size != 0,
specifically 2 -> 3 where remainder = 1 % 2 = 1.
This exercises the remainder handling in sender-receiver pairing.
"""
vllm_serve_args = [
"--trust-remote-code",
"--tensor-parallel-size",
"1",
"--gpu-memory-utilization",
"0.8",
"--max-model-len",
"4096",
"--max-num-seqs",
str(MAX_NUM_SEQS),
"--enable-expert-parallel",
"--all2all-backend",
"allgather_reducescatter",
"--enable-elastic-ep",
"--enable-eplb",
"--eplb-config.num_redundant_experts",
"0",
"--eplb-config.use_async",
"false",
"--data-parallel-backend",
"ray",
"--data-parallel-size",
"2",
"--api-server-count",
"1",
]
if use_async_eplb:
from vllm.distributed.eplb.eplb_communicator import has_nixl
leader_address = os.environ.get("LEADER_ADDRESS")
if leader_address:
vllm_serve_args.extend(["--data-parallel-address", leader_address])
if not has_nixl():
pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
vllm_serve_args = _base_serve_args(use_async_eplb)
with RemoteOpenAIServer(
MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
+122
View File
@@ -782,3 +782,125 @@ def test_rearrange_expert_weights_profile_mode(world_size):
_test_rearrange_expert_weights_profile_mode,
world_size,
)
def _test_nixl_deferred_init_worker(
env,
world_size: int,
num_layers: int,
num_local_experts: int,
num_logical_experts: int,
) -> None:
"""Exercise NixlEplbCommunicator with defer_remote_setup=True (elastic EP path)."""
from vllm.distributed.eplb.eplb_communicator import NixlEplbCommunicator
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.tensor_parallel_size = world_size
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1
)
ep_group_coordinator = get_tp_group()
ep_group = ep_group_coordinator.cpu_group
ep_rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{ep_rank}")
total_physical_experts = world_size * num_local_experts
hidden_sizes = [32, 64]
redundancy_config = create_redundancy_config(
num_logical_experts, total_physical_experts
)
old_indices = create_expert_indices_with_redundancy(
num_layers,
num_logical_experts,
total_physical_experts,
redundancy_config,
)
new_redundancy_config = create_redundancy_config(
num_logical_experts, total_physical_experts
)
new_indices = create_expert_indices_with_redundancy(
num_layers,
num_logical_experts,
total_physical_experts,
new_redundancy_config,
)
expert_weights = create_expert_weights(
num_layers, num_local_experts, hidden_sizes, ep_rank, device, old_indices
)
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
communicator = NixlEplbCommunicator(
cpu_group=ep_group_coordinator.cpu_group,
all_expert_weights=expert_weights,
expert_buffer=expert_buffer,
defer_remote_setup=True,
)
assert not communicator._remote_state_initialized
rearrange_expert_weights_inplace(
old_indices,
new_indices,
expert_weights,
expert_buffer,
ep_group,
communicator,
)
assert communicator._remote_state_initialized
local_ok = verify_expert_weights_after_shuffle(
expert_weights,
new_indices,
hidden_sizes,
ep_rank,
num_local_experts,
)
local_ok = (
verify_redundant_experts_have_same_weights(
expert_weights,
new_indices,
hidden_sizes,
ep_rank,
world_size,
num_local_experts,
)
and local_ok
)
assert_verification_synced(
local_ok,
"Deferred NIXL init verification failed on at least one rank.",
)
@pytest.mark.skipif(not has_nixl(), reason="NIXL is not available")
@pytest.mark.parametrize(
"world_size,num_layers,num_local_experts,num_logical_experts",
[(2, 2, 3, 4)],
)
def test_nixl_deferred_init(
world_size,
num_layers,
num_local_experts,
num_logical_experts,
):
"""Test NixlEplbCommunicator with defer_remote_setup=True (elastic EP path)."""
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
distributed_run(
_test_nixl_deferred_init_worker,
world_size,
num_layers,
num_local_experts,
num_logical_experts,
)
+22 -22
View File
@@ -803,13 +803,6 @@ class ParallelConfig:
if self.enable_elastic_ep:
if not self.enable_eplb:
raise ValueError("Elastic EP is only supported with enable_eplb=True.")
if self.eplb_config.use_async:
raise ValueError(
"Elastic EP requires the pynccl communicator, which is "
"incompatible with async EPLB due to NCCL multi-stream "
"conflicts. Disable async EPLB (eplb_config.use_async=False) "
"to use elastic EP."
)
if self.pipeline_parallel_size > 1:
raise ValueError(
"Elastic EP is not supported with pipeline parallelism "
@@ -821,6 +814,15 @@ class ParallelConfig:
"or data_parallel_hybrid_lb. Elastic EP relies on a single API "
"server and core client to coordinate scale up/down."
)
if self.eplb_config.use_async:
from vllm.distributed.nixl_utils import is_nixl_available
if not is_nixl_available():
raise ValueError(
"Elastic EP with async EPLB requires the NIXL "
"package. Either install NIXL or set "
"--eplb-config.use_async=false."
)
if self.data_parallel_size > 1 or self.data_parallel_size_local == 0:
# Data parallel was specified in the engine args.
@@ -929,23 +931,21 @@ class ParallelConfig:
)
if self.enable_eplb and self.eplb_config.communicator is None:
if self.enable_elastic_ep:
# Elastic EP requires stateless mode
# (torch.distributed.batch_isend_irecv doesn't
# support stateless mode), so we use PyNCCL backend
# Prefer NIXL when available: zero-copy RDMA reads, compatible
# with both async EPLB and elastic EP (deferred remote setup).
# Fallbacks: pynccl for elastic EP (stateless groups need it),
# torch_gloo for static EP. torch_nccl is avoided because NCCL
# is incompatible with async EPLB (multi-stream conflicts) and
# batched isend/irecv hangs under high load.
# See https://github.com/pytorch/pytorch/issues/174288
from vllm.distributed.nixl_utils import is_nixl_available
if is_nixl_available():
self.eplb_config.communicator = "nixl"
elif self.enable_elastic_ep:
self.eplb_config.communicator = "pynccl"
else:
# Avoid torch_nccl: NCCL is fundamentally incompatible
# with async EPLB due to multi-stream conflicts, and
# batched isend/irecv hangs under high load.
# See https://github.com/pytorch/pytorch/issues/174288
# Prefer nixl when available; fall back to torch_gloo.
from vllm.distributed.nixl_utils import is_nixl_available
if is_nixl_available():
self.eplb_config.communicator = "nixl"
else:
self.eplb_config.communicator = "torch_gloo"
self.eplb_config.communicator = "torch_gloo"
@property
def use_ray(self) -> bool:
@@ -207,6 +207,9 @@ class ElasticEPScalingExecutor:
)
if new_dp_size > old_dp_size:
self._set_eplb_suppressed(True)
eplb_state = self.worker.model_runner.eplb_state
if eplb_state is not None:
eplb_state.drain_async()
elif new_dp_size < old_dp_size:
self._stage_standby_moe_quant_methods()
@@ -540,6 +543,11 @@ class ElasticEPScalingExecutor:
eplb_model_state.physical_to_logical_map.shape[1]
)
eplb_state.is_async = is_async_enabled
# Start the async worker thread if it doesn't exist yet (idempotent).
# This is needed for new workers after scale-up: they create EplbState
# in setup_eplb_from_mapping() but don't start the thread there because
# groups aren't ready yet.
eplb_state.start_async_loop()
if get_ep_group().rank == 0:
logger.info("[Elastic EP] Expert resharding completed")
@@ -549,6 +557,9 @@ class ElasticEPScalingExecutor:
def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None:
self._set_eplb_suppressed(True)
eplb_state = self.worker.model_runner.eplb_state
if eplb_state is not None:
eplb_state.drain_async()
parallel_config = self.worker.vllm_config.parallel_config
tp_size = parallel_config.tensor_parallel_size
old_ep_size = parallel_config.data_parallel_size * tp_size
+27 -13
View File
@@ -8,7 +8,6 @@ import threading
from typing import TYPE_CHECKING
import torch
from torch.distributed import ProcessGroup
from vllm.distributed.parallel_state import get_eplb_group
from vllm.logger import init_logger
@@ -26,8 +25,7 @@ def start_async_worker(
state: "EplbState",
is_profile: bool = False,
) -> threading.Thread:
eplb_group = get_eplb_group().device_group
rank = eplb_group.rank()
rank = get_eplb_group().device_group.rank()
device_index = state.cuda_device_index
assert state.is_async
@@ -38,7 +36,6 @@ def start_async_worker(
try:
transfer_run_periodically(
state=state,
eplb_group=eplb_group,
cuda_stream=cuda_stream,
is_profile=is_profile,
)
@@ -78,13 +75,15 @@ def run_rebalance_experts(
def transfer_run_periodically(
state: "EplbState",
eplb_group: ProcessGroup,
cuda_stream: torch.cuda.Stream,
is_profile: bool = False,
) -> None:
while True:
state.rearrange_event.wait(stream=cuda_stream)
logger.info("async worker woke up for EPLB transfer")
eplb_group = get_eplb_group().device_group
eplb_cpu_group = get_eplb_group().cpu_group
ep_rank = eplb_group.rank()
assert state.is_async
for model_state in state.model_states.values():
@@ -101,16 +100,32 @@ def transfer_run_periodically(
new_physical_to_logical_map = run_rebalance_experts(
model_state, state, physical_to_logical_map_cpu, cuda_stream
)
logger.info(
"Async worker computed new indices for model %s",
model_state.model_name,
)
# Execute one EPLB layer transfer per model forward pass. Each iteration
# of this loop will copy the new set of expert weights into
# model_state.expert_buffer, which will be consumed by the main thread in
# move_to_workspace
while model_state.rebalanced and layer_idx < num_layers:
# move_to_workspace.
# We sync the rebalanced flag across ranks before each iteration so
# all ranks make a coordinated decision to continue or stop.
while layer_idx < num_layers:
flag = torch.tensor(
[int(model_state.rebalanced)],
dtype=torch.int32,
device="cpu",
)
torch.distributed.all_reduce(flag, group=eplb_cpu_group)
if int(flag.item()) != eplb_cpu_group.size():
logger.warning(
"async worker (rank=%d): layer %d coordinated stop "
"(flag_sum=%d, group_size=%d)",
ep_rank,
layer_idx,
int(flag.item()),
eplb_cpu_group.size(),
)
model_state.rebalanced = False
break
transfer_metadata = transfer_layer(
old_layer_indices=physical_to_logical_map_cpu[layer_idx],
new_layer_indices=new_physical_to_logical_map[layer_idx],
@@ -143,6 +158,5 @@ def transfer_run_periodically(
# finish copying model_state.expert_buffer into
# model_state.model.expert_weights[layer_idx]
consumed_event.wait(stream=cuda_stream)
logger.debug("Layer %d transfer complete", layer_idx)
assert model_state.pending_result is None
layer_idx += 1
+71 -25
View File
@@ -246,7 +246,20 @@ class NixlEplbCommunicator(EplbCommunicator):
cpu_group: ProcessGroup,
all_expert_weights: Sequence[Sequence[torch.Tensor]],
expert_buffer: Sequence[torch.Tensor],
defer_remote_setup: bool = False,
) -> None:
"""Create a NIXL-backed EPLB communicator.
Args:
cpu_group: CPU process group for metadata exchange.
all_expert_weights: Expert weight tensors for all MoE layers.
expert_buffer: Pre-allocated receive buffer tensors.
defer_remote_setup: If True, postpone the collective
all-gather of NIXL agent metadata until the first
``set_transfer_context`` call. Required for elastic EP
where ranks join asynchronously and cannot participate
in collectives at construction time.
"""
assert all_expert_weights, (
"NixlEplbCommunicator requires non-empty all_expert_weights."
)
@@ -302,10 +315,29 @@ class NixlEplbCommunicator(EplbCommunicator):
] = {}
self._cuda_device_id = int(self._device.index or 0)
self._remote_state_initialized = False
self._init_step("buffers", self._init_registered_buffers)
if defer_remote_setup:
logger.info_once("NIXL EPLB: deferring remote agent setup (elastic EP).")
else:
self._init_remote_state()
self._log_initialized()
def _init_remote_state(self) -> None:
"""Exchange NIXL agent metadata and RDMA pointer info with all peers.
This is a collective operation (uses ``all_gather_object`` twice).
Under elastic EP the call is deferred to the first
``set_transfer_context`` invocation, where all ranks are
guaranteed to be synchronized.
"""
self._init_step("agents", self._init_remote_agents)
self._init_step("send meta", self._exchange_remote_send_meta)
self._log_initialized()
self._remote_state_initialized = True
def _ensure_remote_state(self) -> None:
if not self._remote_state_initialized:
self._init_remote_state()
@property
def needs_profile_buffer_reservation(self) -> bool:
@@ -339,8 +371,7 @@ class NixlEplbCommunicator(EplbCommunicator):
pass
def set_transfer_context(self, old_indices: np.ndarray, layer_idx: int) -> None:
# Pre-compute expert_id -> src_row mapping for every rank so that
# add_recv can immediately issue NIXL READs.
self._ensure_remote_state()
assert not self._xfer_entries, (
f"set_transfer_context() called with {len(self._xfer_entries)} "
f"pending transfers from layer {self._layer_idx}; "
@@ -523,6 +554,21 @@ class NixlEplbCommunicator(EplbCommunicator):
)
return (local_handle, remote_handle, xfer_handle)
def _post_read_barrier(self) -> None:
"""Correctness fence: prevents overwrite-while-remote-read race.
We avoid ``torch.distributed.monitored_barrier`` because it
calls ``get_backend(group)`` which fails for stateless groups
(elastic EP). An async ``all_reduce`` + ``wait(timeout)``
works with both regular and stateless groups and provides
equivalent timeout detection.
"""
_dummy = torch.zeros(1, dtype=torch.int32)
work = torch.distributed.all_reduce(
_dummy, group=self._cpu_group, async_op=True
)
work.wait(timeout=timedelta(minutes=5))
def execute(self) -> None:
assert self._layer_idx is not None or not self._xfer_entries, (
"set_transfer_context() must be called before execute() "
@@ -531,13 +577,7 @@ class NixlEplbCommunicator(EplbCommunicator):
try:
self._wait_for_all_transfers([x[2] for x in self._xfer_entries])
# Post-READ barrier.
# Correctness fence for zero-copy: prevents overwrite-while-
# remote-read race.
torch.distributed.monitored_barrier(
group=self._cpu_group,
timeout=timedelta(minutes=5),
)
self._post_read_barrier()
finally:
for local_h, remote_h, xfer_h in self._xfer_entries:
with contextlib.suppress(Exception):
@@ -628,11 +668,13 @@ def create_eplb_communicator(
device and CPU communication groups.
backend: Communicator backend name (``"torch_nccl"``,
``"torch_gloo"``, ``"pynccl"``, or ``"nixl"``).
Stateless (elastic EP) groups only support ``"torch_nccl"``
and ``"pynccl"``; ``"torch_nccl"`` is silently promoted to
``"pynccl"`` in that case. When tensors reside on CPU,
``"torch_gloo"`` or ``"torch_nccl"`` are used via the CPU
process group.
Falls back to ``"torch_nccl"`` when *None*.
Stateless (elastic EP) groups support ``"torch_nccl"``,
``"pynccl"``, and ``"nixl"``; ``"torch_nccl"`` is silently
promoted to ``"pynccl"``. ``"nixl"`` uses deferred remote
agent setup to avoid collective deadlocks during elastic
scaling. When tensors reside on CPU, ``"torch_gloo"`` or
``"torch_nccl"`` are used via the CPU process group.
expert_weights: Expert weight tensors for *all* MoE layers.
Shape ``(num_layers)(num_tensors_per_layer)``.
NixlEplbCommunicator registers all layers with NIXL for
@@ -686,18 +728,21 @@ def create_eplb_communicator(
is_stateless = isinstance(group_coordinator, StatelessGroupCoordinator)
if is_stateless:
if backend not in ("torch_nccl", "pynccl"):
if backend == "nixl":
pass # handled below with defer_remote_setup=True
elif backend not in ("torch_nccl", "pynccl"):
raise ValueError(
f"Elastic EP requires 'torch_nccl' or 'pynccl' EPLB communicator "
f"(got '{backend}')."
f"Elastic EP requires 'torch_nccl', 'pynccl', or 'nixl' "
f"EPLB communicator (got '{backend}')."
)
if backend == "torch_nccl":
logger.warning(
"Stateless elastic EP requires PyNCCL backend. "
"Forcing EPLB communicator to 'pynccl'."
)
backend = "pynccl"
return _create_pynccl()
else:
if backend == "torch_nccl":
logger.warning(
"Stateless elastic EP requires PyNCCL backend. "
"Forcing EPLB communicator to 'pynccl'."
)
backend = "pynccl"
return _create_pynccl()
if backend == "nixl":
if not has_nixl():
@@ -714,6 +759,7 @@ def create_eplb_communicator(
cpu_group=group_coordinator.cpu_group,
all_expert_weights=expert_weights,
expert_buffer=expert_buffer,
defer_remote_setup=is_stateless,
)
except Exception as exc:
raise RuntimeError(
+42 -2
View File
@@ -27,6 +27,7 @@ physical experts.
"""
import threading
import time
from collections.abc import Sequence
from dataclasses import dataclass
@@ -825,6 +826,45 @@ class EplbState:
is_profile=is_profile,
)
def drain_async(self) -> None:
"""Drain in-flight async EPLB by consuming all remaining layer results.
Each pending result is acknowledged (consumed_event recorded) so the
async worker can proceed, but the transferred weights are intentionally
NOT applied — a full synchronous rearrange is expected to follow.
Ranks are kept in lockstep via _all_ranks_result_ready (all_reduce
on the EP CPU group). The async worker's coordinated-stop collectives
use the separate EPLB group, so the two sets of collectives do not
interfere.
No-op when no async cycle is in progress (rebalanced=False).
"""
if not self.is_async:
return
for model_key, ms in self.model_states.items():
needs_drain = ms.rebalanced
if needs_drain:
logger.info(
"Draining async EPLB worker for model %s",
model_key,
)
while ms.rebalanced:
if self._all_ranks_result_ready(ms):
result = ms.pending_result
assert result is not None
if result.layer_idx == ms.model.num_moe_layers - 1:
ms.rebalanced = False
ms.pending_result = None
result.consumed_event.record()
else:
time.sleep(0.001)
if needs_drain:
logger.info(
"Async EPLB worker drained for model %s",
model_key,
)
def _all_ranks_result_ready(self, model_state: EplbModelState) -> bool:
parallel_state = get_ep_group()
has_result = int(model_state.pending_result is not None)
@@ -850,8 +890,9 @@ class EplbState:
"""
All-reduce a list of tensors.
"""
ep_group = get_ep_group().device_group
if len(tensor_list) == 1:
all_reduce(tensor_list[0], group=get_ep_group().device_group)
all_reduce(tensor_list[0], group=ep_group)
return tensor_list
assert all(t.dim() == 2 for t in tensor_list), "All tensors must be 2D."
assert all(t.shape[1] == tensor_list[0].shape[1] for t in tensor_list), (
@@ -863,7 +904,6 @@ class EplbState:
shapes = [t.shape for t in tensor_list]
concat_tensor = torch.cat(tensor_list, dim=0)
ep_group = get_ep_group().device_group
all_reduce(concat_tensor, group=ep_group)
all_reduce_list = []