From ac614587f514e9032bff75eb04c86ddeb3c9dbef Mon Sep 17 00:00:00 2001 From: Ilya Markov Date: Mon, 22 Jun 2026 19:54:08 +0200 Subject: [PATCH] [EPLB] Enable nixl eplb communicator for elastic ep (#45013) Signed-off-by: Markov Ilya Signed-off-by: Markov Ilya Co-authored-by: Markov Ilya --- tests/distributed/test_elastic_ep.py | 69 +++++----- tests/distributed/test_eplb_execute.py | 122 ++++++++++++++++++ vllm/config/parallel.py | 44 +++---- .../distributed/elastic_ep/elastic_execute.py | 11 ++ vllm/distributed/eplb/async_worker.py | 40 ++++-- vllm/distributed/eplb/eplb_communicator.py | 96 ++++++++++---- vllm/distributed/eplb/eplb_state.py | 44 ++++++- 7 files changed, 329 insertions(+), 97 deletions(-) diff --git a/tests/distributed/test_elastic_ep.py b/tests/distributed/test_elastic_ep.py index 7c59d9dca5..4ce7497598 100644 --- a/tests/distributed/test_elastic_ep.py +++ b/tests/distributed/test_elastic_ep.py @@ -59,9 +59,8 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float: return accuracy -@multi_gpu_test(num_gpus=4) -def test_elastic_ep_scaling(): - vllm_serve_args = [ +def _base_serve_args(use_async_eplb: bool = False) -> list[str]: + args = [ "--trust-remote-code", "--tensor-parallel-size", "1", @@ -79,7 +78,11 @@ def test_elastic_ep_scaling(): "--eplb-config.num_redundant_experts", "0", "--eplb-config.use_async", - "false", + "true" if use_async_eplb else "false", + "--eplb-config.step_interval", + "10", + "--eplb-config.window_size", + "5", "--data-parallel-backend", "ray", "--data-parallel-size", @@ -90,7 +93,23 @@ def test_elastic_ep_scaling(): leader_address = os.environ.get("LEADER_ADDRESS") if leader_address: - vllm_serve_args.extend(["--data-parallel-address", leader_address]) + args.extend(["--data-parallel-address", leader_address]) + + return args + + +@pytest.mark.parametrize( + "use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"] +) +@multi_gpu_test(num_gpus=4) +def test_elastic_ep_scaling(use_async_eplb: bool): + if use_async_eplb: + from vllm.distributed.eplb.eplb_communicator import has_nixl + + if not has_nixl(): + pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)") + + vllm_serve_args = _base_serve_args(use_async_eplb) with RemoteOpenAIServer( MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200 @@ -128,44 +147,24 @@ def test_elastic_ep_scaling(): print(f" Tolerance: {ACCURACY_TOL:.3f}") +@pytest.mark.parametrize( + "use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"] +) @multi_gpu_test(num_gpus=4) -def test_elastic_ep_scaling_uneven(): +def test_elastic_ep_scaling_uneven(use_async_eplb: bool): """Test scale up with uneven worker distribution. This tests the case where num_new_workers % old_dp_size != 0, specifically 2 -> 3 where remainder = 1 % 2 = 1. This exercises the remainder handling in sender-receiver pairing. """ - vllm_serve_args = [ - "--trust-remote-code", - "--tensor-parallel-size", - "1", - "--gpu-memory-utilization", - "0.8", - "--max-model-len", - "4096", - "--max-num-seqs", - str(MAX_NUM_SEQS), - "--enable-expert-parallel", - "--all2all-backend", - "allgather_reducescatter", - "--enable-elastic-ep", - "--enable-eplb", - "--eplb-config.num_redundant_experts", - "0", - "--eplb-config.use_async", - "false", - "--data-parallel-backend", - "ray", - "--data-parallel-size", - "2", - "--api-server-count", - "1", - ] + if use_async_eplb: + from vllm.distributed.eplb.eplb_communicator import has_nixl - leader_address = os.environ.get("LEADER_ADDRESS") - if leader_address: - vllm_serve_args.extend(["--data-parallel-address", leader_address]) + if not has_nixl(): + pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)") + + vllm_serve_args = _base_serve_args(use_async_eplb) with RemoteOpenAIServer( MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200 diff --git a/tests/distributed/test_eplb_execute.py b/tests/distributed/test_eplb_execute.py index 21fa057fd2..4c9b98b62c 100644 --- a/tests/distributed/test_eplb_execute.py +++ b/tests/distributed/test_eplb_execute.py @@ -782,3 +782,125 @@ def test_rearrange_expert_weights_profile_mode(world_size): _test_rearrange_expert_weights_profile_mode, world_size, ) + + +def _test_nixl_deferred_init_worker( + env, + world_size: int, + num_layers: int, + num_local_experts: int, + num_logical_experts: int, +) -> None: + """Exercise NixlEplbCommunicator with defer_remote_setup=True (elastic EP path).""" + from vllm.distributed.eplb.eplb_communicator import NixlEplbCommunicator + + set_env_vars_and_device(env) + + vllm_config = VllmConfig() + vllm_config.parallel_config.tensor_parallel_size = world_size + + with set_current_vllm_config(vllm_config): + ensure_model_parallel_initialized( + tensor_model_parallel_size=world_size, pipeline_model_parallel_size=1 + ) + + ep_group_coordinator = get_tp_group() + ep_group = ep_group_coordinator.cpu_group + ep_rank = torch.distributed.get_rank() + device = torch.device(f"cuda:{ep_rank}") + + total_physical_experts = world_size * num_local_experts + hidden_sizes = [32, 64] + + redundancy_config = create_redundancy_config( + num_logical_experts, total_physical_experts + ) + old_indices = create_expert_indices_with_redundancy( + num_layers, + num_logical_experts, + total_physical_experts, + redundancy_config, + ) + + new_redundancy_config = create_redundancy_config( + num_logical_experts, total_physical_experts + ) + new_indices = create_expert_indices_with_redundancy( + num_layers, + num_logical_experts, + total_physical_experts, + new_redundancy_config, + ) + + expert_weights = create_expert_weights( + num_layers, num_local_experts, hidden_sizes, ep_rank, device, old_indices + ) + + expert_buffer = [torch.empty_like(w) for w in expert_weights[0]] + + communicator = NixlEplbCommunicator( + cpu_group=ep_group_coordinator.cpu_group, + all_expert_weights=expert_weights, + expert_buffer=expert_buffer, + defer_remote_setup=True, + ) + assert not communicator._remote_state_initialized + + rearrange_expert_weights_inplace( + old_indices, + new_indices, + expert_weights, + expert_buffer, + ep_group, + communicator, + ) + + assert communicator._remote_state_initialized + + local_ok = verify_expert_weights_after_shuffle( + expert_weights, + new_indices, + hidden_sizes, + ep_rank, + num_local_experts, + ) + + local_ok = ( + verify_redundant_experts_have_same_weights( + expert_weights, + new_indices, + hidden_sizes, + ep_rank, + world_size, + num_local_experts, + ) + and local_ok + ) + assert_verification_synced( + local_ok, + "Deferred NIXL init verification failed on at least one rank.", + ) + + +@pytest.mark.skipif(not has_nixl(), reason="NIXL is not available") +@pytest.mark.parametrize( + "world_size,num_layers,num_local_experts,num_logical_experts", + [(2, 2, 3, 4)], +) +def test_nixl_deferred_init( + world_size, + num_layers, + num_local_experts, + num_logical_experts, +): + """Test NixlEplbCommunicator with defer_remote_setup=True (elastic EP path).""" + + if torch.accelerator.device_count() < world_size: + pytest.skip(f"Need at least {world_size} GPUs to run the test") + distributed_run( + _test_nixl_deferred_init_worker, + world_size, + num_layers, + num_local_experts, + num_logical_experts, + ) diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 2ae773d79c..5bd528f4c9 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -803,13 +803,6 @@ class ParallelConfig: if self.enable_elastic_ep: if not self.enable_eplb: raise ValueError("Elastic EP is only supported with enable_eplb=True.") - if self.eplb_config.use_async: - raise ValueError( - "Elastic EP requires the pynccl communicator, which is " - "incompatible with async EPLB due to NCCL multi-stream " - "conflicts. Disable async EPLB (eplb_config.use_async=False) " - "to use elastic EP." - ) if self.pipeline_parallel_size > 1: raise ValueError( "Elastic EP is not supported with pipeline parallelism " @@ -821,6 +814,15 @@ class ParallelConfig: "or data_parallel_hybrid_lb. Elastic EP relies on a single API " "server and core client to coordinate scale up/down." ) + if self.eplb_config.use_async: + from vllm.distributed.nixl_utils import is_nixl_available + + if not is_nixl_available(): + raise ValueError( + "Elastic EP with async EPLB requires the NIXL " + "package. Either install NIXL or set " + "--eplb-config.use_async=false." + ) if self.data_parallel_size > 1 or self.data_parallel_size_local == 0: # Data parallel was specified in the engine args. @@ -929,23 +931,21 @@ class ParallelConfig: ) if self.enable_eplb and self.eplb_config.communicator is None: - if self.enable_elastic_ep: - # Elastic EP requires stateless mode - # (torch.distributed.batch_isend_irecv doesn't - # support stateless mode), so we use PyNCCL backend + # Prefer NIXL when available: zero-copy RDMA reads, compatible + # with both async EPLB and elastic EP (deferred remote setup). + # Fallbacks: pynccl for elastic EP (stateless groups need it), + # torch_gloo for static EP. torch_nccl is avoided because NCCL + # is incompatible with async EPLB (multi-stream conflicts) and + # batched isend/irecv hangs under high load. + # See https://github.com/pytorch/pytorch/issues/174288 + from vllm.distributed.nixl_utils import is_nixl_available + + if is_nixl_available(): + self.eplb_config.communicator = "nixl" + elif self.enable_elastic_ep: self.eplb_config.communicator = "pynccl" else: - # Avoid torch_nccl: NCCL is fundamentally incompatible - # with async EPLB due to multi-stream conflicts, and - # batched isend/irecv hangs under high load. - # See https://github.com/pytorch/pytorch/issues/174288 - # Prefer nixl when available; fall back to torch_gloo. - from vllm.distributed.nixl_utils import is_nixl_available - - if is_nixl_available(): - self.eplb_config.communicator = "nixl" - else: - self.eplb_config.communicator = "torch_gloo" + self.eplb_config.communicator = "torch_gloo" @property def use_ray(self) -> bool: diff --git a/vllm/distributed/elastic_ep/elastic_execute.py b/vllm/distributed/elastic_ep/elastic_execute.py index 5aff5567d7..3cb0d603e3 100644 --- a/vllm/distributed/elastic_ep/elastic_execute.py +++ b/vllm/distributed/elastic_ep/elastic_execute.py @@ -207,6 +207,9 @@ class ElasticEPScalingExecutor: ) if new_dp_size > old_dp_size: self._set_eplb_suppressed(True) + eplb_state = self.worker.model_runner.eplb_state + if eplb_state is not None: + eplb_state.drain_async() elif new_dp_size < old_dp_size: self._stage_standby_moe_quant_methods() @@ -540,6 +543,11 @@ class ElasticEPScalingExecutor: eplb_model_state.physical_to_logical_map.shape[1] ) eplb_state.is_async = is_async_enabled + # Start the async worker thread if it doesn't exist yet (idempotent). + # This is needed for new workers after scale-up: they create EplbState + # in setup_eplb_from_mapping() but don't start the thread there because + # groups aren't ready yet. + eplb_state.start_async_loop() if get_ep_group().rank == 0: logger.info("[Elastic EP] Expert resharding completed") @@ -549,6 +557,9 @@ class ElasticEPScalingExecutor: def perform_scale_down_eplb_reshuffle(self, new_dp_size: int) -> None: self._set_eplb_suppressed(True) + eplb_state = self.worker.model_runner.eplb_state + if eplb_state is not None: + eplb_state.drain_async() parallel_config = self.worker.vllm_config.parallel_config tp_size = parallel_config.tensor_parallel_size old_ep_size = parallel_config.data_parallel_size * tp_size diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index eb2ec26090..ff872c8c22 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -8,7 +8,6 @@ import threading from typing import TYPE_CHECKING import torch -from torch.distributed import ProcessGroup from vllm.distributed.parallel_state import get_eplb_group from vllm.logger import init_logger @@ -26,8 +25,7 @@ def start_async_worker( state: "EplbState", is_profile: bool = False, ) -> threading.Thread: - eplb_group = get_eplb_group().device_group - rank = eplb_group.rank() + rank = get_eplb_group().device_group.rank() device_index = state.cuda_device_index assert state.is_async @@ -38,7 +36,6 @@ def start_async_worker( try: transfer_run_periodically( state=state, - eplb_group=eplb_group, cuda_stream=cuda_stream, is_profile=is_profile, ) @@ -78,13 +75,15 @@ def run_rebalance_experts( def transfer_run_periodically( state: "EplbState", - eplb_group: ProcessGroup, cuda_stream: torch.cuda.Stream, is_profile: bool = False, ) -> None: while True: state.rearrange_event.wait(stream=cuda_stream) - logger.info("async worker woke up for EPLB transfer") + + eplb_group = get_eplb_group().device_group + eplb_cpu_group = get_eplb_group().cpu_group + ep_rank = eplb_group.rank() assert state.is_async for model_state in state.model_states.values(): @@ -101,16 +100,32 @@ def transfer_run_periodically( new_physical_to_logical_map = run_rebalance_experts( model_state, state, physical_to_logical_map_cpu, cuda_stream ) - logger.info( - "Async worker computed new indices for model %s", - model_state.model_name, - ) # Execute one EPLB layer transfer per model forward pass. Each iteration # of this loop will copy the new set of expert weights into # model_state.expert_buffer, which will be consumed by the main thread in - # move_to_workspace - while model_state.rebalanced and layer_idx < num_layers: + # move_to_workspace. + # We sync the rebalanced flag across ranks before each iteration so + # all ranks make a coordinated decision to continue or stop. + while layer_idx < num_layers: + flag = torch.tensor( + [int(model_state.rebalanced)], + dtype=torch.int32, + device="cpu", + ) + torch.distributed.all_reduce(flag, group=eplb_cpu_group) + if int(flag.item()) != eplb_cpu_group.size(): + logger.warning( + "async worker (rank=%d): layer %d coordinated stop " + "(flag_sum=%d, group_size=%d)", + ep_rank, + layer_idx, + int(flag.item()), + eplb_cpu_group.size(), + ) + model_state.rebalanced = False + break + transfer_metadata = transfer_layer( old_layer_indices=physical_to_logical_map_cpu[layer_idx], new_layer_indices=new_physical_to_logical_map[layer_idx], @@ -143,6 +158,5 @@ def transfer_run_periodically( # finish copying model_state.expert_buffer into # model_state.model.expert_weights[layer_idx] consumed_event.wait(stream=cuda_stream) - logger.debug("Layer %d transfer complete", layer_idx) assert model_state.pending_result is None layer_idx += 1 diff --git a/vllm/distributed/eplb/eplb_communicator.py b/vllm/distributed/eplb/eplb_communicator.py index 6bd20c460e..891b57bcf1 100644 --- a/vllm/distributed/eplb/eplb_communicator.py +++ b/vllm/distributed/eplb/eplb_communicator.py @@ -246,7 +246,20 @@ class NixlEplbCommunicator(EplbCommunicator): cpu_group: ProcessGroup, all_expert_weights: Sequence[Sequence[torch.Tensor]], expert_buffer: Sequence[torch.Tensor], + defer_remote_setup: bool = False, ) -> None: + """Create a NIXL-backed EPLB communicator. + + Args: + cpu_group: CPU process group for metadata exchange. + all_expert_weights: Expert weight tensors for all MoE layers. + expert_buffer: Pre-allocated receive buffer tensors. + defer_remote_setup: If True, postpone the collective + all-gather of NIXL agent metadata until the first + ``set_transfer_context`` call. Required for elastic EP + where ranks join asynchronously and cannot participate + in collectives at construction time. + """ assert all_expert_weights, ( "NixlEplbCommunicator requires non-empty all_expert_weights." ) @@ -302,10 +315,29 @@ class NixlEplbCommunicator(EplbCommunicator): ] = {} self._cuda_device_id = int(self._device.index or 0) + self._remote_state_initialized = False self._init_step("buffers", self._init_registered_buffers) + if defer_remote_setup: + logger.info_once("NIXL EPLB: deferring remote agent setup (elastic EP).") + else: + self._init_remote_state() + self._log_initialized() + + def _init_remote_state(self) -> None: + """Exchange NIXL agent metadata and RDMA pointer info with all peers. + + This is a collective operation (uses ``all_gather_object`` twice). + Under elastic EP the call is deferred to the first + ``set_transfer_context`` invocation, where all ranks are + guaranteed to be synchronized. + """ self._init_step("agents", self._init_remote_agents) self._init_step("send meta", self._exchange_remote_send_meta) - self._log_initialized() + self._remote_state_initialized = True + + def _ensure_remote_state(self) -> None: + if not self._remote_state_initialized: + self._init_remote_state() @property def needs_profile_buffer_reservation(self) -> bool: @@ -339,8 +371,7 @@ class NixlEplbCommunicator(EplbCommunicator): pass def set_transfer_context(self, old_indices: np.ndarray, layer_idx: int) -> None: - # Pre-compute expert_id -> src_row mapping for every rank so that - # add_recv can immediately issue NIXL READs. + self._ensure_remote_state() assert not self._xfer_entries, ( f"set_transfer_context() called with {len(self._xfer_entries)} " f"pending transfers from layer {self._layer_idx}; " @@ -523,6 +554,21 @@ class NixlEplbCommunicator(EplbCommunicator): ) return (local_handle, remote_handle, xfer_handle) + def _post_read_barrier(self) -> None: + """Correctness fence: prevents overwrite-while-remote-read race. + + We avoid ``torch.distributed.monitored_barrier`` because it + calls ``get_backend(group)`` which fails for stateless groups + (elastic EP). An async ``all_reduce`` + ``wait(timeout)`` + works with both regular and stateless groups and provides + equivalent timeout detection. + """ + _dummy = torch.zeros(1, dtype=torch.int32) + work = torch.distributed.all_reduce( + _dummy, group=self._cpu_group, async_op=True + ) + work.wait(timeout=timedelta(minutes=5)) + def execute(self) -> None: assert self._layer_idx is not None or not self._xfer_entries, ( "set_transfer_context() must be called before execute() " @@ -531,13 +577,7 @@ class NixlEplbCommunicator(EplbCommunicator): try: self._wait_for_all_transfers([x[2] for x in self._xfer_entries]) - # Post-READ barrier. - # Correctness fence for zero-copy: prevents overwrite-while- - # remote-read race. - torch.distributed.monitored_barrier( - group=self._cpu_group, - timeout=timedelta(minutes=5), - ) + self._post_read_barrier() finally: for local_h, remote_h, xfer_h in self._xfer_entries: with contextlib.suppress(Exception): @@ -628,11 +668,13 @@ def create_eplb_communicator( device and CPU communication groups. backend: Communicator backend name (``"torch_nccl"``, ``"torch_gloo"``, ``"pynccl"``, or ``"nixl"``). - Stateless (elastic EP) groups only support ``"torch_nccl"`` - and ``"pynccl"``; ``"torch_nccl"`` is silently promoted to - ``"pynccl"`` in that case. When tensors reside on CPU, - ``"torch_gloo"`` or ``"torch_nccl"`` are used via the CPU - process group. + Falls back to ``"torch_nccl"`` when *None*. + Stateless (elastic EP) groups support ``"torch_nccl"``, + ``"pynccl"``, and ``"nixl"``; ``"torch_nccl"`` is silently + promoted to ``"pynccl"``. ``"nixl"`` uses deferred remote + agent setup to avoid collective deadlocks during elastic + scaling. When tensors reside on CPU, ``"torch_gloo"`` or + ``"torch_nccl"`` are used via the CPU process group. expert_weights: Expert weight tensors for *all* MoE layers. Shape ``(num_layers)(num_tensors_per_layer)``. NixlEplbCommunicator registers all layers with NIXL for @@ -686,18 +728,21 @@ def create_eplb_communicator( is_stateless = isinstance(group_coordinator, StatelessGroupCoordinator) if is_stateless: - if backend not in ("torch_nccl", "pynccl"): + if backend == "nixl": + pass # handled below with defer_remote_setup=True + elif backend not in ("torch_nccl", "pynccl"): raise ValueError( - f"Elastic EP requires 'torch_nccl' or 'pynccl' EPLB communicator " - f"(got '{backend}')." + f"Elastic EP requires 'torch_nccl', 'pynccl', or 'nixl' " + f"EPLB communicator (got '{backend}')." ) - if backend == "torch_nccl": - logger.warning( - "Stateless elastic EP requires PyNCCL backend. " - "Forcing EPLB communicator to 'pynccl'." - ) - backend = "pynccl" - return _create_pynccl() + else: + if backend == "torch_nccl": + logger.warning( + "Stateless elastic EP requires PyNCCL backend. " + "Forcing EPLB communicator to 'pynccl'." + ) + backend = "pynccl" + return _create_pynccl() if backend == "nixl": if not has_nixl(): @@ -714,6 +759,7 @@ def create_eplb_communicator( cpu_group=group_coordinator.cpu_group, all_expert_weights=expert_weights, expert_buffer=expert_buffer, + defer_remote_setup=is_stateless, ) except Exception as exc: raise RuntimeError( diff --git a/vllm/distributed/eplb/eplb_state.py b/vllm/distributed/eplb/eplb_state.py index 1eb3a8feac..74f357fbdb 100644 --- a/vllm/distributed/eplb/eplb_state.py +++ b/vllm/distributed/eplb/eplb_state.py @@ -27,6 +27,7 @@ physical experts. """ import threading +import time from collections.abc import Sequence from dataclasses import dataclass @@ -825,6 +826,45 @@ class EplbState: is_profile=is_profile, ) + def drain_async(self) -> None: + """Drain in-flight async EPLB by consuming all remaining layer results. + + Each pending result is acknowledged (consumed_event recorded) so the + async worker can proceed, but the transferred weights are intentionally + NOT applied — a full synchronous rearrange is expected to follow. + + Ranks are kept in lockstep via _all_ranks_result_ready (all_reduce + on the EP CPU group). The async worker's coordinated-stop collectives + use the separate EPLB group, so the two sets of collectives do not + interfere. + + No-op when no async cycle is in progress (rebalanced=False). + """ + if not self.is_async: + return + for model_key, ms in self.model_states.items(): + needs_drain = ms.rebalanced + if needs_drain: + logger.info( + "Draining async EPLB worker for model %s", + model_key, + ) + while ms.rebalanced: + if self._all_ranks_result_ready(ms): + result = ms.pending_result + assert result is not None + if result.layer_idx == ms.model.num_moe_layers - 1: + ms.rebalanced = False + ms.pending_result = None + result.consumed_event.record() + else: + time.sleep(0.001) + if needs_drain: + logger.info( + "Async EPLB worker drained for model %s", + model_key, + ) + def _all_ranks_result_ready(self, model_state: EplbModelState) -> bool: parallel_state = get_ep_group() has_result = int(model_state.pending_result is not None) @@ -850,8 +890,9 @@ class EplbState: """ All-reduce a list of tensors. """ + ep_group = get_ep_group().device_group if len(tensor_list) == 1: - all_reduce(tensor_list[0], group=get_ep_group().device_group) + all_reduce(tensor_list[0], group=ep_group) return tensor_list assert all(t.dim() == 2 for t in tensor_list), "All tensors must be 2D." assert all(t.shape[1] == tensor_list[0].shape[1] for t in tensor_list), ( @@ -863,7 +904,6 @@ class EplbState: shapes = [t.shape for t in tensor_list] concat_tensor = torch.cat(tensor_list, dim=0) - ep_group = get_ep_group().device_group all_reduce(concat_tensor, group=ep_group) all_reduce_list = []