[Elastic EP] Reduce eager-mode reconfiguration downtime (#51885)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
Itay Alroy
2026-08-19 02:13:06 +00:00
committed by GitHub
parent d36cc4254e
commit a2257f95b7
12 changed files with 194 additions and 303 deletions
+12 -36
View File
@@ -248,7 +248,6 @@ class NixlEplbCommunicator(EplbCommunicator):
cpu_group: ProcessGroup,
all_expert_weights: Sequence[Sequence[torch.Tensor]],
expert_buffer: Sequence[torch.Tensor],
defer_remote_setup: bool = False,
) -> None:
"""Create a NIXL-backed EPLB communicator.
@@ -256,11 +255,6 @@ class NixlEplbCommunicator(EplbCommunicator):
cpu_group: CPU process group for metadata exchange.
all_expert_weights: Expert weight tensors for all MoE layers.
expert_buffer: Pre-allocated receive buffer tensors.
defer_remote_setup: If True, postpone the collective
all-gather of NIXL agent metadata until the first
``set_transfer_context`` call. Required for elastic EP
where ranks join asynchronously and cannot participate
in collectives at construction time.
"""
assert all_expert_weights, (
"NixlEplbCommunicator requires non-empty all_expert_weights."
@@ -317,29 +311,17 @@ class NixlEplbCommunicator(EplbCommunicator):
] = {}
self._cuda_device_id = int(self._device.index or 0)
self._remote_state_initialized = False
self._init_step("buffers", self._init_registered_buffers)
if defer_remote_setup:
logger.info_once("NIXL EPLB: deferring remote agent setup (elastic EP).")
else:
self._init_remote_state()
self._init_remote_state()
self._log_initialized()
def _init_remote_state(self) -> None:
"""Exchange NIXL agent metadata and RDMA pointer info with all peers.
This is a collective operation (uses ``all_gather_object`` twice).
Under elastic EP the call is deferred to the first
``set_transfer_context`` invocation, where all ranks are
guaranteed to be synchronized.
"""
self._init_step("agents", self._init_remote_agents)
self._init_step("send meta", self._exchange_remote_send_meta)
self._remote_state_initialized = True
def _ensure_remote_state(self) -> None:
if not self._remote_state_initialized:
self._init_remote_state()
@property
def needs_profile_buffer_reservation(self) -> bool:
@@ -373,7 +355,6 @@ class NixlEplbCommunicator(EplbCommunicator):
pass
def set_transfer_context(self, old_indices: np.ndarray, layer_idx: int) -> None:
self._ensure_remote_state()
assert not self._xfer_entries, (
f"set_transfer_context() called with {len(self._xfer_entries)} "
f"pending transfers from layer {self._layer_idx}; "
@@ -673,9 +654,8 @@ def create_eplb_communicator(
Falls back to ``"torch_nccl"`` when *None*.
Stateless (elastic EP) groups support ``"torch_nccl"``,
``"pynccl"``, and ``"nixl"``; ``"torch_nccl"`` is silently
promoted to ``"pynccl"``. ``"nixl"`` uses deferred remote
agent setup to avoid collective deadlocks during elastic
scaling. When tensors reside on CPU, ``"torch_gloo"`` or
promoted to ``"pynccl"``. When tensors reside on CPU,
``"torch_gloo"`` or
``"torch_nccl"`` are used via the CPU process group.
expert_weights: Expert weight tensors for *all* MoE layers.
Shape ``(num_layers)(num_tensors_per_layer)``.
@@ -729,22 +709,19 @@ def create_eplb_communicator(
) from exc
is_stateless = isinstance(group_coordinator, StatelessGroupCoordinator)
if is_stateless:
if backend == "nixl":
pass # handled below with defer_remote_setup=True
elif backend not in ("torch_nccl", "pynccl"):
if is_stateless and backend != "nixl":
if backend not in ("torch_nccl", "pynccl"):
raise ValueError(
f"Elastic EP requires 'torch_nccl', 'pynccl', or 'nixl' "
f"EPLB communicator (got '{backend}')."
)
else:
if backend == "torch_nccl":
logger.warning(
"Stateless elastic EP requires PyNCCL backend. "
"Forcing EPLB communicator to 'pynccl'."
)
backend = "pynccl"
return _create_pynccl()
if backend == "torch_nccl":
logger.warning(
"Stateless elastic EP requires PyNCCL backend. "
"Forcing EPLB communicator to 'pynccl'."
)
backend = "pynccl"
return _create_pynccl()
if backend == "nixl":
if not has_nixl():
@@ -761,7 +738,6 @@ def create_eplb_communicator(
cpu_group=group_coordinator.cpu_group,
all_expert_weights=expert_weights,
expert_buffer=expert_buffer,
defer_remote_setup=is_stateless,
)
except Exception as exc:
raise RuntimeError(