[Elastic EP] Reduce eager-mode reconfiguration downtime (#51885)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -248,7 +248,6 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
cpu_group: ProcessGroup,
|
||||
all_expert_weights: Sequence[Sequence[torch.Tensor]],
|
||||
expert_buffer: Sequence[torch.Tensor],
|
||||
defer_remote_setup: bool = False,
|
||||
) -> None:
|
||||
"""Create a NIXL-backed EPLB communicator.
|
||||
|
||||
@@ -256,11 +255,6 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
cpu_group: CPU process group for metadata exchange.
|
||||
all_expert_weights: Expert weight tensors for all MoE layers.
|
||||
expert_buffer: Pre-allocated receive buffer tensors.
|
||||
defer_remote_setup: If True, postpone the collective
|
||||
all-gather of NIXL agent metadata until the first
|
||||
``set_transfer_context`` call. Required for elastic EP
|
||||
where ranks join asynchronously and cannot participate
|
||||
in collectives at construction time.
|
||||
"""
|
||||
assert all_expert_weights, (
|
||||
"NixlEplbCommunicator requires non-empty all_expert_weights."
|
||||
@@ -317,29 +311,17 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
] = {}
|
||||
|
||||
self._cuda_device_id = int(self._device.index or 0)
|
||||
self._remote_state_initialized = False
|
||||
self._init_step("buffers", self._init_registered_buffers)
|
||||
if defer_remote_setup:
|
||||
logger.info_once("NIXL EPLB: deferring remote agent setup (elastic EP).")
|
||||
else:
|
||||
self._init_remote_state()
|
||||
self._init_remote_state()
|
||||
self._log_initialized()
|
||||
|
||||
def _init_remote_state(self) -> None:
|
||||
"""Exchange NIXL agent metadata and RDMA pointer info with all peers.
|
||||
|
||||
This is a collective operation (uses ``all_gather_object`` twice).
|
||||
Under elastic EP the call is deferred to the first
|
||||
``set_transfer_context`` invocation, where all ranks are
|
||||
guaranteed to be synchronized.
|
||||
"""
|
||||
self._init_step("agents", self._init_remote_agents)
|
||||
self._init_step("send meta", self._exchange_remote_send_meta)
|
||||
self._remote_state_initialized = True
|
||||
|
||||
def _ensure_remote_state(self) -> None:
|
||||
if not self._remote_state_initialized:
|
||||
self._init_remote_state()
|
||||
|
||||
@property
|
||||
def needs_profile_buffer_reservation(self) -> bool:
|
||||
@@ -373,7 +355,6 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
pass
|
||||
|
||||
def set_transfer_context(self, old_indices: np.ndarray, layer_idx: int) -> None:
|
||||
self._ensure_remote_state()
|
||||
assert not self._xfer_entries, (
|
||||
f"set_transfer_context() called with {len(self._xfer_entries)} "
|
||||
f"pending transfers from layer {self._layer_idx}; "
|
||||
@@ -673,9 +654,8 @@ def create_eplb_communicator(
|
||||
Falls back to ``"torch_nccl"`` when *None*.
|
||||
Stateless (elastic EP) groups support ``"torch_nccl"``,
|
||||
``"pynccl"``, and ``"nixl"``; ``"torch_nccl"`` is silently
|
||||
promoted to ``"pynccl"``. ``"nixl"`` uses deferred remote
|
||||
agent setup to avoid collective deadlocks during elastic
|
||||
scaling. When tensors reside on CPU, ``"torch_gloo"`` or
|
||||
promoted to ``"pynccl"``. When tensors reside on CPU,
|
||||
``"torch_gloo"`` or
|
||||
``"torch_nccl"`` are used via the CPU process group.
|
||||
expert_weights: Expert weight tensors for *all* MoE layers.
|
||||
Shape ``(num_layers)(num_tensors_per_layer)``.
|
||||
@@ -729,22 +709,19 @@ def create_eplb_communicator(
|
||||
) from exc
|
||||
|
||||
is_stateless = isinstance(group_coordinator, StatelessGroupCoordinator)
|
||||
if is_stateless:
|
||||
if backend == "nixl":
|
||||
pass # handled below with defer_remote_setup=True
|
||||
elif backend not in ("torch_nccl", "pynccl"):
|
||||
if is_stateless and backend != "nixl":
|
||||
if backend not in ("torch_nccl", "pynccl"):
|
||||
raise ValueError(
|
||||
f"Elastic EP requires 'torch_nccl', 'pynccl', or 'nixl' "
|
||||
f"EPLB communicator (got '{backend}')."
|
||||
)
|
||||
else:
|
||||
if backend == "torch_nccl":
|
||||
logger.warning(
|
||||
"Stateless elastic EP requires PyNCCL backend. "
|
||||
"Forcing EPLB communicator to 'pynccl'."
|
||||
)
|
||||
backend = "pynccl"
|
||||
return _create_pynccl()
|
||||
if backend == "torch_nccl":
|
||||
logger.warning(
|
||||
"Stateless elastic EP requires PyNCCL backend. "
|
||||
"Forcing EPLB communicator to 'pynccl'."
|
||||
)
|
||||
backend = "pynccl"
|
||||
return _create_pynccl()
|
||||
|
||||
if backend == "nixl":
|
||||
if not has_nixl():
|
||||
@@ -761,7 +738,6 @@ def create_eplb_communicator(
|
||||
cpu_group=group_coordinator.cpu_group,
|
||||
all_expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
defer_remote_setup=is_stateless,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
|
||||
Reference in New Issue
Block a user