[Elastic EP] Reduce eager-mode reconfiguration downtime (#51885)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
Itay Alroy
2026-08-19 02:13:06 +00:00
committed by GitHub
parent d36cc4254e
commit a2257f95b7
12 changed files with 194 additions and 303 deletions
+12 -36
View File
@@ -248,7 +248,6 @@ class NixlEplbCommunicator(EplbCommunicator):
cpu_group: ProcessGroup,
all_expert_weights: Sequence[Sequence[torch.Tensor]],
expert_buffer: Sequence[torch.Tensor],
defer_remote_setup: bool = False,
) -> None:
"""Create a NIXL-backed EPLB communicator.
@@ -256,11 +255,6 @@ class NixlEplbCommunicator(EplbCommunicator):
cpu_group: CPU process group for metadata exchange.
all_expert_weights: Expert weight tensors for all MoE layers.
expert_buffer: Pre-allocated receive buffer tensors.
defer_remote_setup: If True, postpone the collective
all-gather of NIXL agent metadata until the first
``set_transfer_context`` call. Required for elastic EP
where ranks join asynchronously and cannot participate
in collectives at construction time.
"""
assert all_expert_weights, (
"NixlEplbCommunicator requires non-empty all_expert_weights."
@@ -317,29 +311,17 @@ class NixlEplbCommunicator(EplbCommunicator):
] = {}
self._cuda_device_id = int(self._device.index or 0)
self._remote_state_initialized = False
self._init_step("buffers", self._init_registered_buffers)
if defer_remote_setup:
logger.info_once("NIXL EPLB: deferring remote agent setup (elastic EP).")
else:
self._init_remote_state()
self._init_remote_state()
self._log_initialized()
def _init_remote_state(self) -> None:
"""Exchange NIXL agent metadata and RDMA pointer info with all peers.
This is a collective operation (uses ``all_gather_object`` twice).
Under elastic EP the call is deferred to the first
``set_transfer_context`` invocation, where all ranks are
guaranteed to be synchronized.
"""
self._init_step("agents", self._init_remote_agents)
self._init_step("send meta", self._exchange_remote_send_meta)
self._remote_state_initialized = True
def _ensure_remote_state(self) -> None:
if not self._remote_state_initialized:
self._init_remote_state()
@property
def needs_profile_buffer_reservation(self) -> bool:
@@ -373,7 +355,6 @@ class NixlEplbCommunicator(EplbCommunicator):
pass
def set_transfer_context(self, old_indices: np.ndarray, layer_idx: int) -> None:
self._ensure_remote_state()
assert not self._xfer_entries, (
f"set_transfer_context() called with {len(self._xfer_entries)} "
f"pending transfers from layer {self._layer_idx}; "
@@ -673,9 +654,8 @@ def create_eplb_communicator(
Falls back to ``"torch_nccl"`` when *None*.
Stateless (elastic EP) groups support ``"torch_nccl"``,
``"pynccl"``, and ``"nixl"``; ``"torch_nccl"`` is silently
promoted to ``"pynccl"``. ``"nixl"`` uses deferred remote
agent setup to avoid collective deadlocks during elastic
scaling. When tensors reside on CPU, ``"torch_gloo"`` or
promoted to ``"pynccl"``. When tensors reside on CPU,
``"torch_gloo"`` or
``"torch_nccl"`` are used via the CPU process group.
expert_weights: Expert weight tensors for *all* MoE layers.
Shape ``(num_layers)(num_tensors_per_layer)``.
@@ -729,22 +709,19 @@ def create_eplb_communicator(
) from exc
is_stateless = isinstance(group_coordinator, StatelessGroupCoordinator)
if is_stateless:
if backend == "nixl":
pass # handled below with defer_remote_setup=True
elif backend not in ("torch_nccl", "pynccl"):
if is_stateless and backend != "nixl":
if backend not in ("torch_nccl", "pynccl"):
raise ValueError(
f"Elastic EP requires 'torch_nccl', 'pynccl', or 'nixl' "
f"EPLB communicator (got '{backend}')."
)
else:
if backend == "torch_nccl":
logger.warning(
"Stateless elastic EP requires PyNCCL backend. "
"Forcing EPLB communicator to 'pynccl'."
)
backend = "pynccl"
return _create_pynccl()
if backend == "torch_nccl":
logger.warning(
"Stateless elastic EP requires PyNCCL backend. "
"Forcing EPLB communicator to 'pynccl'."
)
backend = "pynccl"
return _create_pynccl()
if backend == "nixl":
if not has_nixl():
@@ -761,7 +738,6 @@ def create_eplb_communicator(
cpu_group=group_coordinator.cpu_group,
all_expert_weights=expert_weights,
expert_buffer=expert_buffer,
defer_remote_setup=is_stateless,
)
except Exception as exc:
raise RuntimeError(
+46 -25
View File
@@ -37,6 +37,7 @@ from torch.distributed import ProcessGroup, all_reduce
from vllm.config import ModelConfig, ParallelConfig
from vllm.config.utils import compute_hash_cached
from vllm.distributed.parallel_state import (
GroupCoordinator,
get_ep_group,
get_eplb_group,
get_node_count,
@@ -289,14 +290,6 @@ class EplbState:
"""
CUDA device index for the async EPLB worker thread.
"""
self.num_valid_physical_experts: int = 0
"""
Number of valid physical experts.
This is the number of physical experts that are
actually mapped to logical experts. In elastic EP,
newly started EP ranks may not have physical experts
mapped yet.
"""
if self.device.type == "cuda":
self.cuda_device_index = self.device.index
if self.cuda_device_index is None and torch.cuda.is_available():
@@ -499,7 +492,6 @@ class EplbState:
num_unpadded_tokens_tensors=num_unpadded_tokens_tensors,
)
self.model_states[model_config.compute_hash()] = model_state
self.num_valid_physical_experts = model.num_physical_experts
def prepare_forward(
self,
@@ -766,28 +758,28 @@ class EplbState:
# Map the physical expert load to global logical experts
global_expert_load_windows = []
for eplb_model_state in self.model_states.values():
expert_load_window = eplb_model_state.expert_load_window[
:, :, : self.num_valid_physical_experts
]
expert_load_window = eplb_model_state.expert_load_window
physical_to_logical = eplb_model_state.physical_to_logical_map
invalid_idx = eplb_model_state.model.num_logical_experts
logical_expert_load_window = torch.zeros(
self.expert_load_window_size,
eplb_model_state.model.num_moe_layers,
eplb_model_state.model.num_logical_experts,
invalid_idx + 1,
dtype=eplb_model_state.expert_load_window.dtype,
device=eplb_model_state.expert_load_window.device,
)
logical_expert_load_window.scatter_add_(
dim=-1,
index=eplb_model_state.physical_to_logical_map[
:, : self.num_valid_physical_experts
]
index=physical_to_logical.masked_fill(
physical_to_logical < 0, invalid_idx
)
.unsqueeze(0)
.expand_as(expert_load_window)
.long(),
src=expert_load_window,
)
global_expert_load_window = logical_expert_load_window.sum(dim=0)
global_expert_load_window = logical_expert_load_window[..., :-1].sum(dim=0)
global_expert_load_windows.append(global_expert_load_window)
# Perform all-reduce to get the expert load across all ranks for each model
global_expert_load_windows = self._allreduce_list(global_expert_load_windows)
@@ -960,7 +952,7 @@ class EplbState:
Each pending result is acknowledged (consumed_event recorded) so the
async worker can proceed, but the transferred weights are intentionally
NOT applied — a full synchronous rearrange is expected to follow.
NOT applied — a full rearrange is expected to follow.
Ranks are kept in lockstep via _all_ranks_result_ready (all_reduce
on the EP CPU group). The async worker's coordinated-stop collectives
@@ -1060,7 +1052,6 @@ class EplbState:
device: torch.device,
parallel_config: ParallelConfig,
expanded_physical_to_logical: torch.Tensor,
num_valid_physical_experts: int,
) -> "EplbState":
eplb_state = cls(
parallel_config=parallel_config,
@@ -1070,12 +1061,24 @@ class EplbState:
model=model,
model_config=model_config,
)
eplb_state.num_valid_physical_experts = num_valid_physical_experts
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
eplb_state.update_mapping(
model_config,
expanded_physical_to_logical,
)
return eplb_state
def update_mapping(
self,
model_config: ModelConfig,
expanded_physical_to_logical: torch.Tensor,
) -> None:
eplb_model_state = self.model_states[model_config.compute_hash()]
eplb_model_state.physical_to_logical_map.copy_(expanded_physical_to_logical)
(logical_to_physical_map_cpu, logical_replica_count_cpu) = compute_logical_maps(
expanded_physical_to_logical.cpu(), model.num_logical_experts
expanded_physical_to_logical.cpu(),
eplb_model_state.model.num_logical_experts,
)
max_num_replicas = eplb_model_state.logical_to_physical_map.shape[-1]
@@ -1087,13 +1090,31 @@ class EplbState:
max_num_replicas - num_replicas,
),
value=-1,
).to(device)
logical_replica_count = logical_replica_count_cpu.to(device)
).to(self.device)
logical_replica_count = logical_replica_count_cpu.to(self.device)
eplb_model_state.logical_to_physical_map.copy_(logical_to_physical_map)
eplb_model_state.logical_replica_count.copy_(logical_replica_count)
return eplb_state
def create_communicator(
self, model_config: ModelConfig, group_coordinator: GroupCoordinator
) -> EplbCommunicator:
model_state = self.model_states[model_config.compute_hash()]
backend = self.parallel_config.eplb_config.communicator
assert backend is not None
return create_eplb_communicator(
group_coordinator,
backend,
model_state.model.expert_weights,
model_state.expert_buffer,
)
def update_communicator(
self,
model_config: ModelConfig,
communicator: EplbCommunicator,
) -> None:
self.model_states[model_config.compute_hash()].communicator = communicator
@dataclass