[Elastic EP] Reduce eager-mode reconfiguration downtime (#51885)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -248,7 +248,6 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
cpu_group: ProcessGroup,
|
||||
all_expert_weights: Sequence[Sequence[torch.Tensor]],
|
||||
expert_buffer: Sequence[torch.Tensor],
|
||||
defer_remote_setup: bool = False,
|
||||
) -> None:
|
||||
"""Create a NIXL-backed EPLB communicator.
|
||||
|
||||
@@ -256,11 +255,6 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
cpu_group: CPU process group for metadata exchange.
|
||||
all_expert_weights: Expert weight tensors for all MoE layers.
|
||||
expert_buffer: Pre-allocated receive buffer tensors.
|
||||
defer_remote_setup: If True, postpone the collective
|
||||
all-gather of NIXL agent metadata until the first
|
||||
``set_transfer_context`` call. Required for elastic EP
|
||||
where ranks join asynchronously and cannot participate
|
||||
in collectives at construction time.
|
||||
"""
|
||||
assert all_expert_weights, (
|
||||
"NixlEplbCommunicator requires non-empty all_expert_weights."
|
||||
@@ -317,29 +311,17 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
] = {}
|
||||
|
||||
self._cuda_device_id = int(self._device.index or 0)
|
||||
self._remote_state_initialized = False
|
||||
self._init_step("buffers", self._init_registered_buffers)
|
||||
if defer_remote_setup:
|
||||
logger.info_once("NIXL EPLB: deferring remote agent setup (elastic EP).")
|
||||
else:
|
||||
self._init_remote_state()
|
||||
self._init_remote_state()
|
||||
self._log_initialized()
|
||||
|
||||
def _init_remote_state(self) -> None:
|
||||
"""Exchange NIXL agent metadata and RDMA pointer info with all peers.
|
||||
|
||||
This is a collective operation (uses ``all_gather_object`` twice).
|
||||
Under elastic EP the call is deferred to the first
|
||||
``set_transfer_context`` invocation, where all ranks are
|
||||
guaranteed to be synchronized.
|
||||
"""
|
||||
self._init_step("agents", self._init_remote_agents)
|
||||
self._init_step("send meta", self._exchange_remote_send_meta)
|
||||
self._remote_state_initialized = True
|
||||
|
||||
def _ensure_remote_state(self) -> None:
|
||||
if not self._remote_state_initialized:
|
||||
self._init_remote_state()
|
||||
|
||||
@property
|
||||
def needs_profile_buffer_reservation(self) -> bool:
|
||||
@@ -373,7 +355,6 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
pass
|
||||
|
||||
def set_transfer_context(self, old_indices: np.ndarray, layer_idx: int) -> None:
|
||||
self._ensure_remote_state()
|
||||
assert not self._xfer_entries, (
|
||||
f"set_transfer_context() called with {len(self._xfer_entries)} "
|
||||
f"pending transfers from layer {self._layer_idx}; "
|
||||
@@ -673,9 +654,8 @@ def create_eplb_communicator(
|
||||
Falls back to ``"torch_nccl"`` when *None*.
|
||||
Stateless (elastic EP) groups support ``"torch_nccl"``,
|
||||
``"pynccl"``, and ``"nixl"``; ``"torch_nccl"`` is silently
|
||||
promoted to ``"pynccl"``. ``"nixl"`` uses deferred remote
|
||||
agent setup to avoid collective deadlocks during elastic
|
||||
scaling. When tensors reside on CPU, ``"torch_gloo"`` or
|
||||
promoted to ``"pynccl"``. When tensors reside on CPU,
|
||||
``"torch_gloo"`` or
|
||||
``"torch_nccl"`` are used via the CPU process group.
|
||||
expert_weights: Expert weight tensors for *all* MoE layers.
|
||||
Shape ``(num_layers)(num_tensors_per_layer)``.
|
||||
@@ -729,22 +709,19 @@ def create_eplb_communicator(
|
||||
) from exc
|
||||
|
||||
is_stateless = isinstance(group_coordinator, StatelessGroupCoordinator)
|
||||
if is_stateless:
|
||||
if backend == "nixl":
|
||||
pass # handled below with defer_remote_setup=True
|
||||
elif backend not in ("torch_nccl", "pynccl"):
|
||||
if is_stateless and backend != "nixl":
|
||||
if backend not in ("torch_nccl", "pynccl"):
|
||||
raise ValueError(
|
||||
f"Elastic EP requires 'torch_nccl', 'pynccl', or 'nixl' "
|
||||
f"EPLB communicator (got '{backend}')."
|
||||
)
|
||||
else:
|
||||
if backend == "torch_nccl":
|
||||
logger.warning(
|
||||
"Stateless elastic EP requires PyNCCL backend. "
|
||||
"Forcing EPLB communicator to 'pynccl'."
|
||||
)
|
||||
backend = "pynccl"
|
||||
return _create_pynccl()
|
||||
if backend == "torch_nccl":
|
||||
logger.warning(
|
||||
"Stateless elastic EP requires PyNCCL backend. "
|
||||
"Forcing EPLB communicator to 'pynccl'."
|
||||
)
|
||||
backend = "pynccl"
|
||||
return _create_pynccl()
|
||||
|
||||
if backend == "nixl":
|
||||
if not has_nixl():
|
||||
@@ -761,7 +738,6 @@ def create_eplb_communicator(
|
||||
cpu_group=group_coordinator.cpu_group,
|
||||
all_expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
defer_remote_setup=is_stateless,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -37,6 +37,7 @@ from torch.distributed import ProcessGroup, all_reduce
|
||||
from vllm.config import ModelConfig, ParallelConfig
|
||||
from vllm.config.utils import compute_hash_cached
|
||||
from vllm.distributed.parallel_state import (
|
||||
GroupCoordinator,
|
||||
get_ep_group,
|
||||
get_eplb_group,
|
||||
get_node_count,
|
||||
@@ -289,14 +290,6 @@ class EplbState:
|
||||
"""
|
||||
CUDA device index for the async EPLB worker thread.
|
||||
"""
|
||||
self.num_valid_physical_experts: int = 0
|
||||
"""
|
||||
Number of valid physical experts.
|
||||
This is the number of physical experts that are
|
||||
actually mapped to logical experts. In elastic EP,
|
||||
newly started EP ranks may not have physical experts
|
||||
mapped yet.
|
||||
"""
|
||||
if self.device.type == "cuda":
|
||||
self.cuda_device_index = self.device.index
|
||||
if self.cuda_device_index is None and torch.cuda.is_available():
|
||||
@@ -499,7 +492,6 @@ class EplbState:
|
||||
num_unpadded_tokens_tensors=num_unpadded_tokens_tensors,
|
||||
)
|
||||
self.model_states[model_config.compute_hash()] = model_state
|
||||
self.num_valid_physical_experts = model.num_physical_experts
|
||||
|
||||
def prepare_forward(
|
||||
self,
|
||||
@@ -766,28 +758,28 @@ class EplbState:
|
||||
# Map the physical expert load to global logical experts
|
||||
global_expert_load_windows = []
|
||||
for eplb_model_state in self.model_states.values():
|
||||
expert_load_window = eplb_model_state.expert_load_window[
|
||||
:, :, : self.num_valid_physical_experts
|
||||
]
|
||||
expert_load_window = eplb_model_state.expert_load_window
|
||||
physical_to_logical = eplb_model_state.physical_to_logical_map
|
||||
invalid_idx = eplb_model_state.model.num_logical_experts
|
||||
logical_expert_load_window = torch.zeros(
|
||||
self.expert_load_window_size,
|
||||
eplb_model_state.model.num_moe_layers,
|
||||
eplb_model_state.model.num_logical_experts,
|
||||
invalid_idx + 1,
|
||||
dtype=eplb_model_state.expert_load_window.dtype,
|
||||
device=eplb_model_state.expert_load_window.device,
|
||||
)
|
||||
logical_expert_load_window.scatter_add_(
|
||||
dim=-1,
|
||||
index=eplb_model_state.physical_to_logical_map[
|
||||
:, : self.num_valid_physical_experts
|
||||
]
|
||||
index=physical_to_logical.masked_fill(
|
||||
physical_to_logical < 0, invalid_idx
|
||||
)
|
||||
.unsqueeze(0)
|
||||
.expand_as(expert_load_window)
|
||||
.long(),
|
||||
src=expert_load_window,
|
||||
)
|
||||
|
||||
global_expert_load_window = logical_expert_load_window.sum(dim=0)
|
||||
global_expert_load_window = logical_expert_load_window[..., :-1].sum(dim=0)
|
||||
global_expert_load_windows.append(global_expert_load_window)
|
||||
# Perform all-reduce to get the expert load across all ranks for each model
|
||||
global_expert_load_windows = self._allreduce_list(global_expert_load_windows)
|
||||
@@ -960,7 +952,7 @@ class EplbState:
|
||||
|
||||
Each pending result is acknowledged (consumed_event recorded) so the
|
||||
async worker can proceed, but the transferred weights are intentionally
|
||||
NOT applied — a full synchronous rearrange is expected to follow.
|
||||
NOT applied — a full rearrange is expected to follow.
|
||||
|
||||
Ranks are kept in lockstep via _all_ranks_result_ready (all_reduce
|
||||
on the EP CPU group). The async worker's coordinated-stop collectives
|
||||
@@ -1060,7 +1052,6 @@ class EplbState:
|
||||
device: torch.device,
|
||||
parallel_config: ParallelConfig,
|
||||
expanded_physical_to_logical: torch.Tensor,
|
||||
num_valid_physical_experts: int,
|
||||
) -> "EplbState":
|
||||
eplb_state = cls(
|
||||
parallel_config=parallel_config,
|
||||
@@ -1070,12 +1061,24 @@ class EplbState:
|
||||
model=model,
|
||||
model_config=model_config,
|
||||
)
|
||||
eplb_state.num_valid_physical_experts = num_valid_physical_experts
|
||||
eplb_model_state = eplb_state.model_states[model_config.compute_hash()]
|
||||
eplb_state.update_mapping(
|
||||
model_config,
|
||||
expanded_physical_to_logical,
|
||||
)
|
||||
|
||||
return eplb_state
|
||||
|
||||
def update_mapping(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
expanded_physical_to_logical: torch.Tensor,
|
||||
) -> None:
|
||||
eplb_model_state = self.model_states[model_config.compute_hash()]
|
||||
eplb_model_state.physical_to_logical_map.copy_(expanded_physical_to_logical)
|
||||
|
||||
(logical_to_physical_map_cpu, logical_replica_count_cpu) = compute_logical_maps(
|
||||
expanded_physical_to_logical.cpu(), model.num_logical_experts
|
||||
expanded_physical_to_logical.cpu(),
|
||||
eplb_model_state.model.num_logical_experts,
|
||||
)
|
||||
|
||||
max_num_replicas = eplb_model_state.logical_to_physical_map.shape[-1]
|
||||
@@ -1087,13 +1090,31 @@ class EplbState:
|
||||
max_num_replicas - num_replicas,
|
||||
),
|
||||
value=-1,
|
||||
).to(device)
|
||||
logical_replica_count = logical_replica_count_cpu.to(device)
|
||||
).to(self.device)
|
||||
logical_replica_count = logical_replica_count_cpu.to(self.device)
|
||||
|
||||
eplb_model_state.logical_to_physical_map.copy_(logical_to_physical_map)
|
||||
eplb_model_state.logical_replica_count.copy_(logical_replica_count)
|
||||
|
||||
return eplb_state
|
||||
def create_communicator(
|
||||
self, model_config: ModelConfig, group_coordinator: GroupCoordinator
|
||||
) -> EplbCommunicator:
|
||||
model_state = self.model_states[model_config.compute_hash()]
|
||||
backend = self.parallel_config.eplb_config.communicator
|
||||
assert backend is not None
|
||||
return create_eplb_communicator(
|
||||
group_coordinator,
|
||||
backend,
|
||||
model_state.model.expert_weights,
|
||||
model_state.expert_buffer,
|
||||
)
|
||||
|
||||
def update_communicator(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
communicator: EplbCommunicator,
|
||||
) -> None:
|
||||
self.model_states[model_config.compute_hash()].communicator = communicator
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in New Issue
Block a user