[EPLB] Add contention-aware expert migration batching (#52641)

Signed-off-by: DOCCA0 <2947323341@qq.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
DOCCA0
2026-10-03 00:13:50 +00:00
committed by GitHub
co-authored by Robert Shaw
parent 8e443871f0
commit 7dfe3338d5
6 changed files with 588 additions and 8 deletions
+26
View File
@@ -11,6 +11,7 @@ from vllm.distributed.parallel_state import get_eplb_group
from vllm.logger import init_logger
from .eplb_utils import CpuGpuEvent, device_stream
from .migration_scheduler import schedule_migration_batches_for_layers
from .rebalance_execute import AsyncEplbLayerResult, transfer_layer
if TYPE_CHECKING:
@@ -99,6 +100,21 @@ def transfer_run_periodically(
model_state, state, physical_to_logical_map_cpu, stream
)
migration_batches_by_layer = None
if state.parallel_config.eplb_config.migration_batching_enabled:
profile_name = "eplb: schedule migration batches"
num_local_experts = model_state.model.expert_weights[0][0].shape[0]
# The placement snapshots stay fixed for this rebalance cycle.
# Precompute all layer schedules once, but execute transfers one
# layer at a time below to preserve the migration lifecycle. This
# profiling range therefore covers scheduling for every MoE layer.
with torch.profiler.record_function(profile_name):
migration_batches_by_layer = schedule_migration_batches_for_layers(
num_local_experts,
physical_to_logical_map_cpu.numpy(),
new_physical_to_logical_map.numpy(),
)
# Execute one EPLB layer transfer per model forward pass. Each iteration
# of this loop will copy the new set of expert weights into
# model_state.expert_buffer, which will be consumed by the main thread in
@@ -134,6 +150,16 @@ def transfer_run_periodically(
is_profile=is_profile,
stream=stream,
layer_idx=layer_idx,
enable_migration_batching=(
state.parallel_config.eplb_config.migration_batching_enabled
),
migration_batches=(
# This is only the current layer's schedule; it does not
# combine transfers from different layers.
migration_batches_by_layer[layer_idx]
if migration_batches_by_layer is not None
else None
),
)
# Wait until all writes to expert_buffer have finished before making the
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Schedule EPLB expert migrations into conflict-free batches."""
from dataclasses import dataclass
import numpy as np
@dataclass(frozen=True, slots=True)
class MigrationFlow:
"""Expert transfers between one directed rank pair."""
src_rank: int
dst_rank: int
# Experts being transferred.
expert_ids: tuple[int, ...]
def schedule_migration_batches(
num_local_experts: int,
old_indices: np.ndarray,
new_indices: np.ndarray,
) -> list[list[MigrationFlow]]:
"""Schedule expert migrations without per-rank communication contention.
Experts assigned to the same source and destination ranks form one flow.
Each flow is placed in the first batch that does not already use either
endpoint, so a rank communicates with at most one peer in each batch while
independent rank pairs can transfer concurrently.
"""
assert old_indices.shape == new_indices.shape
recv_ranks_by_expert: dict[int, list[int]] = {}
old_experts_by_rank: list[set[int]] = []
old_by_rank = old_indices.reshape(-1, num_local_experts)
new_by_rank = new_indices.reshape(-1, num_local_experts)
# First find the ranks that gain each expert. Existing local copies need no
# transfer, even if their physical row changes.
for rank, (old_local, new_local) in enumerate(zip(old_by_rank, new_by_rank)):
old_experts = set(old_local.tolist())
old_experts.discard(-1)
old_experts_by_rank.append(old_experts)
new_experts = set(new_local.tolist())
new_experts.discard(-1)
for expert_id in new_experts - old_experts:
recv_ranks_by_expert.setdefault(expert_id, []).append(rank)
old_ranks_by_expert: dict[int, list[int]] = {}
needed_experts = recv_ranks_by_expert.keys()
# Only index source copies for experts that need a remote transfer.
for rank, old_experts in enumerate(old_experts_by_rank):
for expert_id in old_experts & needed_experts:
old_ranks_by_expert.setdefault(expert_id, []).append(rank)
expert_ids_by_pair: dict[tuple[int, int], list[int]] = {}
for expert_id in sorted(recv_ranks_by_expert):
send_ranks = old_ranks_by_expert.get(expert_id, [])
if not send_ranks:
continue
recv_ranks = recv_ranks_by_expert[expert_id]
num_per_sender, remainder = divmod(len(recv_ranks), len(send_ranks))
remainder_start = len(send_ranks) * num_per_sender
# Preserve the unbatched path's balanced source assignment. Batching
# changes transfer order, not which existing replica supplies a copy.
for sender_idx, src_rank in enumerate(send_ranks):
start = sender_idx * num_per_sender
for dst_rank in recv_ranks[start : start + num_per_sender]:
expert_ids_by_pair.setdefault((src_rank, dst_rank), []).append(
expert_id
)
if sender_idx < remainder:
dst_rank = recv_ranks[remainder_start + sender_idx]
expert_ids_by_pair.setdefault((src_rank, dst_rank), []).append(
expert_id
)
# Transfers with the same directed rank pair have identical contention
# constraints, so coalesce them before grouping. The loop below is a
# deterministic first-fit edge coloring of the rank communication graph.
batches: list[list[MigrationFlow]] = []
endpoints_used: list[int] = []
for (src_rank, dst_rank), expert_ids in expert_ids_by_pair.items():
flow = MigrationFlow(src_rank, dst_rank, tuple(expert_ids))
# Direction is irrelevant for NIC contention. This endpoint mask keeps
# each rank connected to at most one peer in a batch. The conservative
# limit leaves bandwidth headroom for inference during async migration.
endpoints = (1 << src_rank) | (1 << dst_rank)
for batch_idx, (batch, used) in enumerate(zip(batches, endpoints_used)):
if not endpoints & used:
batch.append(flow)
endpoints_used[batch_idx] |= endpoints
break
else:
batches.append([flow])
endpoints_used.append(endpoints)
return batches
def schedule_migration_batches_for_layers(
num_local_experts: int,
old_indices: np.ndarray,
new_indices: np.ndarray,
) -> list[list[list[MigrationFlow]]]:
"""Precompute the independent migration schedule for every MoE layer.
This reduces scheduling entry points from once per layer to once per
rebalance cycle. Only CPU scheduling is grouped; transfers remain ordered
by layer and preserve each layer's batch boundaries.
"""
assert old_indices.shape == new_indices.shape
assert old_indices.ndim == 2
return [
schedule_migration_batches(num_local_experts, old_layer, new_layer)
for old_layer, new_layer in zip(old_indices, new_indices)
]
__all__ = [
"MigrationFlow",
"schedule_migration_batches",
"schedule_migration_batches_for_layers",
]
+142 -8
View File
@@ -14,6 +14,10 @@ from torch.distributed import ProcessGroup, all_gather
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
from vllm.distributed.eplb.migration_scheduler import (
MigrationFlow,
schedule_migration_batches,
)
from vllm.logger import init_logger
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -167,6 +171,107 @@ def get_ep_ranks_with_experts_batch(
return ranks_to_send_map, ranks_to_recv_map
def _execute_migration_batch(
batch: list[MigrationFlow],
old_indices: np.ndarray,
old_rows: dict[int, int],
new_rows: dict[int, int],
ep_rank: int,
expert_weights: Sequence[torch.Tensor],
expert_weights_buffers: Sequence[torch.Tensor],
communicator: EplbCommunicator,
layer_idx: int,
) -> None:
"""Execute one batch of rank-pair-disjoint expert transfers."""
# NIXL drains its transfer context in execute(), so each batch restores it.
communicator.set_transfer_context(old_indices, layer_idx)
for flow in batch:
if flow.src_rank == ep_rank:
for expert_id in flow.expert_ids:
src_row = _get_migration_row(
old_rows, expert_id, flow, ep_rank, mapping_name="old"
)
communicator.add_send(
[weight[src_row] for weight in expert_weights],
flow.dst_rank,
expert_id=expert_id,
)
elif flow.dst_rank == ep_rank:
for expert_id in flow.expert_ids:
dst_row = _get_migration_row(
new_rows, expert_id, flow, ep_rank, mapping_name="new"
)
communicator.add_recv(
[buffer[dst_row] for buffer in expert_weights_buffers],
flow.src_rank,
expert_id=expert_id,
)
# Combining execute calls would erase the scheduler's batch boundaries.
communicator.execute()
def _execute_migration_batches(
num_local_experts: int,
old_indices: np.ndarray,
new_indices: np.ndarray,
ep_rank: int,
expert_weights: Sequence[torch.Tensor],
expert_weights_buffers: Sequence[torch.Tensor],
communicator: EplbCommunicator,
layer_idx: int,
batches: list[list[MigrationFlow]] | None = None,
) -> None:
"""Execute all contention-aware migration batches for one layer."""
if batches is None:
with torch.profiler.record_function("eplb: schedule migration batches"):
batches = schedule_migration_batches(
num_local_experts, old_indices, new_indices
)
base = ep_rank * num_local_experts
old_local = old_indices[base : base + num_local_experts]
new_local = new_indices[base : base + num_local_experts]
old_rows: dict[int, int] = {}
new_rows: dict[int, int] = {}
for row, expert_id in enumerate(old_local):
if expert_id != -1:
old_rows.setdefault(int(expert_id), row)
for row, expert_id in enumerate(new_local):
if expert_id != -1:
new_rows.setdefault(int(expert_id), row)
for batch in batches:
_execute_migration_batch(
batch=batch,
old_indices=old_indices,
old_rows=old_rows,
new_rows=new_rows,
ep_rank=ep_rank,
expert_weights=expert_weights,
expert_weights_buffers=expert_weights_buffers,
communicator=communicator,
layer_idx=layer_idx,
)
def _get_migration_row(
rows: dict[int, int],
expert_id: int,
flow: MigrationFlow,
ep_rank: int,
mapping_name: str,
) -> int:
row = rows.get(expert_id)
if row is not None:
return row
message = (
f"Expert {expert_id} is missing from the {mapping_name} "
f"mapping on rank {ep_rank} for migration "
f"{flow.src_rank} -> {flow.dst_rank}"
)
logger.error(message)
raise RuntimeError(message)
def move_to_buffer(
num_local_experts: int,
old_indices: np.ndarray,
@@ -177,6 +282,8 @@ def move_to_buffer(
ep_rank: int,
communicator: EplbCommunicator,
layer_idx: int = 0,
enable_migration_batching: bool = False,
migration_batches: list[list[MigrationFlow]] | None = None,
) -> TransferMetadata:
"""Rearranges expert weights during EPLB rebalancing.
@@ -192,6 +299,9 @@ def move_to_buffer(
ep_rank: Rank of this process in expert parallel group.
communicator: EplbCommunicator instance for P2P communication.
layer_idx: Index of the MoE layer being transferred.
enable_migration_batching: Schedule remote transfers in batches where
each rank communicates with at most one peer.
migration_batches: Optional batches precomputed for this layer.
Returns:
TransferMetadata: Metadata needed for completing remote weight transfers.
@@ -264,6 +374,30 @@ def move_to_buffer(
with device_stream(stream):
for w, b in zip(expert_weights, expert_weights_buffers):
b[dst].copy_(w[src_local], non_blocking=True)
transfer_metadata = TransferMetadata(
is_unchanged=is_unchanged,
is_received_locally=is_received_locally,
recv_primary_mask=recv_primary_mask,
recv_count=recv_count,
recv_expert_ids=recv_expert_ids,
recv_dst_rows=recv_dst_rows,
)
# Migration batching is disabled by default and must be explicitly enabled.
if enable_migration_batching:
_execute_migration_batches(
num_local_experts=num_local_experts,
old_indices=old_indices,
new_indices=new_indices,
ep_rank=ep_rank,
expert_weights=expert_weights,
expert_weights_buffers=expert_weights_buffers,
communicator=communicator,
layer_idx=layer_idx,
batches=migration_batches,
)
return transfer_metadata
communicator.set_transfer_context(old_indices, layer_idx)
# 2. Post sends
@@ -334,14 +468,7 @@ def move_to_buffer(
# 4. Execute transfers and wait for completion.
communicator.execute()
return TransferMetadata(
is_unchanged=is_unchanged,
is_received_locally=is_received_locally,
recv_primary_mask=recv_primary_mask,
recv_count=recv_count,
recv_expert_ids=recv_expert_ids,
recv_dst_rows=recv_dst_rows,
)
return transfer_metadata
def move_from_buffer(
@@ -432,6 +559,8 @@ def transfer_layer(
stream: torch.Stream | None = None,
rank_mapping: dict[int, int] | None = None,
layer_idx: int = 0,
enable_migration_batching: bool = False,
migration_batches: list[list[MigrationFlow]] | None = None,
) -> TransferMetadata:
"""Rearranges the expert weights in place according to the new expert indices.
@@ -453,6 +582,9 @@ def transfer_layer(
stream: CUDA stream for async copies (can be None for sync mode).
rank_mapping: Optional rank mapping for elastic expert parallelism.
layer_idx: Index of the MoE layer being transferred.
enable_migration_batching: Schedule remote transfers in batches where
each rank communicates with at most one peer.
migration_batches: Optional batches precomputed for this layer.
Returns:
TransferMetadata: Metadata needed for completing remote weight transfers,
@@ -502,6 +634,8 @@ def transfer_layer(
ep_rank=ep_group.rank(),
communicator=communicator,
layer_idx=layer_idx,
enable_migration_batching=enable_migration_batching,
migration_batches=migration_batches,
)