[EPLB] Add contention-aware expert migration batching (#52641)
Signed-off-by: DOCCA0 <2947323341@qq.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -11,6 +11,7 @@ from vllm.distributed.parallel_state import get_eplb_group
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .eplb_utils import CpuGpuEvent, device_stream
|
||||
from .migration_scheduler import schedule_migration_batches_for_layers
|
||||
from .rebalance_execute import AsyncEplbLayerResult, transfer_layer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -99,6 +100,21 @@ def transfer_run_periodically(
|
||||
model_state, state, physical_to_logical_map_cpu, stream
|
||||
)
|
||||
|
||||
migration_batches_by_layer = None
|
||||
if state.parallel_config.eplb_config.migration_batching_enabled:
|
||||
profile_name = "eplb: schedule migration batches"
|
||||
num_local_experts = model_state.model.expert_weights[0][0].shape[0]
|
||||
# The placement snapshots stay fixed for this rebalance cycle.
|
||||
# Precompute all layer schedules once, but execute transfers one
|
||||
# layer at a time below to preserve the migration lifecycle. This
|
||||
# profiling range therefore covers scheduling for every MoE layer.
|
||||
with torch.profiler.record_function(profile_name):
|
||||
migration_batches_by_layer = schedule_migration_batches_for_layers(
|
||||
num_local_experts,
|
||||
physical_to_logical_map_cpu.numpy(),
|
||||
new_physical_to_logical_map.numpy(),
|
||||
)
|
||||
|
||||
# Execute one EPLB layer transfer per model forward pass. Each iteration
|
||||
# of this loop will copy the new set of expert weights into
|
||||
# model_state.expert_buffer, which will be consumed by the main thread in
|
||||
@@ -134,6 +150,16 @@ def transfer_run_periodically(
|
||||
is_profile=is_profile,
|
||||
stream=stream,
|
||||
layer_idx=layer_idx,
|
||||
enable_migration_batching=(
|
||||
state.parallel_config.eplb_config.migration_batching_enabled
|
||||
),
|
||||
migration_batches=(
|
||||
# This is only the current layer's schedule; it does not
|
||||
# combine transfers from different layers.
|
||||
migration_batches_by_layer[layer_idx]
|
||||
if migration_batches_by_layer is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
# Wait until all writes to expert_buffer have finished before making the
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Schedule EPLB expert migrations into conflict-free batches."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class MigrationFlow:
|
||||
"""Expert transfers between one directed rank pair."""
|
||||
|
||||
src_rank: int
|
||||
dst_rank: int
|
||||
# Experts being transferred.
|
||||
expert_ids: tuple[int, ...]
|
||||
|
||||
|
||||
def schedule_migration_batches(
|
||||
num_local_experts: int,
|
||||
old_indices: np.ndarray,
|
||||
new_indices: np.ndarray,
|
||||
) -> list[list[MigrationFlow]]:
|
||||
"""Schedule expert migrations without per-rank communication contention.
|
||||
|
||||
Experts assigned to the same source and destination ranks form one flow.
|
||||
Each flow is placed in the first batch that does not already use either
|
||||
endpoint, so a rank communicates with at most one peer in each batch while
|
||||
independent rank pairs can transfer concurrently.
|
||||
"""
|
||||
assert old_indices.shape == new_indices.shape
|
||||
recv_ranks_by_expert: dict[int, list[int]] = {}
|
||||
old_experts_by_rank: list[set[int]] = []
|
||||
old_by_rank = old_indices.reshape(-1, num_local_experts)
|
||||
new_by_rank = new_indices.reshape(-1, num_local_experts)
|
||||
|
||||
# First find the ranks that gain each expert. Existing local copies need no
|
||||
# transfer, even if their physical row changes.
|
||||
for rank, (old_local, new_local) in enumerate(zip(old_by_rank, new_by_rank)):
|
||||
old_experts = set(old_local.tolist())
|
||||
old_experts.discard(-1)
|
||||
old_experts_by_rank.append(old_experts)
|
||||
|
||||
new_experts = set(new_local.tolist())
|
||||
new_experts.discard(-1)
|
||||
for expert_id in new_experts - old_experts:
|
||||
recv_ranks_by_expert.setdefault(expert_id, []).append(rank)
|
||||
|
||||
old_ranks_by_expert: dict[int, list[int]] = {}
|
||||
needed_experts = recv_ranks_by_expert.keys()
|
||||
|
||||
# Only index source copies for experts that need a remote transfer.
|
||||
for rank, old_experts in enumerate(old_experts_by_rank):
|
||||
for expert_id in old_experts & needed_experts:
|
||||
old_ranks_by_expert.setdefault(expert_id, []).append(rank)
|
||||
|
||||
expert_ids_by_pair: dict[tuple[int, int], list[int]] = {}
|
||||
for expert_id in sorted(recv_ranks_by_expert):
|
||||
send_ranks = old_ranks_by_expert.get(expert_id, [])
|
||||
if not send_ranks:
|
||||
continue
|
||||
recv_ranks = recv_ranks_by_expert[expert_id]
|
||||
|
||||
num_per_sender, remainder = divmod(len(recv_ranks), len(send_ranks))
|
||||
remainder_start = len(send_ranks) * num_per_sender
|
||||
|
||||
# Preserve the unbatched path's balanced source assignment. Batching
|
||||
# changes transfer order, not which existing replica supplies a copy.
|
||||
for sender_idx, src_rank in enumerate(send_ranks):
|
||||
start = sender_idx * num_per_sender
|
||||
for dst_rank in recv_ranks[start : start + num_per_sender]:
|
||||
expert_ids_by_pair.setdefault((src_rank, dst_rank), []).append(
|
||||
expert_id
|
||||
)
|
||||
if sender_idx < remainder:
|
||||
dst_rank = recv_ranks[remainder_start + sender_idx]
|
||||
expert_ids_by_pair.setdefault((src_rank, dst_rank), []).append(
|
||||
expert_id
|
||||
)
|
||||
|
||||
# Transfers with the same directed rank pair have identical contention
|
||||
# constraints, so coalesce them before grouping. The loop below is a
|
||||
# deterministic first-fit edge coloring of the rank communication graph.
|
||||
batches: list[list[MigrationFlow]] = []
|
||||
endpoints_used: list[int] = []
|
||||
for (src_rank, dst_rank), expert_ids in expert_ids_by_pair.items():
|
||||
flow = MigrationFlow(src_rank, dst_rank, tuple(expert_ids))
|
||||
|
||||
# Direction is irrelevant for NIC contention. This endpoint mask keeps
|
||||
# each rank connected to at most one peer in a batch. The conservative
|
||||
# limit leaves bandwidth headroom for inference during async migration.
|
||||
endpoints = (1 << src_rank) | (1 << dst_rank)
|
||||
for batch_idx, (batch, used) in enumerate(zip(batches, endpoints_used)):
|
||||
if not endpoints & used:
|
||||
batch.append(flow)
|
||||
endpoints_used[batch_idx] |= endpoints
|
||||
break
|
||||
else:
|
||||
batches.append([flow])
|
||||
endpoints_used.append(endpoints)
|
||||
|
||||
return batches
|
||||
|
||||
|
||||
def schedule_migration_batches_for_layers(
|
||||
num_local_experts: int,
|
||||
old_indices: np.ndarray,
|
||||
new_indices: np.ndarray,
|
||||
) -> list[list[list[MigrationFlow]]]:
|
||||
"""Precompute the independent migration schedule for every MoE layer.
|
||||
|
||||
This reduces scheduling entry points from once per layer to once per
|
||||
rebalance cycle. Only CPU scheduling is grouped; transfers remain ordered
|
||||
by layer and preserve each layer's batch boundaries.
|
||||
"""
|
||||
assert old_indices.shape == new_indices.shape
|
||||
assert old_indices.ndim == 2
|
||||
return [
|
||||
schedule_migration_batches(num_local_experts, old_layer, new_layer)
|
||||
for old_layer, new_layer in zip(old_indices, new_indices)
|
||||
]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MigrationFlow",
|
||||
"schedule_migration_batches",
|
||||
"schedule_migration_batches_for_layers",
|
||||
]
|
||||
@@ -14,6 +14,10 @@ from torch.distributed import ProcessGroup, all_gather
|
||||
|
||||
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
|
||||
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
|
||||
from vllm.distributed.eplb.migration_scheduler import (
|
||||
MigrationFlow,
|
||||
schedule_migration_batches,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
@@ -167,6 +171,107 @@ def get_ep_ranks_with_experts_batch(
|
||||
return ranks_to_send_map, ranks_to_recv_map
|
||||
|
||||
|
||||
def _execute_migration_batch(
|
||||
batch: list[MigrationFlow],
|
||||
old_indices: np.ndarray,
|
||||
old_rows: dict[int, int],
|
||||
new_rows: dict[int, int],
|
||||
ep_rank: int,
|
||||
expert_weights: Sequence[torch.Tensor],
|
||||
expert_weights_buffers: Sequence[torch.Tensor],
|
||||
communicator: EplbCommunicator,
|
||||
layer_idx: int,
|
||||
) -> None:
|
||||
"""Execute one batch of rank-pair-disjoint expert transfers."""
|
||||
# NIXL drains its transfer context in execute(), so each batch restores it.
|
||||
communicator.set_transfer_context(old_indices, layer_idx)
|
||||
for flow in batch:
|
||||
if flow.src_rank == ep_rank:
|
||||
for expert_id in flow.expert_ids:
|
||||
src_row = _get_migration_row(
|
||||
old_rows, expert_id, flow, ep_rank, mapping_name="old"
|
||||
)
|
||||
communicator.add_send(
|
||||
[weight[src_row] for weight in expert_weights],
|
||||
flow.dst_rank,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
elif flow.dst_rank == ep_rank:
|
||||
for expert_id in flow.expert_ids:
|
||||
dst_row = _get_migration_row(
|
||||
new_rows, expert_id, flow, ep_rank, mapping_name="new"
|
||||
)
|
||||
communicator.add_recv(
|
||||
[buffer[dst_row] for buffer in expert_weights_buffers],
|
||||
flow.src_rank,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
# Combining execute calls would erase the scheduler's batch boundaries.
|
||||
communicator.execute()
|
||||
|
||||
|
||||
def _execute_migration_batches(
|
||||
num_local_experts: int,
|
||||
old_indices: np.ndarray,
|
||||
new_indices: np.ndarray,
|
||||
ep_rank: int,
|
||||
expert_weights: Sequence[torch.Tensor],
|
||||
expert_weights_buffers: Sequence[torch.Tensor],
|
||||
communicator: EplbCommunicator,
|
||||
layer_idx: int,
|
||||
batches: list[list[MigrationFlow]] | None = None,
|
||||
) -> None:
|
||||
"""Execute all contention-aware migration batches for one layer."""
|
||||
if batches is None:
|
||||
with torch.profiler.record_function("eplb: schedule migration batches"):
|
||||
batches = schedule_migration_batches(
|
||||
num_local_experts, old_indices, new_indices
|
||||
)
|
||||
base = ep_rank * num_local_experts
|
||||
old_local = old_indices[base : base + num_local_experts]
|
||||
new_local = new_indices[base : base + num_local_experts]
|
||||
old_rows: dict[int, int] = {}
|
||||
new_rows: dict[int, int] = {}
|
||||
for row, expert_id in enumerate(old_local):
|
||||
if expert_id != -1:
|
||||
old_rows.setdefault(int(expert_id), row)
|
||||
for row, expert_id in enumerate(new_local):
|
||||
if expert_id != -1:
|
||||
new_rows.setdefault(int(expert_id), row)
|
||||
for batch in batches:
|
||||
_execute_migration_batch(
|
||||
batch=batch,
|
||||
old_indices=old_indices,
|
||||
old_rows=old_rows,
|
||||
new_rows=new_rows,
|
||||
ep_rank=ep_rank,
|
||||
expert_weights=expert_weights,
|
||||
expert_weights_buffers=expert_weights_buffers,
|
||||
communicator=communicator,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
|
||||
|
||||
def _get_migration_row(
|
||||
rows: dict[int, int],
|
||||
expert_id: int,
|
||||
flow: MigrationFlow,
|
||||
ep_rank: int,
|
||||
mapping_name: str,
|
||||
) -> int:
|
||||
row = rows.get(expert_id)
|
||||
if row is not None:
|
||||
return row
|
||||
|
||||
message = (
|
||||
f"Expert {expert_id} is missing from the {mapping_name} "
|
||||
f"mapping on rank {ep_rank} for migration "
|
||||
f"{flow.src_rank} -> {flow.dst_rank}"
|
||||
)
|
||||
logger.error(message)
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def move_to_buffer(
|
||||
num_local_experts: int,
|
||||
old_indices: np.ndarray,
|
||||
@@ -177,6 +282,8 @@ def move_to_buffer(
|
||||
ep_rank: int,
|
||||
communicator: EplbCommunicator,
|
||||
layer_idx: int = 0,
|
||||
enable_migration_batching: bool = False,
|
||||
migration_batches: list[list[MigrationFlow]] | None = None,
|
||||
) -> TransferMetadata:
|
||||
"""Rearranges expert weights during EPLB rebalancing.
|
||||
|
||||
@@ -192,6 +299,9 @@ def move_to_buffer(
|
||||
ep_rank: Rank of this process in expert parallel group.
|
||||
communicator: EplbCommunicator instance for P2P communication.
|
||||
layer_idx: Index of the MoE layer being transferred.
|
||||
enable_migration_batching: Schedule remote transfers in batches where
|
||||
each rank communicates with at most one peer.
|
||||
migration_batches: Optional batches precomputed for this layer.
|
||||
|
||||
Returns:
|
||||
TransferMetadata: Metadata needed for completing remote weight transfers.
|
||||
@@ -264,6 +374,30 @@ def move_to_buffer(
|
||||
with device_stream(stream):
|
||||
for w, b in zip(expert_weights, expert_weights_buffers):
|
||||
b[dst].copy_(w[src_local], non_blocking=True)
|
||||
transfer_metadata = TransferMetadata(
|
||||
is_unchanged=is_unchanged,
|
||||
is_received_locally=is_received_locally,
|
||||
recv_primary_mask=recv_primary_mask,
|
||||
recv_count=recv_count,
|
||||
recv_expert_ids=recv_expert_ids,
|
||||
recv_dst_rows=recv_dst_rows,
|
||||
)
|
||||
|
||||
# Migration batching is disabled by default and must be explicitly enabled.
|
||||
if enable_migration_batching:
|
||||
_execute_migration_batches(
|
||||
num_local_experts=num_local_experts,
|
||||
old_indices=old_indices,
|
||||
new_indices=new_indices,
|
||||
ep_rank=ep_rank,
|
||||
expert_weights=expert_weights,
|
||||
expert_weights_buffers=expert_weights_buffers,
|
||||
communicator=communicator,
|
||||
layer_idx=layer_idx,
|
||||
batches=migration_batches,
|
||||
)
|
||||
return transfer_metadata
|
||||
|
||||
communicator.set_transfer_context(old_indices, layer_idx)
|
||||
|
||||
# 2. Post sends
|
||||
@@ -334,14 +468,7 @@ def move_to_buffer(
|
||||
|
||||
# 4. Execute transfers and wait for completion.
|
||||
communicator.execute()
|
||||
return TransferMetadata(
|
||||
is_unchanged=is_unchanged,
|
||||
is_received_locally=is_received_locally,
|
||||
recv_primary_mask=recv_primary_mask,
|
||||
recv_count=recv_count,
|
||||
recv_expert_ids=recv_expert_ids,
|
||||
recv_dst_rows=recv_dst_rows,
|
||||
)
|
||||
return transfer_metadata
|
||||
|
||||
|
||||
def move_from_buffer(
|
||||
@@ -432,6 +559,8 @@ def transfer_layer(
|
||||
stream: torch.Stream | None = None,
|
||||
rank_mapping: dict[int, int] | None = None,
|
||||
layer_idx: int = 0,
|
||||
enable_migration_batching: bool = False,
|
||||
migration_batches: list[list[MigrationFlow]] | None = None,
|
||||
) -> TransferMetadata:
|
||||
"""Rearranges the expert weights in place according to the new expert indices.
|
||||
|
||||
@@ -453,6 +582,9 @@ def transfer_layer(
|
||||
stream: CUDA stream for async copies (can be None for sync mode).
|
||||
rank_mapping: Optional rank mapping for elastic expert parallelism.
|
||||
layer_idx: Index of the MoE layer being transferred.
|
||||
enable_migration_batching: Schedule remote transfers in batches where
|
||||
each rank communicates with at most one peer.
|
||||
migration_batches: Optional batches precomputed for this layer.
|
||||
|
||||
Returns:
|
||||
TransferMetadata: Metadata needed for completing remote weight transfers,
|
||||
@@ -502,6 +634,8 @@ def transfer_layer(
|
||||
ep_rank=ep_group.rank(),
|
||||
communicator=communicator,
|
||||
layer_idx=layer_idx,
|
||||
enable_migration_batching=enable_migration_batching,
|
||||
migration_batches=migration_batches,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user