[EPLB] Add contention-aware expert migration batching (#52641)
Signed-off-by: DOCCA0 <2947323341@qq.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -14,6 +14,10 @@ from torch.distributed import ProcessGroup, all_gather
|
||||
|
||||
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
|
||||
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
|
||||
from vllm.distributed.eplb.migration_scheduler import (
|
||||
MigrationFlow,
|
||||
schedule_migration_batches,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
@@ -167,6 +171,107 @@ def get_ep_ranks_with_experts_batch(
|
||||
return ranks_to_send_map, ranks_to_recv_map
|
||||
|
||||
|
||||
def _execute_migration_batch(
|
||||
batch: list[MigrationFlow],
|
||||
old_indices: np.ndarray,
|
||||
old_rows: dict[int, int],
|
||||
new_rows: dict[int, int],
|
||||
ep_rank: int,
|
||||
expert_weights: Sequence[torch.Tensor],
|
||||
expert_weights_buffers: Sequence[torch.Tensor],
|
||||
communicator: EplbCommunicator,
|
||||
layer_idx: int,
|
||||
) -> None:
|
||||
"""Execute one batch of rank-pair-disjoint expert transfers."""
|
||||
# NIXL drains its transfer context in execute(), so each batch restores it.
|
||||
communicator.set_transfer_context(old_indices, layer_idx)
|
||||
for flow in batch:
|
||||
if flow.src_rank == ep_rank:
|
||||
for expert_id in flow.expert_ids:
|
||||
src_row = _get_migration_row(
|
||||
old_rows, expert_id, flow, ep_rank, mapping_name="old"
|
||||
)
|
||||
communicator.add_send(
|
||||
[weight[src_row] for weight in expert_weights],
|
||||
flow.dst_rank,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
elif flow.dst_rank == ep_rank:
|
||||
for expert_id in flow.expert_ids:
|
||||
dst_row = _get_migration_row(
|
||||
new_rows, expert_id, flow, ep_rank, mapping_name="new"
|
||||
)
|
||||
communicator.add_recv(
|
||||
[buffer[dst_row] for buffer in expert_weights_buffers],
|
||||
flow.src_rank,
|
||||
expert_id=expert_id,
|
||||
)
|
||||
# Combining execute calls would erase the scheduler's batch boundaries.
|
||||
communicator.execute()
|
||||
|
||||
|
||||
def _execute_migration_batches(
|
||||
num_local_experts: int,
|
||||
old_indices: np.ndarray,
|
||||
new_indices: np.ndarray,
|
||||
ep_rank: int,
|
||||
expert_weights: Sequence[torch.Tensor],
|
||||
expert_weights_buffers: Sequence[torch.Tensor],
|
||||
communicator: EplbCommunicator,
|
||||
layer_idx: int,
|
||||
batches: list[list[MigrationFlow]] | None = None,
|
||||
) -> None:
|
||||
"""Execute all contention-aware migration batches for one layer."""
|
||||
if batches is None:
|
||||
with torch.profiler.record_function("eplb: schedule migration batches"):
|
||||
batches = schedule_migration_batches(
|
||||
num_local_experts, old_indices, new_indices
|
||||
)
|
||||
base = ep_rank * num_local_experts
|
||||
old_local = old_indices[base : base + num_local_experts]
|
||||
new_local = new_indices[base : base + num_local_experts]
|
||||
old_rows: dict[int, int] = {}
|
||||
new_rows: dict[int, int] = {}
|
||||
for row, expert_id in enumerate(old_local):
|
||||
if expert_id != -1:
|
||||
old_rows.setdefault(int(expert_id), row)
|
||||
for row, expert_id in enumerate(new_local):
|
||||
if expert_id != -1:
|
||||
new_rows.setdefault(int(expert_id), row)
|
||||
for batch in batches:
|
||||
_execute_migration_batch(
|
||||
batch=batch,
|
||||
old_indices=old_indices,
|
||||
old_rows=old_rows,
|
||||
new_rows=new_rows,
|
||||
ep_rank=ep_rank,
|
||||
expert_weights=expert_weights,
|
||||
expert_weights_buffers=expert_weights_buffers,
|
||||
communicator=communicator,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
|
||||
|
||||
def _get_migration_row(
|
||||
rows: dict[int, int],
|
||||
expert_id: int,
|
||||
flow: MigrationFlow,
|
||||
ep_rank: int,
|
||||
mapping_name: str,
|
||||
) -> int:
|
||||
row = rows.get(expert_id)
|
||||
if row is not None:
|
||||
return row
|
||||
|
||||
message = (
|
||||
f"Expert {expert_id} is missing from the {mapping_name} "
|
||||
f"mapping on rank {ep_rank} for migration "
|
||||
f"{flow.src_rank} -> {flow.dst_rank}"
|
||||
)
|
||||
logger.error(message)
|
||||
raise RuntimeError(message)
|
||||
|
||||
|
||||
def move_to_buffer(
|
||||
num_local_experts: int,
|
||||
old_indices: np.ndarray,
|
||||
@@ -177,6 +282,8 @@ def move_to_buffer(
|
||||
ep_rank: int,
|
||||
communicator: EplbCommunicator,
|
||||
layer_idx: int = 0,
|
||||
enable_migration_batching: bool = False,
|
||||
migration_batches: list[list[MigrationFlow]] | None = None,
|
||||
) -> TransferMetadata:
|
||||
"""Rearranges expert weights during EPLB rebalancing.
|
||||
|
||||
@@ -192,6 +299,9 @@ def move_to_buffer(
|
||||
ep_rank: Rank of this process in expert parallel group.
|
||||
communicator: EplbCommunicator instance for P2P communication.
|
||||
layer_idx: Index of the MoE layer being transferred.
|
||||
enable_migration_batching: Schedule remote transfers in batches where
|
||||
each rank communicates with at most one peer.
|
||||
migration_batches: Optional batches precomputed for this layer.
|
||||
|
||||
Returns:
|
||||
TransferMetadata: Metadata needed for completing remote weight transfers.
|
||||
@@ -264,6 +374,30 @@ def move_to_buffer(
|
||||
with device_stream(stream):
|
||||
for w, b in zip(expert_weights, expert_weights_buffers):
|
||||
b[dst].copy_(w[src_local], non_blocking=True)
|
||||
transfer_metadata = TransferMetadata(
|
||||
is_unchanged=is_unchanged,
|
||||
is_received_locally=is_received_locally,
|
||||
recv_primary_mask=recv_primary_mask,
|
||||
recv_count=recv_count,
|
||||
recv_expert_ids=recv_expert_ids,
|
||||
recv_dst_rows=recv_dst_rows,
|
||||
)
|
||||
|
||||
# Migration batching is disabled by default and must be explicitly enabled.
|
||||
if enable_migration_batching:
|
||||
_execute_migration_batches(
|
||||
num_local_experts=num_local_experts,
|
||||
old_indices=old_indices,
|
||||
new_indices=new_indices,
|
||||
ep_rank=ep_rank,
|
||||
expert_weights=expert_weights,
|
||||
expert_weights_buffers=expert_weights_buffers,
|
||||
communicator=communicator,
|
||||
layer_idx=layer_idx,
|
||||
batches=migration_batches,
|
||||
)
|
||||
return transfer_metadata
|
||||
|
||||
communicator.set_transfer_context(old_indices, layer_idx)
|
||||
|
||||
# 2. Post sends
|
||||
@@ -334,14 +468,7 @@ def move_to_buffer(
|
||||
|
||||
# 4. Execute transfers and wait for completion.
|
||||
communicator.execute()
|
||||
return TransferMetadata(
|
||||
is_unchanged=is_unchanged,
|
||||
is_received_locally=is_received_locally,
|
||||
recv_primary_mask=recv_primary_mask,
|
||||
recv_count=recv_count,
|
||||
recv_expert_ids=recv_expert_ids,
|
||||
recv_dst_rows=recv_dst_rows,
|
||||
)
|
||||
return transfer_metadata
|
||||
|
||||
|
||||
def move_from_buffer(
|
||||
@@ -432,6 +559,8 @@ def transfer_layer(
|
||||
stream: torch.Stream | None = None,
|
||||
rank_mapping: dict[int, int] | None = None,
|
||||
layer_idx: int = 0,
|
||||
enable_migration_batching: bool = False,
|
||||
migration_batches: list[list[MigrationFlow]] | None = None,
|
||||
) -> TransferMetadata:
|
||||
"""Rearranges the expert weights in place according to the new expert indices.
|
||||
|
||||
@@ -453,6 +582,9 @@ def transfer_layer(
|
||||
stream: CUDA stream for async copies (can be None for sync mode).
|
||||
rank_mapping: Optional rank mapping for elastic expert parallelism.
|
||||
layer_idx: Index of the MoE layer being transferred.
|
||||
enable_migration_batching: Schedule remote transfers in batches where
|
||||
each rank communicates with at most one peer.
|
||||
migration_batches: Optional batches precomputed for this layer.
|
||||
|
||||
Returns:
|
||||
TransferMetadata: Metadata needed for completing remote weight transfers,
|
||||
@@ -502,6 +634,8 @@ def transfer_layer(
|
||||
ep_rank=ep_group.rank(),
|
||||
communicator=communicator,
|
||||
layer_idx=layer_idx,
|
||||
enable_migration_batching=enable_migration_batching,
|
||||
migration_batches=migration_batches,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user