[EPLB] Add contention-aware expert migration batching (#52641)

Signed-off-by: DOCCA0 <2947323341@qq.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
DOCCA0
2026-10-03 00:13:50 +00:00
committed by GitHub
co-authored by Robert Shaw
parent 8e443871f0
commit 7dfe3338d5
6 changed files with 588 additions and 8 deletions
@@ -10,9 +10,11 @@ steps:
source_file_dependencies:
- vllm/distributed/eplb
- tests/distributed/test_eplb_algo.py
- tests/distributed/test_eplb_migration_scheduler.py
- tests/distributed/test_eplb_utils.py
commands:
- pytest -v -s distributed/test_eplb_algo.py
- pytest -v -s distributed/test_eplb_migration_scheduler.py
- pytest -v -s distributed/test_eplb_utils.py
mirror:
amd:
@@ -0,0 +1,278 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
from vllm.config.parallel import EPLBConfig
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
from vllm.distributed.eplb.migration_scheduler import (
MigrationFlow,
schedule_migration_batches,
schedule_migration_batches_for_layers,
)
from vllm.distributed.eplb.rebalance_execute import move_from_buffer, move_to_buffer
class _MockEplbCommunicator(EplbCommunicator):
def __init__(self) -> None:
self.send_calls: list[tuple[list[torch.Tensor], int, int]] = []
self.recv_calls: list[tuple[list[torch.Tensor], int, int]] = []
self.context_calls = 0
self.execute_count = 0
def add_send(
self, tensors: list[torch.Tensor], dst_rank: int, expert_id: int
) -> None:
self.send_calls.append((tensors, dst_rank, expert_id))
def add_recv(
self, tensors: list[torch.Tensor], src_rank: int, expert_id: int
) -> None:
self.recv_calls.append((tensors, src_rank, expert_id))
def set_transfer_context(self, old_indices: np.ndarray, layer_idx: int) -> None:
self.context_calls += 1
def execute(self) -> None:
self.execute_count += 1
def test_migration_batching_is_disabled_by_default_and_can_be_enabled() -> None:
assert not EPLBConfig().enable_migration_batching
assert EPLBConfig(enable_migration_batching=True).enable_migration_batching
def test_migration_batching_only_applies_to_async_eplb() -> None:
assert not EPLBConfig(use_async=True).migration_batching_enabled
assert EPLBConfig(
use_async=True, enable_migration_batching=True
).migration_batching_enabled
assert not EPLBConfig(
use_async=True, enable_migration_batching=False
).migration_batching_enabled
assert not EPLBConfig(
use_async=False, enable_migration_batching=True
).migration_batching_enabled
def test_schedule_migration_batches_is_deterministic() -> None:
transfers = [
(1, 3, 0),
(2, 4, 1),
(0, 1, 2),
(0, 2, 3),
(0, 3, 4),
(0, 4, 5),
]
num_local_experts, old_indices, new_indices = _placements_for_transfers(
5, transfers
)
batches = schedule_migration_batches(num_local_experts, old_indices, new_indices)
assert [_endpoints(batch) for batch in batches] == [
[(1, 3), (2, 4)],
[(0, 1)],
[(0, 2)],
[(0, 3)],
[(0, 4)],
]
for batch in batches:
_assert_no_endpoint_conflict(batch)
def test_schedule_migration_batches_coalesces_rank_pair() -> None:
transfers = [
(0, 1, 0),
(0, 1, 1),
(2, 3, 2),
]
num_local_experts, old_indices, new_indices = _placements_for_transfers(
4, transfers
)
assert schedule_migration_batches(num_local_experts, old_indices, new_indices) == [
[
MigrationFlow(0, 1, expert_ids=(0, 1)),
MigrationFlow(2, 3, expert_ids=(2,)),
]
]
def test_schedule_migration_batches_covers_random_instructions() -> None:
rng = np.random.default_rng(42)
transfers = []
for expert_id in range(80):
src_rank, dst_rank = rng.integers(0, 8, size=2).tolist()
if src_rank != dst_rank:
transfers.append((src_rank, dst_rank, expert_id))
num_local_experts, old_indices, new_indices = _placements_for_transfers(
8, transfers
)
batches = schedule_migration_batches(num_local_experts, old_indices, new_indices)
scheduled = _flatten_transfers(batches)
assert len(scheduled) == len(transfers)
assert set(scheduled) == set(transfers)
for batch in batches:
_assert_no_endpoint_conflict(batch)
def test_schedule_migration_batches_excludes_local_copies() -> None:
# Old: rank 0 [0, 1], rank 1 [1, 2]. New: rank 1 needs expert 0.
old_indices = np.array([0, 1, 1, 2], dtype=np.int64)
new_indices = np.array([0, 1, 0, 2], dtype=np.int64)
assert schedule_migration_batches(2, old_indices, new_indices) == [
[MigrationFlow(0, 1, expert_ids=(0,))]
]
def test_schedule_migration_batches_requires_matching_shapes() -> None:
old_indices = np.array([0, 1], dtype=np.int64)
new_indices = np.array([0], dtype=np.int64)
with pytest.raises(AssertionError):
schedule_migration_batches(1, old_indices, new_indices)
def test_schedule_migration_batches_balances_replicas() -> None:
# Rank 0 and rank 1 hold expert 0; ranks 2, 3, and 4 need it.
old_indices = np.array([0, 0, 1, 1, 1], dtype=np.int64)
new_indices = np.array([0, 0, 0, 0, 0], dtype=np.int64)
batches = schedule_migration_batches(1, old_indices, new_indices)
assert set(_flatten_transfers(batches)) == {
(0, 2, 0),
(0, 4, 0),
(1, 3, 0),
}
def test_schedule_migration_batches_for_layers_matches_individual_calls() -> None:
old_indices = np.array([[0, 1, 2, 3], [3, 2, 1, 0]], dtype=np.int64)
new_indices = np.array([[1, 2, 3, 0], [0, 3, 2, 1]], dtype=np.int64)
assert schedule_migration_batches_for_layers(1, old_indices, new_indices) == [
schedule_migration_batches(1, old_indices[0], new_indices[0]),
schedule_migration_batches(1, old_indices[1], new_indices[1]),
]
@pytest.mark.parametrize("use_precomputed", [False, True])
def test_move_to_buffer_uses_multiple_batches(use_precomputed: bool) -> None:
old_indices = np.array([0, 1, 2, 3], dtype=np.int64)
new_indices = np.array([1, 2, 3, 0], dtype=np.int64)
communicator = _MockEplbCommunicator()
migration_batches = (
schedule_migration_batches(1, old_indices, new_indices)
if use_precomputed
else None
)
move_to_buffer(
num_local_experts=1,
old_indices=old_indices,
new_indices=new_indices,
expert_weights=[torch.zeros(1, 1)],
expert_weights_buffers=[torch.zeros(1, 1)],
stream=None,
ep_rank=0,
communicator=communicator,
layer_idx=7,
enable_migration_batching=True,
migration_batches=migration_batches,
)
assert communicator.context_calls == 2
assert communicator.execute_count == 2
assert [(dst, expert) for _, dst, expert in communicator.send_calls] == [(3, 0)]
assert [(src, expert) for _, src, expert in communicator.recv_calls] == [(1, 1)]
def test_move_to_buffer_uses_primary_duplicate_destination() -> None:
old_indices = np.array([0, 1, 2, 3], dtype=np.int64)
new_indices = np.array([2, 2, 0, 3], dtype=np.int64)
expert_weights = [torch.zeros(2, 1)]
recv_buffer = torch.zeros(2, 1)
communicator = _MockEplbCommunicator()
transfer_metadata = move_to_buffer(
num_local_experts=2,
old_indices=old_indices,
new_indices=new_indices,
expert_weights=expert_weights,
expert_weights_buffers=[recv_buffer],
stream=None,
ep_rank=0,
communicator=communicator,
enable_migration_batching=True,
)
assert len(communicator.recv_calls) == 1
recv_tensors, src_rank, expert_id = communicator.recv_calls[0]
assert (src_rank, expert_id) == (1, 2)
assert recv_tensors[0].data_ptr() == recv_buffer[0].data_ptr()
recv_buffer[0].fill_(42)
move_from_buffer(
expert_weights=expert_weights,
expert_weights_buffers=[recv_buffer],
transfer_metadata=transfer_metadata,
new_indices=new_indices,
ep_rank=0,
)
torch.testing.assert_close(expert_weights[0], torch.full((2, 1), 42.0))
def _endpoints(
flows: list[MigrationFlow],
) -> list[tuple[int, int]]:
return [(item.src_rank, item.dst_rank) for item in flows]
def _assert_no_endpoint_conflict(batch: list[MigrationFlow]) -> None:
endpoints: set[int] = set()
for flow in batch:
assert flow.src_rank not in endpoints
assert flow.dst_rank not in endpoints
endpoints.update((flow.src_rank, flow.dst_rank))
def _flatten_transfers(
batches: list[list[MigrationFlow]],
) -> list[tuple[int, int, int]]:
return [
(flow.src_rank, flow.dst_rank, expert_id)
for batch in batches
for flow in batch
for expert_id in flow.expert_ids
]
def _placements_for_transfers(
num_ranks: int,
transfers: list[tuple[int, int, int]],
) -> tuple[int, np.ndarray, np.ndarray]:
old_counts = [0] * num_ranks
new_counts = [0] * num_ranks
for src_rank, dst_rank, _ in transfers:
old_counts[src_rank] += 1
new_counts[dst_rank] += 1
num_local_experts = max(1, *old_counts, *new_counts)
old_indices = np.full(num_ranks * num_local_experts, -1, dtype=np.int64)
new_indices = np.full_like(old_indices, -1)
old_counts = [0] * num_ranks
new_counts = [0] * num_ranks
for src_rank, dst_rank, expert_id in transfers:
old_offset = src_rank * num_local_experts + old_counts[src_rank]
new_offset = dst_rank * num_local_experts + new_counts[dst_rank]
old_indices[old_offset] = expert_id
new_indices[new_offset] = expert_id
old_counts[src_rank] += 1
new_counts[dst_rank] += 1
return num_local_experts, old_indices, new_indices
+11
View File
@@ -106,6 +106,17 @@ class EPLBConfig:
on CUDA, falls back to "torch_gloo")
"""
enable_migration_batching: bool = False
"""Schedule expert migrations in batches where each rank communicates with
at most one peer. This reduces per-rank network contention at the cost of
additional sequential communication steps. This option only applies to
async EPLB and is disabled by default."""
@property
def migration_batching_enabled(self) -> bool:
"""Whether contention-aware batching is active for this configuration."""
return self.use_async and self.enable_migration_batching
@model_validator(mode="after")
def _validate_eplb_config(self) -> Self:
if self.use_async and self.policy != "default":
+26
View File
@@ -11,6 +11,7 @@ from vllm.distributed.parallel_state import get_eplb_group
from vllm.logger import init_logger
from .eplb_utils import CpuGpuEvent, device_stream
from .migration_scheduler import schedule_migration_batches_for_layers
from .rebalance_execute import AsyncEplbLayerResult, transfer_layer
if TYPE_CHECKING:
@@ -99,6 +100,21 @@ def transfer_run_periodically(
model_state, state, physical_to_logical_map_cpu, stream
)
migration_batches_by_layer = None
if state.parallel_config.eplb_config.migration_batching_enabled:
profile_name = "eplb: schedule migration batches"
num_local_experts = model_state.model.expert_weights[0][0].shape[0]
# The placement snapshots stay fixed for this rebalance cycle.
# Precompute all layer schedules once, but execute transfers one
# layer at a time below to preserve the migration lifecycle. This
# profiling range therefore covers scheduling for every MoE layer.
with torch.profiler.record_function(profile_name):
migration_batches_by_layer = schedule_migration_batches_for_layers(
num_local_experts,
physical_to_logical_map_cpu.numpy(),
new_physical_to_logical_map.numpy(),
)
# Execute one EPLB layer transfer per model forward pass. Each iteration
# of this loop will copy the new set of expert weights into
# model_state.expert_buffer, which will be consumed by the main thread in
@@ -134,6 +150,16 @@ def transfer_run_periodically(
is_profile=is_profile,
stream=stream,
layer_idx=layer_idx,
enable_migration_batching=(
state.parallel_config.eplb_config.migration_batching_enabled
),
migration_batches=(
# This is only the current layer's schedule; it does not
# combine transfers from different layers.
migration_batches_by_layer[layer_idx]
if migration_batches_by_layer is not None
else None
),
)
# Wait until all writes to expert_buffer have finished before making the
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Schedule EPLB expert migrations into conflict-free batches."""
from dataclasses import dataclass
import numpy as np
@dataclass(frozen=True, slots=True)
class MigrationFlow:
"""Expert transfers between one directed rank pair."""
src_rank: int
dst_rank: int
# Experts being transferred.
expert_ids: tuple[int, ...]
def schedule_migration_batches(
num_local_experts: int,
old_indices: np.ndarray,
new_indices: np.ndarray,
) -> list[list[MigrationFlow]]:
"""Schedule expert migrations without per-rank communication contention.
Experts assigned to the same source and destination ranks form one flow.
Each flow is placed in the first batch that does not already use either
endpoint, so a rank communicates with at most one peer in each batch while
independent rank pairs can transfer concurrently.
"""
assert old_indices.shape == new_indices.shape
recv_ranks_by_expert: dict[int, list[int]] = {}
old_experts_by_rank: list[set[int]] = []
old_by_rank = old_indices.reshape(-1, num_local_experts)
new_by_rank = new_indices.reshape(-1, num_local_experts)
# First find the ranks that gain each expert. Existing local copies need no
# transfer, even if their physical row changes.
for rank, (old_local, new_local) in enumerate(zip(old_by_rank, new_by_rank)):
old_experts = set(old_local.tolist())
old_experts.discard(-1)
old_experts_by_rank.append(old_experts)
new_experts = set(new_local.tolist())
new_experts.discard(-1)
for expert_id in new_experts - old_experts:
recv_ranks_by_expert.setdefault(expert_id, []).append(rank)
old_ranks_by_expert: dict[int, list[int]] = {}
needed_experts = recv_ranks_by_expert.keys()
# Only index source copies for experts that need a remote transfer.
for rank, old_experts in enumerate(old_experts_by_rank):
for expert_id in old_experts & needed_experts:
old_ranks_by_expert.setdefault(expert_id, []).append(rank)
expert_ids_by_pair: dict[tuple[int, int], list[int]] = {}
for expert_id in sorted(recv_ranks_by_expert):
send_ranks = old_ranks_by_expert.get(expert_id, [])
if not send_ranks:
continue
recv_ranks = recv_ranks_by_expert[expert_id]
num_per_sender, remainder = divmod(len(recv_ranks), len(send_ranks))
remainder_start = len(send_ranks) * num_per_sender
# Preserve the unbatched path's balanced source assignment. Batching
# changes transfer order, not which existing replica supplies a copy.
for sender_idx, src_rank in enumerate(send_ranks):
start = sender_idx * num_per_sender
for dst_rank in recv_ranks[start : start + num_per_sender]:
expert_ids_by_pair.setdefault((src_rank, dst_rank), []).append(
expert_id
)
if sender_idx < remainder:
dst_rank = recv_ranks[remainder_start + sender_idx]
expert_ids_by_pair.setdefault((src_rank, dst_rank), []).append(
expert_id
)
# Transfers with the same directed rank pair have identical contention
# constraints, so coalesce them before grouping. The loop below is a
# deterministic first-fit edge coloring of the rank communication graph.
batches: list[list[MigrationFlow]] = []
endpoints_used: list[int] = []
for (src_rank, dst_rank), expert_ids in expert_ids_by_pair.items():
flow = MigrationFlow(src_rank, dst_rank, tuple(expert_ids))
# Direction is irrelevant for NIC contention. This endpoint mask keeps
# each rank connected to at most one peer in a batch. The conservative
# limit leaves bandwidth headroom for inference during async migration.
endpoints = (1 << src_rank) | (1 << dst_rank)
for batch_idx, (batch, used) in enumerate(zip(batches, endpoints_used)):
if not endpoints & used:
batch.append(flow)
endpoints_used[batch_idx] |= endpoints
break
else:
batches.append([flow])
endpoints_used.append(endpoints)
return batches
def schedule_migration_batches_for_layers(
num_local_experts: int,
old_indices: np.ndarray,
new_indices: np.ndarray,
) -> list[list[list[MigrationFlow]]]:
"""Precompute the independent migration schedule for every MoE layer.
This reduces scheduling entry points from once per layer to once per
rebalance cycle. Only CPU scheduling is grouped; transfers remain ordered
by layer and preserve each layer's batch boundaries.
"""
assert old_indices.shape == new_indices.shape
assert old_indices.ndim == 2
return [
schedule_migration_batches(num_local_experts, old_layer, new_layer)
for old_layer, new_layer in zip(old_indices, new_indices)
]
__all__ = [
"MigrationFlow",
"schedule_migration_batches",
"schedule_migration_batches_for_layers",
]
+142 -8
View File
@@ -14,6 +14,10 @@ from torch.distributed import ProcessGroup, all_gather
from vllm.distributed.eplb.eplb_communicator import EplbCommunicator
from vllm.distributed.eplb.eplb_utils import CpuGpuEvent, device_stream
from vllm.distributed.eplb.migration_scheduler import (
MigrationFlow,
schedule_migration_batches,
)
from vllm.logger import init_logger
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -167,6 +171,107 @@ def get_ep_ranks_with_experts_batch(
return ranks_to_send_map, ranks_to_recv_map
def _execute_migration_batch(
batch: list[MigrationFlow],
old_indices: np.ndarray,
old_rows: dict[int, int],
new_rows: dict[int, int],
ep_rank: int,
expert_weights: Sequence[torch.Tensor],
expert_weights_buffers: Sequence[torch.Tensor],
communicator: EplbCommunicator,
layer_idx: int,
) -> None:
"""Execute one batch of rank-pair-disjoint expert transfers."""
# NIXL drains its transfer context in execute(), so each batch restores it.
communicator.set_transfer_context(old_indices, layer_idx)
for flow in batch:
if flow.src_rank == ep_rank:
for expert_id in flow.expert_ids:
src_row = _get_migration_row(
old_rows, expert_id, flow, ep_rank, mapping_name="old"
)
communicator.add_send(
[weight[src_row] for weight in expert_weights],
flow.dst_rank,
expert_id=expert_id,
)
elif flow.dst_rank == ep_rank:
for expert_id in flow.expert_ids:
dst_row = _get_migration_row(
new_rows, expert_id, flow, ep_rank, mapping_name="new"
)
communicator.add_recv(
[buffer[dst_row] for buffer in expert_weights_buffers],
flow.src_rank,
expert_id=expert_id,
)
# Combining execute calls would erase the scheduler's batch boundaries.
communicator.execute()
def _execute_migration_batches(
num_local_experts: int,
old_indices: np.ndarray,
new_indices: np.ndarray,
ep_rank: int,
expert_weights: Sequence[torch.Tensor],
expert_weights_buffers: Sequence[torch.Tensor],
communicator: EplbCommunicator,
layer_idx: int,
batches: list[list[MigrationFlow]] | None = None,
) -> None:
"""Execute all contention-aware migration batches for one layer."""
if batches is None:
with torch.profiler.record_function("eplb: schedule migration batches"):
batches = schedule_migration_batches(
num_local_experts, old_indices, new_indices
)
base = ep_rank * num_local_experts
old_local = old_indices[base : base + num_local_experts]
new_local = new_indices[base : base + num_local_experts]
old_rows: dict[int, int] = {}
new_rows: dict[int, int] = {}
for row, expert_id in enumerate(old_local):
if expert_id != -1:
old_rows.setdefault(int(expert_id), row)
for row, expert_id in enumerate(new_local):
if expert_id != -1:
new_rows.setdefault(int(expert_id), row)
for batch in batches:
_execute_migration_batch(
batch=batch,
old_indices=old_indices,
old_rows=old_rows,
new_rows=new_rows,
ep_rank=ep_rank,
expert_weights=expert_weights,
expert_weights_buffers=expert_weights_buffers,
communicator=communicator,
layer_idx=layer_idx,
)
def _get_migration_row(
rows: dict[int, int],
expert_id: int,
flow: MigrationFlow,
ep_rank: int,
mapping_name: str,
) -> int:
row = rows.get(expert_id)
if row is not None:
return row
message = (
f"Expert {expert_id} is missing from the {mapping_name} "
f"mapping on rank {ep_rank} for migration "
f"{flow.src_rank} -> {flow.dst_rank}"
)
logger.error(message)
raise RuntimeError(message)
def move_to_buffer(
num_local_experts: int,
old_indices: np.ndarray,
@@ -177,6 +282,8 @@ def move_to_buffer(
ep_rank: int,
communicator: EplbCommunicator,
layer_idx: int = 0,
enable_migration_batching: bool = False,
migration_batches: list[list[MigrationFlow]] | None = None,
) -> TransferMetadata:
"""Rearranges expert weights during EPLB rebalancing.
@@ -192,6 +299,9 @@ def move_to_buffer(
ep_rank: Rank of this process in expert parallel group.
communicator: EplbCommunicator instance for P2P communication.
layer_idx: Index of the MoE layer being transferred.
enable_migration_batching: Schedule remote transfers in batches where
each rank communicates with at most one peer.
migration_batches: Optional batches precomputed for this layer.
Returns:
TransferMetadata: Metadata needed for completing remote weight transfers.
@@ -264,6 +374,30 @@ def move_to_buffer(
with device_stream(stream):
for w, b in zip(expert_weights, expert_weights_buffers):
b[dst].copy_(w[src_local], non_blocking=True)
transfer_metadata = TransferMetadata(
is_unchanged=is_unchanged,
is_received_locally=is_received_locally,
recv_primary_mask=recv_primary_mask,
recv_count=recv_count,
recv_expert_ids=recv_expert_ids,
recv_dst_rows=recv_dst_rows,
)
# Migration batching is disabled by default and must be explicitly enabled.
if enable_migration_batching:
_execute_migration_batches(
num_local_experts=num_local_experts,
old_indices=old_indices,
new_indices=new_indices,
ep_rank=ep_rank,
expert_weights=expert_weights,
expert_weights_buffers=expert_weights_buffers,
communicator=communicator,
layer_idx=layer_idx,
batches=migration_batches,
)
return transfer_metadata
communicator.set_transfer_context(old_indices, layer_idx)
# 2. Post sends
@@ -334,14 +468,7 @@ def move_to_buffer(
# 4. Execute transfers and wait for completion.
communicator.execute()
return TransferMetadata(
is_unchanged=is_unchanged,
is_received_locally=is_received_locally,
recv_primary_mask=recv_primary_mask,
recv_count=recv_count,
recv_expert_ids=recv_expert_ids,
recv_dst_rows=recv_dst_rows,
)
return transfer_metadata
def move_from_buffer(
@@ -432,6 +559,8 @@ def transfer_layer(
stream: torch.Stream | None = None,
rank_mapping: dict[int, int] | None = None,
layer_idx: int = 0,
enable_migration_batching: bool = False,
migration_batches: list[list[MigrationFlow]] | None = None,
) -> TransferMetadata:
"""Rearranges the expert weights in place according to the new expert indices.
@@ -453,6 +582,9 @@ def transfer_layer(
stream: CUDA stream for async copies (can be None for sync mode).
rank_mapping: Optional rank mapping for elastic expert parallelism.
layer_idx: Index of the MoE layer being transferred.
enable_migration_batching: Schedule remote transfers in batches where
each rank communicates with at most one peer.
migration_batches: Optional batches precomputed for this layer.
Returns:
TransferMetadata: Metadata needed for completing remote weight transfers,
@@ -502,6 +634,8 @@ def transfer_layer(
ep_rank=ep_group.rank(),
communicator=communicator,
layer_idx=layer_idx,
enable_migration_batching=enable_migration_batching,
migration_batches=migration_batches,
)