[EPLB] Add contention-aware expert migration batching (#52641)

Signed-off-by: DOCCA0 <2947323341@qq.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
DOCCA0
2026-10-03 00:13:50 +00:00
committed by GitHub
co-authored by Robert Shaw
parent 8e443871f0
commit 7dfe3338d5
6 changed files with 588 additions and 8 deletions
+26
View File
@@ -11,6 +11,7 @@ from vllm.distributed.parallel_state import get_eplb_group
from vllm.logger import init_logger
from .eplb_utils import CpuGpuEvent, device_stream
from .migration_scheduler import schedule_migration_batches_for_layers
from .rebalance_execute import AsyncEplbLayerResult, transfer_layer
if TYPE_CHECKING:
@@ -99,6 +100,21 @@ def transfer_run_periodically(
model_state, state, physical_to_logical_map_cpu, stream
)
migration_batches_by_layer = None
if state.parallel_config.eplb_config.migration_batching_enabled:
profile_name = "eplb: schedule migration batches"
num_local_experts = model_state.model.expert_weights[0][0].shape[0]
# The placement snapshots stay fixed for this rebalance cycle.
# Precompute all layer schedules once, but execute transfers one
# layer at a time below to preserve the migration lifecycle. This
# profiling range therefore covers scheduling for every MoE layer.
with torch.profiler.record_function(profile_name):
migration_batches_by_layer = schedule_migration_batches_for_layers(
num_local_experts,
physical_to_logical_map_cpu.numpy(),
new_physical_to_logical_map.numpy(),
)
# Execute one EPLB layer transfer per model forward pass. Each iteration
# of this loop will copy the new set of expert weights into
# model_state.expert_buffer, which will be consumed by the main thread in
@@ -134,6 +150,16 @@ def transfer_run_periodically(
is_profile=is_profile,
stream=stream,
layer_idx=layer_idx,
enable_migration_batching=(
state.parallel_config.eplb_config.migration_batching_enabled
),
migration_batches=(
# This is only the current layer's schedule; it does not
# combine transfers from different layers.
migration_batches_by_layer[layer_idx]
if migration_batches_by_layer is not None
else None
),
)
# Wait until all writes to expert_buffer have finished before making the