[EPLB] Add contention-aware expert migration batching (#52641)
Signed-off-by: DOCCA0 <2947323341@qq.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -11,6 +11,7 @@ from vllm.distributed.parallel_state import get_eplb_group
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .eplb_utils import CpuGpuEvent, device_stream
|
||||
from .migration_scheduler import schedule_migration_batches_for_layers
|
||||
from .rebalance_execute import AsyncEplbLayerResult, transfer_layer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -99,6 +100,21 @@ def transfer_run_periodically(
|
||||
model_state, state, physical_to_logical_map_cpu, stream
|
||||
)
|
||||
|
||||
migration_batches_by_layer = None
|
||||
if state.parallel_config.eplb_config.migration_batching_enabled:
|
||||
profile_name = "eplb: schedule migration batches"
|
||||
num_local_experts = model_state.model.expert_weights[0][0].shape[0]
|
||||
# The placement snapshots stay fixed for this rebalance cycle.
|
||||
# Precompute all layer schedules once, but execute transfers one
|
||||
# layer at a time below to preserve the migration lifecycle. This
|
||||
# profiling range therefore covers scheduling for every MoE layer.
|
||||
with torch.profiler.record_function(profile_name):
|
||||
migration_batches_by_layer = schedule_migration_batches_for_layers(
|
||||
num_local_experts,
|
||||
physical_to_logical_map_cpu.numpy(),
|
||||
new_physical_to_logical_map.numpy(),
|
||||
)
|
||||
|
||||
# Execute one EPLB layer transfer per model forward pass. Each iteration
|
||||
# of this loop will copy the new set of expert weights into
|
||||
# model_state.expert_buffer, which will be consumed by the main thread in
|
||||
@@ -134,6 +150,16 @@ def transfer_run_periodically(
|
||||
is_profile=is_profile,
|
||||
stream=stream,
|
||||
layer_idx=layer_idx,
|
||||
enable_migration_batching=(
|
||||
state.parallel_config.eplb_config.migration_batching_enabled
|
||||
),
|
||||
migration_batches=(
|
||||
# This is only the current layer's schedule; it does not
|
||||
# combine transfers from different layers.
|
||||
migration_batches_by_layer[layer_idx]
|
||||
if migration_batches_by_layer is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
# Wait until all writes to expert_buffer have finished before making the
|
||||
|
||||
Reference in New Issue
Block a user