[EPLB] Cleanup the transfer logic for the various eplb maps (#34520)

Signed-off-by: Sage Moore <sagmoore@redhat.com>
Signed-off-by: Sage Moore <sage@neuralmagic.com>
This commit is contained in:
Sage Moore
2026-03-27 10:18:46 +01:00
committed by GitHub
parent 6287e7fa20
commit 497e234d38
3 changed files with 247 additions and 76 deletions
+91 -76
View File
@@ -729,13 +729,6 @@ class EplbState:
eplb_model_state.physical_to_logical_map.cpu(),
)
num_logical_experts = global_expert_load_window.shape[-1]
(new_logical_to_physical_map, new_logical_replica_count) = (
compute_logical_maps(
new_physical_to_logical_map, num_logical_experts
)
)
# Update expert weights
rearrange_expert_weights_inplace(
eplb_model_state.physical_to_logical_map,
@@ -747,39 +740,11 @@ class EplbState:
)
if not is_profile:
if (
eplb_model_state.physical_to_logical_map.shape[1]
!= new_physical_to_logical_map.shape[1]
):
eplb_model_state.physical_to_logical_map = (
new_physical_to_logical_map.to(
eplb_model_state.physical_to_logical_map.device
)
)
else:
eplb_model_state.physical_to_logical_map.copy_(
new_physical_to_logical_map
)
max_physical_slots = new_logical_to_physical_map.shape[-1]
assert (
max_physical_slots
<= eplb_model_state.logical_to_physical_map.shape[-1]
)
new_logical_to_physical_map = torch.nn.functional.pad(
new_logical_to_physical_map,
(
0,
eplb_model_state.logical_to_physical_map.shape[-1]
- max_physical_slots,
),
value=-1,
)
eplb_model_state.logical_to_physical_map.copy_(
new_logical_to_physical_map
)
eplb_model_state.logical_replica_count.copy_(
new_logical_replica_count
_commit_eplb_maps(
eplb_model_state,
new_physical_to_logical_map=new_physical_to_logical_map,
)
if is_main_rank:
assert start_event is not None
assert end_event is not None
@@ -829,42 +794,6 @@ class EplbState:
is_profile=is_profile,
)
def _update_layer_mapping_from_new(
self, model_state: EplbModelState, layer: int
) -> None:
if model_state.new_physical_to_logical_map is None:
return
target_device = model_state.physical_to_logical_map.device
new_physical = model_state.new_physical_to_logical_map
# If the number of physical experts has changed, then the new map needs to
# be copied synchronously to avoid a race condition with the async worker
if model_state.physical_to_logical_map.shape[1] != new_physical.shape[1]:
model_state.physical_to_logical_map = new_physical.to(target_device)
else:
model_state.physical_to_logical_map[layer].copy_(
new_physical[layer].to(target_device, non_blocking=True)
)
num_logical_experts = model_state.logical_to_physical_map.shape[1]
new_logical, new_replica_count = compute_logical_maps(
new_physical[layer], num_logical_experts
)
logical_device = model_state.logical_to_physical_map.device
max_slots = model_state.logical_to_physical_map.shape[-1]
slot_delta = max_slots - new_logical.shape[-1]
if slot_delta > 0:
new_logical = torch.nn.functional.pad(
new_logical, (0, slot_delta), value=-1
)
model_state.logical_to_physical_map[layer].copy_(new_logical.to(logical_device))
replica_device = model_state.logical_replica_count.device
model_state.logical_replica_count[layer].copy_(
new_replica_count.to(replica_device)
)
def _all_ranks_buffer_ready(self, model_state: EplbModelState) -> bool:
parallel_state = get_ep_group()
cpu_group = getattr(parallel_state, "cpu_group", None)
@@ -936,7 +865,12 @@ class EplbState:
model_state.buffer_consumed_event = consumed_event
transferred_layer = model_state.layer_to_transfer
self._update_layer_mapping_from_new(model_state, transferred_layer)
assert model_state.new_physical_to_logical_map is not None
_commit_eplb_maps_for_layer(
model_state,
new_physical_to_logical_map=model_state.new_physical_to_logical_map,
layer=transferred_layer,
)
# After the main thread consumes, advance layer_to_transfer
model_state.layer_to_transfer += 1
model_state.ep_buffer_ready = 0
@@ -1175,3 +1109,84 @@ def compute_logical_maps(
if per_layer:
return logical_to_physical_map_out.squeeze(0), logical_replica_count.squeeze(0)
return logical_to_physical_map_out, logical_replica_count
def _pad_out_tensor(src: torch.Tensor, dst: torch.Tensor) -> None:
src_padding = dst.shape[-1] - src.shape[-1]
assert src_padding >= 0
new_src = torch.nn.functional.pad(src, (0, src_padding), value=-1)
dst.copy_(new_src)
def _commit_eplb_maps_for_layer(
model_state: EplbModelState,
new_physical_to_logical_map: torch.Tensor,
layer: int,
) -> None:
"""
Per-layer version of _commit_eplb_maps that's used by the sync portion of EPLB
when running async EPLB. Copies all of the new_* maps into model_state. After this
function completes, the new mappings will become the current mappings and will be
visible to the model.
"""
# Commit physical_to_logical_map
src = new_physical_to_logical_map[layer]
dst = model_state.physical_to_logical_map[layer]
assert src.shape == dst.shape, (
"The number of physical experts must stay the same while running Async EPLB. "
f"Current number of physical experts: {dst.shape[0]}. New number of physical "
f"experts {src.shape[0]}."
)
dst.copy_(src, non_blocking=True)
num_logical_experts = model_state.logical_to_physical_map.shape[1]
new_logical, new_replica_count = compute_logical_maps(src, num_logical_experts)
# Commit logical_to_physical_map
_pad_out_tensor(
src=new_logical,
dst=model_state.logical_to_physical_map[layer],
)
# Commit logical_replica_count
src = new_replica_count
dst = model_state.logical_replica_count[layer]
assert src.shape == dst.shape
dst.copy_(src, non_blocking=True)
def _commit_eplb_maps(
model_state: EplbModelState,
new_physical_to_logical_map: torch.Tensor,
) -> None:
"""
Copies all of the new_* maps into model_state. After this function completes,
the new mappings will become the current mappings and will be visible to the
model.
"""
# Commit physical_to_logical_map
src = new_physical_to_logical_map
dst = model_state.physical_to_logical_map
# Rare Case: When the number of physical experts has changed, discard the old
# physical to logical expert map and use the new one. This only happens when the
# number of GPUs available to vLLM changes while vLLM is running. Otherwise copy the
# new map into the old one.
if src.shape[1] != dst.shape[1]:
model_state.physical_to_logical_map = src.to(dst.device)
else:
dst.copy_(src, non_blocking=True)
num_logical_experts = model_state.logical_to_physical_map.shape[1]
new_logical, new_replica_count = compute_logical_maps(src, num_logical_experts)
# Commit logical_to_physical_map
_pad_out_tensor(
src=new_logical,
dst=model_state.logical_to_physical_map,
)
# Commit logical_replica_count
src = new_replica_count
dst = model_state.logical_replica_count
dst.copy_(src, non_blocking=True)