[EPLB] Cleanup the transfer logic for the various eplb maps (#34520)
Signed-off-by: Sage Moore <sagmoore@redhat.com> Signed-off-by: Sage Moore <sage@neuralmagic.com>
This commit is contained in:
@@ -729,13 +729,6 @@ class EplbState:
|
||||
eplb_model_state.physical_to_logical_map.cpu(),
|
||||
)
|
||||
|
||||
num_logical_experts = global_expert_load_window.shape[-1]
|
||||
(new_logical_to_physical_map, new_logical_replica_count) = (
|
||||
compute_logical_maps(
|
||||
new_physical_to_logical_map, num_logical_experts
|
||||
)
|
||||
)
|
||||
|
||||
# Update expert weights
|
||||
rearrange_expert_weights_inplace(
|
||||
eplb_model_state.physical_to_logical_map,
|
||||
@@ -747,39 +740,11 @@ class EplbState:
|
||||
)
|
||||
|
||||
if not is_profile:
|
||||
if (
|
||||
eplb_model_state.physical_to_logical_map.shape[1]
|
||||
!= new_physical_to_logical_map.shape[1]
|
||||
):
|
||||
eplb_model_state.physical_to_logical_map = (
|
||||
new_physical_to_logical_map.to(
|
||||
eplb_model_state.physical_to_logical_map.device
|
||||
)
|
||||
)
|
||||
else:
|
||||
eplb_model_state.physical_to_logical_map.copy_(
|
||||
new_physical_to_logical_map
|
||||
)
|
||||
max_physical_slots = new_logical_to_physical_map.shape[-1]
|
||||
assert (
|
||||
max_physical_slots
|
||||
<= eplb_model_state.logical_to_physical_map.shape[-1]
|
||||
)
|
||||
new_logical_to_physical_map = torch.nn.functional.pad(
|
||||
new_logical_to_physical_map,
|
||||
(
|
||||
0,
|
||||
eplb_model_state.logical_to_physical_map.shape[-1]
|
||||
- max_physical_slots,
|
||||
),
|
||||
value=-1,
|
||||
)
|
||||
eplb_model_state.logical_to_physical_map.copy_(
|
||||
new_logical_to_physical_map
|
||||
)
|
||||
eplb_model_state.logical_replica_count.copy_(
|
||||
new_logical_replica_count
|
||||
_commit_eplb_maps(
|
||||
eplb_model_state,
|
||||
new_physical_to_logical_map=new_physical_to_logical_map,
|
||||
)
|
||||
|
||||
if is_main_rank:
|
||||
assert start_event is not None
|
||||
assert end_event is not None
|
||||
@@ -829,42 +794,6 @@ class EplbState:
|
||||
is_profile=is_profile,
|
||||
)
|
||||
|
||||
def _update_layer_mapping_from_new(
|
||||
self, model_state: EplbModelState, layer: int
|
||||
) -> None:
|
||||
if model_state.new_physical_to_logical_map is None:
|
||||
return
|
||||
|
||||
target_device = model_state.physical_to_logical_map.device
|
||||
new_physical = model_state.new_physical_to_logical_map
|
||||
# If the number of physical experts has changed, then the new map needs to
|
||||
# be copied synchronously to avoid a race condition with the async worker
|
||||
if model_state.physical_to_logical_map.shape[1] != new_physical.shape[1]:
|
||||
model_state.physical_to_logical_map = new_physical.to(target_device)
|
||||
else:
|
||||
model_state.physical_to_logical_map[layer].copy_(
|
||||
new_physical[layer].to(target_device, non_blocking=True)
|
||||
)
|
||||
|
||||
num_logical_experts = model_state.logical_to_physical_map.shape[1]
|
||||
new_logical, new_replica_count = compute_logical_maps(
|
||||
new_physical[layer], num_logical_experts
|
||||
)
|
||||
|
||||
logical_device = model_state.logical_to_physical_map.device
|
||||
max_slots = model_state.logical_to_physical_map.shape[-1]
|
||||
slot_delta = max_slots - new_logical.shape[-1]
|
||||
if slot_delta > 0:
|
||||
new_logical = torch.nn.functional.pad(
|
||||
new_logical, (0, slot_delta), value=-1
|
||||
)
|
||||
model_state.logical_to_physical_map[layer].copy_(new_logical.to(logical_device))
|
||||
|
||||
replica_device = model_state.logical_replica_count.device
|
||||
model_state.logical_replica_count[layer].copy_(
|
||||
new_replica_count.to(replica_device)
|
||||
)
|
||||
|
||||
def _all_ranks_buffer_ready(self, model_state: EplbModelState) -> bool:
|
||||
parallel_state = get_ep_group()
|
||||
cpu_group = getattr(parallel_state, "cpu_group", None)
|
||||
@@ -936,7 +865,12 @@ class EplbState:
|
||||
model_state.buffer_consumed_event = consumed_event
|
||||
|
||||
transferred_layer = model_state.layer_to_transfer
|
||||
self._update_layer_mapping_from_new(model_state, transferred_layer)
|
||||
assert model_state.new_physical_to_logical_map is not None
|
||||
_commit_eplb_maps_for_layer(
|
||||
model_state,
|
||||
new_physical_to_logical_map=model_state.new_physical_to_logical_map,
|
||||
layer=transferred_layer,
|
||||
)
|
||||
# After the main thread consumes, advance layer_to_transfer
|
||||
model_state.layer_to_transfer += 1
|
||||
model_state.ep_buffer_ready = 0
|
||||
@@ -1175,3 +1109,84 @@ def compute_logical_maps(
|
||||
if per_layer:
|
||||
return logical_to_physical_map_out.squeeze(0), logical_replica_count.squeeze(0)
|
||||
return logical_to_physical_map_out, logical_replica_count
|
||||
|
||||
|
||||
def _pad_out_tensor(src: torch.Tensor, dst: torch.Tensor) -> None:
|
||||
src_padding = dst.shape[-1] - src.shape[-1]
|
||||
assert src_padding >= 0
|
||||
new_src = torch.nn.functional.pad(src, (0, src_padding), value=-1)
|
||||
dst.copy_(new_src)
|
||||
|
||||
|
||||
def _commit_eplb_maps_for_layer(
|
||||
model_state: EplbModelState,
|
||||
new_physical_to_logical_map: torch.Tensor,
|
||||
layer: int,
|
||||
) -> None:
|
||||
"""
|
||||
Per-layer version of _commit_eplb_maps that's used by the sync portion of EPLB
|
||||
when running async EPLB. Copies all of the new_* maps into model_state. After this
|
||||
function completes, the new mappings will become the current mappings and will be
|
||||
visible to the model.
|
||||
"""
|
||||
|
||||
# Commit physical_to_logical_map
|
||||
src = new_physical_to_logical_map[layer]
|
||||
dst = model_state.physical_to_logical_map[layer]
|
||||
assert src.shape == dst.shape, (
|
||||
"The number of physical experts must stay the same while running Async EPLB. "
|
||||
f"Current number of physical experts: {dst.shape[0]}. New number of physical "
|
||||
f"experts {src.shape[0]}."
|
||||
)
|
||||
dst.copy_(src, non_blocking=True)
|
||||
|
||||
num_logical_experts = model_state.logical_to_physical_map.shape[1]
|
||||
new_logical, new_replica_count = compute_logical_maps(src, num_logical_experts)
|
||||
# Commit logical_to_physical_map
|
||||
_pad_out_tensor(
|
||||
src=new_logical,
|
||||
dst=model_state.logical_to_physical_map[layer],
|
||||
)
|
||||
|
||||
# Commit logical_replica_count
|
||||
src = new_replica_count
|
||||
dst = model_state.logical_replica_count[layer]
|
||||
assert src.shape == dst.shape
|
||||
dst.copy_(src, non_blocking=True)
|
||||
|
||||
|
||||
def _commit_eplb_maps(
|
||||
model_state: EplbModelState,
|
||||
new_physical_to_logical_map: torch.Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
Copies all of the new_* maps into model_state. After this function completes,
|
||||
the new mappings will become the current mappings and will be visible to the
|
||||
model.
|
||||
"""
|
||||
|
||||
# Commit physical_to_logical_map
|
||||
src = new_physical_to_logical_map
|
||||
dst = model_state.physical_to_logical_map
|
||||
|
||||
# Rare Case: When the number of physical experts has changed, discard the old
|
||||
# physical to logical expert map and use the new one. This only happens when the
|
||||
# number of GPUs available to vLLM changes while vLLM is running. Otherwise copy the
|
||||
# new map into the old one.
|
||||
if src.shape[1] != dst.shape[1]:
|
||||
model_state.physical_to_logical_map = src.to(dst.device)
|
||||
else:
|
||||
dst.copy_(src, non_blocking=True)
|
||||
|
||||
num_logical_experts = model_state.logical_to_physical_map.shape[1]
|
||||
new_logical, new_replica_count = compute_logical_maps(src, num_logical_experts)
|
||||
# Commit logical_to_physical_map
|
||||
_pad_out_tensor(
|
||||
src=new_logical,
|
||||
dst=model_state.logical_to_physical_map,
|
||||
)
|
||||
|
||||
# Commit logical_replica_count
|
||||
src = new_replica_count
|
||||
dst = model_state.logical_replica_count
|
||||
dst.copy_(src, non_blocking=True)
|
||||
|
||||
Reference in New Issue
Block a user