[EPLB] Optmize eplb mapping and record in router for prefill (#36261)
Signed-off-by: ilmarkov <markovilya197@gmail.com>
This commit is contained in:
@@ -399,6 +399,7 @@ class ElasticEPScalingExecutor:
|
||||
eplb_model_state.logical_to_physical_map,
|
||||
eplb_model_state.logical_replica_count,
|
||||
)
|
||||
eplb_state._init_should_record_tensor(model)
|
||||
model.update_physical_experts_metadata(
|
||||
num_physical_experts=num_physical_experts,
|
||||
num_local_physical_experts=num_local_experts,
|
||||
|
||||
@@ -272,6 +272,13 @@ class EplbState:
|
||||
Interval for expert rearrangement steps.
|
||||
This is a constant and is taken from the config.
|
||||
"""
|
||||
self.should_record_tensor: torch.Tensor | None = None
|
||||
"""
|
||||
Shared scalar bool tensor for all layers. Every
|
||||
:class:`EplbLayerState` holds a reference to the **same** object so
|
||||
a single ``.fill_()`` updates all layers at once. Allocated on the
|
||||
first call to :meth:`_init_should_record_tensor`.
|
||||
"""
|
||||
self.is_async: bool = False
|
||||
"""
|
||||
The flag indicates whether the EPLB is running in async mode.
|
||||
@@ -462,7 +469,7 @@ class EplbState:
|
||||
logical_to_physical_map,
|
||||
logical_replica_count,
|
||||
)
|
||||
|
||||
self._init_should_record_tensor(model)
|
||||
expert_buffer = [torch.empty_like(w) for w in model.expert_weights[0]]
|
||||
|
||||
model_state = EplbModelState(
|
||||
@@ -582,15 +589,18 @@ class EplbState:
|
||||
|
||||
# Update the expert load sliding window
|
||||
if not is_dummy:
|
||||
should_record = self._should_record_current_step(log_stats=log_stats)
|
||||
for eplb_model_state in self.model_states.values():
|
||||
eplb_model_state.expert_load_window[self.expert_load_window_step] = (
|
||||
eplb_model_state.expert_load_pass.clone()
|
||||
)
|
||||
eplb_model_state.expert_load_pass.zero_()
|
||||
if should_record:
|
||||
eplb_model_state.expert_load_window[
|
||||
self.expert_load_window_step
|
||||
].copy_(eplb_model_state.expert_load_pass)
|
||||
eplb_model_state.expert_load_pass.zero_()
|
||||
|
||||
self.expert_load_window_step += 1
|
||||
if self.expert_load_window_step >= self.expert_load_window_size:
|
||||
self.expert_load_window_step = 0
|
||||
if should_record:
|
||||
self.expert_load_window_step += 1
|
||||
if self.expert_load_window_step >= self.expert_load_window_size:
|
||||
self.expert_load_window_step = 0
|
||||
|
||||
# Step the expert rearrangement step
|
||||
# Note that even if this is a dummy step, we still increment the
|
||||
@@ -617,11 +627,66 @@ class EplbState:
|
||||
eplb_model_state.rebalanced
|
||||
for eplb_model_state in self.model_states.values()
|
||||
):
|
||||
# Still performing asynchronous rearrangement
|
||||
# Still performing asynchronous rearrangement; update
|
||||
# should_record (step > step_interval, so always True) and
|
||||
# bail out before the step counter is reset.
|
||||
self._update_layer_should_record(log_stats=log_stats)
|
||||
return
|
||||
self.expert_rearrangement_step = 0
|
||||
self.rearrange()
|
||||
|
||||
self._update_layer_should_record(log_stats=log_stats)
|
||||
|
||||
def _should_record_current_step(self, log_stats: bool = False) -> bool:
|
||||
"""Return whether expert-load recording should be enabled this step.
|
||||
|
||||
Recording is enabled when we are close to either:
|
||||
1) The next rearrangement step, so the sliding window is ready.
|
||||
2) The next balancedness logging step, when log_stats is enabled.
|
||||
"""
|
||||
steps_remaining = (
|
||||
self.expert_rearrangement_step_interval - self.expert_rearrangement_step
|
||||
)
|
||||
should_record_for_rearrange = steps_remaining <= self.expert_load_window_size
|
||||
|
||||
if not log_stats:
|
||||
return should_record_for_rearrange
|
||||
|
||||
log_interval = self.parallel_config.eplb_config.log_balancedness_interval
|
||||
steps_until_next_log = (
|
||||
log_interval - (self.expert_rearrangement_step % log_interval)
|
||||
) % log_interval
|
||||
should_record_for_log = steps_until_next_log <= self.expert_load_window_size
|
||||
return should_record_for_rearrange or should_record_for_log
|
||||
|
||||
def _update_layer_should_record(self, log_stats: bool = False) -> None:
|
||||
"""Update the shared ``should_record_tensor`` for all layers."""
|
||||
if self.should_record_tensor is not None:
|
||||
self.should_record_tensor.fill_(
|
||||
self._should_record_current_step(log_stats=log_stats)
|
||||
)
|
||||
|
||||
def _init_should_record_tensor(self, model: "MixtureOfExperts") -> None: # type: ignore[name-defined]
|
||||
"""Allocate (once) and propagate the shared ``should_record_tensor``.
|
||||
|
||||
Must be called after :meth:`model.set_eplb_state` so that each
|
||||
layer's ``eplb_state`` is already populated with the tensor views.
|
||||
"""
|
||||
layer_states = [
|
||||
layer.eplb_state
|
||||
for layer in model.moe_layers
|
||||
if hasattr(layer, "eplb_state")
|
||||
and isinstance(layer.eplb_state, EplbLayerState)
|
||||
]
|
||||
|
||||
if self.should_record_tensor is None and layer_states:
|
||||
self.should_record_tensor = torch.ones(
|
||||
(), dtype=torch.bool, device=self.device
|
||||
)
|
||||
|
||||
for ls in layer_states:
|
||||
ls.should_record_tensor = self.should_record_tensor
|
||||
|
||||
def rearrange(
|
||||
self,
|
||||
is_profile: bool = False,
|
||||
@@ -993,6 +1058,17 @@ class EplbLayerState:
|
||||
expert_load_view: torch.Tensor | None = None
|
||||
logical_to_physical_map: torch.Tensor | None = None
|
||||
logical_replica_count: torch.Tensor | None = None
|
||||
should_record_tensor: torch.Tensor | None = None
|
||||
"""
|
||||
Shared scalar bool tensor controlling whether to accumulate expert load
|
||||
metrics during this forward pass. All layers reference the **same**
|
||||
tensor object, which is owned and updated by :class:`EplbState`.
|
||||
|
||||
Set to ``False`` for the first ``step_interval - window_size`` steps of
|
||||
each rearrangement period: those steps would be overwritten in the
|
||||
sliding window before the next rearrangement, so recording them wastes
|
||||
GPU work.
|
||||
"""
|
||||
|
||||
|
||||
def _node_count_with_rank_mapping(
|
||||
|
||||
Reference in New Issue
Block a user