[EPLB] Optmize eplb mapping and record in router for prefill (#36261)

Signed-off-by: ilmarkov <markovilya197@gmail.com>
This commit is contained in:
Ilya Markov
2026-03-30 19:48:33 +00:00
committed by GitHub
parent 494636b29d
commit 12701e8af2
6 changed files with 336 additions and 64 deletions
@@ -399,6 +399,7 @@ class ElasticEPScalingExecutor:
eplb_model_state.logical_to_physical_map,
eplb_model_state.logical_replica_count,
)
eplb_state._init_should_record_tensor(model)
model.update_physical_experts_metadata(
num_physical_experts=num_physical_experts,
num_local_physical_experts=num_local_experts,
+85 -9
View File
@@ -272,6 +272,13 @@ class EplbState:
Interval for expert rearrangement steps.
This is a constant and is taken from the config.
"""
self.should_record_tensor: torch.Tensor | None = None
"""
Shared scalar bool tensor for all layers. Every
:class:`EplbLayerState` holds a reference to the **same** object so
a single ``.fill_()`` updates all layers at once. Allocated on the
first call to :meth:`_init_should_record_tensor`.
"""
self.is_async: bool = False
"""
The flag indicates whether the EPLB is running in async mode.
@@ -462,7 +469,7 @@ class EplbState:
logical_to_physical_map,
logical_replica_count,
)
self._init_should_record_tensor(model)
expert_buffer = [torch.empty_like(w) for w in model.expert_weights[0]]
model_state = EplbModelState(
@@ -582,15 +589,18 @@ class EplbState:
# Update the expert load sliding window
if not is_dummy:
should_record = self._should_record_current_step(log_stats=log_stats)
for eplb_model_state in self.model_states.values():
eplb_model_state.expert_load_window[self.expert_load_window_step] = (
eplb_model_state.expert_load_pass.clone()
)
eplb_model_state.expert_load_pass.zero_()
if should_record:
eplb_model_state.expert_load_window[
self.expert_load_window_step
].copy_(eplb_model_state.expert_load_pass)
eplb_model_state.expert_load_pass.zero_()
self.expert_load_window_step += 1
if self.expert_load_window_step >= self.expert_load_window_size:
self.expert_load_window_step = 0
if should_record:
self.expert_load_window_step += 1
if self.expert_load_window_step >= self.expert_load_window_size:
self.expert_load_window_step = 0
# Step the expert rearrangement step
# Note that even if this is a dummy step, we still increment the
@@ -617,11 +627,66 @@ class EplbState:
eplb_model_state.rebalanced
for eplb_model_state in self.model_states.values()
):
# Still performing asynchronous rearrangement
# Still performing asynchronous rearrangement; update
# should_record (step > step_interval, so always True) and
# bail out before the step counter is reset.
self._update_layer_should_record(log_stats=log_stats)
return
self.expert_rearrangement_step = 0
self.rearrange()
self._update_layer_should_record(log_stats=log_stats)
def _should_record_current_step(self, log_stats: bool = False) -> bool:
"""Return whether expert-load recording should be enabled this step.
Recording is enabled when we are close to either:
1) The next rearrangement step, so the sliding window is ready.
2) The next balancedness logging step, when log_stats is enabled.
"""
steps_remaining = (
self.expert_rearrangement_step_interval - self.expert_rearrangement_step
)
should_record_for_rearrange = steps_remaining <= self.expert_load_window_size
if not log_stats:
return should_record_for_rearrange
log_interval = self.parallel_config.eplb_config.log_balancedness_interval
steps_until_next_log = (
log_interval - (self.expert_rearrangement_step % log_interval)
) % log_interval
should_record_for_log = steps_until_next_log <= self.expert_load_window_size
return should_record_for_rearrange or should_record_for_log
def _update_layer_should_record(self, log_stats: bool = False) -> None:
"""Update the shared ``should_record_tensor`` for all layers."""
if self.should_record_tensor is not None:
self.should_record_tensor.fill_(
self._should_record_current_step(log_stats=log_stats)
)
def _init_should_record_tensor(self, model: "MixtureOfExperts") -> None: # type: ignore[name-defined]
"""Allocate (once) and propagate the shared ``should_record_tensor``.
Must be called after :meth:`model.set_eplb_state` so that each
layer's ``eplb_state`` is already populated with the tensor views.
"""
layer_states = [
layer.eplb_state
for layer in model.moe_layers
if hasattr(layer, "eplb_state")
and isinstance(layer.eplb_state, EplbLayerState)
]
if self.should_record_tensor is None and layer_states:
self.should_record_tensor = torch.ones(
(), dtype=torch.bool, device=self.device
)
for ls in layer_states:
ls.should_record_tensor = self.should_record_tensor
def rearrange(
self,
is_profile: bool = False,
@@ -993,6 +1058,17 @@ class EplbLayerState:
expert_load_view: torch.Tensor | None = None
logical_to_physical_map: torch.Tensor | None = None
logical_replica_count: torch.Tensor | None = None
should_record_tensor: torch.Tensor | None = None
"""
Shared scalar bool tensor controlling whether to accumulate expert load
metrics during this forward pass. All layers reference the **same**
tensor object, which is owned and updated by :class:`EplbState`.
Set to ``False`` for the first ``step_interval - window_size`` steps of
each rearrangement period: those steps would be overwritten in the
sliding window before the next rearrangement, so recording them wastes
GPU work.
"""
def _node_count_with_rank_mapping(