[EPLB] Mask padding in EPLB load recording (#38128)
Signed-off-by: ilmarkov <markovilya197@gmail.com> Signed-off-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
co-authored by
Markov Ilya
parent
311ad689ad
commit
a2abce646f
@@ -458,7 +458,9 @@ class ElasticEPScalingExecutor:
|
||||
eplb_model_state.logical_to_physical_map,
|
||||
eplb_model_state.logical_replica_count,
|
||||
)
|
||||
eplb_state._init_should_record_tensor(model)
|
||||
eplb_state._propagate_shared_tensors(
|
||||
model, eplb_model_state.num_unpadded_tokens_tensors
|
||||
)
|
||||
model.update_physical_experts_metadata(
|
||||
num_physical_experts=num_physical_experts,
|
||||
num_local_physical_experts=num_local_experts,
|
||||
|
||||
@@ -35,6 +35,7 @@ import torch
|
||||
from torch.distributed import ProcessGroup, all_reduce
|
||||
|
||||
from vllm.config import ModelConfig, ParallelConfig
|
||||
from vllm.config.utils import compute_hash_cached
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_ep_group,
|
||||
get_eplb_group,
|
||||
@@ -206,6 +207,13 @@ class EplbModelState:
|
||||
pending_result relies on the GIL to synchronize access between the main thread and
|
||||
the async worker.
|
||||
"""
|
||||
num_unpadded_tokens_tensors: list[torch.Tensor] | None = None
|
||||
"""
|
||||
Per-ubatch scalar int32 tensors holding the number of real (non-padding)
|
||||
tokens. Allocated once in :meth:`EplbState.add_model` so that device
|
||||
pointers remain stable across CUDA-graph replays. The router kernel
|
||||
indexes this list with ``dbo_current_ubatch_id()``.
|
||||
"""
|
||||
|
||||
|
||||
class EplbState:
|
||||
@@ -253,7 +261,7 @@ class EplbState:
|
||||
Shared scalar bool tensor for all layers. Every
|
||||
:class:`EplbLayerState` holds a reference to the **same** object so
|
||||
a single ``.fill_()`` updates all layers at once. Allocated on the
|
||||
first call to :meth:`_init_should_record_tensor`.
|
||||
first call to :meth:`_propagate_shared_tensors`.
|
||||
"""
|
||||
self.is_async: bool = False
|
||||
"""
|
||||
@@ -440,12 +448,19 @@ class EplbState:
|
||||
self.policy = EPLB_POLICIES[policy_type]
|
||||
logger.debug("Selected EPLB policy: %s", policy_type)
|
||||
|
||||
# num_ubatches is 0 when DBO is disabled.
|
||||
num_ubatches = max(1, self.parallel_config.num_ubatches)
|
||||
num_unpadded_tokens_tensors = [
|
||||
torch.tensor(0, dtype=torch.int32, device=self.device)
|
||||
for _ in range(num_ubatches)
|
||||
]
|
||||
|
||||
model.set_eplb_state(
|
||||
expert_load_pass,
|
||||
logical_to_physical_map,
|
||||
logical_replica_count,
|
||||
)
|
||||
self._init_should_record_tensor(model)
|
||||
self._propagate_shared_tensors(model, num_unpadded_tokens_tensors)
|
||||
expert_buffer = [torch.empty_like(w) for w in model.expert_weights[0]]
|
||||
|
||||
assert self.parallel_config.eplb_config.communicator is not None, (
|
||||
@@ -471,10 +486,43 @@ class EplbState:
|
||||
eplb_stats=None,
|
||||
cuda_device_index=self.cuda_device_index,
|
||||
communicator=communicator,
|
||||
num_unpadded_tokens_tensors=num_unpadded_tokens_tensors,
|
||||
)
|
||||
self.model_states[model_config.compute_hash()] = model_state
|
||||
self.num_valid_physical_experts = model.num_physical_experts
|
||||
|
||||
def prepare_forward(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
num_unpadded_tokens: int,
|
||||
ubatch_slices: list | None = None,
|
||||
) -> None:
|
||||
"""Fill the per-[u]batch ``num_unpadded_tokens`` tensors before a
|
||||
forward pass.
|
||||
|
||||
Args:
|
||||
model_config: Identifies which ``EplbModelState`` to update.
|
||||
num_unpadded_tokens: Total number of real (non-padding) tokens
|
||||
in the batch.
|
||||
ubatch_slices: When DBO is active, a list of
|
||||
``UBatchSlice`` objects describing each micro-batch's
|
||||
token range. When ``None``, only ``tensors[0]`` is filled.
|
||||
"""
|
||||
model_state = self.model_states.get(compute_hash_cached(model_config))
|
||||
if model_state is None or model_state.num_unpadded_tokens_tensors is None:
|
||||
return
|
||||
tensors = model_state.num_unpadded_tokens_tensors
|
||||
if ubatch_slices is None:
|
||||
tensors[0].fill_(num_unpadded_tokens)
|
||||
else:
|
||||
for i, ubatch_slice in enumerate(ubatch_slices):
|
||||
ts = ubatch_slice.token_slice
|
||||
# Real tokens in this ubatch: clamp the global count into
|
||||
# the slice range so partially-filled ubatches get the
|
||||
# correct count.
|
||||
val = max(0, min(num_unpadded_tokens, ts.stop) - ts.start)
|
||||
tensors[i].fill_(val)
|
||||
|
||||
def step(
|
||||
self,
|
||||
is_dummy: bool = False,
|
||||
@@ -638,11 +686,20 @@ class EplbState:
|
||||
self._should_record_current_step(log_stats=log_stats)
|
||||
)
|
||||
|
||||
def _init_should_record_tensor(self, model: "MixtureOfExperts") -> None: # type: ignore[name-defined]
|
||||
"""Allocate (once) and propagate the shared ``should_record_tensor``.
|
||||
def _propagate_shared_tensors(
|
||||
self,
|
||||
model: "MixtureOfExperts", # type: ignore[name-defined]
|
||||
num_unpadded_tokens_tensors: list[torch.Tensor],
|
||||
) -> None:
|
||||
"""Propagate shared tensors to every :class:`EplbLayerState`.
|
||||
|
||||
Allocates ``should_record_tensor`` on the first call and then
|
||||
assigns both it and ``num_unpadded_tokens_tensors`` to every
|
||||
MoE layer's :class:`EplbLayerState`. All layers reference the
|
||||
**same** objects so a single update is visible everywhere.
|
||||
|
||||
Must be called after :meth:`model.set_eplb_state` so that each
|
||||
layer's ``eplb_state`` is already populated with the tensor views.
|
||||
layer's ``eplb_state`` is already populated.
|
||||
"""
|
||||
layer_states = [
|
||||
layer.eplb_state
|
||||
@@ -659,6 +716,7 @@ class EplbState:
|
||||
for ls in layer_states:
|
||||
if ls is not None:
|
||||
ls.should_record_tensor = self.should_record_tensor
|
||||
ls.num_unpadded_tokens_tensors = num_unpadded_tokens_tensors
|
||||
|
||||
def rearrange(
|
||||
self,
|
||||
@@ -985,6 +1043,11 @@ class EplbLayerState:
|
||||
sliding window before the next rearrangement, so recording them wastes
|
||||
GPU work.
|
||||
"""
|
||||
num_unpadded_tokens_tensors: list[torch.Tensor] | None = None
|
||||
"""
|
||||
Reference to the parent :class:`EplbModelState`'s tensor list so the
|
||||
router can read the correct per-[u]batch unpadded token count.
|
||||
"""
|
||||
|
||||
def set_layer_state(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user