[MoE Refactor] Remove MoE DP chunking (#39107)
Signed-off-by: Bill Nell <bnell@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
+1
-60
@@ -70,27 +70,8 @@ def _compute_sp_num_tokens(
|
||||
return sp_tokens.tolist()
|
||||
|
||||
|
||||
def _compute_chunked_local_num_tokens(
|
||||
num_tokens_across_dp_cpu: torch.Tensor,
|
||||
sequence_parallel_size: int,
|
||||
max_num_tokens: int,
|
||||
chunk_idx: int,
|
||||
) -> list[int]:
|
||||
sp_tokens = _compute_sp_num_tokens(num_tokens_across_dp_cpu, sequence_parallel_size)
|
||||
sp_size = len(sp_tokens)
|
||||
|
||||
local_size = [-1] * sp_size
|
||||
for i in range(sp_size):
|
||||
# Take into account sharding if MoE activation is sequence parallel.
|
||||
local_size[i] = min(max_num_tokens, sp_tokens[i] - (max_num_tokens * chunk_idx))
|
||||
if local_size[i] <= 0:
|
||||
local_size[i] = 1 # ensure lockstep even if done
|
||||
return local_size
|
||||
|
||||
|
||||
@dataclass
|
||||
class DPMetadata:
|
||||
max_tokens_across_dp_cpu: torch.Tensor
|
||||
num_tokens_across_dp_cpu: torch.Tensor
|
||||
|
||||
# NOTE: local_sizes should only be set by the chunked_sizes context manager
|
||||
@@ -113,47 +94,7 @@ class DPMetadata:
|
||||
assert num_tokens_across_dp_cpu[dp_rank] == batchsize, (
|
||||
f"{num_tokens_across_dp_cpu[dp_rank]} {batchsize}"
|
||||
)
|
||||
max_tokens_across_dp_cpu = torch.max(num_tokens_across_dp_cpu)
|
||||
return DPMetadata(max_tokens_across_dp_cpu, num_tokens_across_dp_cpu)
|
||||
|
||||
@contextmanager
|
||||
def chunked_sizes(
|
||||
self, sequence_parallel_size: int, max_chunk_size_per_rank: int, chunk_idx: int
|
||||
):
|
||||
"""
|
||||
Context manager to compute and temporarily set the per-rank local token
|
||||
sizes for a specific chunk during chunked forward execution.
|
||||
|
||||
This is necessary to ensure each DP (data parallel) rank processes its
|
||||
designated portion of tokens in lockstep with others, even when the
|
||||
token counts are uneven or some ranks have completed their input early.
|
||||
|
||||
For chunked execution, we break up the total tokens on each rank into
|
||||
multiple chunks (of at most `max_chunk_size_per_rank`), and for a given
|
||||
`chunk_idx`, this context manager sets `self.local_sizes` to the number
|
||||
of tokens to process in that chunk on each rank.
|
||||
|
||||
`self.local_sizes` is only valid inside the context.
|
||||
|
||||
Args:
|
||||
sequence_parallel_size: When Attn is TP and MoE layers are EP,
|
||||
we use SP between the layers to avoid
|
||||
redundant ops. We need this value to
|
||||
compute the chunked sizes.
|
||||
max_chunk_size_per_rank: The max number of tokens each rank is
|
||||
allowed to process in this chunk.
|
||||
chunk_idx: The index of the chunk to compute sizes for.
|
||||
"""
|
||||
self.local_sizes = _compute_chunked_local_num_tokens(
|
||||
self.num_tokens_across_dp_cpu,
|
||||
sequence_parallel_size,
|
||||
max_chunk_size_per_rank,
|
||||
chunk_idx,
|
||||
)
|
||||
try:
|
||||
yield self.local_sizes
|
||||
finally:
|
||||
self.local_sizes = None
|
||||
return DPMetadata(num_tokens_across_dp_cpu)
|
||||
|
||||
@contextmanager
|
||||
def sp_local_sizes(self, sequence_parallel_size: int):
|
||||
|
||||
Reference in New Issue
Block a user