[Bugfix][R3] Size monolithic routing replay buffer for DP (#50874)

Signed-off-by: tbarnatan <tbarnatan@nvidia.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
TomerBN-Nvidia
2026-08-13 00:18:08 -07:00
committed by GitHub
co-authored by Claude Opus 5 zjy0516
parent 373592ef57
commit 399f97424e
2 changed files with 30 additions and 5 deletions
@@ -1023,8 +1023,15 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
if capture_fn is None:
self._routing_replay_buffer = None
return
# Allocate for per-rank batches gathered across the DP or EP group.
dispatch_group_size = (
self.moe_config.ep_size
if self.moe_config.use_ep
else self.moe_config.dp_size
)
max_num_replay_tokens = self.moe_config.max_num_tokens * dispatch_group_size
self._routing_replay_buffer = torch.empty(
(self.moe_config.max_num_tokens, self.moe_config.experts_per_token),
(max_num_replay_tokens, self.moe_config.experts_per_token),
dtype=torch.int16,
device=self.moe_config.device,
)
@@ -108,15 +108,20 @@ class RoutedExpertsCapturer:
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
"""Capture expert routing decisions for a specific layer.
Under data parallelism, ``topk_ids`` may have three different batch
Under data parallelism, ``topk_ids`` may have four different batch
layouts depending on where the DP combine happens and whether
Sequence Parallelism (SP) is active for the MoE layer:
Expert Parallelism (EP) or Sequence Parallelism (SP) is active for the
MoE layer:
- ``n == total`` (naive dispatch): all DP ranks' tokens are
concatenated before routing; we slice out this rank's span
using the cumulative per-rank counts.
- ``n == token_num_per_dp`` (modular-kernel path): DP combine
happens inside ``quant_method.apply``; ``select_experts`` only
ever sees this rank's tokens, so we take the whole tensor.
- ``n == sum(dp_metadata.local_sizes)`` (naive DP+EP dispatch):
sequence-parallel shards from every DP rank are gathered through
the flattened EP group. The shard sizes include CUDA-graph / SP
padding, so we use them to locate this DP rank's unpadded rows.
- ``n == ceil(token_num_per_dp / tp_size)`` (SP + modular-kernel
path): tokens were split along dim=0 across the TP group by
``_sequence_parallel_context``
@@ -143,6 +148,8 @@ class RoutedExpertsCapturer:
token_num_per_dp = int(num_tokens_dp[self.dp_rank].item())
total = int(num_tokens_dp.sum().item())
n = topk_ids.shape[0]
shard_sizes = getattr(ctx.dp_metadata, "local_sizes", None)
gathered_size = sum(shard_sizes) if shard_sizes is not None else -1
if n == total:
# Naive dispatch: all DP ranks' tokens concatenated
@@ -157,6 +164,17 @@ class RoutedExpertsCapturer:
# rank's tokens, take the whole tensor.
start_loc = 0
end_loc = token_num_per_dp
elif shard_sizes is not None and n == gathered_size:
# Naive DP+EP dispatch gathers the sequence-parallel shards in
# flattened DP-then-TP order. ``local_sizes`` is the exact
# all-gatherv layout, including padding. Select this DP rank's
# contiguous shard group, then trim its trailing padding.
num_dp_ranks = len(num_tokens_dp)
assert len(shard_sizes) % num_dp_ranks == 0
shards_per_dp_rank = len(shard_sizes) // num_dp_ranks
first_shard = self.dp_rank * shards_per_dp_rank
start_loc = sum(shard_sizes[:first_shard])
end_loc = start_loc + token_num_per_dp
elif (
self.tp_size > 1
and n != token_num_per_dp
@@ -189,8 +207,8 @@ class RoutedExpertsCapturer:
raise AssertionError(
"RoutedExpertsCapturer: unexpected topk_ids batch "
f"dim {n} (expected {total}, {token_num_per_dp}, "
f"or {sp_expected} for dp_rank={self.dp_rank}, "
f"tp_size={self.tp_size})"
f"{gathered_size}, or {sp_expected} for "
f"dp_rank={self.dp_rank}, tp_size={self.tp_size})"
)
if layer_id >= self.device_buffer.shape[1]: