[Core] Replace routing replay with device cache and async D2H pipeline (#39917)

Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
TomerBN-Nvidia
2026-05-07 11:24:56 -07:00
committed by GitHub
parent 8eb401134e
commit 8189a15914
15 changed files with 1398 additions and 642 deletions
@@ -246,6 +246,9 @@ class FusedMoE(PluggableLayer):
not supported by the router (or the experts).
"""
# Auto-incrementing layer ID for routing replay buffer binding.
_next_moe_layer_id: int = 0
# --8<-- [end:fused_moe]
def __init__(
@@ -290,6 +293,10 @@ class FusedMoE(PluggableLayer):
):
super().__init__()
# Assign unique layer ID for routing replay buffer binding.
self.moe_layer_id = FusedMoE._next_moe_layer_id
FusedMoE._next_moe_layer_id += 1
if params_dtype is None:
params_dtype = torch.get_default_dtype()
self.params_dtype = params_dtype
File diff suppressed because it is too large Load Diff
@@ -451,6 +451,10 @@ class MoERunner(MoERunnerInterface):
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
)
# Get routing replay buffer from persistent layer attribute
# (set by bind_routing_capture_to_model during capturer init)
routing_replay_out = getattr(layer, "_routing_replay_out", None)
if self._quant_method.is_monolithic:
fused_out = self._quant_method.apply_monolithic(
layer=layer,
@@ -465,6 +469,10 @@ class MoERunner(MoERunnerInterface):
input_ids=input_ids,
)
# Write routing data for non-monolithic path (Triton, etc.)
if routing_replay_out is not None:
routing_replay_out[: topk_ids.shape[0]].copy_(topk_ids.to(torch.int16))
# Passing shared_experts_input in case SharedExpertsOrder is
# MK_INTERNAL_OVERLAPPED.
fused_out = self._quant_method.apply(