[Core] Replace routing replay with device cache and async D2H pipeline (#39917)

Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
TomerBN-Nvidia
2026-05-07 11:24:56 -07:00
committed by GitHub
parent 8eb401134e
commit 8189a15914
15 changed files with 1398 additions and 642 deletions
@@ -451,6 +451,10 @@ class MoERunner(MoERunnerInterface):
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
)
# Get routing replay buffer from persistent layer attribute
# (set by bind_routing_capture_to_model during capturer init)
routing_replay_out = getattr(layer, "_routing_replay_out", None)
if self._quant_method.is_monolithic:
fused_out = self._quant_method.apply_monolithic(
layer=layer,
@@ -465,6 +469,10 @@ class MoERunner(MoERunnerInterface):
input_ids=input_ids,
)
# Write routing data for non-monolithic path (Triton, etc.)
if routing_replay_out is not None:
routing_replay_out[: topk_ids.shape[0]].copy_(topk_ids.to(torch.int16))
# Passing shared_experts_input in case SharedExpertsOrder is
# MK_INTERNAL_OVERLAPPED.
fused_out = self._quant_method.apply(