[Core] Replace routing replay with device cache and async D2H pipeline (#39917)
Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
@@ -451,6 +451,10 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
|
||||
)
|
||||
|
||||
# Get routing replay buffer from persistent layer attribute
|
||||
# (set by bind_routing_capture_to_model during capturer init)
|
||||
routing_replay_out = getattr(layer, "_routing_replay_out", None)
|
||||
|
||||
if self._quant_method.is_monolithic:
|
||||
fused_out = self._quant_method.apply_monolithic(
|
||||
layer=layer,
|
||||
@@ -465,6 +469,10 @@ class MoERunner(MoERunnerInterface):
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
# Write routing data for non-monolithic path (Triton, etc.)
|
||||
if routing_replay_out is not None:
|
||||
routing_replay_out[: topk_ids.shape[0]].copy_(topk_ids.to(torch.int16))
|
||||
|
||||
# Passing shared_experts_input in case SharedExpertsOrder is
|
||||
# MK_INTERNAL_OVERLAPPED.
|
||||
fused_out = self._quant_method.apply(
|
||||
|
||||
Reference in New Issue
Block a user