[Core] Replace routing replay with device cache and async D2H pipeline (#39917)
Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
@@ -246,6 +246,9 @@ class FusedMoE(PluggableLayer):
|
||||
not supported by the router (or the experts).
|
||||
"""
|
||||
|
||||
# Auto-incrementing layer ID for routing replay buffer binding.
|
||||
_next_moe_layer_id: int = 0
|
||||
|
||||
# --8<-- [end:fused_moe]
|
||||
|
||||
def __init__(
|
||||
@@ -290,6 +293,10 @@ class FusedMoE(PluggableLayer):
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
# Assign unique layer ID for routing replay buffer binding.
|
||||
self.moe_layer_id = FusedMoE._next_moe_layer_id
|
||||
FusedMoE._next_moe_layer_id += 1
|
||||
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
self.params_dtype = params_dtype
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -451,6 +451,10 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
|
||||
)
|
||||
|
||||
# Get routing replay buffer from persistent layer attribute
|
||||
# (set by bind_routing_capture_to_model during capturer init)
|
||||
routing_replay_out = getattr(layer, "_routing_replay_out", None)
|
||||
|
||||
if self._quant_method.is_monolithic:
|
||||
fused_out = self._quant_method.apply_monolithic(
|
||||
layer=layer,
|
||||
@@ -465,6 +469,10 @@ class MoERunner(MoERunnerInterface):
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
# Write routing data for non-monolithic path (Triton, etc.)
|
||||
if routing_replay_out is not None:
|
||||
routing_replay_out[: topk_ids.shape[0]].copy_(topk_ids.to(torch.int16))
|
||||
|
||||
# Passing shared_experts_input in case SharedExpertsOrder is
|
||||
# MK_INTERNAL_OVERLAPPED.
|
||||
fused_out = self._quant_method.apply(
|
||||
|
||||
Reference in New Issue
Block a user