[Bugfix] Bind routed-experts capture to the MoE layer, not the kernel (#59455)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
0979892992
commit
4c4003bb5f
@@ -374,6 +374,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
routing_method = self.moe_config.routing_method
|
||||
if routing_method == RoutingMethodType.DeepseekV4:
|
||||
|
||||
@@ -289,6 +289,7 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
|
||||
assert self.moe_config.hidden_dim_unpadded is not None
|
||||
|
||||
@@ -198,6 +198,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the monolithic 4-bit INT MoE forward pass.
|
||||
|
||||
@@ -215,6 +216,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: Bias for expert scores
|
||||
routed_scaling_factor: Scaling factor for routing
|
||||
topk_group: Group size for topk
|
||||
routing_replay_out: Unused; this kernel does not capture routing
|
||||
|
||||
Returns:
|
||||
Output tensor after MoE computation
|
||||
|
||||
@@ -1580,6 +1580,7 @@ class OAITritonMxfp4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return triton_kernel_moe_forward(
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -272,6 +272,7 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
import flashinfer
|
||||
|
||||
@@ -282,10 +283,6 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
# dummy 0-token forward has to keep the finalized (empty) form.
|
||||
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
flashinfer_output = flashinfer.fused_moe.trtllm_bf16_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -312,5 +309,4 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
num_tokens=num_tokens,
|
||||
top_k=self.topk,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
|
||||
return routed_output
|
||||
|
||||
@@ -450,6 +450,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
import flashinfer
|
||||
from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout
|
||||
@@ -487,11 +488,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
n_group = num_expert_group or 0
|
||||
selected_topk_group = topk_group or 0
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
kwargs = dict(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -522,9 +518,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
if is_mxfp8 or activation == MoEActivation.RELU2_NO_MUL:
|
||||
kwargs["activation_type"] = activation_type
|
||||
result = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return result
|
||||
|
||||
def _apply_per_tensor(
|
||||
@@ -543,6 +536,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Delay import for non-CUDA.
|
||||
import flashinfer
|
||||
@@ -558,11 +552,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
else:
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -586,9 +575,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return out
|
||||
|
||||
def apply(
|
||||
@@ -607,6 +593,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.quant_config.block_shape is not None:
|
||||
return self._apply_block_scale(
|
||||
@@ -623,6 +610,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
elif self.quant_config.is_per_tensor:
|
||||
return self._apply_per_tensor(
|
||||
@@ -638,6 +626,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
num_expert_group=num_expert_group,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -215,6 +215,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
from flashinfer import trtllm_fp4_block_scale_moe
|
||||
|
||||
@@ -236,10 +237,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
flashinfer_output = trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -279,7 +276,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
top_k=self.topk,
|
||||
finalized_output=finalized_output,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
|
||||
return routed_output
|
||||
|
||||
|
||||
|
||||
@@ -139,6 +139,7 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import ( # noqa: E501
|
||||
flashinfer_trtllm_mxint4_moe,
|
||||
@@ -147,10 +148,6 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
result = flashinfer_trtllm_mxint4_moe(
|
||||
x=hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -169,7 +166,4 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
routing_method_type=self.routing_method,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -515,6 +515,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
import flashinfer
|
||||
|
||||
@@ -545,10 +546,6 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
# dummy 0-token forward has to keep the finalized (empty) form.
|
||||
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
# Invoke kernel.
|
||||
# NOTE: Activation padding and output
|
||||
# truncation are handled by the MoE runner's
|
||||
@@ -592,5 +589,4 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
num_tokens=num_tokens,
|
||||
top_k=self.topk,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
|
||||
return routed_output
|
||||
|
||||
@@ -151,6 +151,7 @@ class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if apply_router_weight_on_input:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -23,6 +23,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsSink,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
SharedExperts,
|
||||
SharedExpertsOrder,
|
||||
@@ -979,9 +982,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
def is_monolithic() -> bool:
|
||||
return True
|
||||
|
||||
routing_replay_capture_fn: Callable[[torch.Tensor], None] | None = None
|
||||
_routing_replay_buffer: torch.Tensor | None = None
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
"""Whether this expert supports routing replay capture.
|
||||
|
||||
@@ -990,53 +990,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
"""
|
||||
return False
|
||||
|
||||
def set_capture_fn(
|
||||
self,
|
||||
capture_fn: Callable[[torch.Tensor], None] | None,
|
||||
) -> None:
|
||||
self.routing_replay_capture_fn = capture_fn
|
||||
if capture_fn is None:
|
||||
self._routing_replay_buffer = None
|
||||
return
|
||||
# Allocate for per-rank batches gathered across the DP or EP group.
|
||||
dispatch_group_size = (
|
||||
self.moe_config.ep_size
|
||||
if self.moe_config.use_ep
|
||||
else self.moe_config.dp_size
|
||||
)
|
||||
max_num_replay_tokens = self.moe_config.max_num_tokens * dispatch_group_size
|
||||
self._routing_replay_buffer = torch.empty(
|
||||
(max_num_replay_tokens, self.moe_config.experts_per_token),
|
||||
dtype=torch.int16,
|
||||
device=self.moe_config.device,
|
||||
)
|
||||
|
||||
def _maybe_make_routing_replay_buffer(
|
||||
self,
|
||||
num_tokens: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
if self.routing_replay_capture_fn is None:
|
||||
return None
|
||||
buf = self._routing_replay_buffer
|
||||
assert buf is not None
|
||||
if buf.shape[0] < num_tokens or buf.device != device:
|
||||
raise ValueError(
|
||||
"Routing replay buffer was initialized for "
|
||||
f"{buf.shape[0]} tokens on {buf.device}, but the kernel "
|
||||
f"received {num_tokens} tokens on {device}."
|
||||
)
|
||||
return buf
|
||||
|
||||
def _maybe_dispatch_routing_replay(
|
||||
self,
|
||||
routing_replay_out: torch.Tensor | None,
|
||||
num_tokens: int,
|
||||
) -> None:
|
||||
if routing_replay_out is None or self.routing_replay_capture_fn is None:
|
||||
return
|
||||
self.routing_replay_capture_fn(routing_replay_out[:num_tokens])
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -1053,10 +1006,14 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Same as ``FusedMoEExperts.apply``, except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is useful for kernels
|
||||
with fused router and fused_experts (e.g. FLASHINFER_TRTLLM).
|
||||
|
||||
Kernels that ``supports_routing_replay_capture`` write each token's
|
||||
routed expert ids into the leading rows of ``routing_replay_out``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1563,6 +1520,8 @@ class FusedMoEKernelMonolithicImpl:
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Same as forward(), except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is used for kernels
|
||||
@@ -1575,6 +1534,15 @@ class FusedMoEKernelMonolithicImpl:
|
||||
defer_input_quant=self.fused_experts.expects_unquantized_inputs,
|
||||
)
|
||||
|
||||
routing_replay_out = None
|
||||
if routing_sink is not None:
|
||||
# Checked per call: a weight reload may rebuild a different kernel.
|
||||
if not self.fused_experts.supports_routing_replay_capture():
|
||||
raise ValueError(
|
||||
"Routed-experts capture is not supported with monolithic MoE "
|
||||
f"kernel {type(self.fused_experts).__name__}."
|
||||
)
|
||||
routing_replay_out = routing_sink.buffer[: len(a1q)]
|
||||
fused_out = self.fused_experts.apply(
|
||||
hidden_states=a1q,
|
||||
w1=w1,
|
||||
@@ -1590,7 +1558,10 @@ class FusedMoEKernelMonolithicImpl:
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
if routing_sink is not None:
|
||||
routing_sink.capture_fn(routing_replay_out)
|
||||
|
||||
if isinstance(fused_out, UnfinalizedMoEOutput):
|
||||
if not self.prepare_finalize.supports_deferred_moe_finalize():
|
||||
@@ -1698,6 +1669,8 @@ class FusedMoEKernel:
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
|
||||
return self.impl.apply(
|
||||
@@ -1713,6 +1686,7 @@ class FusedMoEKernel:
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_sink=routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -20,6 +20,9 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsSink,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
@@ -138,6 +141,8 @@ class RoutedExperts(PluggableLayer):
|
||||
self.swiglu_alpha = swiglu_alpha
|
||||
self.swiglu_beta = swiglu_beta
|
||||
self.e_score_correction_bias = e_score_correction_bias
|
||||
# Set by bind_routed_experts_capturer for monolithic kernels.
|
||||
self.routing_sink: RoutedExpertsSink | None = None
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
# End random parameters
|
||||
self._loaded_expert_biases: set[str] = set()
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -186,6 +187,23 @@ class RoutedExpertsCapturer:
|
||||
return self.device_buffer[:num_tokens].to(self.output_dtype)
|
||||
|
||||
|
||||
class RoutedExpertsSink:
|
||||
"""Layer-owned buffer and callback for the expert ids a monolithic kernel
|
||||
routes to; it outlives kernel rebuilds on weight reload."""
|
||||
|
||||
def __init__(
|
||||
self, moe_config: FusedMoEConfig, capture_fn: Callable[[torch.Tensor], None]
|
||||
) -> None:
|
||||
# Sized for per-rank batches gathered across the DP or EP group.
|
||||
group_size = moe_config.ep_size if moe_config.use_ep else moe_config.dp_size
|
||||
self.buffer = torch.empty(
|
||||
(moe_config.max_num_tokens * group_size, moe_config.experts_per_token),
|
||||
dtype=torch.int16,
|
||||
device=moe_config.device,
|
||||
)
|
||||
self.capture_fn = capture_fn
|
||||
|
||||
|
||||
def bind_routed_experts_capturer(
|
||||
model: torch.nn.Module,
|
||||
capturer: RoutedExpertsCapturer,
|
||||
@@ -219,7 +237,9 @@ def bind_routed_experts_capturer(
|
||||
"Routed-experts capture is not supported with monolithic "
|
||||
f"MoE kernel {type(fused_experts).__name__}."
|
||||
)
|
||||
fused_experts.set_capture_fn(capture_fn)
|
||||
module.routed_experts.routing_sink = RoutedExpertsSink(
|
||||
fused_experts.moe_config, capture_fn
|
||||
)
|
||||
num_bound += 1
|
||||
elif isinstance(module.router, BaseRouter):
|
||||
module.router.set_capture_fn(capture_fn)
|
||||
|
||||
@@ -362,4 +362,5 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
@@ -790,6 +790,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -779,4 +779,5 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
+1
@@ -293,6 +293,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -287,6 +287,7 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -366,6 +366,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -291,4 +291,5 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
+1
@@ -179,6 +179,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -528,6 +528,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -818,6 +818,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -680,6 +680,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
@@ -1092,6 +1093,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
@@ -1461,6 +1463,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -531,6 +531,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -472,6 +472,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
|
||||
@@ -900,4 +901,5 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
@@ -156,6 +156,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -1980,6 +1980,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user