[Quantization] add humming moe backend to all dense/moe oracles (#41652)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -905,6 +905,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
@@ -951,6 +952,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
a1_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
@@ -1602,7 +1604,9 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
@@ -1616,6 +1620,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
a13_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -2162,6 +2167,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
|
||||
@@ -2207,6 +2213,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
Reference in New Issue
Block a user