[Quantization] add humming moe backend to all dense/moe oracles (#41652)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-07-06 13:36:07 -07:00
committed by GitHub
co-authored by mgoin
parent 04adc8843b
commit d891b9bd51
73 changed files with 1337 additions and 123 deletions
@@ -458,6 +458,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -486,6 +487,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
convert_to_int8_moe_kernel_format,
make_int8_moe_kernel,
make_int8_moe_quant_config,
select_int8_moe_backend,
@@ -92,22 +93,36 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
replace_parameter(layer, "w2_scale", w2_scale)
def _setup_kernel(self, layer: RoutedExperts) -> None:
w13, w2 = convert_to_int8_moe_kernel_format(
int8_backend=self.int8_backend,
w13=layer.w13_weight,
w2=layer.w2_weight,
layer=layer,
w13_scale=layer.w13_scale,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
self.moe_kernel = make_int8_moe_kernel(
int8_backend=self.int8_backend,
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> "FusedMoEQuantConfig | None":
return make_int8_moe_quant_config(
w1_scale=layer.w13_scale,
w2_scale=layer.w2_scale,
int8_backend=self.int8_backend,
w1_scale=getattr(layer, "w13_scale", None),
w2_scale=getattr(layer, "w2_scale", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
layer=layer,
)
@@ -200,6 +200,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -226,6 +227,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
def process_weights_after_loading(self, layer: Module) -> None: