[Quantization] add humming moe backend to all dense/moe oracles (#41652)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -458,6 +458,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
@@ -486,6 +487,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
|
||||
convert_to_int8_moe_kernel_format,
|
||||
make_int8_moe_kernel,
|
||||
make_int8_moe_quant_config,
|
||||
select_int8_moe_backend,
|
||||
@@ -92,22 +93,36 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
replace_parameter(layer, "w2_scale", w2_scale)
|
||||
|
||||
def _setup_kernel(self, layer: RoutedExperts) -> None:
|
||||
w13, w2 = convert_to_int8_moe_kernel_format(
|
||||
int8_backend=self.int8_backend,
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
layer=layer,
|
||||
w13_scale=layer.w13_scale,
|
||||
)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_int8_moe_kernel(
|
||||
int8_backend=self.int8_backend,
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> "FusedMoEQuantConfig | None":
|
||||
return make_int8_moe_quant_config(
|
||||
w1_scale=layer.w13_scale,
|
||||
w2_scale=layer.w2_scale,
|
||||
int8_backend=self.int8_backend,
|
||||
w1_scale=getattr(layer, "w13_scale", None),
|
||||
w2_scale=getattr(layer, "w2_scale", None),
|
||||
w1_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
@@ -200,6 +200,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
@@ -226,6 +227,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
|
||||
Reference in New Issue
Block a user