[Quantization] add humming moe backend to all dense/moe oracles (#41652)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-07-06 13:36:07 -07:00
committed by GitHub
co-authored by mgoin
parent 04adc8843b
commit d891b9bd51
73 changed files with 1337 additions and 123 deletions
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe import (
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
WNA16MoEBackend,
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
make_wna16_moe_quant_config,
@@ -663,6 +664,26 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
layer.workspace = marlin_make_workspace_new(device, 4)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=self.input_dtype,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_qzeros=layer.w13_qzeros,
w2_qzeros=layer.w2_qzeros,
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
if converted is None:
# Backend rewrote the layer's params in place (e.g. Humming).
self._setup_kernel(layer)
return
(
w13,
w2,
@@ -678,20 +699,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
w2_input_global_scale,
w13_bias,
w2_bias,
) = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=self.input_dtype,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_qzeros=layer.w13_qzeros,
w2_qzeros=layer.w2_qzeros,
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
) = converted
replace_parameter(layer, "w13_qweight", w13)
replace_parameter(layer, "w2_qweight", w2)
@@ -734,6 +742,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_moe_backend,
layer=layer,
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
@@ -743,6 +753,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
)
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
return get_humming_moe_quant_config(layer)
return make_wna16_moe_quant_config(
w1_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
@@ -783,8 +799,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
@@ -806,8 +822,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
w1=layer.w13_weight,
w2=layer.w2_weight,
router_logits=router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,