[Quantization] add humming moe backend to all dense/moe oracles (#41652)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-07-06 13:36:07 -07:00
committed by GitHub
co-authored by mgoin
parent 04adc8843b
commit d891b9bd51
73 changed files with 1337 additions and 123 deletions
@@ -658,6 +658,26 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
"W8A8-INT8 is not supported by marlin kernel."
)
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=self.input_dtype,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
if converted is None:
# Backend rewrote the layer's params in place (e.g. Humming).
self._setup_kernel(layer)
return
(
w13,
w2,
@@ -673,20 +693,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2_input_global_scale,
w13_bias,
w2_bias,
) = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=self.input_dtype,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
) = converted
replace_parameter(layer, "w13_qweight", w13)
replace_parameter(layer, "w2_qweight", w2)
@@ -733,6 +740,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
"w2_bias", torch.nn.Parameter(w2_bias, requires_grad=False)
)
# The modular kernel reads w13_weight/w2_weight; marlin keeps *_qweight.
layer.w13_weight = layer.w13_qweight
layer.w2_weight = layer.w2_qweight
self._setup_kernel(layer)
def _setup_kernel(self, layer: RoutedExperts) -> None:
@@ -743,15 +754,24 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_moe_backend,
layer=layer,
is_k_full=self.is_k_full,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
routing_tables=layer._expert_routing_tables(),
)
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
return get_humming_moe_quant_config(layer)
from vllm.model_executor.layers.fused_moe.config import (
gptq_marlin_moe_quant_config,
)
@@ -795,8 +815,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
@@ -818,8 +838,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
w1=layer.w13_weight,
w2=layer.w2_weight,
router_logits=router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,