[Quantization] add humming moe backend to all dense/moe oracles (#41652)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -658,6 +658,26 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
"W8A8-INT8 is not supported by marlin kernel."
|
||||
)
|
||||
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
quant_config=self.quant_config,
|
||||
input_dtype=self.input_dtype,
|
||||
w13=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w13_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
w13_g_idx=layer.w13_g_idx,
|
||||
w2_g_idx=layer.w2_g_idx,
|
||||
w13_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
if converted is None:
|
||||
# Backend rewrote the layer's params in place (e.g. Humming).
|
||||
self._setup_kernel(layer)
|
||||
return
|
||||
|
||||
(
|
||||
w13,
|
||||
w2,
|
||||
@@ -673,20 +693,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
w2_input_global_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
) = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
quant_config=self.quant_config,
|
||||
input_dtype=self.input_dtype,
|
||||
w13=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w13_scale=layer.w13_scales,
|
||||
w2_scale=layer.w2_scales,
|
||||
w13_g_idx=layer.w13_g_idx,
|
||||
w2_g_idx=layer.w2_g_idx,
|
||||
w13_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
) = converted
|
||||
|
||||
replace_parameter(layer, "w13_qweight", w13)
|
||||
replace_parameter(layer, "w2_qweight", w2)
|
||||
@@ -733,6 +740,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
"w2_bias", torch.nn.Parameter(w2_bias, requires_grad=False)
|
||||
)
|
||||
|
||||
# The modular kernel reads w13_weight/w2_weight; marlin keeps *_qweight.
|
||||
layer.w13_weight = layer.w13_qweight
|
||||
layer.w2_weight = layer.w2_qweight
|
||||
|
||||
self._setup_kernel(layer)
|
||||
|
||||
def _setup_kernel(self, layer: RoutedExperts) -> None:
|
||||
@@ -743,15 +754,24 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_moe_backend,
|
||||
layer=layer,
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=layer.w13_g_idx,
|
||||
w2_g_idx=layer.w2_g_idx,
|
||||
w13_g_idx=getattr(layer, "w13_g_idx", None),
|
||||
w2_g_idx=getattr(layer, "w2_g_idx", None),
|
||||
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
|
||||
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
return get_humming_moe_quant_config(layer)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
gptq_marlin_moe_quant_config,
|
||||
)
|
||||
@@ -795,8 +815,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=layer.activation,
|
||||
@@ -818,8 +838,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
router_logits=router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
|
||||
Reference in New Issue
Block a user