[Quantization] add humming moe backend to all dense/moe oracles (#41652)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -19,6 +19,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.dynamic_4bit import (
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.exllama import (
|
||||
ExllamaLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.humming import (
|
||||
HummingLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.machete import (
|
||||
MacheteLinearKernel,
|
||||
)
|
||||
@@ -52,6 +55,7 @@ __all__ = [
|
||||
"CutlassW4A8LinearKernel",
|
||||
"Dynamic4bitLinearKernel",
|
||||
"ExllamaLinearKernel",
|
||||
"HummingLinearKernel",
|
||||
"MacheteLinearKernel",
|
||||
"MarlinLinearKernel",
|
||||
"RDNA3W4A16LinearKernel",
|
||||
|
||||
@@ -23,8 +23,6 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
return False, "Humming is not installed"
|
||||
if c.has_g_idx:
|
||||
return False, "Humming does not support act-order (g_idx)"
|
||||
if c.zero_points:
|
||||
return False, "Humming linear kernel only supports symmetric weights"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
@@ -41,6 +39,11 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
"group_size": 0 if group_size == -1 else group_size,
|
||||
}
|
||||
|
||||
if self.config.zero_points:
|
||||
assert self.w_zp_name is not None
|
||||
name_map["zero_point"] = self.w_zp_name
|
||||
quant_config["has_zero_point"] = True
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user