add 2/3/5/6/7 CUDA support in AutoRound format (#52890)

Signed-off-by: Wenhua Cheng <wenhua.cheng@intel.com>
This commit is contained in:
Wenhua Cheng
2026-09-07 09:59:19 +08:00
committed by GitHub
parent 199cb9b964
commit 294fbb4f59
3 changed files with 152 additions and 2 deletions
@@ -34,9 +34,8 @@ class INCConfig(QuantizationConfig):
Repo: https://github.com/intel/neural-compressor
"""
SUPPORTED_BITS = {2, 3, 4, 5, 6, 7, 8}
DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
SUPPORTED_BITS = {2, 3, 4, 8}
SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
SUPPORTED_FORMATS = {
"auto_round:auto_gptq",
@@ -21,6 +21,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
XPU_WNA16_SUPPORTED_BITS = {2, 4}
# On CUDA the Marlin/GPTQ/AWQ kernels only cover 4/8-bit. The remaining
# widths (2/3/5/6/7) have no dedicated CUDA kernel, so they are dispatched to
# the humming kernel instead. This mirrors compressed-tensors WNA16, whose
# 5/6/7-bit biased scalar types also fall back to humming at kernel selection.
CUDA_HUMMING_SUPPORTED_BITS = {2, 3, 5, 6, 7}
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
@@ -133,6 +138,15 @@ class INCWna16Scheme(INCScheme):
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
# CUDA low-bit (2/3/5/6/7): no Marlin/GPTQ/AWQ kernel, route to humming
# so a single model can mix 4/8-bit (Marlin) and 2/3/5/6/7-bit (humming)
# layers.
if (
current_platform.is_cuda()
and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
):
return _build_humming_linear_method(layer_config)
from .inc_wna16_linear import INCWNA16LinearScheme
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
@@ -152,6 +166,12 @@ class INCWna16Scheme(INCScheme):
)
return UnquantizedFusedMoEMethod(layer.moe_config)
# CUDA low-bit (2/3/5/6/7): route to the humming MoE kernel (see above).
if (
current_platform.is_cuda()
and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
):
return _build_humming_moe_method(layer, layer_config)
if layer_config.is_gptq:
return _resolve_gptq_moe(layer, layer_config)
if layer_config.is_awq:
@@ -251,3 +271,48 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
}
)
return MoeWNA16Method(moe_config, layer.moe_config)
def _humming_weight_config(layer_config: "INCLayerConfig") -> dict:
"""Build the humming weight-schema config for a WNA16 int checkpoint."""
if layer_config.is_gptq:
return {
"quant_method": "gptq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"desc_act": False,
"sym": layer_config.sym,
}
if layer_config.is_awq:
return {
"quant_method": "awq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"zero_point": not layer_config.sym,
}
raise NotImplementedError(
"INC humming dispatch only supports gptq/awq packed int checkpoints, "
f"but found {layer_config}."
)
def _build_humming_quant_config(layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.humming import (
HummingLayerQuantizationConfig,
)
from vllm.utils.humming import BaseWeightSchema
weight_schema = BaseWeightSchema.from_config(_humming_weight_config(layer_config))
return HummingLayerQuantizationConfig(weight_schema=weight_schema)
def _build_humming_linear_method(layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.humming import HummingLinearMethod
return HummingLinearMethod(_build_humming_quant_config(layer_config))
def _build_humming_moe_method(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.humming import HummingMoEMethod
return HummingMoEMethod(_build_humming_quant_config(layer_config), layer.moe_config)