add 2/3/5/6/7 CUDA support in AutoRound format (#52890)

Signed-off-by: Wenhua Cheng <wenhua.cheng@intel.com>

tests/quantization/test_auto_round.py

@@ -666,6 +666,92 @@ def test_inc_resolve_scheme_selects_wna16() -> None:
     assert isinstance(scheme, INCWna16Scheme)
 
 
+@pytest.mark.parametrize("bits", [2, 3])
+@pytest.mark.parametrize(
+    "packing_format", ["auto_round:auto_gptq", "auto_round:auto_awq"]
+)
+def test_wna16_cuda_low_bit_linear_routes_to_humming(
+    monkeypatch, bits, packing_format
+) -> None:
+    expected_method = object()
+    captured = {}
+
+    monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
+    monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
+    monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes."
+        "inc_wna16_scheme._build_humming_linear_method",
+        lambda layer_config: captured.update({"layer_config": layer_config})
+        or expected_method,
+    )
+
+    layer_config = make_layer_config(bits=bits, packing_format=packing_format)
+    method = INCWna16Scheme().get_linear_method(
+        make_config(), object(), "model.layers.0.mlp.down_proj", layer_config
+    )
+
+    assert method is expected_method
+    assert captured["layer_config"] is layer_config
+
+
+@pytest.mark.parametrize("bits", [2, 3])
+def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None:
+    expected_method = object()
+    captured = {}
+
+    class DummyMoeConfig:
+        pass
+
+    monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
+    monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes."
+        "inc_wna16_scheme._build_humming_moe_method",
+        lambda layer, layer_config: captured.update(
+            {"layer": layer, "layer_config": layer_config}
+        )
+        or expected_method,
+    )
+
+    layer = object.__new__(RoutedExperts)
+    layer.moe_config = DummyMoeConfig()
+    layer_config = make_layer_config(bits=bits)
+    method = INCWna16Scheme().get_moe_method(
+        make_config(), layer, "model.layers.0.mlp", layer_config
+    )
+
+    assert method is expected_method
+    assert captured["layer"] is layer
+    assert captured["layer_config"] is layer_config
+
+
+@pytest.mark.parametrize("bits", [4, 8])
+def test_wna16_cuda_high_bit_skips_humming(monkeypatch, bits) -> None:
+    """4/8-bit int stays on the Marlin/GPTQ/AWQ path even on CUDA so a single
+    model can mix high-bit Marlin and low-bit humming layers."""
+    called = {"humming": False}
+
+    monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
+    monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
+    monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
+    monkeypatch.setattr(
+        "vllm.model_executor.layers.quantization.inc.schemes."
+        "inc_wna16_scheme._build_humming_linear_method",
+        lambda layer_config: called.update({"humming": True}),
+    )
+
+    method = INCWna16Scheme().get_linear_method(
+        make_config(),
+        object(),
+        "model.layers.0.mlp.down_proj",
+        make_layer_config(bits=bits),
+    )
+
+    assert called["humming"] is False
+    assert isinstance(method, INCLinearMethod)
+
+
 def test_inc_config_accepts_mxfp_family_llm_compressor() -> None:
     config = INCConfig.from_config(
         {

vllm/model_executor/layers/quantization/inc/inc.py

@@ -34,9 +34,8 @@ class INCConfig(QuantizationConfig):
     Repo: https://github.com/intel/neural-compressor
     """
 
+    SUPPORTED_BITS = {2, 3, 4, 5, 6, 7, 8}
     DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
-
-    SUPPORTED_BITS = {2, 3, 4, 8}
     SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
     SUPPORTED_FORMATS = {
         "auto_round:auto_gptq",

vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py

@@ -21,6 +21,11 @@ if TYPE_CHECKING:
 logger = init_logger(__name__)
 
 XPU_WNA16_SUPPORTED_BITS = {2, 4}
+# On CUDA the Marlin/GPTQ/AWQ kernels only cover 4/8-bit. The remaining
+# widths (2/3/5/6/7) have no dedicated CUDA kernel, so they are dispatched to
+# the humming kernel instead. This mirrors compressed-tensors WNA16, whose
+# 5/6/7-bit biased scalar types also fall back to humming at kernel selection.
+CUDA_HUMMING_SUPPORTED_BITS = {2, 3, 5, 6, 7}
 
 # Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
 # oneDNN int4 GEMMs rather than by ARK. These only cover int4.
@@ -133,6 +138,15 @@ class INCWna16Scheme(INCScheme):
                 return INCLinearMethod(INCWNA16LinearScheme(layer_config))
             raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
 
+        # CUDA low-bit (2/3/5/6/7): no Marlin/GPTQ/AWQ kernel, route to humming
+        # so a single model can mix 4/8-bit (Marlin) and 2/3/5/6/7-bit (humming)
+        # layers.
+        if (
+            current_platform.is_cuda()
+            and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
+        ):
+            return _build_humming_linear_method(layer_config)
+
         from .inc_wna16_linear import INCWNA16LinearScheme
 
         return INCLinearMethod(INCWNA16LinearScheme(layer_config))
@@ -152,6 +166,12 @@ class INCWna16Scheme(INCScheme):
             )
 
             return UnquantizedFusedMoEMethod(layer.moe_config)
+        # CUDA low-bit (2/3/5/6/7): route to the humming MoE kernel (see above).
+        if (
+            current_platform.is_cuda()
+            and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
+        ):
+            return _build_humming_moe_method(layer, layer_config)
         if layer_config.is_gptq:
             return _resolve_gptq_moe(layer, layer_config)
         if layer_config.is_awq:
@@ -251,3 +271,48 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
         }
     )
     return MoeWNA16Method(moe_config, layer.moe_config)
+
+
+def _humming_weight_config(layer_config: "INCLayerConfig") -> dict:
+    """Build the humming weight-schema config for a WNA16 int checkpoint."""
+    if layer_config.is_gptq:
+        return {
+            "quant_method": "gptq",
+            "bits": layer_config.bits,
+            "group_size": layer_config.group_size,
+            "desc_act": False,
+            "sym": layer_config.sym,
+        }
+    if layer_config.is_awq:
+        return {
+            "quant_method": "awq",
+            "bits": layer_config.bits,
+            "group_size": layer_config.group_size,
+            "zero_point": not layer_config.sym,
+        }
+    raise NotImplementedError(
+        "INC humming dispatch only supports gptq/awq packed int checkpoints, "
+        f"but found {layer_config}."
+    )
+
+
+def _build_humming_quant_config(layer_config: "INCLayerConfig"):
+    from vllm.model_executor.layers.quantization.humming import (
+        HummingLayerQuantizationConfig,
+    )
+    from vllm.utils.humming import BaseWeightSchema
+
+    weight_schema = BaseWeightSchema.from_config(_humming_weight_config(layer_config))
+    return HummingLayerQuantizationConfig(weight_schema=weight_schema)
+
+
+def _build_humming_linear_method(layer_config: "INCLayerConfig"):
+    from vllm.model_executor.layers.quantization.humming import HummingLinearMethod
+
+    return HummingLinearMethod(_build_humming_quant_config(layer_config))
+
+
+def _build_humming_moe_method(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
+    from vllm.model_executor.layers.quantization.humming import HummingMoEMethod
+
+    return HummingMoEMethod(_build_humming_quant_config(layer_config), layer.moe_config)