diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index 906f213160..5250f73b82 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -666,6 +666,92 @@ def test_inc_resolve_scheme_selects_wna16() -> None: assert isinstance(scheme, INCWna16Scheme) +@pytest.mark.parametrize("bits", [2, 3]) +@pytest.mark.parametrize( + "packing_format", ["auto_round:auto_gptq", "auto_round:auto_awq"] +) +def test_wna16_cuda_low_bit_linear_routes_to_humming( + monkeypatch, bits, packing_format +) -> None: + expected_method = object() + captured = {} + + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + monkeypatch.setattr(current_platform, "is_xpu", lambda: False) + monkeypatch.setattr(current_platform, "is_cpu", lambda: False) + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes." + "inc_wna16_scheme._build_humming_linear_method", + lambda layer_config: captured.update({"layer_config": layer_config}) + or expected_method, + ) + + layer_config = make_layer_config(bits=bits, packing_format=packing_format) + method = INCWna16Scheme().get_linear_method( + make_config(), object(), "model.layers.0.mlp.down_proj", layer_config + ) + + assert method is expected_method + assert captured["layer_config"] is layer_config + + +@pytest.mark.parametrize("bits", [2, 3]) +def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None: + expected_method = object() + captured = {} + + class DummyMoeConfig: + pass + + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + monkeypatch.setattr(current_platform, "is_cpu", lambda: False) + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes." + "inc_wna16_scheme._build_humming_moe_method", + lambda layer, layer_config: captured.update( + {"layer": layer, "layer_config": layer_config} + ) + or expected_method, + ) + + layer = object.__new__(RoutedExperts) + layer.moe_config = DummyMoeConfig() + layer_config = make_layer_config(bits=bits) + method = INCWna16Scheme().get_moe_method( + make_config(), layer, "model.layers.0.mlp", layer_config + ) + + assert method is expected_method + assert captured["layer"] is layer + assert captured["layer_config"] is layer_config + + +@pytest.mark.parametrize("bits", [4, 8]) +def test_wna16_cuda_high_bit_skips_humming(monkeypatch, bits) -> None: + """4/8-bit int stays on the Marlin/GPTQ/AWQ path even on CUDA so a single + model can mix high-bit Marlin and low-bit humming layers.""" + called = {"humming": False} + + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + monkeypatch.setattr(current_platform, "is_xpu", lambda: False) + monkeypatch.setattr(current_platform, "is_cpu", lambda: False) + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes." + "inc_wna16_scheme._build_humming_linear_method", + lambda layer_config: called.update({"humming": True}), + ) + + method = INCWna16Scheme().get_linear_method( + make_config(), + object(), + "model.layers.0.mlp.down_proj", + make_layer_config(bits=bits), + ) + + assert called["humming"] is False + assert isinstance(method, INCLinearMethod) + + def test_inc_config_accepts_mxfp_family_llm_compressor() -> None: config = INCConfig.from_config( { diff --git a/vllm/model_executor/layers/quantization/inc/inc.py b/vllm/model_executor/layers/quantization/inc/inc.py index 2ffc2add25..ab948957b1 100644 --- a/vllm/model_executor/layers/quantization/inc/inc.py +++ b/vllm/model_executor/layers/quantization/inc/inc.py @@ -34,9 +34,8 @@ class INCConfig(QuantizationConfig): Repo: https://github.com/intel/neural-compressor """ + SUPPORTED_BITS = {2, 3, 4, 5, 6, 7, 8} DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq" - - SUPPORTED_BITS = {2, 3, 4, 8} SUPPORTED_DTYPES = {"int", "mx_fp", "fp"} SUPPORTED_FORMATS = { "auto_round:auto_gptq", diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py index b9e93ad657..82bb4bbb05 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py @@ -21,6 +21,11 @@ if TYPE_CHECKING: logger = init_logger(__name__) XPU_WNA16_SUPPORTED_BITS = {2, 4} +# On CUDA the Marlin/GPTQ/AWQ kernels only cover 4/8-bit. The remaining +# widths (2/3/5/6/7) have no dedicated CUDA kernel, so they are dispatched to +# the humming kernel instead. This mirrors compressed-tensors WNA16, whose +# 5/6/7-bit biased scalar types also fall back to humming at kernel selection. +CUDA_HUMMING_SUPPORTED_BITS = {2, 3, 5, 6, 7} # Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the # oneDNN int4 GEMMs rather than by ARK. These only cover int4. @@ -133,6 +138,15 @@ class INCWna16Scheme(INCScheme): return INCLinearMethod(INCWNA16LinearScheme(layer_config)) raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}") + # CUDA low-bit (2/3/5/6/7): no Marlin/GPTQ/AWQ kernel, route to humming + # so a single model can mix 4/8-bit (Marlin) and 2/3/5/6/7-bit (humming) + # layers. + if ( + current_platform.is_cuda() + and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS + ): + return _build_humming_linear_method(layer_config) + from .inc_wna16_linear import INCWNA16LinearScheme return INCLinearMethod(INCWNA16LinearScheme(layer_config)) @@ -152,6 +166,12 @@ class INCWna16Scheme(INCScheme): ) return UnquantizedFusedMoEMethod(layer.moe_config) + # CUDA low-bit (2/3/5/6/7): route to the humming MoE kernel (see above). + if ( + current_platform.is_cuda() + and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS + ): + return _build_humming_moe_method(layer, layer_config) if layer_config.is_gptq: return _resolve_gptq_moe(layer, layer_config) if layer_config.is_awq: @@ -251,3 +271,48 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"): } ) return MoeWNA16Method(moe_config, layer.moe_config) + + +def _humming_weight_config(layer_config: "INCLayerConfig") -> dict: + """Build the humming weight-schema config for a WNA16 int checkpoint.""" + if layer_config.is_gptq: + return { + "quant_method": "gptq", + "bits": layer_config.bits, + "group_size": layer_config.group_size, + "desc_act": False, + "sym": layer_config.sym, + } + if layer_config.is_awq: + return { + "quant_method": "awq", + "bits": layer_config.bits, + "group_size": layer_config.group_size, + "zero_point": not layer_config.sym, + } + raise NotImplementedError( + "INC humming dispatch only supports gptq/awq packed int checkpoints, " + f"but found {layer_config}." + ) + + +def _build_humming_quant_config(layer_config: "INCLayerConfig"): + from vllm.model_executor.layers.quantization.humming import ( + HummingLayerQuantizationConfig, + ) + from vllm.utils.humming import BaseWeightSchema + + weight_schema = BaseWeightSchema.from_config(_humming_weight_config(layer_config)) + return HummingLayerQuantizationConfig(weight_schema=weight_schema) + + +def _build_humming_linear_method(layer_config: "INCLayerConfig"): + from vllm.model_executor.layers.quantization.humming import HummingLinearMethod + + return HummingLinearMethod(_build_humming_quant_config(layer_config)) + + +def _build_humming_moe_method(layer: "torch.nn.Module", layer_config: "INCLayerConfig"): + from vllm.model_executor.layers.quantization.humming import HummingMoEMethod + + return HummingMoEMethod(_build_humming_quant_config(layer_config), layer.moe_config)