[quant][autoround]Refactor INC quantization into package with INCScheme orchestrator (#40601)

Signed-off-by: yiliu30 <yi4.liu@intel.com>
Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com>
Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com>
Co-authored-by: n1ck-guo <heng.guo@intel.com>
Co-authored-by: Zhenzhong1 <zhenzhong.xu@intel.com>
This commit is contained in:
Yi Liu
2026-06-17 21:51:32 +08:00
committed by GitHub
co-authored by n1ck-guo Zhenzhong1
parent 06e1e0885c
commit bcb518ad7a
13 changed files with 1940 additions and 803 deletions
+17
View File
@@ -87,3 +87,20 @@ steps:
cd tests &&
pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py &&
pytest -v -s benchmarks/test_serve_cli.py'
- label: "XPU quantization test"
depends_on:
- image-build-xpu
timeout_in_minutes: 30
device: intel_gpu
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
source_file_dependencies:
- vllm/
- .buildkite/intel_jobs/test-intel.yaml
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pytest -v -s quantization/test_auto_round.py'
+1 -1
View File
@@ -16,5 +16,5 @@ torch==2.12.0
torchaudio
torchvision
auto_round_lib>=0.13.0
auto_round_lib>=0.13.3
vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.9.1/vllm_xpu_kernels-0.1.9.1-cp38-abi3-manylinux_2_28_x86_64.whl
+750 -8
View File
@@ -10,23 +10,765 @@ Run `pytest tests/quantization/test_auto_round.py`.
import pytest
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.inc import INCConfig
from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
from vllm.model_executor.layers.quantization.inc.schemes import (
INCWna16Scheme,
resolve_scheme,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import (
INCLinearScheme,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear import (
INCARKLinearMethod,
INCWNA16LinearScheme,
INCXPULinearMethod,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_scheme import (
_resolve_awq_moe,
_resolve_gptq_moe,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.platforms import current_platform
MODELS = [
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", ##auto_round:auto_gptq
"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound", ##auto_round:auto_awq
pytest.param(
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
id="auto_round:auto_gptq",
),
pytest.param(
"Intel/Qwen2-0.5B-Instruct-int4-sym-AutoRound",
marks=pytest.mark.skipif(
not current_platform.is_cuda(),
reason="AWQ AutoRound model only supports CUDA backend for now.",
),
id="auto_round:auto_awq",
),
]
@pytest.mark.skipif(
not current_platform.is_cpu()
and not current_platform.is_xpu()
and not current_platform.is_cuda(),
reason="only supports CPU/XPU/CUDA backend.",
not (
current_platform.is_cpu()
or current_platform.is_xpu()
or current_platform.is_cuda()
),
reason="Only supports CPU/XPU/CUDA backend.",
)
@pytest.mark.parametrize("model", MODELS)
def test_auto_round(vllm_runner, model):
def test_auto_round_model(vllm_runner, model):
with vllm_runner(model, enforce_eager=True) as llm:
output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
assert output
print(f"{output[0][1]}")
print(output[0][1])
# ---------------------------------------------------------------------------
# Unit tests for INCConfig and related classes
# ---------------------------------------------------------------------------
class DummyLayer:
pass
class DummyFusedMoE:
pass
def make_config(**overrides) -> INCConfig:
kwargs = {
"weight_bits": 4,
"group_size": 128,
"sym": True,
"packing_format": "auto_round:auto_gptq",
"block_name_to_quantize": None,
"extra_config": None,
"data_type": "int",
"backend": "auto",
}
kwargs.update(overrides)
return INCConfig(**kwargs)
def make_layer_config(**overrides) -> INCLayerConfig:
kwargs = {
"bits": 4,
"group_size": 128,
"sym": True,
"packing_format": "auto_round:auto_gptq",
"backend": "auto",
"data_type": "int",
"quantized": True,
}
kwargs.update(overrides)
return INCLayerConfig(**kwargs)
def test_inc_config_parser_exact_match() -> None:
config = make_config(
extra_config={
"layers.0.self_attn.q_proj": {
"bits": 8,
"group_size": 64,
"sym": False,
}
}
)
layer_config = config.config_parser.resolve(
DummyLayer(), "layers.0.self_attn.q_proj"
)
assert layer_config.bits == 8
assert layer_config.group_size == 64
assert layer_config.sym is False
assert layer_config.quantized is True
def test_inc_model_prefix_early_exit() -> None:
"""extra_config keys with model. prefix trigger early unquantized return."""
config = make_config(
extra_config={
"model.layers.1.mlp.gate_proj": {
"bits": 16,
},
}
)
# get_quant_method checks model. prefix for unquantized early-exit
result = config.get_quant_method(DummyLayer(), "layers.1.mlp.gate_proj")
assert isinstance(result, UnquantizedLinearMethod)
def test_inc_config_parser_regex_match() -> None:
config = make_config(
extra_config={
r"layers\.\d+\.self_attn\.(q|k|v)_proj": {
"bits": 8,
"group_size": 64,
"sym": False,
}
}
)
layer_config = config.config_parser.resolve(
DummyLayer(), "layers.3.self_attn.q_proj"
)
assert layer_config.bits == 8
assert layer_config.group_size == 64
assert layer_config.sym is False
def test_inc_config_parser_invalid_regex_ignored() -> None:
config = make_config(
extra_config={
"[invalid": {
"bits": 8,
"group_size": 64,
"sym": False,
}
}
)
layer_config = config.config_parser.resolve(
DummyLayer(), "layers.0.self_attn.q_proj"
)
assert layer_config.bits == 4
assert layer_config.group_size == 128
assert layer_config.sym is True
def test_inc_config_parser_block_name_to_quantize_marks_unquantized() -> None:
config = make_config(block_name_to_quantize=["layers.1"])
layer_config = config.config_parser.resolve(
DummyLayer(), "layers.0.self_attn.q_proj"
)
assert layer_config.bits == 16
assert layer_config.group_size == -1
assert layer_config.sym is True
assert layer_config.quantized is False
def test_inc_config_parser_parallel_lm_head_defaults_to_unquantized() -> None:
layer = object.__new__(ParallelLMHead)
config = make_config()
layer_config = config.config_parser.resolve(layer, "lm_head")
assert layer_config.quantized is False
assert layer_config.bits == 16
def test_inc_config_parser_fused_moe_requires_consistent_configs() -> None:
config = make_config(
extra_config={
"layers.0.block_sparse_moe.experts.0.w1": {
"bits": 4,
"group_size": 128,
"sym": True,
},
"layers.0.block_sparse_moe.experts.0.w2": {
"bits": 8,
"group_size": 128,
"sym": True,
},
}
)
with pytest.raises(ValueError, match="requires consistent quant config"):
config.config_parser.resolve(DummyFusedMoE(), "layers.0.block_sparse_moe")
def test_inc_config_parser_fused_module_requires_consistent_configs() -> None:
config = make_config(
extra_config={
"layers.0.self_attn.q_proj": {
"bits": 4,
"group_size": 128,
"sym": True,
},
"layers.0.self_attn.k_proj": {
"bits": 8,
"group_size": 128,
"sym": True,
},
"layers.0.self_attn.v_proj": {
"bits": 4,
"group_size": 128,
"sym": True,
},
}
)
config.packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
with pytest.raises(ValueError, match="requires consistent quant config"):
config.config_parser.resolve(DummyLayer(), "layers.0.self_attn.qkv_proj")
def test_inc_layer_config_mx_fp_helpers() -> None:
layer_config = INCLayerConfig(
bits=4,
group_size=32,
sym=True,
packing_format="",
backend="",
data_type="mx_fp",
quantized=True,
)
assert layer_config.is_mxfp4 is True
assert layer_config.is_mxfp8 is False
def test_inc_resolve_scheme_selects_wna16() -> None:
layer_config = INCLayerConfig(
bits=4,
group_size=128,
sym=True,
packing_format="auto_round:auto_gptq",
backend="auto",
data_type="int",
quantized=True,
)
scheme = resolve_scheme(layer_config)
assert isinstance(scheme, INCWna16Scheme)
class DummyLinearScheme(INCLinearScheme):
def __init__(self) -> None:
self.calls: list[tuple] = []
@classmethod
def get_min_capability(cls) -> int:
return 0
def create_weights(self, *args, **kwargs) -> None:
self.calls.append(("create_weights", args, kwargs))
def process_weights_after_loading(self, layer) -> None:
self.calls.append(("process_weights_after_loading", layer))
def apply_weights(self, layer, x, bias=None):
self.calls.append(("apply_weights", layer, x, bias))
return "applied"
def test_inc_linear_method_delegates() -> None:
scheme = DummyLinearScheme()
method = INCLinearMethod(scheme)
layer = DummyLayer()
method.create_weights(
layer,
input_size_per_partition=1,
output_partition_sizes=[2],
input_size=1,
output_size=2,
params_dtype=None,
)
method.process_weights_after_loading(layer)
result = method.apply(layer, "x", "b")
assert result == "applied"
assert [call[0] for call in scheme.calls] == [
"create_weights",
"process_weights_after_loading",
"apply_weights",
]
def test_wna16_xpu_prefers_ark_when_available(monkeypatch) -> None:
class DummyQuantLinear:
pass
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.get_ark_state",
lambda: (True, None, object(), DummyQuantLinear),
)
method = INCWna16Scheme().get_linear_method(
make_config(),
object(),
"layer",
make_layer_config(),
)
assert isinstance(method, INCLinearMethod)
assert isinstance(method.scheme, INCARKLinearMethod)
def test_wna16_xpu_falls_back_when_ark_unavailable(monkeypatch) -> None:
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.get_ark_state",
lambda: (False, "missing", None, None),
)
method = INCWna16Scheme().get_linear_method(
make_config(),
object(),
"layer",
make_layer_config(),
)
assert isinstance(method, INCLinearMethod)
assert isinstance(method.scheme, INCXPULinearMethod)
def test_wna16_cpu_gptq_prefers_ark_when_available(monkeypatch) -> None:
class DummyQuantLinear:
pass
monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
monkeypatch.setattr(current_platform, "is_cpu", lambda: True)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.get_ark_state",
lambda: (True, None, object(), DummyQuantLinear),
)
method = INCWna16Scheme().get_linear_method(
make_config(),
object(),
"layer",
make_layer_config(),
)
assert isinstance(method, INCLinearMethod)
assert isinstance(method.scheme, INCARKLinearMethod)
def test_wna16_cpu_gptq_raises_when_ark_and_marlin_unavailable(
monkeypatch,
) -> None:
monkeypatch.setattr(current_platform, "is_xpu", lambda: False)
monkeypatch.setattr(current_platform, "is_cpu", lambda: True)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.get_ark_state",
lambda: (False, "missing", None, None),
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear.check_marlin_supported",
lambda *args, **kwargs: False,
)
with pytest.raises(NotImplementedError, match="Only 4-bit and 8-bit symmetric"):
INCWna16Scheme().get_linear_method(
make_config(),
object(),
"layer",
make_layer_config(),
)
def test_wna16_linear_gptq_uses_auto_gptq_when_supported(monkeypatch) -> None:
captured = {}
class DummyMethod:
def __init__(self, cfg):
captured["cfg"] = cfg
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_wna16_linear."
"check_marlin_supported",
lambda *args, **kwargs: True,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.auto_gptq.AutoGPTQLinearMethod",
DummyMethod,
)
scheme = INCWNA16LinearScheme(make_layer_config())
assert isinstance(scheme.inner_method, DummyMethod)
assert isinstance(captured["cfg"], AutoGPTQConfig)
assert captured["cfg"].weight_bits == 4
assert captured["cfg"].group_size == 128
assert captured["cfg"].is_sym is True
def test_wna16_linear_gptq_unsupported_config_raises() -> None:
with pytest.raises(NotImplementedError, match="Only 4-bit and 8-bit symmetric"):
INCWNA16LinearScheme(make_layer_config(sym=False))
def test_wna16_xpu_unsupported_config_still_raises(monkeypatch) -> None:
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
with pytest.raises(NotImplementedError, match="unsupported config"):
INCWna16Scheme().get_linear_method(
make_config(sym=False),
object(),
"layer",
make_layer_config(sym=False),
)
def test_inc_get_quant_method_unquantized_linear_returns_unquantized() -> None:
config = make_config(extra_config={"layer": {"bits": 16}})
layer = object.__new__(LinearBase)
method = config.get_quant_method(layer, "layer")
assert isinstance(method, UnquantizedLinearMethod)
def test_inc_get_quant_method_unquantized_moe_returns_unquantized(
monkeypatch,
) -> None:
"""Early-exit returns UnquantizedFusedMoEMethod for FusedMoE layers
when extra_config has bits >= 16."""
config = make_config(extra_config={"layer": {"bits": 16}})
layer = object.__new__(RoutedExperts)
layer.moe_config = None # UnquantizedFusedMoEMethod accepts moe_config
class DummyUnquantizedFusedMoEMethod:
def __init__(self, moe_config) -> None:
self.moe_config = moe_config
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.inc.UnquantizedFusedMoEMethod",
DummyUnquantizedFusedMoEMethod,
)
method = config.get_quant_method(layer, "layer")
assert isinstance(method, DummyUnquantizedFusedMoEMethod)
assert method.moe_config is None
def test_inc_get_quant_method_linear_uses_resolved_scheme(monkeypatch) -> None:
config = make_config()
layer = object.__new__(LinearBase)
sentinel = object()
class DummyScheme:
def get_linear_method(self, _config, _layer, _prefix, _layer_config):
return sentinel
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.factory.resolve_scheme",
lambda _layer_config: DummyScheme(),
)
method = config.get_quant_method(layer, "layer")
assert method is sentinel
def test_inc_get_quant_method_moe_uses_resolved_scheme(monkeypatch) -> None:
config = make_config()
layer = object.__new__(RoutedExperts)
sentinel = object()
class DummyScheme:
def get_moe_method(self, _config, _layer, _prefix, _layer_config):
return sentinel
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.factory.resolve_scheme",
lambda _layer_config: DummyScheme(),
)
method = config.get_quant_method(layer, "layer")
assert method is sentinel
def test_resolve_gptq_moe_falls_back_to_moe_wna16(monkeypatch) -> None:
captured = {}
class DummyMoeConfig:
pass
class DummyLayer:
moe_config = DummyMoeConfig()
class DummyBuiltConfig:
pass
built_config = DummyBuiltConfig()
class DummyMethod:
def __init__(self, cfg, moe):
captured["cfg"] = cfg
captured["moe"] = moe
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
lambda *args, **kwargs: False,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.moe_wna16.MoeWNA16Config.from_config",
lambda cfg: captured.update({"from_config": cfg}) or built_config,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.moe_wna16.MoeWNA16Method",
DummyMethod,
)
layer_config = INCLayerConfig(
bits=4,
group_size=128,
sym=True,
packing_format="auto_round:auto_gptq",
backend="auto",
data_type="int",
quantized=True,
)
_resolve_gptq_moe(DummyLayer(), layer_config)
assert captured["from_config"] == {
"quant_method": "gptq",
"bits": 4,
"group_size": 128,
"sym": True,
"lm_head": False,
}
assert captured["cfg"] is built_config
assert captured["moe"] is DummyLayer.moe_config
def test_resolve_gptq_moe_uses_auto_gptq_when_supported(monkeypatch) -> None:
captured = {}
class DummyMoeConfig:
pass
class DummyLayer:
moe_config = DummyMoeConfig()
class DummyMethod:
def __init__(self, cfg, moe):
captured["cfg"] = cfg
captured["moe"] = moe
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
lambda *args, **kwargs: True,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.utils.marlin_utils."
"check_moe_marlin_supports_layer",
lambda *args, **kwargs: True,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.auto_gptq.AutoGPTQMoEMethod",
DummyMethod,
)
_resolve_gptq_moe(DummyLayer(), make_layer_config())
assert isinstance(captured["cfg"], AutoGPTQConfig)
assert captured["cfg"].weight_bits == 4
assert captured["cfg"].group_size == 128
assert captured["cfg"].is_sym is True
assert captured["moe"] is DummyLayer.moe_config
def test_resolve_awq_moe_uses_marlin_when_supported(monkeypatch) -> None:
captured = {}
class DummyMoeConfig:
pass
class DummyLayer:
moe_config = DummyMoeConfig()
class DummyMethod:
def __init__(self, cfg, moe):
captured["cfg"] = cfg
captured["moe"] = moe
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
lambda *args, **kwargs: True,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.utils.marlin_utils.check_moe_marlin_supports_layer",
lambda *args, **kwargs: True,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.awq_marlin.verify_marlin_supported",
lambda *args, **kwargs: None,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.awq_marlin.AWQMarlinMoEMethod",
DummyMethod,
)
layer_config = INCLayerConfig(
bits=4,
group_size=128,
sym=False,
packing_format="auto_round:auto_awq",
backend="auto",
data_type="int",
quantized=True,
)
_resolve_awq_moe(DummyLayer(), layer_config)
assert captured["cfg"].weight_bits == 4
assert captured["cfg"].zero_point is True
assert captured["moe"] is DummyLayer.moe_config
# ---------------------------------------------------------------------------
# Tests for get_layer_config step 4 (fused QKV / packed_modules_mapping)
# ---------------------------------------------------------------------------
class TestGetLayerConfigFusedQKV:
"""Tests for step-4 (fused QKV / packed_modules_mapping) logic.
Focused on preventing false-positive substring matches.
"""
def test_exact_fusion_key_match(self):
"""A layer whose name contains 'qkv' maps to its extra_config entry."""
config = make_config(
extra_config={
"model.layers.0.self_attn.qkv_proj": {"bits": 8},
}
)
config.packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
}
bits, _, _ = config.get_layer_config(
DummyLayer(), "model.layers.0.self_attn.qkv_proj"
)
assert bits == 8
def test_false_substring_match_does_not_override(self):
"""Regression test for the false-substring-match bug.
Scenario (Qwen3.6-35B-A3B VLM):
- packed_modules_mapping has "qkv" → ["qkv"] (from vision encoder).
- The GDN text-attention layer is named "in_proj_qkvz".
- "qkv" is a substring of "in_proj_qkvz", so old code would enter
step 4 and generate sub_name "in_proj_qkvz" (replacing "qkv" with
"qkv"). That name is NOT in extra_config, so get_config() falls
back to the global default (bits=4), even though correct is 16.
- Fix: skip the fusion key when none of the generated sub_names
actually exist in extra_config.
"""
config = make_config(
extra_config={
"model.layers.0.in_proj_qkv": {"bits": 16},
"model.layers.0.in_proj_z": {"bits": 16},
}
)
config.packed_modules_mapping = {
"qkv": ["qkv"],
}
bits, _, _ = config.get_layer_config(
DummyLayer(), "model.layers.0.in_proj_qkvz"
)
# bits should be the global default (4) – no erroneous fusion match
assert bits == 4
def test_real_qkv_fusion_key_still_resolves(self):
"""The true "qkv" fusion (vision encoder) still resolves correctly."""
config = make_config(
extra_config={
"vision_model.encoder.layers.0.self_attn.qkv": {"bits": 8},
}
)
config.packed_modules_mapping = {
"qkv": ["qkv"],
}
bits, _, _ = config.get_layer_config(
DummyLayer(), "vision_model.encoder.layers.0.self_attn.qkv"
)
assert bits == 8
def test_mixed_fp16_and_int4_fused_layer(self):
"""All sub-keys must agree; inconsistent configs raise ValueError."""
config = make_config(
extra_config={
"model.layers.0.self_attn.q_proj": {"bits": 16},
"model.layers.0.self_attn.k_proj": {"bits": 4},
"model.layers.0.self_attn.v_proj": {"bits": 4},
}
)
config.packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
}
with pytest.raises(ValueError, match="consistent quant config"):
config.get_layer_config(DummyLayer(), "model.layers.0.self_attn.qkv_proj")
def test_fusion_triggered_by_regex_configured_sub_name(self):
"""Fusion step 4 is still triggered when sub_names match via regex.
Ensures the guard does not regress when extra_config uses regex
patterns instead of exact keys to configure sub-modules.
"""
config = make_config(
extra_config={
r"model\.layers\.\d+\.self_attn\.(q|k|v)_proj": {"bits": 8},
}
)
config.packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
}
bits, _, _ = config.get_layer_config(
DummyLayer(), "model.layers.0.self_attn.qkv_proj"
)
assert bits == 8
@@ -1,794 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fractions import Fraction
from functools import lru_cache
from typing import TYPE_CHECKING, Any
import regex as re
import torch
from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.linear import (
LinearBase,
LinearMethodBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
QuantizationMethods,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
RowvLLMParameter,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
if TYPE_CHECKING:
from vllm.model_executor.models.utils import WeightsMapper
logger = init_logger(__name__)
class INCConfig(QuantizationConfig):
"""Config class for Intel Neural Compressor (INC).
Repo: https://github.com/intel/neural-compressor
"""
SUPPORTED_BITS = {2, 3, 4, 8}
SUPPORTED_DTYPES = {"int"}
SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq"}
SUPPORTED_BACKENDS = {
"auto",
"gptq",
"gptq:marlin",
"awq",
"awq:marlin",
"marlin",
}
def __init__(
self,
weight_bits: int,
group_size: int,
sym: bool = True,
packing_format: str = "auto_round:auto_gptq",
block_name_to_quantize: str | list[str] | None = None,
extra_config: dict[str, Any] | None = None,
data_type: str = "int",
backend: str = "auto",
) -> None:
super().__init__()
if weight_bits not in self.SUPPORTED_BITS:
raise ValueError(
f"Unsupported weight_bits: {weight_bits}, "
f"currently only support {self.SUPPORTED_BITS}."
)
if data_type not in self.SUPPORTED_DTYPES:
raise ValueError(
f"Unsupported data_type: {data_type},"
f" currently only support {self.SUPPORTED_DTYPES}."
)
if packing_format not in self.SUPPORTED_FORMATS:
raise ValueError(
f"Unsupported packing_format: {packing_format}, "
f"currently only support {self.SUPPORTED_FORMATS}."
)
if backend not in self.SUPPORTED_BACKENDS:
raise ValueError(
f"Unsupported backend: {backend}, "
f"currently only support {self.SUPPORTED_BACKENDS}."
)
self.weight_bits = weight_bits
self.group_size = group_size
self.sym = sym
self.packing_format = packing_format
self.block_name_to_quantize = (
block_name_to_quantize.split(",")
if isinstance(block_name_to_quantize, str)
else block_name_to_quantize
)
self.extra_config = extra_config
self.data_type = data_type
self.backend = backend
self.pack_factor = Fraction(32, weight_bits)
def __repr__(self) -> str:
return (
f"INCConfig(weight_bits={self.weight_bits}, "
f"group_size={self.group_size}, sym={self.sym})"
)
@classmethod
def get_name(cls) -> QuantizationMethods:
return "inc"
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
return [torch.half, torch.bfloat16]
@classmethod
def get_min_capability(cls) -> int:
return 60
@classmethod
def get_config_filenames(cls) -> list[str]:
return ["quantization_config.json"]
@classmethod
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
return cls(
weight_bits=cls.get_from_keys(config, ["bits"]),
group_size=cls.get_from_keys(config, ["group_size"]),
sym=cls.get_from_keys(config, ["sym"]),
packing_format=cls.get_from_keys_or(
config, ["packing_format"], "auto_round:auto_gptq"
),
block_name_to_quantize=cls.get_from_keys_or(
config, ["block_name_to_quantize", "to_quant_block_names"], None
),
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
)
def get_layer_config(self, layer, layer_name: str):
def get_config(name: str, quantized: bool = True):
if not self.extra_config:
return (
self.weight_bits if quantized else 16,
self.group_size if quantized else -1,
self.sym if quantized else True,
)
# exact match first
if name in self.extra_config:
cfg = self.extra_config[name]
return (
cfg.get("bits", self.weight_bits if quantized else 16),
cfg.get("group_size", self.group_size if quantized else -1),
cfg.get("sym", self.sym if quantized else True),
)
REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
for pattern, cfg in self.extra_config.items():
if not isinstance(pattern, str) or not any(
c in REGEX_SPECIAL_CHARS for c in pattern
):
continue
try:
if re.search(re.compile(pattern), name) is not None:
return (
cfg.get("bits", self.weight_bits if quantized else 16),
cfg.get("group_size", self.group_size if quantized else -1),
cfg.get("sym", self.sym if quantized else True),
)
except re.error:
# Invalid regex, ignore.
continue
return (
self.weight_bits if quantized else 16,
self.group_size if quantized else -1,
self.sym if quantized else True,
)
# 1. Exact match from config
if self.extra_config and layer_name in self.extra_config:
return get_config(layer_name)
# 2. Determine whether layer should be quantized
quantized = not isinstance(layer, ParallelLMHead)
if self.block_name_to_quantize:
quantized = any(
layer_name.startswith(name) for name in self.block_name_to_quantize
)
# 3. Handle fused MoE
if self.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
moe_configs = [
get_config(name, quantized)
for name in self.extra_config
if name.startswith(layer_name)
]
if moe_configs:
if len(set(moe_configs)) == 1:
return moe_configs[0]
raise ValueError(
f"Fused MoE layer '{layer_name}' requires "
f"consistent quant config for all sub-layers"
)
# 4. Handle fused QKV or other patterns
if self.extra_config:
for fusion_key, sub_keys in self.packed_modules_mapping.items():
if fusion_key in layer_name and layer_name.count(fusion_key) == 1:
sub_names = [
layer_name.replace(fusion_key, sub_key) for sub_key in sub_keys
]
sub_configs = [get_config(name, quantized) for name in sub_names]
if len(set(sub_configs)) == 1:
return sub_configs[0]
raise ValueError(
f"Fused module '{layer_name}' requires "
f"consistent quant config for {sub_names}"
)
# 5. Fallback or try a regular expression match
return get_config(layer_name, quantized)
def check_quantized(self, weight_bits: int) -> bool:
return weight_bits < 16
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
if self.block_name_to_quantize is not None:
self.block_name_to_quantize = hf_to_vllm_mapper.apply_list(
self.block_name_to_quantize
)
if self.extra_config is not None:
self.extra_config = hf_to_vllm_mapper.apply_dict(self.extra_config)
def apply_awq_quant_layer(self, layer, prefix: str, backend: str = "auto"):
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supported,
check_moe_marlin_supports_layer,
)
weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
if not self.check_quantized(weight_bits):
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
else:
return None
logger.debug(
"[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
prefix,
layer.__class__.__name__,
weight_bits,
group_size,
sym,
)
if backend == "auto" or "marlin" in backend:
AWQ_TYPE_MAP = {
4: scalar_types.uint4,
8: scalar_types.uint8,
}
use_marlin = (weight_bits in AWQ_TYPE_MAP) and check_marlin_supported(
AWQ_TYPE_MAP[weight_bits], group_size, not sym
)
if isinstance(layer, RoutedExperts):
use_marlin = use_marlin and check_moe_marlin_supports_layer(
layer, group_size
)
else:
use_marlin = False
if use_marlin:
from vllm.model_executor.layers.quantization.awq_marlin import (
AWQMarlinConfig,
AWQMarlinLinearMethod,
AWQMarlinMoEMethod,
)
quant_args_marlin = AWQMarlinConfig(
weight_bits=weight_bits,
group_size=group_size,
zero_point=not sym,
lm_head_quantized=False,
full_config={},
modules_to_not_convert=[],
)
else:
from vllm.model_executor.layers.quantization.awq import (
AWQConfig,
AWQLinearMethod,
)
quant_args = AWQConfig(
weight_bits=weight_bits,
group_size=group_size,
zero_point=not sym,
)
if isinstance(layer, RoutedExperts):
if use_marlin:
return AWQMarlinMoEMethod(quant_args_marlin, layer.moe_config)
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
config = {
"quant_method": "awq",
"bits": weight_bits,
"group_size": group_size,
"zero_point": not sym,
"lm_head": False,
}
return MoeWNA16Config.from_config(config).get_quant_method(layer, prefix)
if isinstance(layer, (LinearBase, ParallelLMHead)):
if use_marlin:
return AWQMarlinLinearMethod(quant_args_marlin)
else:
return AWQLinearMethod(quant_args)
return None
def apply_gptq_quant_layer(self, layer, prefix: str, backend: str = "auto"):
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supported,
check_moe_marlin_supports_layer,
)
weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
if not self.check_quantized(weight_bits):
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
else:
return None
logger.debug(
"[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
prefix,
layer.__class__.__name__,
weight_bits,
group_size,
sym,
)
if backend == "auto" or "marlin" in backend:
GPTQ_TYPE_MAP = {
(4, True): scalar_types.uint4b8,
(8, True): scalar_types.uint8b128,
}
use_marlin = (weight_bits, sym) in GPTQ_TYPE_MAP and check_marlin_supported(
GPTQ_TYPE_MAP[(weight_bits, sym)], group_size, has_zp=not sym
)
if isinstance(layer, RoutedExperts):
use_marlin = use_marlin and check_moe_marlin_supports_layer(
layer, group_size
)
else:
use_marlin = False
if use_marlin:
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQConfig,
AutoGPTQLinearMethod,
AutoGPTQMoEMethod,
)
quant_args_marlin = AutoGPTQConfig(
weight_bits=weight_bits,
group_size=group_size,
is_sym=sym,
lm_head_quantized=False,
desc_act=False,
dynamic={},
full_config={},
)
if isinstance(layer, RoutedExperts):
if use_marlin:
return AutoGPTQMoEMethod(quant_args_marlin, layer.moe_config)
else:
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
)
config = {
"quant_method": "gptq",
"bits": weight_bits,
"group_size": group_size,
"sym": sym,
"lm_head": False,
}
return MoeWNA16Config.from_config(config).get_quant_method(
layer, prefix
)
if isinstance(layer, (LinearBase, ParallelLMHead)):
if use_marlin:
return AutoGPTQLinearMethod(quant_args_marlin)
else:
raise NotImplementedError(
f"INC quantization with bits={weight_bits}, sym={sym} "
"is not supported. Only 4-bit and 8-bit symmetric "
"quantization is supported with Marlin kernels."
)
return None
def apply_xpu_w4a16_quant_layer(self, layer, prefix: str):
weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
if not self.check_quantized(weight_bits):
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
else:
return None
if weight_bits != 4:
raise NotImplementedError(
f"INC on XPU only supports 4-bit quantization, "
f"got weight_bits={weight_bits}."
)
if not sym:
raise NotImplementedError(
"INC W4A16 on XPU only supports symmetric quantization for now."
)
if isinstance(layer, (LinearBase, ParallelLMHead)):
is_ark_available, ark_error, _, _ = get_ark_state()
if is_ark_available:
return INCARKLinearMethod(
weight_bits=weight_bits,
group_size=group_size,
sym=sym,
)
logger.debug(
"ARK backend is unavailable for layer %s; "
"falling back to the default XPU INC path. Error: %s",
prefix,
ark_error or "unknown error",
)
return INCXPULinearMethod(
weight_bits=weight_bits,
group_size=group_size,
sym=sym,
)
return None
def apply_cpu_w4a16_quant_layer(self, layer, prefix: str):
weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
if not self.check_quantized(weight_bits):
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
else:
return None
if weight_bits != 4:
raise NotImplementedError(
f"INC on CPU only supports 4-bit quantization, "
f"got weight_bits={weight_bits}."
)
if not sym:
raise NotImplementedError(
"INC W4A16 on CPU only supports symmetric quantization for now."
)
if isinstance(layer, (LinearBase, ParallelLMHead)):
is_ark_available, ark_error, _, _ = get_ark_state()
if is_ark_available:
return INCARKLinearMethod(
weight_bits=weight_bits,
group_size=group_size,
sym=sym,
)
logger.debug(
"ARK backend is unavailable for layer %s; "
"falling back to the default CPU INC path. Error: %s",
prefix,
ark_error or "unknown error",
)
return self.apply_gptq_quant_layer(layer, prefix)
return None
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
if prefix and self.extra_config:
for layer_name in self.extra_config:
if (
layer_name == prefix or layer_name == f"model.{prefix}"
) and self.extra_config[layer_name].get("bits", 16) >= 16:
return UnquantizedLinearMethod()
if current_platform.is_xpu():
return self.apply_xpu_w4a16_quant_layer(layer, prefix)
is_gptq = "gptq" in self.packing_format or "gptq" in self.backend
if current_platform.is_cpu() and is_gptq:
return self.apply_cpu_w4a16_quant_layer(layer, prefix)
if is_gptq:
return self.apply_gptq_quant_layer(layer, prefix)
if "awq" in self.packing_format or "awq" in self.backend:
return self.apply_awq_quant_layer(layer, prefix)
raise NotImplementedError(
f"Unsupported quantization configuration for layer '{prefix}'. "
f"Platform: CPU={current_platform.is_cpu()}. "
f"Platform: XPU={current_platform.is_xpu()}. "
f"Format: {self.packing_format}, Backend: {self.backend}."
)
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant, hf_config=None
) -> "QuantizationMethods | None":
"""Override the `auto-round` method to `inc`."""
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
if is_auto_round_format:
return cls.get_name()
return None
class INCXPULinearBase(LinearMethodBase):
def __init__(self, weight_bits: int, group_size: int, sym: bool):
self.weight_bits = weight_bits
self.group_size = group_size
self.sym = sym
self.pack_factor = 32 // weight_bits
def _create_inc_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
params_dtype: torch.dtype,
weight_loader: Any,
group_size: int,
pack_factor: int,
) -> None:
output_size_per_partition = sum(output_partition_sizes)
scales_and_zp_size = input_size_per_partition // group_size
qweight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition // pack_factor,
output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=0,
packed_factor=pack_factor,
weight_loader=weight_loader,
)
scales = GroupQuantScaleParameter(
data=torch.empty(
scales_and_zp_size,
output_size_per_partition,
dtype=params_dtype,
),
input_dim=0,
output_dim=1,
weight_loader=weight_loader,
)
qzeros = PackedvLLMParameter(
data=torch.empty(
scales_and_zp_size,
output_size_per_partition // pack_factor,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=pack_factor,
weight_loader=weight_loader,
)
layer.register_parameter("qweight", qweight)
layer.register_parameter("scales", scales)
layer.register_parameter("qzeros", qzeros)
g_idx = RowvLLMParameter(
data=torch.tensor(
[i // group_size for i in range(input_size_per_partition)],
dtype=torch.int32,
),
input_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("g_idx", g_idx)
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
self._create_inc_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
params_dtype=params_dtype,
weight_loader=extra_weight_attrs.get("weight_loader"),
group_size=self.group_size,
pack_factor=self.pack_factor,
)
@lru_cache(maxsize=1)
def get_ark_state() -> tuple[bool, str | None, Any | None, Any | None]:
"""Return ARK availability, error details, cached instance, and QuantLinear."""
try:
import auto_round_kernel
from auto_round_kernel.qlinear import QuantLinear
logger.info("Successfully imported auto_round_kernel.")
except ImportError as error:
return False, str(error), None, None
ark_loader = getattr(auto_round_kernel, "_ark_instance", None)
if not callable(ark_loader):
return False, "auto_round_kernel does not expose _ark_instance().", None, None
try:
ark_instance = ark_loader()
except Exception as error:
return False, str(error), None, None
if ark_instance is None:
return False, "auto_round_kernel._ark_instance() returned None.", None, None
return True, None, ark_instance, QuantLinear
class INCXPULinearMethod(INCXPULinearBase):
"""XPU linear method for INC w4a16 GPTQ quantization (symmetric only).
Repacks GPTQ weights from [in_packed, out] to oneDNN [out, in_packed]
layout and calls torch.ops._xpu_C.int4_gemm_w4a16.
GPTQ format: qweight [in_packed, out] with sequential nibble order.
Note: Asymmetric quantization (sym=false) is not for now.
FIXME(yiliu30): Refine the implementation to reuse XPUwNa16LinearKernel.
"""
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Repack GPTQ weights into kernel-ready NT layout."""
device = layer.qweight.data.device
# oneDNN int4 kernel requires strides[0]==1 ("NT format"), but GPTQ
# checkpoint is [K_packed, N] contiguous with strides (N, 1).
# Two transposes are needed — neither alone can achieve this:
# 1. .t().contiguous() → [N, K_packed] contiguous in memory
# 2. .t() → [K_packed, N] view with strides (1, K_packed)
# The result has the same logical shape but strides[0]==1 as required.
qweight_ct = layer.qweight.data.t().contiguous()
layer.qweight = Parameter(qweight_ct.t(), requires_grad=False)
# Scales: [num_groups, out] — no change needed
layer.scales = Parameter(layer.scales.data, requires_grad=False)
# Symmetric: GPTQ v1 stores qzeros=7, effective zp = 7+1 = 8
# Kernel expects int8 scalar = 8
layer.qzeros = Parameter(
torch.tensor([8], dtype=torch.int8, device=device),
requires_grad=False,
)
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
# qweight is already in NT layout [K_packed, N] (strides (1, K_packed))
# from process_weights_after_loading — pass directly to kernel.
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
reshaped_x = x.reshape(-1, x.shape[-1])
out = torch.ops._xpu_C.int4_gemm_w4a16(
reshaped_x,
layer.qweight,
bias,
layer.scales,
layer.qzeros,
self.group_size,
None, # g_idx not needed: desc_act is always False for INC models
)
return out.reshape(out_shape)
class INCARKLinearMethod(INCXPULinearBase):
"""XPU & CPU w4a16 linear method for INC quantization utilizing the ARK backend.
See: https://github.com/intel/auto-round/blob/main/auto_round_extension/ark/README.md
Repacks GPTQ/INC weights into ARK's layout.
"""
def __init__(self, weight_bits: int, group_size: int, sym: bool):
super().__init__(weight_bits=weight_bits, group_size=group_size, sym=sym)
is_available, error_str, _, quant_linear_cls = get_ark_state()
if not is_available or quant_linear_cls is None:
reason = error_str or "unknown error"
raise ImportError(f"Failed to import auto_round_kernel. {reason}")
self.QuantLinear = quant_linear_cls
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
super().create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
layer.in_features = input_size_per_partition
layer.out_features = sum(output_partition_sizes)
layer.params_dtype = params_dtype
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if hasattr(layer, "input_size_per_partition"):
in_features = layer.input_size_per_partition
elif hasattr(layer, "input_size"):
in_features = layer.input_size
else:
raise AttributeError("Cannot determine in_features for layer.")
if hasattr(layer, "output_partition_sizes"):
out_features = sum(layer.output_partition_sizes)
elif hasattr(layer, "output_size_per_partition"):
out_features = layer.output_size_per_partition
elif hasattr(layer, "output_size"):
out_features = layer.output_size
else:
out_features = layer.scales.shape[-1]
ark_linear = self.QuantLinear(
bits=self.weight_bits,
group_size=self.group_size,
sym=self.sym,
in_features=in_features,
out_features=out_features,
bias=layer.bias is not None,
weight_dtype=layer.params_dtype,
)
ark_linear.to(layer.qweight.device)
with torch.no_grad():
ark_linear.qweight.copy_(layer.qweight.detach())
if hasattr(layer, "qzeros") and layer.qzeros is not None:
ark_linear.qzeros.copy_(layer.qzeros.detach())
else:
ark_linear.qzeros = None
ark_linear.scales.copy_(layer.scales.detach())
if hasattr(layer, "bias") and layer.bias is not None:
ark_linear.bias.copy_(layer.bias.detach())
ark_linear.post_init()
layer.ark_linear = ark_linear
del layer.qweight
if hasattr(layer, "qzeros"):
del layer.qzeros
del layer.scales
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return layer.ark_linear.forward(x)
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .inc import INCConfig
__all__ = ["INCConfig"]
@@ -0,0 +1,188 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING
import regex as re
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
if TYPE_CHECKING:
import torch
from .inc import INCConfig
@dataclass(frozen=True)
class INCLayerConfig:
bits: int
group_size: int
sym: bool
packing_format: str
backend: str
data_type: str
quantized: bool
@property
def is_gptq(self) -> bool:
return "gptq" in self.packing_format or "gptq" in self.backend
@property
def is_awq(self) -> bool:
return "awq" in self.packing_format or "awq" in self.backend
@property
def is_wna16_int(self) -> bool:
return self.data_type == "int" and self.quantized
@property
def is_mxfp4(self) -> bool:
return self.data_type == "mx_fp" and self.bits == 4
@property
def is_mxfp8(self) -> bool:
return self.data_type == "mx_fp" and self.bits == 8
class INCConfigParser:
def __init__(self, config: "INCConfig") -> None:
self._config = config
def resolve(self, layer: "torch.nn.Module", layer_name: str) -> INCLayerConfig:
bits, group_size, sym = self._resolve_raw(layer, layer_name)
return INCLayerConfig(
bits=bits,
group_size=group_size,
sym=sym,
packing_format=self._config.packing_format,
backend=self._config.backend,
data_type=self._config.data_type,
quantized=bits < 16,
)
def get_layer_config(
self, layer: "torch.nn.Module", layer_name: str
) -> tuple[int, int, bool]:
layer_config = self.resolve(layer, layer_name)
return layer_config.bits, layer_config.group_size, layer_config.sym
def _resolve_raw(
self, layer: "torch.nn.Module", layer_name: str
) -> tuple[int, int, bool]:
REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
def is_explicitly_configured(name: str) -> bool:
"""Return True if *name* has an explicit entry in extra_config,
either via exact key match or via a regex pattern key."""
if not self._config.extra_config:
return False
if name in self._config.extra_config:
return True
for pattern in self._config.extra_config:
if not isinstance(pattern, str) or not any(
c in REGEX_SPECIAL_CHARS for c in pattern
):
continue
try:
if re.search(re.compile(pattern), name) is not None:
return True
except re.error:
continue
return False
def get_config(name: str, quantized: bool = True) -> tuple[int, int, bool]:
if not self._config.extra_config:
return (
self._config.weight_bits if quantized else 16,
self._config.group_size if quantized else -1,
self._config.sym if quantized else True,
)
if name in self._config.extra_config:
cfg = self._config.extra_config[name]
return (
cfg.get("bits", self._config.weight_bits if quantized else 16),
cfg.get(
"group_size",
self._config.group_size if quantized else -1,
),
cfg.get("sym", self._config.sym if quantized else True),
)
regex_special_chars = set(r"*+?^$()[]{}|\\")
for pattern, cfg in self._config.extra_config.items():
if not isinstance(pattern, str) or not any(
c in regex_special_chars for c in pattern
):
continue
try:
if re.search(re.compile(pattern), name) is not None:
return (
cfg.get(
"bits",
self._config.weight_bits if quantized else 16,
),
cfg.get(
"group_size",
self._config.group_size if quantized else -1,
),
cfg.get("sym", self._config.sym if quantized else True),
)
except re.error:
continue
return (
self._config.weight_bits if quantized else 16,
self._config.group_size if quantized else -1,
self._config.sym if quantized else True,
)
if self._config.extra_config and layer_name in self._config.extra_config:
return get_config(layer_name)
quantized = not isinstance(layer, ParallelLMHead)
if self._config.block_name_to_quantize:
quantized = any(
layer_name.startswith(name)
for name in self._config.block_name_to_quantize
)
if self._config.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
moe_configs = [
get_config(name, quantized)
for name in self._config.extra_config
if name.startswith(layer_name)
]
if moe_configs:
if len(set(moe_configs)) == 1:
return moe_configs[0]
raise ValueError(
f"Fused MoE layer '{layer_name}' requires "
f"consistent quant config for all sub-layers"
)
if self._config.extra_config:
for fusion_key, sub_keys in self._config.packed_modules_mapping.items():
if fusion_key in layer_name and layer_name.count(fusion_key) == 1:
sub_names = [
layer_name.replace(fusion_key, sub_key) for sub_key in sub_keys
]
# Only trigger if at least one sub_name is explicitly
# configured in extra_config (via exact match or regex).
# This prevents false matches when a short fusion_key
# (e.g. "qkv") is merely a substring of a longer layer
# name (e.g. "in_proj_qkvz") and none of the generated
# sub_names are actually configured.
if not any(is_explicitly_configured(n) for n in sub_names):
continue
sub_configs = [get_config(name, quantized) for name in sub_names]
if len(set(sub_configs)) == 1:
return sub_configs[0]
raise ValueError(
f"Fused module '{layer_name}' requires "
f"consistent quant config for {sub_names}"
)
return get_config(layer_name, quantized)
@@ -0,0 +1,192 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fractions import Fraction
from typing import TYPE_CHECKING, Any
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
QuantizationMethods,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from .config_parser import INCConfigParser
if TYPE_CHECKING:
from vllm.model_executor.models.utils import WeightsMapper
logger = init_logger(__name__)
class INCConfig(QuantizationConfig):
"""Config class for Intel Neural Compressor (INC).
Repo: https://github.com/intel/neural-compressor
"""
SUPPORTED_BITS = {2, 3, 4, 8}
SUPPORTED_DTYPES = {"int"}
SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq"}
SUPPORTED_BACKENDS = {
"auto",
"gptq",
"gptq:marlin",
"awq",
"awq:marlin",
"marlin",
}
def __init__(
self,
weight_bits: int,
group_size: int,
sym: bool = True,
packing_format: str = "auto_round:auto_gptq",
block_name_to_quantize: str | list[str] | None = None,
extra_config: dict[str, Any] | None = None,
data_type: str = "int",
backend: str = "auto",
) -> None:
super().__init__()
if weight_bits not in self.SUPPORTED_BITS:
raise ValueError(
f"Unsupported weight_bits: {weight_bits}, "
f"currently only support {self.SUPPORTED_BITS}."
)
if data_type not in self.SUPPORTED_DTYPES:
raise ValueError(
f"Unsupported data_type: {data_type},"
f" currently only support {self.SUPPORTED_DTYPES}."
)
if packing_format not in self.SUPPORTED_FORMATS:
raise ValueError(
f"Unsupported packing_format: {packing_format}, "
f"currently only support {self.SUPPORTED_FORMATS}."
)
if backend not in self.SUPPORTED_BACKENDS:
raise ValueError(
f"Unsupported backend: {backend}, "
f"currently only support {self.SUPPORTED_BACKENDS}."
)
self.weight_bits = weight_bits
self.group_size = group_size
self.sym = sym
self.packing_format = packing_format
self.block_name_to_quantize = (
block_name_to_quantize.split(",")
if isinstance(block_name_to_quantize, str)
else block_name_to_quantize
)
self.extra_config = extra_config
self.data_type = data_type
self.backend = backend
self.pack_factor = Fraction(32, weight_bits)
self.config_parser = INCConfigParser(self)
def __repr__(self) -> str:
return (
f"INCConfig(weight_bits={self.weight_bits}, "
f"group_size={self.group_size}, sym={self.sym})"
)
@classmethod
def get_name(cls) -> QuantizationMethods:
return "inc"
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
return [torch.half, torch.bfloat16]
@classmethod
def get_min_capability(cls) -> int:
return 60
@classmethod
def get_config_filenames(cls) -> list[str]:
return ["quantization_config.json"]
@classmethod
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
return cls(
weight_bits=cls.get_from_keys(config, ["bits"]),
group_size=cls.get_from_keys(config, ["group_size"]),
sym=cls.get_from_keys(config, ["sym"]),
packing_format=cls.get_from_keys_or(
config, ["packing_format"], "auto_round:auto_gptq"
),
block_name_to_quantize=cls.get_from_keys_or(
config, ["block_name_to_quantize", "to_quant_block_names"], None
),
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
)
def get_layer_config(self, layer, layer_name: str):
return self.config_parser.get_layer_config(layer, layer_name)
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
if self.block_name_to_quantize is not None:
self.block_name_to_quantize = hf_to_vllm_mapper.apply_list(
self.block_name_to_quantize
)
if self.extra_config is not None:
self.extra_config = hf_to_vllm_mapper.apply_dict(self.extra_config)
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
from .schemes.factory import resolve_scheme
# Match original: check model.-prefixed names for unquantized layers
if prefix and self.extra_config:
for layer_name in self.extra_config:
if (
layer_name == prefix or layer_name == f"model.{prefix}"
) and self.extra_config[layer_name].get("bits", 16) >= 16:
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return UnquantizedLinearMethod()
layer_config = self.config_parser.resolve(layer, prefix)
if not layer_config.quantized:
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
logger.debug(
"[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
prefix,
layer.__class__.__name__,
layer_config.bits,
layer_config.group_size,
layer_config.sym,
)
scheme = resolve_scheme(layer_config)
if isinstance(layer, (LinearBase, ParallelLMHead)):
return scheme.get_linear_method(self, layer, prefix, layer_config)
if isinstance(layer, RoutedExperts):
return scheme.get_moe_method(self, layer, prefix, layer_config)
return None
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant, hf_config=None
) -> "QuantizationMethods | None":
"""Override the `auto-round` method to `inc`."""
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
if is_auto_round_format:
return cls.get_name()
return None
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
import torch
from vllm.model_executor.layers.linear import LinearMethodBase
if TYPE_CHECKING:
from .schemes.inc_scheme import INCLinearScheme
class INCLinearMethod(LinearMethodBase):
def __init__(self, scheme: "INCLinearScheme") -> None:
self.scheme = scheme
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
return self.scheme.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
return self.scheme.process_weights_after_loading(layer)
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.scheme.apply_weights(layer, x, bias)
@@ -0,0 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .factory import resolve_scheme
from .inc_scheme import INCLinearScheme, INCScheme
from .inc_wna16_scheme import INCWna16Scheme
__all__ = [
"INCScheme",
"INCLinearScheme",
"INCWna16Scheme",
"resolve_scheme",
]
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from ..config_parser import INCLayerConfig
from .inc_scheme import INCScheme
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
from .inc_wna16_scheme import INCWna16Scheme
scheme_list: list[type[INCScheme]] = [
INCWna16Scheme,
]
for scheme_cls in scheme_list:
if scheme_cls.can_handle(layer_config):
return scheme_cls()
raise NotImplementedError(f"No INC scheme found for layer config: {layer_config}")
@@ -0,0 +1,104 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
if TYPE_CHECKING:
import torch
from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
from vllm.model_executor.layers.linear import LinearMethodBase
from vllm.model_executor.layers.quantization import QuantizationMethods
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
class INCScheme(ABC):
"""One class per quant type. Single registration point for the factory.
Each subclass defines:
- can_handle(): when does this scheme apply?
- get_linear_method(): required — how to quantize Linear layers
- get_moe_method(): optional — how to quantize MoE layers
- get_kvcache_method(): optional — how to quantize KV cache
Schemes that don't support MoE/KVCache inherit the default raise.
"""
@staticmethod
@abstractmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
raise NotImplementedError
@abstractmethod
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
) -> "LinearMethodBase":
raise NotImplementedError
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
) -> "FusedMoEMethodBase | None":
"""Optional. Override if this scheme supports MoE.
Default raises NotImplementedError."""
raise NotImplementedError(
f"{type(self).__name__} does not support MoE layers. "
f"Layer config: {layer_config}"
)
def get_kvcache_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
) -> "QuantizationMethods":
"""Optional. Override if this scheme supports KV cache quantization.
Default raises NotImplementedError."""
raise NotImplementedError(
f"{type(self).__name__} does not support KV cache quantization. "
f"Layer config: {layer_config}"
)
class INCLinearScheme(ABC):
@classmethod
@abstractmethod
def get_min_capability(cls) -> int:
raise NotImplementedError
@abstractmethod
def create_weights(
self,
layer: "torch.nn.Module",
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: "torch.dtype",
**extra_weight_attrs,
) -> None:
raise NotImplementedError
@abstractmethod
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
raise NotImplementedError
@abstractmethod
def apply_weights(
self,
layer: "torch.nn.Module",
x: "torch.Tensor",
bias: "torch.Tensor | None" = None,
) -> "torch.Tensor":
raise NotImplementedError
@@ -0,0 +1,399 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import lru_cache
from typing import TYPE_CHECKING, Any
import torch
from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supported,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
RowvLLMParameter,
)
from vllm.scalar_type import scalar_types
from .inc_scheme import INCLinearScheme
logger = init_logger(__name__)
if TYPE_CHECKING:
from ..config_parser import INCLayerConfig
@lru_cache(maxsize=1)
def get_ark_state() -> tuple[bool, str | None, Any | None, Any | None]:
"""Return ARK availability, error details, cached module, and QuantLinear."""
try:
import auto_round_kernel as ark
from auto_round_kernel.qlinear import QuantLinear
logger.info("Successfully imported auto_round_kernel.")
except ImportError as error:
return False, str(error), None, None
if getattr(ark, "cpu_lib", None) is None and getattr(ark, "xpu_lib", None) is None:
return (
False,
"No ARK backend library is available.",
None,
None,
)
logger.info("Successfully loaded auto_round_kernel backend library.")
return True, None, ark, QuantLinear
class INCWNA16LinearScheme(INCLinearScheme):
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.layer_config = layer_config
self.inner_method = self._build_inner_method()
@classmethod
def get_min_capability(cls) -> int:
return 60
def _build_inner_method(self):
if self.layer_config.is_gptq:
return self._build_gptq_method()
if self.layer_config.is_awq:
return self._build_awq_method()
raise NotImplementedError(
f"WNA16 linear scheme does not support {self.layer_config}"
)
def _build_gptq_method(self):
gptq_type_map = {
(4, True): scalar_types.uint4b8,
(8, True): scalar_types.uint8b128,
}
use_marlin = (
self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
) and (self.layer_config.bits, self.layer_config.sym) in gptq_type_map
if use_marlin:
use_marlin = check_marlin_supported(
gptq_type_map[(self.layer_config.bits, self.layer_config.sym)],
self.layer_config.group_size,
has_zp=not self.layer_config.sym,
)
if use_marlin:
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQLinearMethod,
)
return AutoGPTQLinearMethod(
AutoGPTQConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
desc_act=False,
is_sym=self.layer_config.sym,
lm_head_quantized=False,
dynamic={},
full_config={},
)
)
raise NotImplementedError(
f"INC quantization with bits={self.layer_config.bits}, "
f"sym={self.layer_config.sym} is not supported. "
"Only 4-bit and 8-bit symmetric quantization is supported "
"with Marlin kernels."
)
def _build_awq_method(self):
awq_type_map = {
4: scalar_types.uint4,
8: scalar_types.uint8,
}
use_marlin = (
self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
) and self.layer_config.bits in awq_type_map
if use_marlin:
use_marlin = check_marlin_supported(
awq_type_map[self.layer_config.bits],
self.layer_config.group_size,
not self.layer_config.sym,
)
if use_marlin:
from vllm.model_executor.layers.quantization.awq_marlin import (
AWQMarlinLinearMethod,
)
return AWQMarlinLinearMethod(
AWQMarlinConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
zero_point=not self.layer_config.sym,
lm_head_quantized=False,
modules_to_not_convert=[],
full_config={},
)
)
from vllm.model_executor.layers.quantization.awq import AWQLinearMethod
return AWQLinearMethod(
AWQConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
zero_point=not self.layer_config.sym,
)
)
def create_weights(
self,
layer: "torch.nn.Module",
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: "torch.dtype",
**extra_weight_attrs,
) -> None:
return self.inner_method.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
return self.inner_method.process_weights_after_loading(layer)
def apply_weights(
self,
layer: "torch.nn.Module",
x: "torch.Tensor",
bias: "torch.Tensor | None" = None,
) -> "torch.Tensor":
return self.inner_method.apply(layer, x, bias)
class INCXPULinearBase(INCLinearScheme):
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.weight_bits = layer_config.bits
self.group_size = layer_config.group_size
self.sym = layer_config.sym
self.pack_factor = 32 // self.weight_bits
@classmethod
def get_min_capability(cls) -> int:
return 0
def _create_inc_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
params_dtype: torch.dtype,
weight_loader: Any,
) -> None:
output_size_per_partition = sum(output_partition_sizes)
scales_and_zp_size = input_size_per_partition // self.group_size
qweight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition // self.pack_factor,
output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
scales = GroupQuantScaleParameter(
data=torch.empty(
scales_and_zp_size,
output_size_per_partition,
dtype=params_dtype,
),
input_dim=0,
output_dim=1,
weight_loader=weight_loader,
)
qzeros = PackedvLLMParameter(
data=torch.empty(
scales_and_zp_size,
output_size_per_partition // self.pack_factor,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
layer.register_parameter("qweight", qweight)
layer.register_parameter("scales", scales)
layer.register_parameter("qzeros", qzeros)
g_idx = RowvLLMParameter(
data=torch.tensor(
[i // self.group_size for i in range(input_size_per_partition)],
dtype=torch.int32,
),
input_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("g_idx", g_idx)
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size
self._create_inc_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
params_dtype=params_dtype,
weight_loader=extra_weight_attrs.get("weight_loader"),
)
class INCXPULinearMethod(INCXPULinearBase):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
device = layer.qweight.data.device
qweight_ct = layer.qweight.data.t().contiguous()
layer.qweight = Parameter(qweight_ct.t(), requires_grad=False)
layer.scales = Parameter(layer.scales.data, requires_grad=False)
layer.qzeros = Parameter(
torch.tensor([8], dtype=torch.int8, device=device),
requires_grad=False,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
reshaped_x = x.reshape(-1, x.shape[-1])
out = torch.ops._xpu_C.int4_gemm_w4a16(
reshaped_x,
layer.qweight,
bias,
layer.scales,
layer.qzeros,
self.group_size,
None,
)
return out.reshape(out_shape)
class INCARKLinearMethod(INCXPULinearBase):
def __init__(self, layer_config: "INCLayerConfig") -> None:
super().__init__(layer_config)
is_available, error_str, _, quant_linear_cls = get_ark_state()
if not is_available or quant_linear_cls is None:
reason = error_str or "unknown error"
raise ImportError(f"Failed to import auto_round_kernel. {reason}")
self.quant_linear_cls = quant_linear_cls
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
super().create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
layer.in_features = input_size_per_partition
layer.out_features = sum(output_partition_sizes)
layer.params_dtype = params_dtype
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if hasattr(layer, "input_size_per_partition"):
in_features = layer.input_size_per_partition
elif hasattr(layer, "input_size"):
in_features = layer.input_size
else:
raise AttributeError("Cannot determine in_features for layer.")
if hasattr(layer, "output_partition_sizes"):
out_features = sum(layer.output_partition_sizes)
elif hasattr(layer, "output_size_per_partition"):
out_features = layer.output_size_per_partition
elif hasattr(layer, "output_size"):
out_features = layer.output_size
else:
out_features = layer.scales.shape[-1]
ark_linear = self.quant_linear_cls(
bits=self.weight_bits,
group_size=self.group_size,
sym=self.sym,
in_features=in_features,
out_features=out_features,
bias=layer.bias is not None,
weight_dtype=layer.params_dtype,
)
ark_linear.to(layer.qweight.device)
with torch.no_grad():
ark_linear.qweight.copy_(layer.qweight.detach())
if hasattr(layer, "qzeros") and layer.qzeros is not None:
ark_linear.qzeros.copy_(layer.qzeros.detach())
else:
ark_linear.qzeros = None
ark_linear.scales.copy_(layer.scales.detach())
if hasattr(layer, "bias") and layer.bias is not None:
ark_linear.bias.copy_(layer.bias.detach())
ark_linear.post_init()
layer.ark_linear = ark_linear
del layer.qweight
if hasattr(layer, "qzeros"):
del layer.qzeros
del layer.scales
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
del bias
return layer.ark_linear.forward(x)
class INCXPUW4A16LinearScheme(INCXPULinearMethod):
pass
@@ -0,0 +1,201 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
logger = init_logger(__name__)
class INCWna16Scheme(INCScheme):
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_wna16_int
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer
if current_platform.is_xpu():
if layer_config.bits == 4 and layer_config.sym:
from .inc_wna16_linear import (
INCARKLinearMethod,
INCXPULinearMethod,
get_ark_state,
)
is_ark_available, ark_error, _, _ = get_ark_state()
if is_ark_available:
return INCLinearMethod(INCARKLinearMethod(layer_config))
logger.debug(
"ARK backend is unavailable for layer %s; "
"falling back to the default XPU INC path. Error: %s",
prefix,
ark_error or "unknown error",
)
return INCLinearMethod(INCXPULinearMethod(layer_config))
raise NotImplementedError(f"INC on XPU: unsupported config {layer_config}")
if current_platform.is_cpu() and layer_config.is_gptq:
if layer_config.bits == 4 and layer_config.sym:
from .inc_wna16_linear import (
INCARKLinearMethod,
INCWNA16LinearScheme,
get_ark_state,
)
is_ark_available, ark_error, _, _ = get_ark_state()
if is_ark_available:
return INCLinearMethod(INCARKLinearMethod(layer_config))
logger.debug(
"ARK backend is unavailable for layer %s; "
"falling back to the default CPU INC path. Error: %s",
prefix,
ark_error or "unknown error",
)
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
from .inc_wna16_linear import INCWNA16LinearScheme
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, prefix
# XPU and CPU do not support MoE quantization yet
if current_platform.is_xpu() or current_platform.is_cpu():
from vllm.model_executor.layers.fused_moe import (
UnquantizedFusedMoEMethod,
)
return UnquantizedFusedMoEMethod(layer.moe_config)
if layer_config.is_gptq:
return _resolve_gptq_moe(layer, layer_config)
if layer_config.is_awq:
return _resolve_awq_moe(layer, layer_config)
raise NotImplementedError(f"WNA16 MoE does not support config {layer_config}")
def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQMoEMethod,
)
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
MoeWNA16Method,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supported,
check_moe_marlin_supports_layer,
)
gptq_type_map = {
(4, True): scalar_types.uint4b8,
(8, True): scalar_types.uint8b128,
}
use_marlin = (layer_config.bits, layer_config.sym) in gptq_type_map
if use_marlin:
use_marlin = check_marlin_supported(
gptq_type_map[(layer_config.bits, layer_config.sym)],
layer_config.group_size,
has_zp=not layer_config.sym,
) and check_moe_marlin_supports_layer(layer, layer_config.group_size)
if use_marlin:
return AutoGPTQMoEMethod(
AutoGPTQConfig(
weight_bits=layer_config.bits,
group_size=layer_config.group_size,
desc_act=False,
is_sym=layer_config.sym,
lm_head_quantized=False,
dynamic={},
full_config={},
),
layer.moe_config,
)
moe_config = MoeWNA16Config.from_config(
{
"quant_method": "gptq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"sym": layer_config.sym,
"lm_head": False,
}
)
return MoeWNA16Method(moe_config, layer.moe_config)
def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinMoEMethod
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
MoeWNA16Method,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supported,
check_moe_marlin_supports_layer,
)
awq_type_map = {
4: scalar_types.uint4,
8: scalar_types.uint8,
}
use_marlin = layer_config.bits in awq_type_map
if use_marlin:
use_marlin = check_marlin_supported(
awq_type_map[layer_config.bits],
layer_config.group_size,
not layer_config.sym,
) and check_moe_marlin_supports_layer(layer, layer_config.group_size)
if use_marlin:
return AWQMarlinMoEMethod(
AWQMarlinConfig(
weight_bits=layer_config.bits,
group_size=layer_config.group_size,
zero_point=not layer_config.sym,
lm_head_quantized=False,
modules_to_not_convert=[],
full_config={},
),
layer.moe_config,
)
moe_config = MoeWNA16Config.from_config(
{
"quant_method": "awq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"zero_point": not layer_config.sym,
"lm_head": False,
}
)
return MoeWNA16Method(moe_config, layer.moe_config)