[AutoRound] Support AutoRound Format Block-Wise FP8 in vLLM (#47434)

Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com>
Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Zhenzhong Xu
2026-08-31 22:56:20 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent dafbef15a1
commit bd575a0d0b
10 changed files with 503 additions and 62 deletions
+167 -1
View File
@@ -22,11 +22,15 @@ from vllm.model_executor.layers.quantization.inc import INCConfig
from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
from vllm.model_executor.layers.quantization.inc.schemes import (
INCFp8Scheme,
INCMxfp4Scheme,
INCMxfp8Scheme,
INCWna16Scheme,
resolve_scheme,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear import (
INCFp8LinearScheme,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import (
INCMxfp8LinearScheme,
)
@@ -77,6 +81,14 @@ MODELS = [
),
id="auto_round:llm_compressor_mxfp8",
),
pytest.param(
"INC4AI/Qwen3-8B-fp-w8g128x128-for-ut",
marks=pytest.mark.skipif(
not (current_platform.is_cuda()),
reason="Block-wise AutoRound model only supports CUDA backend for now.",
),
id="auto_round:block_wise_fp8_on_cuda",
),
]
QWEN3_AUTOROUND_MODELS = [
@@ -128,6 +140,10 @@ MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = {
"gpu_memory_utilization": 0.8,
"max_model_len": 512,
},
"INC4AI/Qwen3-8B-fp-w8g128x128-for-ut": {
"gpu_memory_utilization": 0.8,
"max_model_len": 512,
},
}
@@ -190,6 +206,27 @@ def make_layer_config(**overrides) -> INCLayerConfig:
return INCLayerConfig(**kwargs)
def make_block_fp8_raw_config(**overrides) -> dict[str, object]:
kwargs = {
"bits": 8,
"group_size": [128, 128],
"sym": True,
"packing_format": "auto_round:fp8",
"quant_method": "auto-round",
"data_type": "fp",
"act_bits": 8,
"act_data_type": "fp",
"act_group_size": 128,
"act_sym": True,
"act_dynamic": True,
"activation_scheme": "dynamic",
"fmt": "e4m3",
"weight_block_size": [128, 128],
}
kwargs.update(overrides)
return kwargs
def make_qwen3_autoround_config(kind: str) -> INCConfig:
configs = {
"qwen3_1p7b_mxfp4": {
@@ -480,6 +517,52 @@ def test_inc_layer_config_mx_fp_helpers() -> None:
assert layer_config.is_mxfp8 is False
def test_inc_config_accepts_block_fp8() -> None:
config = INCConfig.from_config(make_block_fp8_raw_config())
assert config.weight_bits == 8
assert config.group_size == (128, 128)
assert config.sym is True
assert config.data_type == "fp"
assert config.packing_format == "auto_round:fp8"
def test_inc_config_rejects_invalid_block_fp8_activation_config() -> None:
with pytest.raises(AssertionError, match="act_dynamic=True"):
INCConfig.from_config(make_block_fp8_raw_config(act_dynamic=False))
def test_inc_layer_config_fp8_block_helper() -> None:
layer_config = INCLayerConfig(
bits=8,
group_size=(128, 128),
sym=True,
packing_format="auto_round:fp8",
backend="auto",
data_type="fp",
quantized=True,
)
assert layer_config.is_fp8_block is True
assert layer_config.is_wna16_int is False
def test_inc_resolve_scheme_selects_fp8_block() -> None:
layer_config = INCLayerConfig(
bits=8,
group_size=(128, 128),
sym=True,
packing_format="auto_round:fp8",
backend="auto",
data_type="fp",
quantized=True,
)
scheme = resolve_scheme(layer_config)
assert isinstance(scheme, INCFp8Scheme)
def test_inc_resolve_scheme_selects_wna16() -> None:
layer_config = INCLayerConfig(
bits=4,
@@ -1053,6 +1136,89 @@ def test_inc_linear_method_delegates() -> None:
]
def test_inc_fp8_linear_scheme_delegates_to_fp8_linear_method(
monkeypatch,
) -> None:
captured = {}
class DummyMethod:
def __init__(self, quant_config) -> None:
captured["quant_config"] = quant_config
captured["method"] = self
self.calls: list[tuple] = []
self.marlin_input_dtype = None
def create_weights(
self,
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
) -> None:
self.calls.append(
(
"create_weights",
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
extra_weight_attrs,
)
)
def process_weights_after_loading(self, layer) -> None:
self.calls.append(("process_weights_after_loading", layer))
def apply(self, layer, x, bias=None):
self.calls.append(("apply", layer, x, bias))
return "applied"
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear."
"Fp8LinearMethod",
DummyMethod,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear."
"get_marlin_input_dtype",
lambda _prefix: "fp8-marlin",
)
scheme = INCFp8LinearScheme(
prefix="layers.0.mlp.down_proj",
weight_block_size=(128, 128),
)
layer = torch.nn.Module()
scheme.create_weights(
layer=layer,
input_size_per_partition=128,
output_partition_sizes=[64, 64],
input_size=128,
output_size=128,
params_dtype=torch.bfloat16,
weight_loader=lambda *args, **kwargs: None,
)
scheme.process_weights_after_loading(layer)
result = scheme.apply_weights(layer, torch.randn(1, 128), None)
assert captured["quant_config"].is_checkpoint_fp8_serialized is True
assert captured["quant_config"].activation_scheme == "dynamic"
assert captured["quant_config"].weight_block_size == [128, 128]
assert captured["method"].marlin_input_dtype == "fp8-marlin"
assert result == "applied"
assert [call[0] for call in captured["method"].calls] == [
"create_weights",
"process_weights_after_loading",
"apply",
]
def test_wna16_xpu_prefers_ark_when_available(monkeypatch) -> None:
class DummyQuantLinear:
pass
@@ -1284,7 +1450,7 @@ def test_wna16_linear_gptq_uses_auto_gptq_when_supported(monkeypatch) -> None:
scheme = INCWNA16LinearScheme(make_layer_config())
assert isinstance(scheme.inner_method, DummyMethod)
assert isinstance(scheme.linear_method, DummyMethod)
assert isinstance(captured["cfg"], AutoGPTQConfig)
assert captured["cfg"].weight_bits == 4
assert captured["cfg"].group_size == 128
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
@dataclass(frozen=True)
class INCLayerConfig:
bits: int
group_size: int
group_size: int | tuple[int, int]
sym: bool
packing_format: str
backend: str
@@ -44,6 +44,18 @@ class INCLayerConfig:
def is_mxfp8(self) -> bool:
return "mx_fp" in self.data_type and self.bits == 8
@property
def is_fp8_block(self) -> bool:
return (
self.data_type == "fp"
and self.bits == 8
and self.sym
and self.quantized
and self.packing_format == "auto_round:fp8"
and isinstance(self.group_size, tuple)
and len(self.group_size) == 2
)
class INCConfigParser:
def __init__(self, config: "INCConfig") -> None:
@@ -63,13 +75,13 @@ class INCConfigParser:
def get_layer_config(
self, layer: "torch.nn.Module", layer_name: str
) -> tuple[int, int, bool]:
) -> tuple[int, int | tuple[int, int], bool]:
layer_config = self.resolve(layer, layer_name)
return layer_config.bits, layer_config.group_size, layer_config.sym
def _resolve_raw(
self, layer: "torch.nn.Module", layer_name: str
) -> tuple[int, int, bool]:
) -> tuple[int, int | tuple[int, int], bool]:
REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
def is_explicitly_configured(name: str) -> bool:
@@ -91,7 +103,9 @@ class INCConfigParser:
continue
return False
def get_config(name: str, quantized: bool = True) -> tuple[int, int, bool]:
def get_config(
name: str, quantized: bool = True
) -> tuple[int, int | tuple[int, int], bool]:
if not self._config.extra_config:
return (
self._config.weight_bits if quantized else 16,
@@ -103,9 +117,11 @@ class INCConfigParser:
cfg = self._config.extra_config[name]
return (
cfg.get("bits", self._config.weight_bits if quantized else 16),
cfg.get(
"group_size",
self._config.group_size if quantized else -1,
self._normalize_group_size(
cfg.get(
"group_size",
self._config.group_size if quantized else -1,
)
),
cfg.get("sym", self._config.sym if quantized else True),
)
@@ -124,9 +140,11 @@ class INCConfigParser:
"bits",
self._config.weight_bits if quantized else 16,
),
cfg.get(
"group_size",
self._config.group_size if quantized else -1,
self._normalize_group_size(
cfg.get(
"group_size",
self._config.group_size if quantized else -1,
)
),
cfg.get("sym", self._config.sym if quantized else True),
)
@@ -194,3 +212,26 @@ class INCConfigParser:
)
return get_config(layer_name, quantized)
@staticmethod
def _normalize_group_size(
group_size: int | list[int] | tuple[int, int],
) -> int | tuple[int, int]:
"""Normalize INC group_size into either an int or a 2-D int tuple."""
if isinstance(group_size, (list, tuple)):
if len(group_size) != 2 or not all(
isinstance(value, int) for value in group_size
):
raise ValueError(
"INC block-wise FP8 requires group_size to be a 2-D "
f"integer sequence, but found {group_size!r}."
)
return (group_size[0], group_size[1])
if not isinstance(group_size, int):
raise ValueError(
"INC group_size must be an int or a 2-D integer sequence, "
f"but found {type(group_size).__name__}."
)
return group_size
@@ -34,11 +34,14 @@ class INCConfig(QuantizationConfig):
Repo: https://github.com/intel/neural-compressor
"""
DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
SUPPORTED_BITS = {2, 3, 4, 8}
SUPPORTED_DTYPES = {"int", "mx_fp"}
SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
SUPPORTED_FORMATS = {
"auto_round:auto_gptq",
"auto_round:auto_awq",
"auto_round:fp8",
"auto_round:llm_compressor",
}
SUPPORTED_BACKENDS = {
@@ -49,23 +52,39 @@ class INCConfig(QuantizationConfig):
"awq:marlin",
"marlin",
}
FP8_BLOCK_BITS = 8
FP8_BLOCK_DATA_TYPE = "fp"
FP8_BLOCK_PACKING_FORMAT = "auto_round:fp8"
FP8_BLOCK_FMT = "e4m3"
FP8_BLOCK_ACTIVATION_SCHEME = "dynamic"
MXFP8_BITS = 8
MXFP8_GROUP_SIZE = 32
MXFP8_DATA_TYPE = "mx_fp"
MXFP8_PACKING_FORMAT = "auto_round:llm_compressor"
@staticmethod
def _normalize_data_type(data_type: str) -> str:
# Keep a small alias for branches that already experimented with "fp8".
return "fp" if data_type == "fp8" else data_type
def __init__(
self,
weight_bits: int,
group_size: int,
group_size: int | tuple[int, int],
sym: bool = True,
packing_format: str = "auto_round:auto_gptq",
packing_format: str = DEFAULT_INT_PACKING_FORMAT,
block_name_to_quantize: str | list[str] | None = None,
extra_config: dict[str, Any] | None = None,
data_type: str = "int",
backend: str = "auto",
) -> None:
super().__init__()
data_type = self._normalize_data_type(data_type)
group_size = INCConfigParser._normalize_group_size(group_size)
if weight_bits not in self.SUPPORTED_BITS:
raise ValueError(
f"Unsupported weight_bits: {weight_bits}, "
@@ -124,48 +143,118 @@ class INCConfig(QuantizationConfig):
# MXFP4 and MXFP8 share data_type "mx_fp" and differ only by bit width.
return self.is_mxfp and self.weight_bits == self.MXFP8_BITS
def _validate_supported_quantization(self) -> None:
if self.is_mxfp8:
assert self.group_size == self.MXFP8_GROUP_SIZE, (
"INC MXFP8 only supports group_size=32, "
f"but found group_size={self.group_size}."
)
assert self.sym, "INC MXFP8 only supports symmetric weights."
assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
"INC MXFP8 only supports "
f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
f"but found {self.packing_format!r}."
)
assert self.backend == "auto", (
"INC MXFP8 only supports backend='auto', "
f"but found backend={self.backend!r}."
)
elif self.packing_format == self.MXFP8_PACKING_FORMAT and not self.is_mxfp:
raise ValueError(
f"packing_format={self.MXFP8_PACKING_FORMAT!r} requires "
f"an {self.MXFP8_DATA_TYPE!r} data_type."
)
def _validate_raw_config(self, config: dict[str, Any]) -> None:
if not self.is_mxfp8:
return
expected_fields = {
"act_bits": self.MXFP8_BITS,
"act_data_type": self.MXFP8_DATA_TYPE,
"act_group_size": self.MXFP8_GROUP_SIZE,
"act_sym": True,
"act_dynamic": True,
"enable_quanted_input": False,
}
@classmethod
def _check_expected(
cls,
values: dict[str, Any],
expected_fields: dict[str, Any],
quantization_name: str,
) -> None:
for field_name, expected_value in expected_fields.items():
actual_value = self.get_from_keys_or(config, [field_name], expected_value)
actual_value = cls.get_from_keys_or(values, [field_name], expected_value)
assert actual_value == expected_value, (
"INC MXFP8 only supports "
f"{quantization_name} only supports "
f"{field_name}={expected_value!r}, "
f"but found {field_name}={actual_value!r}."
)
def _validate_int_quantization(self) -> None:
int_packing_formats = {
self.DEFAULT_INT_PACKING_FORMAT,
"auto_round:auto_awq",
}
assert isinstance(self.group_size, int), (
"INC int quantization requires scalar group_size, "
f"but found group_size={self.group_size!r}."
)
assert self.packing_format in int_packing_formats, (
"INC int quantization only supports "
f"packing_format in {sorted(int_packing_formats)!r}, "
f"but found packing_format={self.packing_format!r}."
)
def _validate_fp8_block_quantization(self) -> None:
assert self.weight_bits == self.FP8_BLOCK_BITS, (
"INC block-wise FP8 only supports bits=8, "
f"but found bits={self.weight_bits}."
)
assert isinstance(self.group_size, tuple), (
"INC block-wise FP8 requires a 2-D group_size, "
f"but found group_size={self.group_size!r}."
)
assert self.sym, "INC block-wise FP8 only supports symmetric weights."
assert self.packing_format == self.FP8_BLOCK_PACKING_FORMAT, (
"INC block-wise FP8 only supports "
f"packing_format={self.FP8_BLOCK_PACKING_FORMAT!r}, "
f"but found {self.packing_format!r}."
)
assert self.backend == "auto", (
"INC block-wise FP8 only supports backend='auto', "
f"but found backend={self.backend!r}."
)
def _validate_mxfp_quantization(self) -> None:
assert self.weight_bits in {4, self.MXFP8_BITS}, (
"INC MXFP only supports bits=4 or bits=8, "
f"but found bits={self.weight_bits}."
)
assert self.group_size == self.MXFP8_GROUP_SIZE, (
"INC MXFP only supports group_size=32, "
f"but found group_size={self.group_size}."
)
assert self.sym, "INC MXFP only supports symmetric weights."
assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
"INC MXFP only supports "
f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
f"but found {self.packing_format!r}."
)
assert self.backend == "auto", (
"INC MXFP only supports backend='auto', "
f"but found backend={self.backend!r}."
)
def _validate_supported_quantization(self) -> None:
if self.data_type == "int":
self._validate_int_quantization()
elif self.data_type == self.FP8_BLOCK_DATA_TYPE:
self._validate_fp8_block_quantization()
elif self.data_type == self.MXFP8_DATA_TYPE:
self._validate_mxfp_quantization()
def _validate_raw_config(self, config: dict[str, Any]) -> None:
if self.data_type == self.FP8_BLOCK_DATA_TYPE:
assert isinstance(self.group_size, tuple), (
"INC block-wise FP8 group_size must be a tuple."
)
expected_fields = {
"act_bits": self.FP8_BLOCK_BITS,
"act_group_size": self.group_size[0],
"act_data_type": self.FP8_BLOCK_DATA_TYPE,
"act_sym": True,
"act_dynamic": True,
"activation_scheme": self.FP8_BLOCK_ACTIVATION_SCHEME,
"fmt": self.FP8_BLOCK_FMT,
}
self._check_expected(
config,
expected_fields,
"INC block-wise FP8",
)
elif self.is_mxfp8:
expected_fields = {
"act_bits": self.MXFP8_BITS,
"act_data_type": self.MXFP8_DATA_TYPE,
"act_group_size": self.MXFP8_GROUP_SIZE,
"act_sym": True,
"act_dynamic": True,
}
self._check_expected(
config,
expected_fields,
"INC MXFP8",
)
@classmethod
def get_name(cls) -> QuantizationMethods:
return "inc"
@@ -184,18 +273,37 @@ class INCConfig(QuantizationConfig):
@classmethod
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
data_type = cls._normalize_data_type(
cls.get_from_keys_or(config, ["data_type"], "int")
)
weight_bits = cls.get_from_keys(config, ["bits"])
group_size = INCConfigParser._normalize_group_size(
cls.get_from_keys(config, ["group_size"])
)
packing_format = cls.get_from_keys_or(config, ["packing_format"], None)
if packing_format is None:
if (
data_type == cls.FP8_BLOCK_DATA_TYPE
and weight_bits == cls.FP8_BLOCK_BITS
and isinstance(group_size, tuple)
):
packing_format = cls.FP8_BLOCK_PACKING_FORMAT
elif "mx_fp" in data_type:
packing_format = cls.MXFP8_PACKING_FORMAT
else:
packing_format = cls.DEFAULT_INT_PACKING_FORMAT
quant_config = cls(
weight_bits=cls.get_from_keys(config, ["bits"]),
group_size=cls.get_from_keys(config, ["group_size"]),
sym=cls.get_from_keys(config, ["sym"]),
packing_format=cls.get_from_keys_or(
config, ["packing_format"], "auto_round:auto_gptq"
),
weight_bits=weight_bits,
group_size=group_size,
sym=cls.get_from_keys_or(config, ["sym"], True),
packing_format=packing_format,
block_name_to_quantize=cls.get_from_keys_or(
config, ["block_name_to_quantize", "to_quant_block_names"], None
),
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
data_type=data_type,
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
)
quant_config._validate_raw_config(config)
@@ -215,7 +323,6 @@ class INCConfig(QuantizationConfig):
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
from .schemes.factory import resolve_scheme
# Match original: check model.-prefixed names for unquantized layers
if prefix and self.extra_config:
for layer_name in self.extra_config:
if (
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .factory import resolve_scheme
from .inc_fp8_scheme import INCFp8Scheme
from .inc_mxfp4_scheme import INCMxfp4Scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_scheme import INCLinearScheme, INCScheme
@@ -11,6 +12,7 @@ __all__ = [
"INCScheme",
"INCLinearScheme",
"INCMxfp8Scheme",
"INCFp8Scheme",
"INCWna16Scheme",
"INCMxfp4Scheme",
"resolve_scheme",
@@ -9,11 +9,13 @@ if TYPE_CHECKING:
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
from .inc_fp8_scheme import INCFp8Scheme
from .inc_mxfp4_scheme import INCMxfp4Scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_wna16_scheme import INCWna16Scheme
scheme_list: list[type[INCScheme]] = [
INCFp8Scheme,
INCMxfp8Scheme,
INCWna16Scheme,
INCMxfp4Scheme,
@@ -0,0 +1,57 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
get_marlin_input_dtype,
)
from .inc_scheme import INCLinearScheme
class INCFp8LinearScheme(INCLinearScheme):
def __init__(self, prefix: str, weight_block_size: tuple[int, int]) -> None:
self.quant_config = Fp8Config(
is_checkpoint_fp8_serialized=True,
activation_scheme="dynamic",
weight_block_size=list(weight_block_size),
)
self.linear_method = Fp8LinearMethod(self.quant_config)
self.linear_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
@classmethod
def get_min_capability(cls) -> int:
return Fp8Config.get_min_capability()
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
self.linear_method.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.linear_method.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.linear_method.apply(layer, x, bias)
@@ -0,0 +1,39 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
class INCFp8Scheme(INCScheme):
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_fp8_block
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer
assert isinstance(layer_config.group_size, tuple)
from .inc_fp8_linear import INCFp8LinearScheme
return INCLinearMethod(
INCFp8LinearScheme(
prefix=prefix,
weight_block_size=layer_config.group_size,
)
)
@@ -29,6 +29,11 @@ class INCMxfp4LinearMethod(INCLinearScheme):
"""
def __init__(self, layer_config: "INCLayerConfig") -> None:
if not isinstance(layer_config.group_size, int):
raise ValueError(
"INC MXFP4 requires scalar group_size, "
f"but found group_size={layer_config.group_size!r}."
)
self.group_size = layer_config.group_size or 32
self.kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic)
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
class INCWNA16LinearScheme(INCLinearScheme):
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.layer_config = layer_config
self.inner_method = self._build_inner_method()
self.linear_method = self._build_inner_method()
@classmethod
def get_min_capability(cls) -> int:
@@ -43,6 +43,10 @@ class INCWNA16LinearScheme(INCLinearScheme):
)
def _build_gptq_method(self):
assert isinstance(self.layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
gptq_type_map = {
(4, True): scalar_types.uint4b8,
(8, True): scalar_types.uint8b128,
@@ -82,6 +86,10 @@ class INCWNA16LinearScheme(INCLinearScheme):
)
def _build_awq_method(self):
assert isinstance(self.layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
awq_type_map = {
4: scalar_types.uint4,
8: scalar_types.uint8,
@@ -135,7 +143,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
params_dtype: "torch.dtype",
**extra_weight_attrs,
) -> None:
return self.inner_method.create_weights(
return self.linear_method.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
@@ -146,7 +154,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
)
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
return self.inner_method.process_weights_after_loading(layer)
return self.linear_method.process_weights_after_loading(layer)
def apply_weights(
self,
@@ -154,7 +162,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
x: "torch.Tensor",
bias: "torch.Tensor | None" = None,
) -> "torch.Tensor":
return self.inner_method.apply(layer, x, bias)
return self.linear_method.apply(layer, x, bias)
class INCXPULinearBase(INCLinearScheme):
@@ -165,6 +173,9 @@ class INCXPULinearBase(INCLinearScheme):
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.weight_bits = layer_config.bits
assert isinstance(layer_config.group_size, int), (
"INCXPULinearBase requires integer group_size."
)
self.group_size = layer_config.group_size
self.sym = layer_config.sym
@@ -41,6 +41,9 @@ def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> No
"which this build of vllm-xpu-kernels does not provide. "
f"Layer: {prefix}."
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
@@ -168,6 +171,10 @@ def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
check_moe_marlin_supports_layer,
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
# AutoGPTQMoEMethod selects its fused-MoE backend through the WNA16 oracle
# (Marlin on CUDA, XPUExpertsWNA16 on XPU). Gate only on the layer-shape
# check like compressed-tensors does; the capability-based
@@ -213,6 +220,10 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
check_moe_marlin_supports_layer,
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
use_marlin = layer_config.bits in (4, 8) and check_moe_marlin_supports_layer(
layer, layer_config.group_size
)