[AutoRound] Support AutoRound Format Block-Wise FP8 in vLLM (#47434)
Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com> Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
dafbef15a1
commit
bd575a0d0b
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
|
||||
@dataclass(frozen=True)
|
||||
class INCLayerConfig:
|
||||
bits: int
|
||||
group_size: int
|
||||
group_size: int | tuple[int, int]
|
||||
sym: bool
|
||||
packing_format: str
|
||||
backend: str
|
||||
@@ -44,6 +44,18 @@ class INCLayerConfig:
|
||||
def is_mxfp8(self) -> bool:
|
||||
return "mx_fp" in self.data_type and self.bits == 8
|
||||
|
||||
@property
|
||||
def is_fp8_block(self) -> bool:
|
||||
return (
|
||||
self.data_type == "fp"
|
||||
and self.bits == 8
|
||||
and self.sym
|
||||
and self.quantized
|
||||
and self.packing_format == "auto_round:fp8"
|
||||
and isinstance(self.group_size, tuple)
|
||||
and len(self.group_size) == 2
|
||||
)
|
||||
|
||||
|
||||
class INCConfigParser:
|
||||
def __init__(self, config: "INCConfig") -> None:
|
||||
@@ -63,13 +75,13 @@ class INCConfigParser:
|
||||
|
||||
def get_layer_config(
|
||||
self, layer: "torch.nn.Module", layer_name: str
|
||||
) -> tuple[int, int, bool]:
|
||||
) -> tuple[int, int | tuple[int, int], bool]:
|
||||
layer_config = self.resolve(layer, layer_name)
|
||||
return layer_config.bits, layer_config.group_size, layer_config.sym
|
||||
|
||||
def _resolve_raw(
|
||||
self, layer: "torch.nn.Module", layer_name: str
|
||||
) -> tuple[int, int, bool]:
|
||||
) -> tuple[int, int | tuple[int, int], bool]:
|
||||
REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
|
||||
|
||||
def is_explicitly_configured(name: str) -> bool:
|
||||
@@ -91,7 +103,9 @@ class INCConfigParser:
|
||||
continue
|
||||
return False
|
||||
|
||||
def get_config(name: str, quantized: bool = True) -> tuple[int, int, bool]:
|
||||
def get_config(
|
||||
name: str, quantized: bool = True
|
||||
) -> tuple[int, int | tuple[int, int], bool]:
|
||||
if not self._config.extra_config:
|
||||
return (
|
||||
self._config.weight_bits if quantized else 16,
|
||||
@@ -103,9 +117,11 @@ class INCConfigParser:
|
||||
cfg = self._config.extra_config[name]
|
||||
return (
|
||||
cfg.get("bits", self._config.weight_bits if quantized else 16),
|
||||
cfg.get(
|
||||
"group_size",
|
||||
self._config.group_size if quantized else -1,
|
||||
self._normalize_group_size(
|
||||
cfg.get(
|
||||
"group_size",
|
||||
self._config.group_size if quantized else -1,
|
||||
)
|
||||
),
|
||||
cfg.get("sym", self._config.sym if quantized else True),
|
||||
)
|
||||
@@ -124,9 +140,11 @@ class INCConfigParser:
|
||||
"bits",
|
||||
self._config.weight_bits if quantized else 16,
|
||||
),
|
||||
cfg.get(
|
||||
"group_size",
|
||||
self._config.group_size if quantized else -1,
|
||||
self._normalize_group_size(
|
||||
cfg.get(
|
||||
"group_size",
|
||||
self._config.group_size if quantized else -1,
|
||||
)
|
||||
),
|
||||
cfg.get("sym", self._config.sym if quantized else True),
|
||||
)
|
||||
@@ -194,3 +212,26 @@ class INCConfigParser:
|
||||
)
|
||||
|
||||
return get_config(layer_name, quantized)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_group_size(
|
||||
group_size: int | list[int] | tuple[int, int],
|
||||
) -> int | tuple[int, int]:
|
||||
"""Normalize INC group_size into either an int or a 2-D int tuple."""
|
||||
if isinstance(group_size, (list, tuple)):
|
||||
if len(group_size) != 2 or not all(
|
||||
isinstance(value, int) for value in group_size
|
||||
):
|
||||
raise ValueError(
|
||||
"INC block-wise FP8 requires group_size to be a 2-D "
|
||||
f"integer sequence, but found {group_size!r}."
|
||||
)
|
||||
return (group_size[0], group_size[1])
|
||||
|
||||
if not isinstance(group_size, int):
|
||||
raise ValueError(
|
||||
"INC group_size must be an int or a 2-D integer sequence, "
|
||||
f"but found {type(group_size).__name__}."
|
||||
)
|
||||
|
||||
return group_size
|
||||
|
||||
@@ -34,11 +34,14 @@ class INCConfig(QuantizationConfig):
|
||||
Repo: https://github.com/intel/neural-compressor
|
||||
"""
|
||||
|
||||
DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
|
||||
|
||||
SUPPORTED_BITS = {2, 3, 4, 8}
|
||||
SUPPORTED_DTYPES = {"int", "mx_fp"}
|
||||
SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
|
||||
SUPPORTED_FORMATS = {
|
||||
"auto_round:auto_gptq",
|
||||
"auto_round:auto_awq",
|
||||
"auto_round:fp8",
|
||||
"auto_round:llm_compressor",
|
||||
}
|
||||
SUPPORTED_BACKENDS = {
|
||||
@@ -49,23 +52,39 @@ class INCConfig(QuantizationConfig):
|
||||
"awq:marlin",
|
||||
"marlin",
|
||||
}
|
||||
|
||||
FP8_BLOCK_BITS = 8
|
||||
FP8_BLOCK_DATA_TYPE = "fp"
|
||||
FP8_BLOCK_PACKING_FORMAT = "auto_round:fp8"
|
||||
FP8_BLOCK_FMT = "e4m3"
|
||||
FP8_BLOCK_ACTIVATION_SCHEME = "dynamic"
|
||||
|
||||
MXFP8_BITS = 8
|
||||
MXFP8_GROUP_SIZE = 32
|
||||
MXFP8_DATA_TYPE = "mx_fp"
|
||||
MXFP8_PACKING_FORMAT = "auto_round:llm_compressor"
|
||||
|
||||
@staticmethod
|
||||
def _normalize_data_type(data_type: str) -> str:
|
||||
# Keep a small alias for branches that already experimented with "fp8".
|
||||
return "fp" if data_type == "fp8" else data_type
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_bits: int,
|
||||
group_size: int,
|
||||
group_size: int | tuple[int, int],
|
||||
sym: bool = True,
|
||||
packing_format: str = "auto_round:auto_gptq",
|
||||
packing_format: str = DEFAULT_INT_PACKING_FORMAT,
|
||||
block_name_to_quantize: str | list[str] | None = None,
|
||||
extra_config: dict[str, Any] | None = None,
|
||||
data_type: str = "int",
|
||||
backend: str = "auto",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
data_type = self._normalize_data_type(data_type)
|
||||
group_size = INCConfigParser._normalize_group_size(group_size)
|
||||
|
||||
if weight_bits not in self.SUPPORTED_BITS:
|
||||
raise ValueError(
|
||||
f"Unsupported weight_bits: {weight_bits}, "
|
||||
@@ -124,48 +143,118 @@ class INCConfig(QuantizationConfig):
|
||||
# MXFP4 and MXFP8 share data_type "mx_fp" and differ only by bit width.
|
||||
return self.is_mxfp and self.weight_bits == self.MXFP8_BITS
|
||||
|
||||
def _validate_supported_quantization(self) -> None:
|
||||
if self.is_mxfp8:
|
||||
assert self.group_size == self.MXFP8_GROUP_SIZE, (
|
||||
"INC MXFP8 only supports group_size=32, "
|
||||
f"but found group_size={self.group_size}."
|
||||
)
|
||||
assert self.sym, "INC MXFP8 only supports symmetric weights."
|
||||
assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
|
||||
"INC MXFP8 only supports "
|
||||
f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
|
||||
f"but found {self.packing_format!r}."
|
||||
)
|
||||
assert self.backend == "auto", (
|
||||
"INC MXFP8 only supports backend='auto', "
|
||||
f"but found backend={self.backend!r}."
|
||||
)
|
||||
elif self.packing_format == self.MXFP8_PACKING_FORMAT and not self.is_mxfp:
|
||||
raise ValueError(
|
||||
f"packing_format={self.MXFP8_PACKING_FORMAT!r} requires "
|
||||
f"an {self.MXFP8_DATA_TYPE!r} data_type."
|
||||
)
|
||||
|
||||
def _validate_raw_config(self, config: dict[str, Any]) -> None:
|
||||
if not self.is_mxfp8:
|
||||
return
|
||||
|
||||
expected_fields = {
|
||||
"act_bits": self.MXFP8_BITS,
|
||||
"act_data_type": self.MXFP8_DATA_TYPE,
|
||||
"act_group_size": self.MXFP8_GROUP_SIZE,
|
||||
"act_sym": True,
|
||||
"act_dynamic": True,
|
||||
"enable_quanted_input": False,
|
||||
}
|
||||
@classmethod
|
||||
def _check_expected(
|
||||
cls,
|
||||
values: dict[str, Any],
|
||||
expected_fields: dict[str, Any],
|
||||
quantization_name: str,
|
||||
) -> None:
|
||||
for field_name, expected_value in expected_fields.items():
|
||||
actual_value = self.get_from_keys_or(config, [field_name], expected_value)
|
||||
actual_value = cls.get_from_keys_or(values, [field_name], expected_value)
|
||||
assert actual_value == expected_value, (
|
||||
"INC MXFP8 only supports "
|
||||
f"{quantization_name} only supports "
|
||||
f"{field_name}={expected_value!r}, "
|
||||
f"but found {field_name}={actual_value!r}."
|
||||
)
|
||||
|
||||
def _validate_int_quantization(self) -> None:
|
||||
int_packing_formats = {
|
||||
self.DEFAULT_INT_PACKING_FORMAT,
|
||||
"auto_round:auto_awq",
|
||||
}
|
||||
assert isinstance(self.group_size, int), (
|
||||
"INC int quantization requires scalar group_size, "
|
||||
f"but found group_size={self.group_size!r}."
|
||||
)
|
||||
assert self.packing_format in int_packing_formats, (
|
||||
"INC int quantization only supports "
|
||||
f"packing_format in {sorted(int_packing_formats)!r}, "
|
||||
f"but found packing_format={self.packing_format!r}."
|
||||
)
|
||||
|
||||
def _validate_fp8_block_quantization(self) -> None:
|
||||
assert self.weight_bits == self.FP8_BLOCK_BITS, (
|
||||
"INC block-wise FP8 only supports bits=8, "
|
||||
f"but found bits={self.weight_bits}."
|
||||
)
|
||||
assert isinstance(self.group_size, tuple), (
|
||||
"INC block-wise FP8 requires a 2-D group_size, "
|
||||
f"but found group_size={self.group_size!r}."
|
||||
)
|
||||
assert self.sym, "INC block-wise FP8 only supports symmetric weights."
|
||||
assert self.packing_format == self.FP8_BLOCK_PACKING_FORMAT, (
|
||||
"INC block-wise FP8 only supports "
|
||||
f"packing_format={self.FP8_BLOCK_PACKING_FORMAT!r}, "
|
||||
f"but found {self.packing_format!r}."
|
||||
)
|
||||
assert self.backend == "auto", (
|
||||
"INC block-wise FP8 only supports backend='auto', "
|
||||
f"but found backend={self.backend!r}."
|
||||
)
|
||||
|
||||
def _validate_mxfp_quantization(self) -> None:
|
||||
assert self.weight_bits in {4, self.MXFP8_BITS}, (
|
||||
"INC MXFP only supports bits=4 or bits=8, "
|
||||
f"but found bits={self.weight_bits}."
|
||||
)
|
||||
assert self.group_size == self.MXFP8_GROUP_SIZE, (
|
||||
"INC MXFP only supports group_size=32, "
|
||||
f"but found group_size={self.group_size}."
|
||||
)
|
||||
assert self.sym, "INC MXFP only supports symmetric weights."
|
||||
assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
|
||||
"INC MXFP only supports "
|
||||
f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
|
||||
f"but found {self.packing_format!r}."
|
||||
)
|
||||
assert self.backend == "auto", (
|
||||
"INC MXFP only supports backend='auto', "
|
||||
f"but found backend={self.backend!r}."
|
||||
)
|
||||
|
||||
def _validate_supported_quantization(self) -> None:
|
||||
if self.data_type == "int":
|
||||
self._validate_int_quantization()
|
||||
elif self.data_type == self.FP8_BLOCK_DATA_TYPE:
|
||||
self._validate_fp8_block_quantization()
|
||||
elif self.data_type == self.MXFP8_DATA_TYPE:
|
||||
self._validate_mxfp_quantization()
|
||||
|
||||
def _validate_raw_config(self, config: dict[str, Any]) -> None:
|
||||
if self.data_type == self.FP8_BLOCK_DATA_TYPE:
|
||||
assert isinstance(self.group_size, tuple), (
|
||||
"INC block-wise FP8 group_size must be a tuple."
|
||||
)
|
||||
|
||||
expected_fields = {
|
||||
"act_bits": self.FP8_BLOCK_BITS,
|
||||
"act_group_size": self.group_size[0],
|
||||
"act_data_type": self.FP8_BLOCK_DATA_TYPE,
|
||||
"act_sym": True,
|
||||
"act_dynamic": True,
|
||||
"activation_scheme": self.FP8_BLOCK_ACTIVATION_SCHEME,
|
||||
"fmt": self.FP8_BLOCK_FMT,
|
||||
}
|
||||
self._check_expected(
|
||||
config,
|
||||
expected_fields,
|
||||
"INC block-wise FP8",
|
||||
)
|
||||
elif self.is_mxfp8:
|
||||
expected_fields = {
|
||||
"act_bits": self.MXFP8_BITS,
|
||||
"act_data_type": self.MXFP8_DATA_TYPE,
|
||||
"act_group_size": self.MXFP8_GROUP_SIZE,
|
||||
"act_sym": True,
|
||||
"act_dynamic": True,
|
||||
}
|
||||
self._check_expected(
|
||||
config,
|
||||
expected_fields,
|
||||
"INC MXFP8",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "inc"
|
||||
@@ -184,18 +273,37 @@ class INCConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
|
||||
data_type = cls._normalize_data_type(
|
||||
cls.get_from_keys_or(config, ["data_type"], "int")
|
||||
)
|
||||
weight_bits = cls.get_from_keys(config, ["bits"])
|
||||
group_size = INCConfigParser._normalize_group_size(
|
||||
cls.get_from_keys(config, ["group_size"])
|
||||
)
|
||||
packing_format = cls.get_from_keys_or(config, ["packing_format"], None)
|
||||
|
||||
if packing_format is None:
|
||||
if (
|
||||
data_type == cls.FP8_BLOCK_DATA_TYPE
|
||||
and weight_bits == cls.FP8_BLOCK_BITS
|
||||
and isinstance(group_size, tuple)
|
||||
):
|
||||
packing_format = cls.FP8_BLOCK_PACKING_FORMAT
|
||||
elif "mx_fp" in data_type:
|
||||
packing_format = cls.MXFP8_PACKING_FORMAT
|
||||
else:
|
||||
packing_format = cls.DEFAULT_INT_PACKING_FORMAT
|
||||
|
||||
quant_config = cls(
|
||||
weight_bits=cls.get_from_keys(config, ["bits"]),
|
||||
group_size=cls.get_from_keys(config, ["group_size"]),
|
||||
sym=cls.get_from_keys(config, ["sym"]),
|
||||
packing_format=cls.get_from_keys_or(
|
||||
config, ["packing_format"], "auto_round:auto_gptq"
|
||||
),
|
||||
weight_bits=weight_bits,
|
||||
group_size=group_size,
|
||||
sym=cls.get_from_keys_or(config, ["sym"], True),
|
||||
packing_format=packing_format,
|
||||
block_name_to_quantize=cls.get_from_keys_or(
|
||||
config, ["block_name_to_quantize", "to_quant_block_names"], None
|
||||
),
|
||||
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
|
||||
data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
|
||||
data_type=data_type,
|
||||
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
|
||||
)
|
||||
quant_config._validate_raw_config(config)
|
||||
@@ -215,7 +323,6 @@ class INCConfig(QuantizationConfig):
|
||||
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
|
||||
from .schemes.factory import resolve_scheme
|
||||
|
||||
# Match original: check model.-prefixed names for unquantized layers
|
||||
if prefix and self.extra_config:
|
||||
for layer_name in self.extra_config:
|
||||
if (
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .factory import resolve_scheme
|
||||
from .inc_fp8_scheme import INCFp8Scheme
|
||||
from .inc_mxfp4_scheme import INCMxfp4Scheme
|
||||
from .inc_mxfp8_scheme import INCMxfp8Scheme
|
||||
from .inc_scheme import INCLinearScheme, INCScheme
|
||||
@@ -11,6 +12,7 @@ __all__ = [
|
||||
"INCScheme",
|
||||
"INCLinearScheme",
|
||||
"INCMxfp8Scheme",
|
||||
"INCFp8Scheme",
|
||||
"INCWna16Scheme",
|
||||
"INCMxfp4Scheme",
|
||||
"resolve_scheme",
|
||||
|
||||
@@ -9,11 +9,13 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
|
||||
from .inc_fp8_scheme import INCFp8Scheme
|
||||
from .inc_mxfp4_scheme import INCMxfp4Scheme
|
||||
from .inc_mxfp8_scheme import INCMxfp8Scheme
|
||||
from .inc_wna16_scheme import INCWna16Scheme
|
||||
|
||||
scheme_list: list[type[INCScheme]] = [
|
||||
INCFp8Scheme,
|
||||
INCMxfp8Scheme,
|
||||
INCWna16Scheme,
|
||||
INCMxfp4Scheme,
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_input_dtype,
|
||||
)
|
||||
|
||||
from .inc_scheme import INCLinearScheme
|
||||
|
||||
|
||||
class INCFp8LinearScheme(INCLinearScheme):
|
||||
def __init__(self, prefix: str, weight_block_size: tuple[int, int]) -> None:
|
||||
self.quant_config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
activation_scheme="dynamic",
|
||||
weight_block_size=list(weight_block_size),
|
||||
)
|
||||
self.linear_method = Fp8LinearMethod(self.quant_config)
|
||||
self.linear_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return Fp8Config.get_min_capability()
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
self.linear_method.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.linear_method.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.linear_method.apply(layer, x, bias)
|
||||
@@ -0,0 +1,39 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..inc_linear import INCLinearMethod
|
||||
from .inc_scheme import INCScheme
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from ..config_parser import INCLayerConfig
|
||||
from ..inc import INCConfig
|
||||
|
||||
|
||||
class INCFp8Scheme(INCScheme):
|
||||
@staticmethod
|
||||
def can_handle(layer_config: "INCLayerConfig") -> bool:
|
||||
return layer_config.is_fp8_block
|
||||
|
||||
def get_linear_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, layer
|
||||
|
||||
assert isinstance(layer_config.group_size, tuple)
|
||||
|
||||
from .inc_fp8_linear import INCFp8LinearScheme
|
||||
|
||||
return INCLinearMethod(
|
||||
INCFp8LinearScheme(
|
||||
prefix=prefix,
|
||||
weight_block_size=layer_config.group_size,
|
||||
)
|
||||
)
|
||||
@@ -29,6 +29,11 @@ class INCMxfp4LinearMethod(INCLinearScheme):
|
||||
"""
|
||||
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
if not isinstance(layer_config.group_size, int):
|
||||
raise ValueError(
|
||||
"INC MXFP4 requires scalar group_size, "
|
||||
f"but found group_size={layer_config.group_size!r}."
|
||||
)
|
||||
self.group_size = layer_config.group_size or 32
|
||||
self.kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic)
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
|
||||
class INCWNA16LinearScheme(INCLinearScheme):
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
self.layer_config = layer_config
|
||||
self.inner_method = self._build_inner_method()
|
||||
self.linear_method = self._build_inner_method()
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
@@ -43,6 +43,10 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
)
|
||||
|
||||
def _build_gptq_method(self):
|
||||
assert isinstance(self.layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
gptq_type_map = {
|
||||
(4, True): scalar_types.uint4b8,
|
||||
(8, True): scalar_types.uint8b128,
|
||||
@@ -82,6 +86,10 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
)
|
||||
|
||||
def _build_awq_method(self):
|
||||
assert isinstance(self.layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
awq_type_map = {
|
||||
4: scalar_types.uint4,
|
||||
8: scalar_types.uint8,
|
||||
@@ -135,7 +143,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
params_dtype: "torch.dtype",
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
return self.inner_method.create_weights(
|
||||
return self.linear_method.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
@@ -146,7 +154,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
|
||||
return self.inner_method.process_weights_after_loading(layer)
|
||||
return self.linear_method.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -154,7 +162,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
x: "torch.Tensor",
|
||||
bias: "torch.Tensor | None" = None,
|
||||
) -> "torch.Tensor":
|
||||
return self.inner_method.apply(layer, x, bias)
|
||||
return self.linear_method.apply(layer, x, bias)
|
||||
|
||||
|
||||
class INCXPULinearBase(INCLinearScheme):
|
||||
@@ -165,6 +173,9 @@ class INCXPULinearBase(INCLinearScheme):
|
||||
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
self.weight_bits = layer_config.bits
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"INCXPULinearBase requires integer group_size."
|
||||
)
|
||||
self.group_size = layer_config.group_size
|
||||
|
||||
self.sym = layer_config.sym
|
||||
|
||||
@@ -41,6 +41,9 @@ def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> No
|
||||
"which this build of vllm-xpu-kernels does not provide. "
|
||||
f"Layer: {prefix}."
|
||||
)
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
|
||||
@@ -168,6 +171,10 @@ def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
# AutoGPTQMoEMethod selects its fused-MoE backend through the WNA16 oracle
|
||||
# (Marlin on CUDA, XPUExpertsWNA16 on XPU). Gate only on the layer-shape
|
||||
# check like compressed-tensors does; the capability-based
|
||||
@@ -213,6 +220,10 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
use_marlin = layer_config.bits in (4, 8) and check_moe_marlin_supports_layer(
|
||||
layer, layer_config.group_size
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user