[AutoRound] Support AutoRound Format Block-Wise FP8 in vLLM (#47434)
Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com> Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
dafbef15a1
commit
bd575a0d0b
@@ -34,11 +34,14 @@ class INCConfig(QuantizationConfig):
|
||||
Repo: https://github.com/intel/neural-compressor
|
||||
"""
|
||||
|
||||
DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
|
||||
|
||||
SUPPORTED_BITS = {2, 3, 4, 8}
|
||||
SUPPORTED_DTYPES = {"int", "mx_fp"}
|
||||
SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
|
||||
SUPPORTED_FORMATS = {
|
||||
"auto_round:auto_gptq",
|
||||
"auto_round:auto_awq",
|
||||
"auto_round:fp8",
|
||||
"auto_round:llm_compressor",
|
||||
}
|
||||
SUPPORTED_BACKENDS = {
|
||||
@@ -49,23 +52,39 @@ class INCConfig(QuantizationConfig):
|
||||
"awq:marlin",
|
||||
"marlin",
|
||||
}
|
||||
|
||||
FP8_BLOCK_BITS = 8
|
||||
FP8_BLOCK_DATA_TYPE = "fp"
|
||||
FP8_BLOCK_PACKING_FORMAT = "auto_round:fp8"
|
||||
FP8_BLOCK_FMT = "e4m3"
|
||||
FP8_BLOCK_ACTIVATION_SCHEME = "dynamic"
|
||||
|
||||
MXFP8_BITS = 8
|
||||
MXFP8_GROUP_SIZE = 32
|
||||
MXFP8_DATA_TYPE = "mx_fp"
|
||||
MXFP8_PACKING_FORMAT = "auto_round:llm_compressor"
|
||||
|
||||
@staticmethod
|
||||
def _normalize_data_type(data_type: str) -> str:
|
||||
# Keep a small alias for branches that already experimented with "fp8".
|
||||
return "fp" if data_type == "fp8" else data_type
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_bits: int,
|
||||
group_size: int,
|
||||
group_size: int | tuple[int, int],
|
||||
sym: bool = True,
|
||||
packing_format: str = "auto_round:auto_gptq",
|
||||
packing_format: str = DEFAULT_INT_PACKING_FORMAT,
|
||||
block_name_to_quantize: str | list[str] | None = None,
|
||||
extra_config: dict[str, Any] | None = None,
|
||||
data_type: str = "int",
|
||||
backend: str = "auto",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
data_type = self._normalize_data_type(data_type)
|
||||
group_size = INCConfigParser._normalize_group_size(group_size)
|
||||
|
||||
if weight_bits not in self.SUPPORTED_BITS:
|
||||
raise ValueError(
|
||||
f"Unsupported weight_bits: {weight_bits}, "
|
||||
@@ -124,48 +143,118 @@ class INCConfig(QuantizationConfig):
|
||||
# MXFP4 and MXFP8 share data_type "mx_fp" and differ only by bit width.
|
||||
return self.is_mxfp and self.weight_bits == self.MXFP8_BITS
|
||||
|
||||
def _validate_supported_quantization(self) -> None:
|
||||
if self.is_mxfp8:
|
||||
assert self.group_size == self.MXFP8_GROUP_SIZE, (
|
||||
"INC MXFP8 only supports group_size=32, "
|
||||
f"but found group_size={self.group_size}."
|
||||
)
|
||||
assert self.sym, "INC MXFP8 only supports symmetric weights."
|
||||
assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
|
||||
"INC MXFP8 only supports "
|
||||
f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
|
||||
f"but found {self.packing_format!r}."
|
||||
)
|
||||
assert self.backend == "auto", (
|
||||
"INC MXFP8 only supports backend='auto', "
|
||||
f"but found backend={self.backend!r}."
|
||||
)
|
||||
elif self.packing_format == self.MXFP8_PACKING_FORMAT and not self.is_mxfp:
|
||||
raise ValueError(
|
||||
f"packing_format={self.MXFP8_PACKING_FORMAT!r} requires "
|
||||
f"an {self.MXFP8_DATA_TYPE!r} data_type."
|
||||
)
|
||||
|
||||
def _validate_raw_config(self, config: dict[str, Any]) -> None:
|
||||
if not self.is_mxfp8:
|
||||
return
|
||||
|
||||
expected_fields = {
|
||||
"act_bits": self.MXFP8_BITS,
|
||||
"act_data_type": self.MXFP8_DATA_TYPE,
|
||||
"act_group_size": self.MXFP8_GROUP_SIZE,
|
||||
"act_sym": True,
|
||||
"act_dynamic": True,
|
||||
"enable_quanted_input": False,
|
||||
}
|
||||
@classmethod
|
||||
def _check_expected(
|
||||
cls,
|
||||
values: dict[str, Any],
|
||||
expected_fields: dict[str, Any],
|
||||
quantization_name: str,
|
||||
) -> None:
|
||||
for field_name, expected_value in expected_fields.items():
|
||||
actual_value = self.get_from_keys_or(config, [field_name], expected_value)
|
||||
actual_value = cls.get_from_keys_or(values, [field_name], expected_value)
|
||||
assert actual_value == expected_value, (
|
||||
"INC MXFP8 only supports "
|
||||
f"{quantization_name} only supports "
|
||||
f"{field_name}={expected_value!r}, "
|
||||
f"but found {field_name}={actual_value!r}."
|
||||
)
|
||||
|
||||
def _validate_int_quantization(self) -> None:
|
||||
int_packing_formats = {
|
||||
self.DEFAULT_INT_PACKING_FORMAT,
|
||||
"auto_round:auto_awq",
|
||||
}
|
||||
assert isinstance(self.group_size, int), (
|
||||
"INC int quantization requires scalar group_size, "
|
||||
f"but found group_size={self.group_size!r}."
|
||||
)
|
||||
assert self.packing_format in int_packing_formats, (
|
||||
"INC int quantization only supports "
|
||||
f"packing_format in {sorted(int_packing_formats)!r}, "
|
||||
f"but found packing_format={self.packing_format!r}."
|
||||
)
|
||||
|
||||
def _validate_fp8_block_quantization(self) -> None:
|
||||
assert self.weight_bits == self.FP8_BLOCK_BITS, (
|
||||
"INC block-wise FP8 only supports bits=8, "
|
||||
f"but found bits={self.weight_bits}."
|
||||
)
|
||||
assert isinstance(self.group_size, tuple), (
|
||||
"INC block-wise FP8 requires a 2-D group_size, "
|
||||
f"but found group_size={self.group_size!r}."
|
||||
)
|
||||
assert self.sym, "INC block-wise FP8 only supports symmetric weights."
|
||||
assert self.packing_format == self.FP8_BLOCK_PACKING_FORMAT, (
|
||||
"INC block-wise FP8 only supports "
|
||||
f"packing_format={self.FP8_BLOCK_PACKING_FORMAT!r}, "
|
||||
f"but found {self.packing_format!r}."
|
||||
)
|
||||
assert self.backend == "auto", (
|
||||
"INC block-wise FP8 only supports backend='auto', "
|
||||
f"but found backend={self.backend!r}."
|
||||
)
|
||||
|
||||
def _validate_mxfp_quantization(self) -> None:
|
||||
assert self.weight_bits in {4, self.MXFP8_BITS}, (
|
||||
"INC MXFP only supports bits=4 or bits=8, "
|
||||
f"but found bits={self.weight_bits}."
|
||||
)
|
||||
assert self.group_size == self.MXFP8_GROUP_SIZE, (
|
||||
"INC MXFP only supports group_size=32, "
|
||||
f"but found group_size={self.group_size}."
|
||||
)
|
||||
assert self.sym, "INC MXFP only supports symmetric weights."
|
||||
assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
|
||||
"INC MXFP only supports "
|
||||
f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
|
||||
f"but found {self.packing_format!r}."
|
||||
)
|
||||
assert self.backend == "auto", (
|
||||
"INC MXFP only supports backend='auto', "
|
||||
f"but found backend={self.backend!r}."
|
||||
)
|
||||
|
||||
def _validate_supported_quantization(self) -> None:
|
||||
if self.data_type == "int":
|
||||
self._validate_int_quantization()
|
||||
elif self.data_type == self.FP8_BLOCK_DATA_TYPE:
|
||||
self._validate_fp8_block_quantization()
|
||||
elif self.data_type == self.MXFP8_DATA_TYPE:
|
||||
self._validate_mxfp_quantization()
|
||||
|
||||
def _validate_raw_config(self, config: dict[str, Any]) -> None:
|
||||
if self.data_type == self.FP8_BLOCK_DATA_TYPE:
|
||||
assert isinstance(self.group_size, tuple), (
|
||||
"INC block-wise FP8 group_size must be a tuple."
|
||||
)
|
||||
|
||||
expected_fields = {
|
||||
"act_bits": self.FP8_BLOCK_BITS,
|
||||
"act_group_size": self.group_size[0],
|
||||
"act_data_type": self.FP8_BLOCK_DATA_TYPE,
|
||||
"act_sym": True,
|
||||
"act_dynamic": True,
|
||||
"activation_scheme": self.FP8_BLOCK_ACTIVATION_SCHEME,
|
||||
"fmt": self.FP8_BLOCK_FMT,
|
||||
}
|
||||
self._check_expected(
|
||||
config,
|
||||
expected_fields,
|
||||
"INC block-wise FP8",
|
||||
)
|
||||
elif self.is_mxfp8:
|
||||
expected_fields = {
|
||||
"act_bits": self.MXFP8_BITS,
|
||||
"act_data_type": self.MXFP8_DATA_TYPE,
|
||||
"act_group_size": self.MXFP8_GROUP_SIZE,
|
||||
"act_sym": True,
|
||||
"act_dynamic": True,
|
||||
}
|
||||
self._check_expected(
|
||||
config,
|
||||
expected_fields,
|
||||
"INC MXFP8",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "inc"
|
||||
@@ -184,18 +273,37 @@ class INCConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
|
||||
data_type = cls._normalize_data_type(
|
||||
cls.get_from_keys_or(config, ["data_type"], "int")
|
||||
)
|
||||
weight_bits = cls.get_from_keys(config, ["bits"])
|
||||
group_size = INCConfigParser._normalize_group_size(
|
||||
cls.get_from_keys(config, ["group_size"])
|
||||
)
|
||||
packing_format = cls.get_from_keys_or(config, ["packing_format"], None)
|
||||
|
||||
if packing_format is None:
|
||||
if (
|
||||
data_type == cls.FP8_BLOCK_DATA_TYPE
|
||||
and weight_bits == cls.FP8_BLOCK_BITS
|
||||
and isinstance(group_size, tuple)
|
||||
):
|
||||
packing_format = cls.FP8_BLOCK_PACKING_FORMAT
|
||||
elif "mx_fp" in data_type:
|
||||
packing_format = cls.MXFP8_PACKING_FORMAT
|
||||
else:
|
||||
packing_format = cls.DEFAULT_INT_PACKING_FORMAT
|
||||
|
||||
quant_config = cls(
|
||||
weight_bits=cls.get_from_keys(config, ["bits"]),
|
||||
group_size=cls.get_from_keys(config, ["group_size"]),
|
||||
sym=cls.get_from_keys(config, ["sym"]),
|
||||
packing_format=cls.get_from_keys_or(
|
||||
config, ["packing_format"], "auto_round:auto_gptq"
|
||||
),
|
||||
weight_bits=weight_bits,
|
||||
group_size=group_size,
|
||||
sym=cls.get_from_keys_or(config, ["sym"], True),
|
||||
packing_format=packing_format,
|
||||
block_name_to_quantize=cls.get_from_keys_or(
|
||||
config, ["block_name_to_quantize", "to_quant_block_names"], None
|
||||
),
|
||||
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
|
||||
data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
|
||||
data_type=data_type,
|
||||
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
|
||||
)
|
||||
quant_config._validate_raw_config(config)
|
||||
@@ -215,7 +323,6 @@ class INCConfig(QuantizationConfig):
|
||||
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
|
||||
from .schemes.factory import resolve_scheme
|
||||
|
||||
# Match original: check model.-prefixed names for unquantized layers
|
||||
if prefix and self.extra_config:
|
||||
for layer_name in self.extra_config:
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user