[Quantization][INC]Add MXFP8 Linear Support (#47514)
Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com> Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com> Co-authored-by: Yi Liu <yi4.liu@intel.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Yi Liu
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d742856610
commit
5d07e268b1
@@ -18,9 +18,13 @@ from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
|
||||
from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
|
||||
from vllm.model_executor.layers.quantization.inc.schemes import (
|
||||
INCMxfp8Scheme,
|
||||
INCWna16Scheme,
|
||||
resolve_scheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import (
|
||||
INCMxfp8LinearScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import (
|
||||
INCLinearScheme,
|
||||
)
|
||||
@@ -52,12 +56,19 @@ MODELS = [
|
||||
pytest.param(
|
||||
"Intel/Qwen3-8B-w2g64-for-ut",
|
||||
marks=pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_xpu())
|
||||
or current_platform.device_count() < 2,
|
||||
reason="72B INT2 AutoRound model requires XPU with at least 2 devices.",
|
||||
not (current_platform.is_xpu()),
|
||||
reason="INC int2 on XPU requires the ARK backend.",
|
||||
),
|
||||
id="auto_round:auto_gptq_int2_tp2",
|
||||
),
|
||||
pytest.param(
|
||||
"INC4AI/Qwen3-8B-MXFP8-AR",
|
||||
marks=pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_xpu()),
|
||||
reason="MXFP8 AutoRound model only supports CUDA/XPU backend for now.",
|
||||
),
|
||||
id="auto_round:llm_compressor_mxfp8",
|
||||
),
|
||||
]
|
||||
|
||||
MODEL_RUNNER_KWARGS = {
|
||||
@@ -66,6 +77,11 @@ MODEL_RUNNER_KWARGS = {
|
||||
"gpu_memory_utilization": 0.8,
|
||||
"max_model_len": 512,
|
||||
},
|
||||
"INC4AI/Qwen3-8B-MXFP8-AR": {
|
||||
"block_size": 64,
|
||||
"gpu_memory_utilization": 0.8,
|
||||
"max_model_len": 512,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -273,6 +289,39 @@ def test_inc_config_parser_fused_module_requires_consistent_configs() -> None:
|
||||
config.config_parser.resolve(DummyLayer(), "layers.0.self_attn.qkv_proj")
|
||||
|
||||
|
||||
def test_inc_mxfp8() -> None:
|
||||
config = make_config(
|
||||
weight_bits=8,
|
||||
group_size=32,
|
||||
sym=True,
|
||||
packing_format="auto_round:llm_compressor",
|
||||
data_type="mx_fp",
|
||||
)
|
||||
|
||||
assert config.weight_bits == 8
|
||||
assert config.group_size == 32
|
||||
assert config.data_type == "mx_fp"
|
||||
assert config.packing_format == "auto_round:llm_compressor"
|
||||
|
||||
|
||||
def test_inc_config_rejects_invalid_mxfp8_activation_config() -> None:
|
||||
with pytest.raises(AssertionError, match="act_dynamic=True"):
|
||||
INCConfig.from_config(
|
||||
{
|
||||
"bits": 8,
|
||||
"group_size": 32,
|
||||
"sym": True,
|
||||
"packing_format": "auto_round:llm_compressor",
|
||||
"data_type": "mx_fp",
|
||||
"act_bits": 8,
|
||||
"act_data_type": "mx_fp",
|
||||
"act_group_size": 32,
|
||||
"act_sym": True,
|
||||
"act_dynamic": False,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_inc_layer_config_mx_fp_helpers() -> None:
|
||||
layer_config = INCLayerConfig(
|
||||
bits=4,
|
||||
@@ -304,6 +353,22 @@ def test_inc_resolve_scheme_selects_wna16() -> None:
|
||||
assert isinstance(scheme, INCWna16Scheme)
|
||||
|
||||
|
||||
def test_inc_resolve_scheme_selects_mxfp8() -> None:
|
||||
layer_config = INCLayerConfig(
|
||||
bits=8,
|
||||
group_size=32,
|
||||
sym=True,
|
||||
packing_format="auto_round:llm_compressor",
|
||||
backend="auto",
|
||||
data_type="mx_fp",
|
||||
quantized=True,
|
||||
)
|
||||
|
||||
scheme = resolve_scheme(layer_config)
|
||||
|
||||
assert isinstance(scheme, INCMxfp8Scheme)
|
||||
|
||||
|
||||
class DummyLinearScheme(INCLinearScheme):
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple] = []
|
||||
@@ -323,6 +388,76 @@ class DummyLinearScheme(INCLinearScheme):
|
||||
return "applied"
|
||||
|
||||
|
||||
def test_inc_mxfp8_linear_scheme_delegates_to_kernel(monkeypatch) -> None:
|
||||
class DummyKernel:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple] = []
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
self.calls.append(("process", layer))
|
||||
|
||||
def apply_weights(self, layer, x, bias=None):
|
||||
self.calls.append(("apply", layer, x, bias))
|
||||
return "applied"
|
||||
|
||||
kernel = DummyKernel()
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel",
|
||||
lambda: kernel,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.ModelWeightParameter",
|
||||
lambda **kwargs: torch.nn.Parameter(kwargs["data"], requires_grad=False),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.GroupQuantScaleParameter",
|
||||
lambda **kwargs: torch.nn.Parameter(kwargs["data"], requires_grad=False),
|
||||
)
|
||||
|
||||
scheme = INCMxfp8LinearScheme()
|
||||
layer = torch.nn.Module()
|
||||
|
||||
scheme.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=64,
|
||||
output_partition_sizes=[48, 16],
|
||||
input_size=64,
|
||||
output_size=64,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=lambda *args, **kwargs: None,
|
||||
)
|
||||
|
||||
assert layer.weight.shape == (64, 64)
|
||||
assert layer.weight.dtype == torch.float8_e4m3fn
|
||||
assert layer.weight_scale.shape == (64, 2)
|
||||
assert layer.weight_scale.dtype == torch.uint8
|
||||
|
||||
scheme.process_weights_after_loading(layer)
|
||||
result = scheme.apply_weights(layer, torch.randn(1, 64), None)
|
||||
|
||||
assert result == "applied"
|
||||
assert [call[0] for call in kernel.calls] == ["process", "apply"]
|
||||
|
||||
|
||||
def test_inc_mxfp8_linear_scheme_requires_block_32_input(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel",
|
||||
lambda: object(),
|
||||
)
|
||||
scheme = INCMxfp8LinearScheme()
|
||||
|
||||
with pytest.raises(ValueError, match="divisible by 32"):
|
||||
scheme.create_weights(
|
||||
layer=torch.nn.Module(),
|
||||
input_size_per_partition=48,
|
||||
output_partition_sizes=[32],
|
||||
input_size=48,
|
||||
output_size=32,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=lambda *args, **kwargs: None,
|
||||
)
|
||||
|
||||
|
||||
def test_inc_linear_method_delegates() -> None:
|
||||
scheme = DummyLinearScheme()
|
||||
method = INCLinearMethod(scheme)
|
||||
|
||||
@@ -35,8 +35,12 @@ class INCConfig(QuantizationConfig):
|
||||
"""
|
||||
|
||||
SUPPORTED_BITS = {2, 3, 4, 8}
|
||||
SUPPORTED_DTYPES = {"int"}
|
||||
SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq"}
|
||||
SUPPORTED_DTYPES = {"int", "mx_fp"}
|
||||
SUPPORTED_FORMATS = {
|
||||
"auto_round:auto_gptq",
|
||||
"auto_round:auto_awq",
|
||||
"auto_round:llm_compressor",
|
||||
}
|
||||
SUPPORTED_BACKENDS = {
|
||||
"auto",
|
||||
"gptq",
|
||||
@@ -45,6 +49,11 @@ class INCConfig(QuantizationConfig):
|
||||
"awq:marlin",
|
||||
"marlin",
|
||||
}
|
||||
MXFP8_BITS = 8
|
||||
MXFP8_GROUP_SIZE = 32
|
||||
MXFP8_DATA_TYPE = "mx_fp"
|
||||
MXFP8_PACKING_FORMAT = "auto_round:llm_compressor"
|
||||
MXFP8_SUPPORTED_ACT_DTYPES = {"mx_fp", "mx_fp_rceil"}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -65,8 +74,8 @@ class INCConfig(QuantizationConfig):
|
||||
)
|
||||
if data_type not in self.SUPPORTED_DTYPES:
|
||||
raise ValueError(
|
||||
f"Unsupported data_type: {data_type},"
|
||||
f" currently only support {self.SUPPORTED_DTYPES}."
|
||||
f"Unsupported data_type: {data_type}, "
|
||||
f"currently only support {self.SUPPORTED_DTYPES}."
|
||||
)
|
||||
if packing_format not in self.SUPPORTED_FORMATS:
|
||||
raise ValueError(
|
||||
@@ -75,7 +84,7 @@ class INCConfig(QuantizationConfig):
|
||||
)
|
||||
if backend not in self.SUPPORTED_BACKENDS:
|
||||
raise ValueError(
|
||||
f"Unsupported backend: {backend}, "
|
||||
f"Unsupported backend: {backend}, "
|
||||
f"currently only support {self.SUPPORTED_BACKENDS}."
|
||||
)
|
||||
|
||||
@@ -94,12 +103,59 @@ class INCConfig(QuantizationConfig):
|
||||
self.pack_factor = Fraction(32, weight_bits)
|
||||
self.config_parser = INCConfigParser(self)
|
||||
|
||||
self._validate_supported_quantization()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"INCConfig(weight_bits={self.weight_bits}, "
|
||||
f"group_size={self.group_size}, sym={self.sym})"
|
||||
)
|
||||
|
||||
def _validate_supported_quantization(self) -> None:
|
||||
if self.data_type == self.MXFP8_DATA_TYPE:
|
||||
assert self.weight_bits == self.MXFP8_BITS, (
|
||||
f"INC MXFP8 only supports bits=8, but found bits={self.weight_bits}."
|
||||
)
|
||||
assert self.group_size == self.MXFP8_GROUP_SIZE, (
|
||||
"INC MXFP8 only supports group_size=32, "
|
||||
f"but found group_size={self.group_size}."
|
||||
)
|
||||
assert self.sym, "INC MXFP8 only supports symmetric weights."
|
||||
assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
|
||||
"INC MXFP8 only supports "
|
||||
f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
|
||||
f"but found {self.packing_format!r}."
|
||||
)
|
||||
assert self.backend == "auto", (
|
||||
"INC MXFP8 only supports backend='auto', "
|
||||
f"but found backend={self.backend!r}."
|
||||
)
|
||||
elif self.packing_format == self.MXFP8_PACKING_FORMAT:
|
||||
raise ValueError(
|
||||
f"packing_format={self.MXFP8_PACKING_FORMAT!r} requires "
|
||||
f"data_type={self.MXFP8_DATA_TYPE!r}."
|
||||
)
|
||||
|
||||
def _validate_raw_config(self, config: dict[str, Any]) -> None:
|
||||
if self.data_type != self.MXFP8_DATA_TYPE:
|
||||
return
|
||||
|
||||
expected_fields = {
|
||||
"act_bits": self.MXFP8_BITS,
|
||||
"act_data_type": self.MXFP8_DATA_TYPE,
|
||||
"act_group_size": self.MXFP8_GROUP_SIZE,
|
||||
"act_sym": True,
|
||||
"act_dynamic": True,
|
||||
"enable_quanted_input": False,
|
||||
}
|
||||
for field_name, expected_value in expected_fields.items():
|
||||
actual_value = self.get_from_keys_or(config, [field_name], expected_value)
|
||||
assert actual_value == expected_value, (
|
||||
"INC MXFP8 only supports "
|
||||
f"{field_name}={expected_value!r}, "
|
||||
f"but found {field_name}={actual_value!r}."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "inc"
|
||||
@@ -118,7 +174,7 @@ class INCConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
|
||||
return cls(
|
||||
quant_config = cls(
|
||||
weight_bits=cls.get_from_keys(config, ["bits"]),
|
||||
group_size=cls.get_from_keys(config, ["group_size"]),
|
||||
sym=cls.get_from_keys(config, ["sym"]),
|
||||
@@ -132,6 +188,8 @@ class INCConfig(QuantizationConfig):
|
||||
data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
|
||||
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
|
||||
)
|
||||
quant_config._validate_raw_config(config)
|
||||
return quant_config
|
||||
|
||||
def get_layer_config(self, layer, layer_name: str):
|
||||
return self.config_parser.get_layer_config(layer, layer_name)
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .factory import resolve_scheme
|
||||
from .inc_mxfp8_scheme import INCMxfp8Scheme
|
||||
from .inc_scheme import INCLinearScheme, INCScheme
|
||||
from .inc_wna16_scheme import INCWna16Scheme
|
||||
|
||||
__all__ = [
|
||||
"INCScheme",
|
||||
"INCLinearScheme",
|
||||
"INCMxfp8Scheme",
|
||||
"INCWna16Scheme",
|
||||
"resolve_scheme",
|
||||
]
|
||||
|
||||
@@ -9,9 +9,11 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
|
||||
from .inc_mxfp8_scheme import INCMxfp8Scheme
|
||||
from .inc_wna16_scheme import INCWna16Scheme
|
||||
|
||||
scheme_list: list[type[INCScheme]] = [
|
||||
INCMxfp8Scheme,
|
||||
INCWna16Scheme,
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
)
|
||||
|
||||
from .inc_scheme import INCLinearScheme
|
||||
|
||||
|
||||
class INCMxfp8LinearScheme(INCLinearScheme):
|
||||
def __init__(self) -> None:
|
||||
self.kernel = init_mxfp8_linear_kernel()
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 75
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
del input_size, output_size
|
||||
if input_size_per_partition % MXFP8_BLOCK_SIZE != 0:
|
||||
raise ValueError(
|
||||
"INC MXFP8 requires input_size_per_partition "
|
||||
f"({input_size_per_partition}) to be divisible by "
|
||||
f"{MXFP8_BLOCK_SIZE}."
|
||||
)
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=extra_weight_attrs.get("weight_loader"),
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=extra_weight_attrs.get("weight_loader"),
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
@@ -0,0 +1,31 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..inc_linear import INCLinearMethod
|
||||
from .inc_scheme import INCScheme
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from ..config_parser import INCLayerConfig
|
||||
from ..inc import INCConfig
|
||||
|
||||
|
||||
class INCMxfp8Scheme(INCScheme):
|
||||
@staticmethod
|
||||
def can_handle(layer_config: "INCLayerConfig") -> bool:
|
||||
return layer_config.is_mxfp8
|
||||
|
||||
def get_linear_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, layer, prefix, layer_config
|
||||
from .inc_mxfp8_linear import INCMxfp8LinearScheme
|
||||
|
||||
return INCLinearMethod(INCMxfp8LinearScheme())
|
||||
@@ -166,6 +166,7 @@ class INCXPULinearBase(INCLinearScheme):
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
self.weight_bits = layer_config.bits
|
||||
self.group_size = layer_config.group_size
|
||||
|
||||
self.sym = layer_config.sym
|
||||
self.pack_factor = 32 // self.weight_bits
|
||||
self.is_awq_packed = layer_config.is_awq
|
||||
|
||||
Reference in New Issue
Block a user