From 5d07e268b11ae0240d3af2582516861dfe52e86a Mon Sep 17 00:00:00 2001 From: Zhenzhong Xu Date: Mon, 27 Jul 2026 14:26:31 +0800 Subject: [PATCH] [Quantization][INC]Add MXFP8 Linear Support (#47514) Signed-off-by: Zhenzhong1 Signed-off-by: Zhenzhong Xu Co-authored-by: Yi Liu Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/quantization/test_auto_round.py | 141 +++++++++++++++++- .../layers/quantization/inc/inc.py | 70 ++++++++- .../quantization/inc/schemes/__init__.py | 2 + .../quantization/inc/schemes/factory.py | 2 + .../inc/schemes/inc_mxfp8_linear.py | 84 +++++++++++ .../inc/schemes/inc_mxfp8_scheme.py | 31 ++++ .../inc/schemes/inc_wna16_linear.py | 1 + 7 files changed, 322 insertions(+), 9 deletions(-) create mode 100644 vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_linear.py create mode 100644 vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_scheme.py diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index 6df7211549..732080fc96 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -18,9 +18,13 @@ from vllm.model_executor.layers.quantization.inc import INCConfig from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod from vllm.model_executor.layers.quantization.inc.schemes import ( + INCMxfp8Scheme, INCWna16Scheme, resolve_scheme, ) +from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import ( + INCMxfp8LinearScheme, +) from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import ( INCLinearScheme, ) @@ -52,12 +56,19 @@ MODELS = [ pytest.param( "Intel/Qwen3-8B-w2g64-for-ut", marks=pytest.mark.skipif( - not (current_platform.is_cuda() or current_platform.is_xpu()) - or current_platform.device_count() < 2, - reason="72B INT2 AutoRound model requires XPU with at least 2 devices.", + not (current_platform.is_xpu()), + reason="INC int2 on XPU requires the ARK backend.", ), id="auto_round:auto_gptq_int2_tp2", ), + pytest.param( + "INC4AI/Qwen3-8B-MXFP8-AR", + marks=pytest.mark.skipif( + not (current_platform.is_cuda() or current_platform.is_xpu()), + reason="MXFP8 AutoRound model only supports CUDA/XPU backend for now.", + ), + id="auto_round:llm_compressor_mxfp8", + ), ] MODEL_RUNNER_KWARGS = { @@ -66,6 +77,11 @@ MODEL_RUNNER_KWARGS = { "gpu_memory_utilization": 0.8, "max_model_len": 512, }, + "INC4AI/Qwen3-8B-MXFP8-AR": { + "block_size": 64, + "gpu_memory_utilization": 0.8, + "max_model_len": 512, + }, } @@ -273,6 +289,39 @@ def test_inc_config_parser_fused_module_requires_consistent_configs() -> None: config.config_parser.resolve(DummyLayer(), "layers.0.self_attn.qkv_proj") +def test_inc_mxfp8() -> None: + config = make_config( + weight_bits=8, + group_size=32, + sym=True, + packing_format="auto_round:llm_compressor", + data_type="mx_fp", + ) + + assert config.weight_bits == 8 + assert config.group_size == 32 + assert config.data_type == "mx_fp" + assert config.packing_format == "auto_round:llm_compressor" + + +def test_inc_config_rejects_invalid_mxfp8_activation_config() -> None: + with pytest.raises(AssertionError, match="act_dynamic=True"): + INCConfig.from_config( + { + "bits": 8, + "group_size": 32, + "sym": True, + "packing_format": "auto_round:llm_compressor", + "data_type": "mx_fp", + "act_bits": 8, + "act_data_type": "mx_fp", + "act_group_size": 32, + "act_sym": True, + "act_dynamic": False, + } + ) + + def test_inc_layer_config_mx_fp_helpers() -> None: layer_config = INCLayerConfig( bits=4, @@ -304,6 +353,22 @@ def test_inc_resolve_scheme_selects_wna16() -> None: assert isinstance(scheme, INCWna16Scheme) +def test_inc_resolve_scheme_selects_mxfp8() -> None: + layer_config = INCLayerConfig( + bits=8, + group_size=32, + sym=True, + packing_format="auto_round:llm_compressor", + backend="auto", + data_type="mx_fp", + quantized=True, + ) + + scheme = resolve_scheme(layer_config) + + assert isinstance(scheme, INCMxfp8Scheme) + + class DummyLinearScheme(INCLinearScheme): def __init__(self) -> None: self.calls: list[tuple] = [] @@ -323,6 +388,76 @@ class DummyLinearScheme(INCLinearScheme): return "applied" +def test_inc_mxfp8_linear_scheme_delegates_to_kernel(monkeypatch) -> None: + class DummyKernel: + def __init__(self) -> None: + self.calls: list[tuple] = [] + + def process_weights_after_loading(self, layer) -> None: + self.calls.append(("process", layer)) + + def apply_weights(self, layer, x, bias=None): + self.calls.append(("apply", layer, x, bias)) + return "applied" + + kernel = DummyKernel() + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel", + lambda: kernel, + ) + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.ModelWeightParameter", + lambda **kwargs: torch.nn.Parameter(kwargs["data"], requires_grad=False), + ) + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.GroupQuantScaleParameter", + lambda **kwargs: torch.nn.Parameter(kwargs["data"], requires_grad=False), + ) + + scheme = INCMxfp8LinearScheme() + layer = torch.nn.Module() + + scheme.create_weights( + layer=layer, + input_size_per_partition=64, + output_partition_sizes=[48, 16], + input_size=64, + output_size=64, + params_dtype=torch.bfloat16, + weight_loader=lambda *args, **kwargs: None, + ) + + assert layer.weight.shape == (64, 64) + assert layer.weight.dtype == torch.float8_e4m3fn + assert layer.weight_scale.shape == (64, 2) + assert layer.weight_scale.dtype == torch.uint8 + + scheme.process_weights_after_loading(layer) + result = scheme.apply_weights(layer, torch.randn(1, 64), None) + + assert result == "applied" + assert [call[0] for call in kernel.calls] == ["process", "apply"] + + +def test_inc_mxfp8_linear_scheme_requires_block_32_input(monkeypatch) -> None: + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel", + lambda: object(), + ) + scheme = INCMxfp8LinearScheme() + + with pytest.raises(ValueError, match="divisible by 32"): + scheme.create_weights( + layer=torch.nn.Module(), + input_size_per_partition=48, + output_partition_sizes=[32], + input_size=48, + output_size=32, + params_dtype=torch.bfloat16, + weight_loader=lambda *args, **kwargs: None, + ) + + def test_inc_linear_method_delegates() -> None: scheme = DummyLinearScheme() method = INCLinearMethod(scheme) diff --git a/vllm/model_executor/layers/quantization/inc/inc.py b/vllm/model_executor/layers/quantization/inc/inc.py index 86fa7cefcf..ad8389a8b9 100644 --- a/vllm/model_executor/layers/quantization/inc/inc.py +++ b/vllm/model_executor/layers/quantization/inc/inc.py @@ -35,8 +35,12 @@ class INCConfig(QuantizationConfig): """ SUPPORTED_BITS = {2, 3, 4, 8} - SUPPORTED_DTYPES = {"int"} - SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq"} + SUPPORTED_DTYPES = {"int", "mx_fp"} + SUPPORTED_FORMATS = { + "auto_round:auto_gptq", + "auto_round:auto_awq", + "auto_round:llm_compressor", + } SUPPORTED_BACKENDS = { "auto", "gptq", @@ -45,6 +49,11 @@ class INCConfig(QuantizationConfig): "awq:marlin", "marlin", } + MXFP8_BITS = 8 + MXFP8_GROUP_SIZE = 32 + MXFP8_DATA_TYPE = "mx_fp" + MXFP8_PACKING_FORMAT = "auto_round:llm_compressor" + MXFP8_SUPPORTED_ACT_DTYPES = {"mx_fp", "mx_fp_rceil"} def __init__( self, @@ -65,8 +74,8 @@ class INCConfig(QuantizationConfig): ) if data_type not in self.SUPPORTED_DTYPES: raise ValueError( - f"Unsupported data_type: {data_type}," - f" currently only support {self.SUPPORTED_DTYPES}." + f"Unsupported data_type: {data_type}, " + f"currently only support {self.SUPPORTED_DTYPES}." ) if packing_format not in self.SUPPORTED_FORMATS: raise ValueError( @@ -75,7 +84,7 @@ class INCConfig(QuantizationConfig): ) if backend not in self.SUPPORTED_BACKENDS: raise ValueError( - f"Unsupported backend: {backend}, " + f"Unsupported backend: {backend}, " f"currently only support {self.SUPPORTED_BACKENDS}." ) @@ -94,12 +103,59 @@ class INCConfig(QuantizationConfig): self.pack_factor = Fraction(32, weight_bits) self.config_parser = INCConfigParser(self) + self._validate_supported_quantization() + def __repr__(self) -> str: return ( f"INCConfig(weight_bits={self.weight_bits}, " f"group_size={self.group_size}, sym={self.sym})" ) + def _validate_supported_quantization(self) -> None: + if self.data_type == self.MXFP8_DATA_TYPE: + assert self.weight_bits == self.MXFP8_BITS, ( + f"INC MXFP8 only supports bits=8, but found bits={self.weight_bits}." + ) + assert self.group_size == self.MXFP8_GROUP_SIZE, ( + "INC MXFP8 only supports group_size=32, " + f"but found group_size={self.group_size}." + ) + assert self.sym, "INC MXFP8 only supports symmetric weights." + assert self.packing_format == self.MXFP8_PACKING_FORMAT, ( + "INC MXFP8 only supports " + f"packing_format={self.MXFP8_PACKING_FORMAT!r}, " + f"but found {self.packing_format!r}." + ) + assert self.backend == "auto", ( + "INC MXFP8 only supports backend='auto', " + f"but found backend={self.backend!r}." + ) + elif self.packing_format == self.MXFP8_PACKING_FORMAT: + raise ValueError( + f"packing_format={self.MXFP8_PACKING_FORMAT!r} requires " + f"data_type={self.MXFP8_DATA_TYPE!r}." + ) + + def _validate_raw_config(self, config: dict[str, Any]) -> None: + if self.data_type != self.MXFP8_DATA_TYPE: + return + + expected_fields = { + "act_bits": self.MXFP8_BITS, + "act_data_type": self.MXFP8_DATA_TYPE, + "act_group_size": self.MXFP8_GROUP_SIZE, + "act_sym": True, + "act_dynamic": True, + "enable_quanted_input": False, + } + for field_name, expected_value in expected_fields.items(): + actual_value = self.get_from_keys_or(config, [field_name], expected_value) + assert actual_value == expected_value, ( + "INC MXFP8 only supports " + f"{field_name}={expected_value!r}, " + f"but found {field_name}={actual_value!r}." + ) + @classmethod def get_name(cls) -> QuantizationMethods: return "inc" @@ -118,7 +174,7 @@ class INCConfig(QuantizationConfig): @classmethod def from_config(cls, config: dict[str, Any]) -> "INCConfig": - return cls( + quant_config = cls( weight_bits=cls.get_from_keys(config, ["bits"]), group_size=cls.get_from_keys(config, ["group_size"]), sym=cls.get_from_keys(config, ["sym"]), @@ -132,6 +188,8 @@ class INCConfig(QuantizationConfig): data_type=cls.get_from_keys_or(config, ["data_type"], "int"), backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"), ) + quant_config._validate_raw_config(config) + return quant_config def get_layer_config(self, layer, layer_name: str): return self.config_parser.get_layer_config(layer, layer_name) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/__init__.py b/vllm/model_executor/layers/quantization/inc/schemes/__init__.py index ea6c0a00d8..e4dc5cb5dc 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/__init__.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/__init__.py @@ -2,12 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from .factory import resolve_scheme +from .inc_mxfp8_scheme import INCMxfp8Scheme from .inc_scheme import INCLinearScheme, INCScheme from .inc_wna16_scheme import INCWna16Scheme __all__ = [ "INCScheme", "INCLinearScheme", + "INCMxfp8Scheme", "INCWna16Scheme", "resolve_scheme", ] diff --git a/vllm/model_executor/layers/quantization/inc/schemes/factory.py b/vllm/model_executor/layers/quantization/inc/schemes/factory.py index 4ae85ed9a8..30ce420b50 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/factory.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/factory.py @@ -9,9 +9,11 @@ if TYPE_CHECKING: def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme": + from .inc_mxfp8_scheme import INCMxfp8Scheme from .inc_wna16_scheme import INCWna16Scheme scheme_list: list[type[INCScheme]] = [ + INCMxfp8Scheme, INCWna16Scheme, ] diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_linear.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_linear.py new file mode 100644 index 0000000000..775b5f5bba --- /dev/null +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_linear.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch + +from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + MXFP8_BLOCK_SIZE, + MXFP8_SCALE_DTYPE, + MXFP8_VALUE_DTYPE, +) +from vllm.model_executor.parameter import ( + GroupQuantScaleParameter, + ModelWeightParameter, +) + +from .inc_scheme import INCLinearScheme + + +class INCMxfp8LinearScheme(INCLinearScheme): + def __init__(self) -> None: + self.kernel = init_mxfp8_linear_kernel() + + @classmethod + def get_min_capability(cls) -> int: + return 75 + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + del input_size, output_size + if input_size_per_partition % MXFP8_BLOCK_SIZE != 0: + raise ValueError( + "INC MXFP8 requires input_size_per_partition " + f"({input_size_per_partition}) to be divisible by " + f"{MXFP8_BLOCK_SIZE}." + ) + + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.params_dtype = params_dtype + + weight = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=MXFP8_VALUE_DTYPE, + ), + input_dim=1, + output_dim=0, + weight_loader=extra_weight_attrs.get("weight_loader"), + ) + layer.register_parameter("weight", weight) + + weight_scale = GroupQuantScaleParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // MXFP8_BLOCK_SIZE, + dtype=MXFP8_SCALE_DTYPE, + ), + input_dim=1, + output_dim=0, + weight_loader=extra_weight_attrs.get("weight_loader"), + ) + layer.register_parameter("weight_scale", weight_scale) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + self.kernel.process_weights_after_loading(layer) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + return self.kernel.apply_weights(layer, x, bias) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_scheme.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_scheme.py new file mode 100644 index 0000000000..2e251a8a54 --- /dev/null +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_scheme.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import TYPE_CHECKING + +from ..inc_linear import INCLinearMethod +from .inc_scheme import INCScheme + +if TYPE_CHECKING: + import torch + + from ..config_parser import INCLayerConfig + from ..inc import INCConfig + + +class INCMxfp8Scheme(INCScheme): + @staticmethod + def can_handle(layer_config: "INCLayerConfig") -> bool: + return layer_config.is_mxfp8 + + def get_linear_method( + self, + config: "INCConfig", + layer: "torch.nn.Module", + prefix: str, + layer_config: "INCLayerConfig", + ): + del config, layer, prefix, layer_config + from .inc_mxfp8_linear import INCMxfp8LinearScheme + + return INCLinearMethod(INCMxfp8LinearScheme()) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py index dd6c2fa2ea..5c99fd98b5 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py @@ -166,6 +166,7 @@ class INCXPULinearBase(INCLinearScheme): def __init__(self, layer_config: "INCLayerConfig") -> None: self.weight_bits = layer_config.bits self.group_size = layer_config.group_size + self.sym = layer_config.sym self.pack_factor = 32 // self.weight_bits self.is_awq_packed = layer_config.is_awq