[Quantization][INC]Add MXFP8 Linear Support (#47514)

Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com>
Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com>
Co-authored-by: Yi Liu <yi4.liu@intel.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Zhenzhong Xu
2026-07-27 14:26:31 +08:00
committed by GitHub
co-authored by Yi Liu mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d742856610
commit 5d07e268b1
7 changed files with 322 additions and 9 deletions
+138 -3
View File
@@ -18,9 +18,13 @@ from vllm.model_executor.layers.quantization.inc import INCConfig
from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
from vllm.model_executor.layers.quantization.inc.schemes import (
INCMxfp8Scheme,
INCWna16Scheme,
resolve_scheme,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import (
INCMxfp8LinearScheme,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_scheme import (
INCLinearScheme,
)
@@ -52,12 +56,19 @@ MODELS = [
pytest.param(
"Intel/Qwen3-8B-w2g64-for-ut",
marks=pytest.mark.skipif(
not (current_platform.is_cuda() or current_platform.is_xpu())
or current_platform.device_count() < 2,
reason="72B INT2 AutoRound model requires XPU with at least 2 devices.",
not (current_platform.is_xpu()),
reason="INC int2 on XPU requires the ARK backend.",
),
id="auto_round:auto_gptq_int2_tp2",
),
pytest.param(
"INC4AI/Qwen3-8B-MXFP8-AR",
marks=pytest.mark.skipif(
not (current_platform.is_cuda() or current_platform.is_xpu()),
reason="MXFP8 AutoRound model only supports CUDA/XPU backend for now.",
),
id="auto_round:llm_compressor_mxfp8",
),
]
MODEL_RUNNER_KWARGS = {
@@ -66,6 +77,11 @@ MODEL_RUNNER_KWARGS = {
"gpu_memory_utilization": 0.8,
"max_model_len": 512,
},
"INC4AI/Qwen3-8B-MXFP8-AR": {
"block_size": 64,
"gpu_memory_utilization": 0.8,
"max_model_len": 512,
},
}
@@ -273,6 +289,39 @@ def test_inc_config_parser_fused_module_requires_consistent_configs() -> None:
config.config_parser.resolve(DummyLayer(), "layers.0.self_attn.qkv_proj")
def test_inc_mxfp8() -> None:
config = make_config(
weight_bits=8,
group_size=32,
sym=True,
packing_format="auto_round:llm_compressor",
data_type="mx_fp",
)
assert config.weight_bits == 8
assert config.group_size == 32
assert config.data_type == "mx_fp"
assert config.packing_format == "auto_round:llm_compressor"
def test_inc_config_rejects_invalid_mxfp8_activation_config() -> None:
with pytest.raises(AssertionError, match="act_dynamic=True"):
INCConfig.from_config(
{
"bits": 8,
"group_size": 32,
"sym": True,
"packing_format": "auto_round:llm_compressor",
"data_type": "mx_fp",
"act_bits": 8,
"act_data_type": "mx_fp",
"act_group_size": 32,
"act_sym": True,
"act_dynamic": False,
}
)
def test_inc_layer_config_mx_fp_helpers() -> None:
layer_config = INCLayerConfig(
bits=4,
@@ -304,6 +353,22 @@ def test_inc_resolve_scheme_selects_wna16() -> None:
assert isinstance(scheme, INCWna16Scheme)
def test_inc_resolve_scheme_selects_mxfp8() -> None:
layer_config = INCLayerConfig(
bits=8,
group_size=32,
sym=True,
packing_format="auto_round:llm_compressor",
backend="auto",
data_type="mx_fp",
quantized=True,
)
scheme = resolve_scheme(layer_config)
assert isinstance(scheme, INCMxfp8Scheme)
class DummyLinearScheme(INCLinearScheme):
def __init__(self) -> None:
self.calls: list[tuple] = []
@@ -323,6 +388,76 @@ class DummyLinearScheme(INCLinearScheme):
return "applied"
def test_inc_mxfp8_linear_scheme_delegates_to_kernel(monkeypatch) -> None:
class DummyKernel:
def __init__(self) -> None:
self.calls: list[tuple] = []
def process_weights_after_loading(self, layer) -> None:
self.calls.append(("process", layer))
def apply_weights(self, layer, x, bias=None):
self.calls.append(("apply", layer, x, bias))
return "applied"
kernel = DummyKernel()
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel",
lambda: kernel,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.ModelWeightParameter",
lambda **kwargs: torch.nn.Parameter(kwargs["data"], requires_grad=False),
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.GroupQuantScaleParameter",
lambda **kwargs: torch.nn.Parameter(kwargs["data"], requires_grad=False),
)
scheme = INCMxfp8LinearScheme()
layer = torch.nn.Module()
scheme.create_weights(
layer=layer,
input_size_per_partition=64,
output_partition_sizes=[48, 16],
input_size=64,
output_size=64,
params_dtype=torch.bfloat16,
weight_loader=lambda *args, **kwargs: None,
)
assert layer.weight.shape == (64, 64)
assert layer.weight.dtype == torch.float8_e4m3fn
assert layer.weight_scale.shape == (64, 2)
assert layer.weight_scale.dtype == torch.uint8
scheme.process_weights_after_loading(layer)
result = scheme.apply_weights(layer, torch.randn(1, 64), None)
assert result == "applied"
assert [call[0] for call in kernel.calls] == ["process", "apply"]
def test_inc_mxfp8_linear_scheme_requires_block_32_input(monkeypatch) -> None:
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear.init_mxfp8_linear_kernel",
lambda: object(),
)
scheme = INCMxfp8LinearScheme()
with pytest.raises(ValueError, match="divisible by 32"):
scheme.create_weights(
layer=torch.nn.Module(),
input_size_per_partition=48,
output_partition_sizes=[32],
input_size=48,
output_size=32,
params_dtype=torch.bfloat16,
weight_loader=lambda *args, **kwargs: None,
)
def test_inc_linear_method_delegates() -> None:
scheme = DummyLinearScheme()
method = INCLinearMethod(scheme)