[AutoRound] Support AutoRound Format Block-Wise FP8 in vLLM (#47434)

Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com>
Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Zhenzhong Xu
2026-08-31 22:56:20 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent dafbef15a1
commit bd575a0d0b
10 changed files with 503 additions and 62 deletions
+167 -1
View File
@@ -22,11 +22,15 @@ from vllm.model_executor.layers.quantization.inc import INCConfig
from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
from vllm.model_executor.layers.quantization.inc.schemes import (
INCFp8Scheme,
INCMxfp4Scheme,
INCMxfp8Scheme,
INCWna16Scheme,
resolve_scheme,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear import (
INCFp8LinearScheme,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import (
INCMxfp8LinearScheme,
)
@@ -77,6 +81,14 @@ MODELS = [
),
id="auto_round:llm_compressor_mxfp8",
),
pytest.param(
"INC4AI/Qwen3-8B-fp-w8g128x128-for-ut",
marks=pytest.mark.skipif(
not (current_platform.is_cuda()),
reason="Block-wise AutoRound model only supports CUDA backend for now.",
),
id="auto_round:block_wise_fp8_on_cuda",
),
]
QWEN3_AUTOROUND_MODELS = [
@@ -128,6 +140,10 @@ MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = {
"gpu_memory_utilization": 0.8,
"max_model_len": 512,
},
"INC4AI/Qwen3-8B-fp-w8g128x128-for-ut": {
"gpu_memory_utilization": 0.8,
"max_model_len": 512,
},
}
@@ -190,6 +206,27 @@ def make_layer_config(**overrides) -> INCLayerConfig:
return INCLayerConfig(**kwargs)
def make_block_fp8_raw_config(**overrides) -> dict[str, object]:
kwargs = {
"bits": 8,
"group_size": [128, 128],
"sym": True,
"packing_format": "auto_round:fp8",
"quant_method": "auto-round",
"data_type": "fp",
"act_bits": 8,
"act_data_type": "fp",
"act_group_size": 128,
"act_sym": True,
"act_dynamic": True,
"activation_scheme": "dynamic",
"fmt": "e4m3",
"weight_block_size": [128, 128],
}
kwargs.update(overrides)
return kwargs
def make_qwen3_autoround_config(kind: str) -> INCConfig:
configs = {
"qwen3_1p7b_mxfp4": {
@@ -480,6 +517,52 @@ def test_inc_layer_config_mx_fp_helpers() -> None:
assert layer_config.is_mxfp8 is False
def test_inc_config_accepts_block_fp8() -> None:
config = INCConfig.from_config(make_block_fp8_raw_config())
assert config.weight_bits == 8
assert config.group_size == (128, 128)
assert config.sym is True
assert config.data_type == "fp"
assert config.packing_format == "auto_round:fp8"
def test_inc_config_rejects_invalid_block_fp8_activation_config() -> None:
with pytest.raises(AssertionError, match="act_dynamic=True"):
INCConfig.from_config(make_block_fp8_raw_config(act_dynamic=False))
def test_inc_layer_config_fp8_block_helper() -> None:
layer_config = INCLayerConfig(
bits=8,
group_size=(128, 128),
sym=True,
packing_format="auto_round:fp8",
backend="auto",
data_type="fp",
quantized=True,
)
assert layer_config.is_fp8_block is True
assert layer_config.is_wna16_int is False
def test_inc_resolve_scheme_selects_fp8_block() -> None:
layer_config = INCLayerConfig(
bits=8,
group_size=(128, 128),
sym=True,
packing_format="auto_round:fp8",
backend="auto",
data_type="fp",
quantized=True,
)
scheme = resolve_scheme(layer_config)
assert isinstance(scheme, INCFp8Scheme)
def test_inc_resolve_scheme_selects_wna16() -> None:
layer_config = INCLayerConfig(
bits=4,
@@ -1053,6 +1136,89 @@ def test_inc_linear_method_delegates() -> None:
]
def test_inc_fp8_linear_scheme_delegates_to_fp8_linear_method(
monkeypatch,
) -> None:
captured = {}
class DummyMethod:
def __init__(self, quant_config) -> None:
captured["quant_config"] = quant_config
captured["method"] = self
self.calls: list[tuple] = []
self.marlin_input_dtype = None
def create_weights(
self,
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
) -> None:
self.calls.append(
(
"create_weights",
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
extra_weight_attrs,
)
)
def process_weights_after_loading(self, layer) -> None:
self.calls.append(("process_weights_after_loading", layer))
def apply(self, layer, x, bias=None):
self.calls.append(("apply", layer, x, bias))
return "applied"
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear."
"Fp8LinearMethod",
DummyMethod,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear."
"get_marlin_input_dtype",
lambda _prefix: "fp8-marlin",
)
scheme = INCFp8LinearScheme(
prefix="layers.0.mlp.down_proj",
weight_block_size=(128, 128),
)
layer = torch.nn.Module()
scheme.create_weights(
layer=layer,
input_size_per_partition=128,
output_partition_sizes=[64, 64],
input_size=128,
output_size=128,
params_dtype=torch.bfloat16,
weight_loader=lambda *args, **kwargs: None,
)
scheme.process_weights_after_loading(layer)
result = scheme.apply_weights(layer, torch.randn(1, 128), None)
assert captured["quant_config"].is_checkpoint_fp8_serialized is True
assert captured["quant_config"].activation_scheme == "dynamic"
assert captured["quant_config"].weight_block_size == [128, 128]
assert captured["method"].marlin_input_dtype == "fp8-marlin"
assert result == "applied"
assert [call[0] for call in captured["method"].calls] == [
"create_weights",
"process_weights_after_loading",
"apply",
]
def test_wna16_xpu_prefers_ark_when_available(monkeypatch) -> None:
class DummyQuantLinear:
pass
@@ -1284,7 +1450,7 @@ def test_wna16_linear_gptq_uses_auto_gptq_when_supported(monkeypatch) -> None:
scheme = INCWNA16LinearScheme(make_layer_config())
assert isinstance(scheme.inner_method, DummyMethod)
assert isinstance(scheme.linear_method, DummyMethod)
assert isinstance(captured["cfg"], AutoGPTQConfig)
assert captured["cfg"].weight_bits == 4
assert captured["cfg"].group_size == 128