[Quantization][Autoround][XPU] Add W4A16(moe) / MXFP4(linear/moe) Support (#47124)
Signed-off-by: lkk12014402 <kaokao.lv@intel.com> Signed-off-by: lkk <33276950+lkk12014402@users.noreply.github.com> Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -8,16 +8,20 @@ Validating the configuration and printing results for manual checking.
|
||||
Run `pytest tests/quantization/test_auto_round.py`.
|
||||
"""
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig
|
||||
from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod
|
||||
from vllm.model_executor.layers.quantization.inc.schemes import (
|
||||
INCMxfp4Scheme,
|
||||
INCMxfp8Scheme,
|
||||
INCWna16Scheme,
|
||||
resolve_scheme,
|
||||
@@ -71,7 +75,36 @@ MODELS = [
|
||||
),
|
||||
]
|
||||
|
||||
MODEL_RUNNER_KWARGS = {
|
||||
QWEN3_AUTOROUND_MODELS = [
|
||||
pytest.param(
|
||||
"INCModel/Qwen3-1.7B-AutoRound-MXFP4-W4A4",
|
||||
marks=pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_xpu()),
|
||||
reason="Qwen3-1.7B MXFP4 AutoRound model requires CUDA/XPU.",
|
||||
),
|
||||
id="auto_round:mxfp4:qwen3-1p7b",
|
||||
),
|
||||
pytest.param(
|
||||
"INCModel/Qwen3-30B-A3B-12L-W4A16-test",
|
||||
marks=pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_xpu()),
|
||||
reason="Qwen3-30B-A3B W4A16 AutoRound model requires CUDA/XPU.",
|
||||
),
|
||||
id="auto_round:w4a16:qwen3-30b-a3b",
|
||||
),
|
||||
pytest.param(
|
||||
"INCModel/Qwen3-30B-A3B-12L-MXFP4-test",
|
||||
marks=pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_xpu()),
|
||||
reason="Qwen3-30B-A3B MXFP4 AutoRound model requires CUDA/XPU.",
|
||||
),
|
||||
id="auto_round:mxfp4:qwen3-30b-a3b",
|
||||
),
|
||||
]
|
||||
|
||||
MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = {
|
||||
"INCModel/Qwen3-1.7B-AutoRound-MXFP4-W4A4": {"enforce_eager": True},
|
||||
"INCModel/Qwen3-30B-A3B-12L-MXFP4-test": {"enforce_eager": True},
|
||||
"Intel/Qwen3-8B-w2g64-for-ut": {
|
||||
"block_size": 64,
|
||||
"gpu_memory_utilization": 0.8,
|
||||
@@ -93,7 +126,7 @@ MODEL_RUNNER_KWARGS = {
|
||||
),
|
||||
reason="Only supports CPU/XPU/CUDA backend.",
|
||||
)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("model", MODELS + QWEN3_AUTOROUND_MODELS)
|
||||
def test_auto_round_model(vllm_runner, model):
|
||||
with vllm_runner(model, **MODEL_RUNNER_KWARGS.get(model, {})) as llm:
|
||||
output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
|
||||
@@ -144,6 +177,65 @@ def make_layer_config(**overrides) -> INCLayerConfig:
|
||||
return INCLayerConfig(**kwargs)
|
||||
|
||||
|
||||
def make_qwen3_autoround_config(kind: str) -> INCConfig:
|
||||
configs = {
|
||||
"qwen3_1p7b_mxfp4": {
|
||||
"quant_method": "auto-round",
|
||||
"bits": 4,
|
||||
"group_size": 32,
|
||||
"sym": True,
|
||||
"packing_format": "auto_round:llm_compressor",
|
||||
"data_type": "mx_fp",
|
||||
"extra_config": {
|
||||
"model.layers.0.self_attn.q_proj": {
|
||||
"bits": 16,
|
||||
"data_type": "float",
|
||||
},
|
||||
},
|
||||
},
|
||||
"qwen3_30b_a3b_w4a16": {
|
||||
"quant_method": "auto-round",
|
||||
"bits": 4,
|
||||
"group_size": 32,
|
||||
"sym": True,
|
||||
"packing_format": "auto_round:auto_gptq",
|
||||
"data_type": "int",
|
||||
"extra_config": {
|
||||
"model.layers.0.mlp.gate": {
|
||||
"bits": 16,
|
||||
"data_type": "float",
|
||||
},
|
||||
},
|
||||
},
|
||||
"qwen3_30b_a3b_mxfp4": {
|
||||
"quant_method": "auto-round",
|
||||
"bits": 4,
|
||||
"group_size": 32,
|
||||
"sym": True,
|
||||
"packing_format": "auto_round:llm_compressor",
|
||||
"data_type": "mx_fp",
|
||||
"act_bits": 4,
|
||||
"act_group_size": 32,
|
||||
"act_data_type": "mx_fp",
|
||||
"extra_config": {
|
||||
"model.layers.0.mlp.gate": {
|
||||
"bits": 16,
|
||||
"data_type": "float",
|
||||
},
|
||||
"model.layers.0.self_attn.q_proj": {
|
||||
"bits": 16,
|
||||
"data_type": "float",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
try:
|
||||
config = configs[kind]
|
||||
except KeyError as err:
|
||||
raise AssertionError(f"unknown qwen3 autoround config: {kind}") from err
|
||||
return INCConfig.from_config(config)
|
||||
|
||||
|
||||
def test_inc_config_parser_exact_match() -> None:
|
||||
config = make_config(
|
||||
extra_config={
|
||||
@@ -374,6 +466,301 @@ def test_inc_resolve_scheme_selects_wna16() -> None:
|
||||
assert isinstance(scheme, INCWna16Scheme)
|
||||
|
||||
|
||||
def test_inc_config_accepts_mxfp_family_llm_compressor() -> None:
|
||||
config = INCConfig.from_config(
|
||||
{
|
||||
"quant_method": "auto-round",
|
||||
"bits": 4,
|
||||
"group_size": 32,
|
||||
"sym": True,
|
||||
"packing_format": "auto_round:llm_compressor",
|
||||
"data_type": "mx_fp4e2m1",
|
||||
}
|
||||
)
|
||||
|
||||
layer_config = config.config_parser.resolve(
|
||||
DummyLayer(), "model.layers.0.mlp.down_proj"
|
||||
)
|
||||
|
||||
assert config.sym is True
|
||||
assert layer_config.is_mxfp4 is True
|
||||
assert isinstance(resolve_scheme(layer_config), INCMxfp4Scheme)
|
||||
|
||||
|
||||
def test_qwen3_1p7b_mxfp4_autoround_uses_mxfp4_linear_scheme(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
class DummyKernel:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||
"inc_mxfp4_linear.init_mxfp4_linear_kernel",
|
||||
lambda: DummyKernel(),
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_linear import ( # noqa: E501
|
||||
INCMxfp4LinearMethod,
|
||||
)
|
||||
|
||||
config = make_qwen3_autoround_config("qwen3_1p7b_mxfp4")
|
||||
|
||||
assert (
|
||||
INCConfig.override_quantization_method(
|
||||
{"quant_method": "auto-round"}, user_quant=None
|
||||
)
|
||||
== "inc"
|
||||
)
|
||||
ignored_method = config.get_quant_method(
|
||||
object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
|
||||
)
|
||||
layer_config = config.config_parser.resolve(
|
||||
DummyLayer(), "model.layers.0.mlp.gate_proj"
|
||||
)
|
||||
method = INCMxfp4Scheme().get_linear_method(
|
||||
config,
|
||||
object.__new__(LinearBase),
|
||||
"model.layers.0.mlp.gate_proj",
|
||||
layer_config,
|
||||
)
|
||||
|
||||
assert isinstance(ignored_method, UnquantizedLinearMethod)
|
||||
assert layer_config.bits == 4
|
||||
assert layer_config.group_size == 32
|
||||
assert layer_config.is_mxfp4 is True
|
||||
assert isinstance(resolve_scheme(layer_config), INCMxfp4Scheme)
|
||||
assert isinstance(method, INCLinearMethod)
|
||||
assert isinstance(method.scheme, INCMxfp4LinearMethod)
|
||||
assert isinstance(method.scheme.kernel, DummyKernel)
|
||||
|
||||
|
||||
def test_qwen3_30b_a3b_w4a16_autoround_routes_to_gptq_moe(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
captured = {}
|
||||
expected_method = object()
|
||||
|
||||
class DummyMoeConfig:
|
||||
pass
|
||||
|
||||
def fake_resolve_gptq_moe(layer, layer_config):
|
||||
captured["layer"] = layer
|
||||
captured["layer_config"] = layer_config
|
||||
return expected_method
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||
"inc_wna16_scheme._resolve_gptq_moe",
|
||||
fake_resolve_gptq_moe,
|
||||
)
|
||||
|
||||
config = make_qwen3_autoround_config("qwen3_30b_a3b_w4a16")
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
|
||||
method = config.get_quant_method(layer, "model.layers.0.mlp")
|
||||
|
||||
assert method is expected_method
|
||||
assert captured["layer"] is layer
|
||||
assert captured["layer_config"].bits == 4
|
||||
assert captured["layer_config"].group_size == 32
|
||||
assert captured["layer_config"].is_gptq is True
|
||||
assert captured["layer_config"].is_wna16_int is True
|
||||
|
||||
|
||||
def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
class DummyMoeConfig:
|
||||
pass
|
||||
|
||||
class DummyMxfp4MoEMethod:
|
||||
def __init__(self, moe_config) -> None:
|
||||
self.moe_config = moe_config
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe.INCMxfp4MoEMethod",
|
||||
DummyMxfp4MoEMethod,
|
||||
)
|
||||
|
||||
config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp4")
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
|
||||
ignored_method = config.get_quant_method(
|
||||
object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
|
||||
)
|
||||
method = config.get_quant_method(layer, "model.layers.0.mlp")
|
||||
layer_config = config.config_parser.resolve(DummyLayer(), "model.layers.0.mlp")
|
||||
|
||||
assert isinstance(ignored_method, UnquantizedLinearMethod)
|
||||
assert layer_config.bits == 4
|
||||
assert layer_config.group_size == 32
|
||||
assert layer_config.is_mxfp4 is True
|
||||
assert isinstance(resolve_scheme(layer_config), INCMxfp4Scheme)
|
||||
assert isinstance(method, DummyMxfp4MoEMethod)
|
||||
assert method.moe_config is layer.moe_config
|
||||
|
||||
|
||||
def test_inc_mxfp4_linear_method_registers_and_processes_weights(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
captured = {}
|
||||
|
||||
class DummyKernel:
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
captured["processed_layer"] = layer
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||
"inc_mxfp4_linear.init_mxfp4_linear_kernel",
|
||||
lambda: DummyKernel(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_linear import ( # noqa: E501
|
||||
INCMxfp4LinearMethod,
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
method = INCMxfp4LinearMethod(
|
||||
make_layer_config(group_size=32, data_type="mx_fp4e2m1")
|
||||
)
|
||||
|
||||
method.create_weights(
|
||||
layer,
|
||||
input_size_per_partition=64,
|
||||
output_partition_sizes=[16, 32],
|
||||
input_size=64,
|
||||
output_size=48,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
assert layer.weight_packed.shape == (48, 32)
|
||||
assert layer.weight_packed.dtype is torch.uint8
|
||||
assert layer.weight_scale.shape == (48, 2)
|
||||
assert layer.weight_scale.dtype is torch.uint8
|
||||
assert layer.logical_widths == [16, 32]
|
||||
assert layer.input_size_per_partition == 64
|
||||
assert layer.output_size_per_partition == 48
|
||||
|
||||
packed_data = layer.weight_packed.data
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer.weight.data.data_ptr() == packed_data.data_ptr()
|
||||
assert not hasattr(layer, "weight_packed")
|
||||
assert captured["processed_layer"] is layer
|
||||
|
||||
|
||||
def test_inc_mxfp4_moe_method_registers_weights_and_builds_kernel(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
captured = {}
|
||||
expected_quant_config = object()
|
||||
expected_kernel = object()
|
||||
expected_experts_cls = object()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
|
||||
"CutlassExpertsMxfp4._supports_current_device",
|
||||
lambda: False,
|
||||
)
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
|
||||
"select_mxfp4_moe_backend",
|
||||
lambda moe: (Mxfp4MoeBackend.XPU, expected_experts_cls),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
|
||||
"make_mxfp4_moe_quant_config",
|
||||
lambda **kwargs: captured.update({"quant_config_kwargs": kwargs})
|
||||
or expected_quant_config,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe."
|
||||
"make_mxfp4_moe_kernel",
|
||||
lambda **kwargs: captured.update({"kernel_kwargs": kwargs}) or expected_kernel,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp4_moe import (
|
||||
INCMxfp4MoEMethod,
|
||||
)
|
||||
|
||||
method = INCMxfp4MoEMethod(moe=cast(Any, "moe-config"))
|
||||
layer = torch.nn.Module()
|
||||
layer._expert_routing_tables = lambda: "routing-tables"
|
||||
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=2,
|
||||
hidden_size=64,
|
||||
intermediate_size_per_partition=32,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
assert method.experts_cls is expected_experts_cls
|
||||
assert layer.w13_weight_packed.shape == (2, 64, 32)
|
||||
assert layer.w2_weight_packed.shape == (2, 64, 16)
|
||||
assert layer.w13_weight_scale.shape == (2, 64, 2)
|
||||
assert layer.w2_weight_scale.shape == (2, 64, 1)
|
||||
|
||||
w13_packed_data = layer.w13_weight_packed.data
|
||||
w2_packed_data = layer.w2_weight_packed.data
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer.w13_weight.data.data_ptr() == w13_packed_data.data_ptr()
|
||||
assert layer.w2_weight.data.data_ptr() == w2_packed_data.data_ptr()
|
||||
assert not hasattr(layer, "w13_weight_packed")
|
||||
assert not hasattr(layer, "w2_weight_packed")
|
||||
assert captured["quant_config_kwargs"]["w1_scale"] is layer.w13_weight_scale
|
||||
assert captured["quant_config_kwargs"]["w2_scale"] is layer.w2_weight_scale
|
||||
assert captured["kernel_kwargs"]["moe_quant_config"] is expected_quant_config
|
||||
assert captured["kernel_kwargs"]["moe_config"] == "moe-config"
|
||||
assert captured["kernel_kwargs"]["experts_cls"] is expected_experts_cls
|
||||
assert captured["kernel_kwargs"]["routing_tables"] == "routing-tables"
|
||||
assert method.moe_kernel is expected_kernel
|
||||
|
||||
|
||||
def test_wna16_xpu_moe_routes_to_gptq_moe(monkeypatch) -> None:
|
||||
captured = {}
|
||||
expected_method = object()
|
||||
|
||||
class DummyMoeConfig:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "is_cpu", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||
"inc_wna16_scheme._resolve_gptq_moe",
|
||||
lambda layer, layer_config: captured.update(
|
||||
{"layer": layer, "layer_config": layer_config}
|
||||
)
|
||||
or expected_method,
|
||||
)
|
||||
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = DummyMoeConfig()
|
||||
method = INCWna16Scheme().get_moe_method(
|
||||
make_config(),
|
||||
layer,
|
||||
"model.layers.0.mlp",
|
||||
make_layer_config(group_size=32),
|
||||
)
|
||||
|
||||
assert method is expected_method
|
||||
assert captured["layer"] is layer
|
||||
assert captured["layer_config"].is_gptq is True
|
||||
|
||||
|
||||
def test_inc_resolve_scheme_selects_mxfp8() -> None:
|
||||
layer_config = INCLayerConfig(
|
||||
bits=8,
|
||||
@@ -878,7 +1265,8 @@ def test_resolve_gptq_moe_falls_back_to_moe_wna16(monkeypatch) -> None:
|
||||
captured["moe"] = moe
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
|
||||
"vllm.model_executor.layers.quantization.utils.marlin_utils."
|
||||
"check_moe_marlin_supports_layer",
|
||||
lambda *args, **kwargs: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
@@ -927,10 +1315,6 @@ def test_resolve_gptq_moe_uses_auto_gptq_when_supported(monkeypatch) -> None:
|
||||
captured["cfg"] = cfg
|
||||
captured["moe"] = moe
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
|
||||
lambda *args, **kwargs: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.utils.marlin_utils."
|
||||
"check_moe_marlin_supports_layer",
|
||||
@@ -964,10 +1348,6 @@ def test_resolve_awq_moe_uses_marlin_when_supported(monkeypatch) -> None:
|
||||
captured["cfg"] = cfg
|
||||
captured["moe"] = moe
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.utils.marlin_utils.check_marlin_supported",
|
||||
lambda *args, **kwargs: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.utils.marlin_utils.check_moe_marlin_supports_layer",
|
||||
lambda *args, **kwargs: True,
|
||||
|
||||
Reference in New Issue
Block a user