[Quantization][Autoround][XPU] Support AutoRound MXFP8 MoE models (#51248)
Signed-off-by: jl9876 <jia.li@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -104,11 +104,20 @@ QWEN3_AUTOROUND_MODELS = [
|
||||
),
|
||||
id="auto_round:mxfp4:qwen3-30b-a3b",
|
||||
),
|
||||
pytest.param(
|
||||
"INCModel/Qwen3-30B-A3B-12L-MXFP8-test",
|
||||
marks=pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_xpu()),
|
||||
reason="Qwen3-30B-A3B MXFP8 AutoRound model requires CUDA/XPU.",
|
||||
),
|
||||
id="auto_round:mxfp8:qwen3-30b-a3b",
|
||||
),
|
||||
]
|
||||
|
||||
MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = {
|
||||
"INCModel/Qwen3-1.7B-AutoRound-MXFP4-W4A4": {"enforce_eager": True},
|
||||
"INCModel/Qwen3-30B-A3B-12L-MXFP4-test": {"enforce_eager": True},
|
||||
"INCModel/Qwen3-30B-A3B-12L-MXFP8-test": {"enforce_eager": True},
|
||||
"Intel/Qwen3-8B-w2g64-for-ut": {
|
||||
"block_size": 64,
|
||||
"gpu_memory_utilization": 0.8,
|
||||
@@ -232,6 +241,23 @@ def make_qwen3_autoround_config(kind: str) -> INCConfig:
|
||||
},
|
||||
},
|
||||
},
|
||||
"qwen3_30b_a3b_mxfp8": {
|
||||
"quant_method": "auto-round",
|
||||
"bits": 8,
|
||||
"group_size": 32,
|
||||
"sym": True,
|
||||
"packing_format": "auto_round:llm_compressor",
|
||||
"data_type": "mx_fp",
|
||||
"act_bits": 8,
|
||||
"act_group_size": 32,
|
||||
"act_data_type": "mx_fp",
|
||||
"extra_config": {
|
||||
"model.layers.0.mlp.gate": {
|
||||
"bits": 16,
|
||||
"data_type": "float",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
try:
|
||||
config = configs[kind]
|
||||
@@ -606,6 +632,113 @@ def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe(
|
||||
assert method.moe_config is layer.moe_config
|
||||
|
||||
|
||||
def test_qwen3_30b_a3b_mxfp8_autoround_routes_to_mxfp8_moe(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
expected_backend = object()
|
||||
expected_experts_cls = object()
|
||||
captured = {}
|
||||
|
||||
def fake_select_mxfp8_moe_backend(*, config):
|
||||
captured["moe_config"] = config
|
||||
return expected_backend, expected_experts_cls
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||
"inc_mxfp8_moe.select_mxfp8_moe_backend",
|
||||
fake_select_mxfp8_moe_backend,
|
||||
)
|
||||
|
||||
config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp8")
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = cast(Any, "moe-config")
|
||||
|
||||
linear_method = config.get_quant_method(
|
||||
object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
|
||||
)
|
||||
moe_method = config.get_quant_method(layer, "model.layers.0.mlp")
|
||||
gate_method = config.get_quant_method(
|
||||
object.__new__(LinearBase), "model.layers.0.mlp.gate"
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_moe import (
|
||||
INCMxfp8MoEMethod,
|
||||
)
|
||||
|
||||
assert isinstance(linear_method, INCLinearMethod)
|
||||
assert isinstance(linear_method.scheme, INCMxfp8LinearScheme)
|
||||
assert isinstance(moe_method, INCMxfp8MoEMethod)
|
||||
assert moe_method.mxfp8_backend is expected_backend
|
||||
assert moe_method.experts_cls is expected_experts_cls
|
||||
assert captured["moe_config"] == layer.moe_config
|
||||
assert isinstance(gate_method, UnquantizedLinearMethod)
|
||||
|
||||
|
||||
def test_inc_mxfp8_moe_method_registers_block_scaled_weights(monkeypatch) -> None:
|
||||
expected_backend = object()
|
||||
expected_experts_cls = object()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||
"inc_mxfp8_moe.select_mxfp8_moe_backend",
|
||||
lambda *, config: (expected_backend, expected_experts_cls),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_moe import (
|
||||
INCMxfp8MoEMethod,
|
||||
)
|
||||
|
||||
class DummyMoeConfig:
|
||||
is_act_and_mul = True
|
||||
|
||||
method = INCMxfp8MoEMethod(cast(Any, DummyMoeConfig()))
|
||||
layer = torch.nn.Module()
|
||||
layer.hidden_size = 64
|
||||
layer.intermediate_size_per_partition = 32
|
||||
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=2,
|
||||
hidden_size=64,
|
||||
intermediate_size_per_partition=32,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
assert method.mxfp8_backend is expected_backend
|
||||
assert method.experts_cls is expected_experts_cls
|
||||
assert method.weight_block_size == [1, 32]
|
||||
assert layer.w13_weight.shape == (2, 64, 64)
|
||||
assert layer.w2_weight.shape == (2, 64, 32)
|
||||
assert layer.w13_weight.dtype is torch.float8_e4m3fn
|
||||
assert layer.w2_weight.dtype is torch.float8_e4m3fn
|
||||
assert layer.w13_weight_scale.shape == (2, 64, 2)
|
||||
assert layer.w2_weight_scale.shape == (2, 64, 1)
|
||||
assert layer.w13_weight_scale.dtype is torch.uint8
|
||||
assert layer.w2_weight_scale.dtype is torch.uint8
|
||||
assert layer.w13_weight_scale.quant_method == "block"
|
||||
assert layer.w2_weight_scale.quant_method == "block"
|
||||
|
||||
misaligned_layer = torch.nn.Module()
|
||||
misaligned_layer.hidden_size = 65
|
||||
misaligned_layer.intermediate_size_per_partition = 32
|
||||
with pytest.raises(ValueError, match="hidden_size divisible by 32"):
|
||||
method.create_weights(
|
||||
misaligned_layer,
|
||||
num_experts=2,
|
||||
hidden_size=65,
|
||||
intermediate_size_per_partition=32,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
def test_inc_mxfp4_linear_method_registers_and_processes_weights(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user