[Quantization][Autoround][XPU] Support AutoRound MXFP8 MoE models (#51248)

Signed-off-by: jl9876 <jia.li@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
jl9876
2026-08-31 16:37:01 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent c6c33f2b1f
commit 1b9539d37c
3 changed files with 341 additions and 0 deletions
+133
View File
@@ -104,11 +104,20 @@ QWEN3_AUTOROUND_MODELS = [
),
id="auto_round:mxfp4:qwen3-30b-a3b",
),
pytest.param(
"INCModel/Qwen3-30B-A3B-12L-MXFP8-test",
marks=pytest.mark.skipif(
not (current_platform.is_cuda() or current_platform.is_xpu()),
reason="Qwen3-30B-A3B MXFP8 AutoRound model requires CUDA/XPU.",
),
id="auto_round:mxfp8:qwen3-30b-a3b",
),
]
MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = {
"INCModel/Qwen3-1.7B-AutoRound-MXFP4-W4A4": {"enforce_eager": True},
"INCModel/Qwen3-30B-A3B-12L-MXFP4-test": {"enforce_eager": True},
"INCModel/Qwen3-30B-A3B-12L-MXFP8-test": {"enforce_eager": True},
"Intel/Qwen3-8B-w2g64-for-ut": {
"block_size": 64,
"gpu_memory_utilization": 0.8,
@@ -232,6 +241,23 @@ def make_qwen3_autoround_config(kind: str) -> INCConfig:
},
},
},
"qwen3_30b_a3b_mxfp8": {
"quant_method": "auto-round",
"bits": 8,
"group_size": 32,
"sym": True,
"packing_format": "auto_round:llm_compressor",
"data_type": "mx_fp",
"act_bits": 8,
"act_group_size": 32,
"act_data_type": "mx_fp",
"extra_config": {
"model.layers.0.mlp.gate": {
"bits": 16,
"data_type": "float",
},
},
},
}
try:
config = configs[kind]
@@ -606,6 +632,113 @@ def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe(
assert method.moe_config is layer.moe_config
def test_qwen3_30b_a3b_mxfp8_autoround_routes_to_mxfp8_moe(
monkeypatch,
) -> None:
expected_backend = object()
expected_experts_cls = object()
captured = {}
def fake_select_mxfp8_moe_backend(*, config):
captured["moe_config"] = config
return expected_backend, expected_experts_cls
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes."
"inc_mxfp8_moe.select_mxfp8_moe_backend",
fake_select_mxfp8_moe_backend,
)
config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp8")
layer = object.__new__(RoutedExperts)
layer.moe_config = cast(Any, "moe-config")
linear_method = config.get_quant_method(
object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
)
moe_method = config.get_quant_method(layer, "model.layers.0.mlp")
gate_method = config.get_quant_method(
object.__new__(LinearBase), "model.layers.0.mlp.gate"
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_moe import (
INCMxfp8MoEMethod,
)
assert isinstance(linear_method, INCLinearMethod)
assert isinstance(linear_method.scheme, INCMxfp8LinearScheme)
assert isinstance(moe_method, INCMxfp8MoEMethod)
assert moe_method.mxfp8_backend is expected_backend
assert moe_method.experts_cls is expected_experts_cls
assert captured["moe_config"] == layer.moe_config
assert isinstance(gate_method, UnquantizedLinearMethod)
def test_inc_mxfp8_moe_method_registers_block_scaled_weights(monkeypatch) -> None:
expected_backend = object()
expected_experts_cls = object()
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.inc.schemes."
"inc_mxfp8_moe.select_mxfp8_moe_backend",
lambda *, config: (expected_backend, expected_experts_cls),
)
monkeypatch.setattr(
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
lambda: 0,
)
monkeypatch.setattr(
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
lambda: 1,
)
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_moe import (
INCMxfp8MoEMethod,
)
class DummyMoeConfig:
is_act_and_mul = True
method = INCMxfp8MoEMethod(cast(Any, DummyMoeConfig()))
layer = torch.nn.Module()
layer.hidden_size = 64
layer.intermediate_size_per_partition = 32
method.create_weights(
layer,
num_experts=2,
hidden_size=64,
intermediate_size_per_partition=32,
params_dtype=torch.bfloat16,
)
assert method.mxfp8_backend is expected_backend
assert method.experts_cls is expected_experts_cls
assert method.weight_block_size == [1, 32]
assert layer.w13_weight.shape == (2, 64, 64)
assert layer.w2_weight.shape == (2, 64, 32)
assert layer.w13_weight.dtype is torch.float8_e4m3fn
assert layer.w2_weight.dtype is torch.float8_e4m3fn
assert layer.w13_weight_scale.shape == (2, 64, 2)
assert layer.w2_weight_scale.shape == (2, 64, 1)
assert layer.w13_weight_scale.dtype is torch.uint8
assert layer.w2_weight_scale.dtype is torch.uint8
assert layer.w13_weight_scale.quant_method == "block"
assert layer.w2_weight_scale.quant_method == "block"
misaligned_layer = torch.nn.Module()
misaligned_layer.hidden_size = 65
misaligned_layer.intermediate_size_per_partition = 32
with pytest.raises(ValueError, match="hidden_size divisible by 32"):
method.create_weights(
misaligned_layer,
num_experts=2,
hidden_size=65,
intermediate_size_per_partition=32,
params_dtype=torch.bfloat16,
)
def test_inc_mxfp4_linear_method_registers_and_processes_weights(
monkeypatch,
) -> None:
@@ -0,0 +1,196 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fused_moe import (
FusedMoEQuantConfig,
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.parameter import ModelWeightParameter
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
class INCMxfp8MoEMethod(FusedMoEMethodBase):
"""W8A8 MXFP8 MoE method for serialized AutoRound checkpoints."""
def __init__(self, moe) -> None:
super().__init__(moe)
self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
self.mxfp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
def create_weights(
self,
layer: RoutedExperts,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
assert layer.intermediate_size_per_partition == intermediate_size_per_partition
assert layer.hidden_size == hidden_size
layer.orig_dtype = params_dtype
if hidden_size % MXFP8_BLOCK_SIZE != 0:
raise ValueError(
f"MXFP8 MoE requires hidden_size divisible by {MXFP8_BLOCK_SIZE}, "
f"got {hidden_size}."
)
if intermediate_size_per_partition % MXFP8_BLOCK_SIZE != 0:
raise ValueError(
"MXFP8 MoE requires intermediate_size_per_partition divisible by "
f"{MXFP8_BLOCK_SIZE}, got {intermediate_size_per_partition}."
)
layer.num_experts = num_experts
weight_loader = extra_weight_attrs.get("weight_loader")
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_weight = ModelWeightParameter(
data=torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w13_weight", w13_weight)
w2_weight = ModelWeightParameter(
data=torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w2_weight", w2_weight)
w13_weight_scale = ModelWeightParameter(
data=torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
w2_weight_scale = ModelWeightParameter(
data=torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(
layer.w13_weight_scale,
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
)
set_weight_attrs(
layer.w2_weight_scale,
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
layer.weight_block_size = self.weight_block_size
w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
fp8_backend=self.mxfp8_backend,
layer=layer,
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_input_scale=None,
w2_input_scale=None,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
self.moe_kernel = make_fp8_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
fp8_backend=self.mxfp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
)
def get_fused_moe_quant_config(
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
return make_fp8_moe_quant_config(
fp8_backend=self.mxfp8_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a1_scale=None,
a2_scale=None,
block_shape=self.weight_block_size,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -29,3 +29,15 @@ class INCMxfp8Scheme(INCScheme):
from .inc_mxfp8_linear import INCMxfp8LinearScheme
return INCLinearMethod(INCMxfp8LinearScheme())
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, prefix, layer_config
from .inc_mxfp8_moe import INCMxfp8MoEMethod
return INCMxfp8MoEMethod(layer.moe_config)