[Quantization][Autoround][XPU] Support AutoRound MXFP8 MoE models (#51248)
Signed-off-by: jl9876 <jia.li@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -104,11 +104,20 @@ QWEN3_AUTOROUND_MODELS = [
|
||||
),
|
||||
id="auto_round:mxfp4:qwen3-30b-a3b",
|
||||
),
|
||||
pytest.param(
|
||||
"INCModel/Qwen3-30B-A3B-12L-MXFP8-test",
|
||||
marks=pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_xpu()),
|
||||
reason="Qwen3-30B-A3B MXFP8 AutoRound model requires CUDA/XPU.",
|
||||
),
|
||||
id="auto_round:mxfp8:qwen3-30b-a3b",
|
||||
),
|
||||
]
|
||||
|
||||
MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = {
|
||||
"INCModel/Qwen3-1.7B-AutoRound-MXFP4-W4A4": {"enforce_eager": True},
|
||||
"INCModel/Qwen3-30B-A3B-12L-MXFP4-test": {"enforce_eager": True},
|
||||
"INCModel/Qwen3-30B-A3B-12L-MXFP8-test": {"enforce_eager": True},
|
||||
"Intel/Qwen3-8B-w2g64-for-ut": {
|
||||
"block_size": 64,
|
||||
"gpu_memory_utilization": 0.8,
|
||||
@@ -232,6 +241,23 @@ def make_qwen3_autoround_config(kind: str) -> INCConfig:
|
||||
},
|
||||
},
|
||||
},
|
||||
"qwen3_30b_a3b_mxfp8": {
|
||||
"quant_method": "auto-round",
|
||||
"bits": 8,
|
||||
"group_size": 32,
|
||||
"sym": True,
|
||||
"packing_format": "auto_round:llm_compressor",
|
||||
"data_type": "mx_fp",
|
||||
"act_bits": 8,
|
||||
"act_group_size": 32,
|
||||
"act_data_type": "mx_fp",
|
||||
"extra_config": {
|
||||
"model.layers.0.mlp.gate": {
|
||||
"bits": 16,
|
||||
"data_type": "float",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
try:
|
||||
config = configs[kind]
|
||||
@@ -606,6 +632,113 @@ def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe(
|
||||
assert method.moe_config is layer.moe_config
|
||||
|
||||
|
||||
def test_qwen3_30b_a3b_mxfp8_autoround_routes_to_mxfp8_moe(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
expected_backend = object()
|
||||
expected_experts_cls = object()
|
||||
captured = {}
|
||||
|
||||
def fake_select_mxfp8_moe_backend(*, config):
|
||||
captured["moe_config"] = config
|
||||
return expected_backend, expected_experts_cls
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||
"inc_mxfp8_moe.select_mxfp8_moe_backend",
|
||||
fake_select_mxfp8_moe_backend,
|
||||
)
|
||||
|
||||
config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp8")
|
||||
layer = object.__new__(RoutedExperts)
|
||||
layer.moe_config = cast(Any, "moe-config")
|
||||
|
||||
linear_method = config.get_quant_method(
|
||||
object.__new__(LinearBase), "model.layers.0.self_attn.q_proj"
|
||||
)
|
||||
moe_method = config.get_quant_method(layer, "model.layers.0.mlp")
|
||||
gate_method = config.get_quant_method(
|
||||
object.__new__(LinearBase), "model.layers.0.mlp.gate"
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_moe import (
|
||||
INCMxfp8MoEMethod,
|
||||
)
|
||||
|
||||
assert isinstance(linear_method, INCLinearMethod)
|
||||
assert isinstance(linear_method.scheme, INCMxfp8LinearScheme)
|
||||
assert isinstance(moe_method, INCMxfp8MoEMethod)
|
||||
assert moe_method.mxfp8_backend is expected_backend
|
||||
assert moe_method.experts_cls is expected_experts_cls
|
||||
assert captured["moe_config"] == layer.moe_config
|
||||
assert isinstance(gate_method, UnquantizedLinearMethod)
|
||||
|
||||
|
||||
def test_inc_mxfp8_moe_method_registers_block_scaled_weights(monkeypatch) -> None:
|
||||
expected_backend = object()
|
||||
expected_experts_cls = object()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.inc.schemes."
|
||||
"inc_mxfp8_moe.select_mxfp8_moe_backend",
|
||||
lambda *, config: (expected_backend, expected_experts_cls),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_moe import (
|
||||
INCMxfp8MoEMethod,
|
||||
)
|
||||
|
||||
class DummyMoeConfig:
|
||||
is_act_and_mul = True
|
||||
|
||||
method = INCMxfp8MoEMethod(cast(Any, DummyMoeConfig()))
|
||||
layer = torch.nn.Module()
|
||||
layer.hidden_size = 64
|
||||
layer.intermediate_size_per_partition = 32
|
||||
|
||||
method.create_weights(
|
||||
layer,
|
||||
num_experts=2,
|
||||
hidden_size=64,
|
||||
intermediate_size_per_partition=32,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
assert method.mxfp8_backend is expected_backend
|
||||
assert method.experts_cls is expected_experts_cls
|
||||
assert method.weight_block_size == [1, 32]
|
||||
assert layer.w13_weight.shape == (2, 64, 64)
|
||||
assert layer.w2_weight.shape == (2, 64, 32)
|
||||
assert layer.w13_weight.dtype is torch.float8_e4m3fn
|
||||
assert layer.w2_weight.dtype is torch.float8_e4m3fn
|
||||
assert layer.w13_weight_scale.shape == (2, 64, 2)
|
||||
assert layer.w2_weight_scale.shape == (2, 64, 1)
|
||||
assert layer.w13_weight_scale.dtype is torch.uint8
|
||||
assert layer.w2_weight_scale.dtype is torch.uint8
|
||||
assert layer.w13_weight_scale.quant_method == "block"
|
||||
assert layer.w2_weight_scale.quant_method == "block"
|
||||
|
||||
misaligned_layer = torch.nn.Module()
|
||||
misaligned_layer.hidden_size = 65
|
||||
misaligned_layer.intermediate_size_per_partition = 32
|
||||
with pytest.raises(ValueError, match="hidden_size divisible by 32"):
|
||||
method.create_weights(
|
||||
misaligned_layer,
|
||||
num_experts=2,
|
||||
hidden_size=65,
|
||||
intermediate_size_per_partition=32,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
def test_inc_mxfp4_linear_method_registers_and_processes_weights(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.parameter import ModelWeightParameter
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
|
||||
|
||||
class INCMxfp8MoEMethod(FusedMoEMethodBase):
|
||||
"""W8A8 MXFP8 MoE method for serialized AutoRound checkpoints."""
|
||||
|
||||
def __init__(self, moe) -> None:
|
||||
super().__init__(moe)
|
||||
self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
|
||||
self.mxfp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
assert layer.intermediate_size_per_partition == intermediate_size_per_partition
|
||||
assert layer.hidden_size == hidden_size
|
||||
layer.orig_dtype = params_dtype
|
||||
|
||||
if hidden_size % MXFP8_BLOCK_SIZE != 0:
|
||||
raise ValueError(
|
||||
f"MXFP8 MoE requires hidden_size divisible by {MXFP8_BLOCK_SIZE}, "
|
||||
f"got {hidden_size}."
|
||||
)
|
||||
if intermediate_size_per_partition % MXFP8_BLOCK_SIZE != 0:
|
||||
raise ValueError(
|
||||
"MXFP8 MoE requires intermediate_size_per_partition divisible by "
|
||||
f"{MXFP8_BLOCK_SIZE}, got {intermediate_size_per_partition}."
|
||||
)
|
||||
|
||||
layer.num_experts = num_experts
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
|
||||
w13_weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
|
||||
w2_weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
|
||||
w13_weight_scale = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
hidden_size // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
|
||||
w2_weight_scale = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
input_dim=2,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
|
||||
set_weight_attrs(
|
||||
layer.w13_weight_scale,
|
||||
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
|
||||
)
|
||||
set_weight_attrs(
|
||||
layer.w2_weight_scale,
|
||||
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
layer=layer,
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
self.moe_kernel = make_fp8_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: RoutedExperts
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
return make_fp8_moe_quant_config(
|
||||
fp8_backend=self.mxfp8_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
@@ -29,3 +29,15 @@ class INCMxfp8Scheme(INCScheme):
|
||||
from .inc_mxfp8_linear import INCMxfp8LinearScheme
|
||||
|
||||
return INCLinearMethod(INCMxfp8LinearScheme())
|
||||
|
||||
def get_moe_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, prefix, layer_config
|
||||
from .inc_mxfp8_moe import INCMxfp8MoEMethod
|
||||
|
||||
return INCMxfp8MoEMethod(layer.moe_config)
|
||||
|
||||
Reference in New Issue
Block a user