[Quantization][Autoround][XPU] Add W4A16(moe) / MXFP4(linear/moe) Support (#47124)

Signed-off-by: lkk12014402 <kaokao.lv@intel.com>
Signed-off-by: lkk <33276950+lkk12014402@users.noreply.github.com>
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
lkk
2026-07-30 09:52:13 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent 4e582c5b54
commit e5f48dfda2
9 changed files with 816 additions and 48 deletions
@@ -38,11 +38,11 @@ class INCLayerConfig:
@property
def is_mxfp4(self) -> bool:
return self.data_type == "mx_fp" and self.bits == 4
return "mx_fp" in self.data_type and self.bits == 4
@property
def is_mxfp8(self) -> bool:
return self.data_type == "mx_fp" and self.bits == 8
return "mx_fp" in self.data_type and self.bits == 8
class INCConfigParser:
@@ -72,7 +72,10 @@ class INCConfig(QuantizationConfig):
f"Unsupported weight_bits: {weight_bits}, "
f"currently only support {self.SUPPORTED_BITS}."
)
if data_type not in self.SUPPORTED_DTYPES:
# auto-round mxfp data_type is e.g. "mx_fp4" / "mx_fp4e2m1"; match the
# "mx_fp" family by substring like auto_round.compressors.is_mx_fp.
is_mxfp = "mx_fp" in data_type
if data_type not in self.SUPPORTED_DTYPES and not is_mxfp:
raise ValueError(
f"Unsupported data_type: {data_type}, "
f"currently only support {self.SUPPORTED_DTYPES}."
@@ -111,11 +114,19 @@ class INCConfig(QuantizationConfig):
f"group_size={self.group_size}, sym={self.sym})"
)
@property
def is_mxfp(self) -> bool:
# auto-round mxfp data_type is the "mx_fp" family (e.g. "mx_fp",
# "mx_fp4e2m1"); match by substring like auto_round.compressors.is_mx_fp.
return "mx_fp" in self.data_type
@property
def is_mxfp8(self) -> bool:
# MXFP4 and MXFP8 share data_type "mx_fp" and differ only by bit width.
return self.is_mxfp and self.weight_bits == self.MXFP8_BITS
def _validate_supported_quantization(self) -> None:
if self.data_type == self.MXFP8_DATA_TYPE:
assert self.weight_bits == self.MXFP8_BITS, (
f"INC MXFP8 only supports bits=8, but found bits={self.weight_bits}."
)
if self.is_mxfp8:
assert self.group_size == self.MXFP8_GROUP_SIZE, (
"INC MXFP8 only supports group_size=32, "
f"but found group_size={self.group_size}."
@@ -130,14 +141,14 @@ class INCConfig(QuantizationConfig):
"INC MXFP8 only supports backend='auto', "
f"but found backend={self.backend!r}."
)
elif self.packing_format == self.MXFP8_PACKING_FORMAT:
elif self.packing_format == self.MXFP8_PACKING_FORMAT and not self.is_mxfp:
raise ValueError(
f"packing_format={self.MXFP8_PACKING_FORMAT!r} requires "
f"data_type={self.MXFP8_DATA_TYPE!r}."
f"an {self.MXFP8_DATA_TYPE!r} data_type."
)
def _validate_raw_config(self, config: dict[str, Any]) -> None:
if self.data_type != self.MXFP8_DATA_TYPE:
if not self.is_mxfp8:
return
expected_fields = {
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .factory import resolve_scheme
from .inc_mxfp4_scheme import INCMxfp4Scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_scheme import INCLinearScheme, INCScheme
from .inc_wna16_scheme import INCWna16Scheme
@@ -11,5 +12,6 @@ __all__ = [
"INCLinearScheme",
"INCMxfp8Scheme",
"INCWna16Scheme",
"INCMxfp4Scheme",
"resolve_scheme",
]
@@ -9,12 +9,14 @@ if TYPE_CHECKING:
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
from .inc_mxfp4_scheme import INCMxfp4Scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_wna16_scheme import INCWna16Scheme
scheme_list: list[type[INCScheme]] = [
INCMxfp8Scheme,
INCWna16Scheme,
INCMxfp4Scheme,
]
for scheme_cls in scheme_list:
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
import torch
from torch.nn.parameter import Parameter
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
)
from .inc_scheme import INCLinearScheme
if TYPE_CHECKING:
from ..config_parser import INCLayerConfig
class INCMxfp4LinearMethod(INCLinearScheme):
"""MXFP4 (W4A4) linear method for AutoRound checkpoints.
E2M1 weights packed two per byte with per-group E8M0 scales
(group_size=32, no global scale). The platform kernel is selected by
``init_mxfp4_linear_kernel`` (FlashInfer / Marlin on CUDA, ``fp4_gemm``
on XPU).
"""
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.group_size = layer_config.group_size or 32
self.kernel = init_mxfp4_linear_kernel()
@classmethod
def get_min_capability(cls) -> int:
return 80
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs: Any,
) -> None:
del input_size, output_size
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.params_dtype = params_dtype
weight_loader = extra_weight_attrs.get("weight_loader")
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // 2,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_packed", weight)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // self.group_size,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight = Parameter(layer.weight_packed.data, requires_grad=False)
del layer.weight_packed
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
@@ -0,0 +1,242 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
MarlinExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_moe_fp4_layer_for_marlin,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
logger = init_logger(__name__)
class INCMxfp4MoEMethod(FusedMoEMethodBase):
"""W4A4 MXFP4 group MoE for AutoRound ``auto_round:llm_compressor`` exports.
Registers the packed MXFP4 layout (uint8 ``weight_packed`` + uint8 E8M0
``weight_scale``, ``group_size=32``) and dispatches the fused MoE to the
best backend for the current device: CUTLASS (true W4A4 on supported
GPUs), the native XPU kernel, or Marlin weight-only as a fallback. The
per-expert ``gate_proj`` / ``up_proj`` / ``down_proj`` tensors are folded
into the stacked ``w13`` / ``w2`` parameters by ``make_expert_params_mapping``.
"""
def __init__(self, moe) -> None:
super().__init__(moe)
self.group_size = 32
# Backend selection must stay consistent with the weight preparation in
# process_weights_after_loading / get_fused_moe_quant_config, which only
# implement three layouts: CUTLASS swizzle (true W4A4), the native XPU
# kernel (packed passthrough), and Marlin weight-only. XPU dispatch is
# deferred to the shared oracle; every other non-CUTLASS device falls
# back to Marlin (mirroring CompressedTensorsW4A4Mxfp4MoEMethod).
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
self.experts_cls: type[mk.FusedMoEExperts] | None = None
if self.use_cutlass_mxfp4:
self.experts_cls = CutlassExpertsMxfp4
logger.info_once("Using CutlassExpertsMxfp4 for AutoRound MXFP4 MoE")
elif current_platform.is_xpu():
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
else:
self.experts_cls = MarlinExperts
logger.info_once(
"Using MarlinExperts (weight-only FP4) for AutoRound MXFP4 MoE"
)
def create_weights(
self,
layer: torch.nn.Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
layer.num_experts = num_experts
layer.params_dtype = params_dtype
# gate + up fused on the output dim; two FP4 packed per input byte.
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
hidden_size // 2,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_packed", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // 2,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
# Per-group E8M0 block scales (group_size=32), stored as uint8.
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
hidden_size // self.group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
extra_weight_attrs.update(
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
w2_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // self.group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
if self.use_cutlass_mxfp4:
# W4A4: both weights and activations quantized to MXFP4.
return mxfp4_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
# Weight-only (Marlin) or native XPU kernel.
return make_mxfp4_moe_quant_config(
mxfp4_backend=self.mxfp4_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
layer.w13_weight = torch.nn.Parameter(
layer.w13_weight_packed.data, requires_grad=False
)
delattr(layer, "w13_weight_packed")
layer.w2_weight = torch.nn.Parameter(
layer.w2_weight_packed.data, requires_grad=False
)
delattr(layer, "w2_weight_packed")
if self.use_cutlass_mxfp4:
# Swizzle weight scales from flat checkpoint layout [E, N, K//32]
# to the CUTLASS tiled layout.
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
swizzle_mxfp4_scales,
)
E = layer.w13_weight_scale.shape[0]
w13_N = layer.w13_weight_scale.shape[1]
w13_scale_K = layer.w13_weight_scale.shape[2]
w13_K = w13_scale_K * 32
w2_M = layer.w2_weight_scale.shape[1]
w2_scale_N = layer.w2_weight_scale.shape[2]
w2_N = w2_scale_N * 32
swizzled_w13 = []
swizzled_w2 = []
for e_idx in range(E):
s13 = layer.w13_weight_scale[e_idx]
sw13 = swizzle_mxfp4_scales(s13, w13_N, w13_K)
swizzled_w13.append(sw13.reshape(w13_N, w13_scale_K))
s2 = layer.w2_weight_scale[e_idx]
sw2 = swizzle_mxfp4_scales(s2, w2_M, w2_N)
swizzled_w2.append(sw2.reshape(w2_M, w2_scale_N))
layer.w13_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w13), requires_grad=False
)
layer.w2_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w2), requires_grad=False
)
elif current_platform.is_xpu():
# The XPU fused-MoE kernel consumes the packed layout directly; no
# swizzle / repack / transpose is required.
pass
else:
logger.warning_once(
"This device lacks native FP4 compute; using weight-only FP4 "
"via the Marlin kernel, which may reduce performance for "
"compute-heavy workloads."
)
prepare_moe_fp4_layer_for_marlin(layer)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None:
assert self.experts_cls is not None
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
mxfp4_backend=self.mxfp4_backend,
routing_tables=layer._expert_routing_tables(),
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
logger = init_logger(__name__)
class INCMxfp4Scheme(INCScheme):
"""MXFP4 (W4A4) scheme for AutoRound checkpoints.
Dispatches to :class:`INCMxfp4LinearMethod` for linear layers and
:class:`INCMxfp4MoEMethod` for fused MoE layers; see those classes for the
per-module weight layout and kernel-selection details.
"""
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_mxfp4
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer, prefix
from .inc_mxfp4_linear import INCMxfp4LinearMethod
return INCLinearMethod(INCMxfp4LinearMethod(layer_config))
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, prefix, layer_config
from .inc_mxfp4_moe import INCMxfp4MoEMethod
return INCMxfp4MoEMethod(layer.moe_config)
@@ -7,7 +7,6 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
@@ -97,8 +96,8 @@ class INCWna16Scheme(INCScheme):
layer_config: "INCLayerConfig",
):
del config, prefix
# XPU and CPU do not support MoE quantization yet
if current_platform.is_xpu() or current_platform.is_cpu():
# CPU does not support quantized MoE yet.
if current_platform.is_cpu():
from vllm.model_executor.layers.fused_moe import (
UnquantizedFusedMoEMethod,
)
@@ -120,21 +119,17 @@ def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
MoeWNA16Method,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supported,
check_moe_marlin_supports_layer,
)
gptq_type_map = {
(4, True): scalar_types.uint4b8,
(8, True): scalar_types.uint8b128,
}
use_marlin = (layer_config.bits, layer_config.sym) in gptq_type_map
if use_marlin:
use_marlin = check_marlin_supported(
gptq_type_map[(layer_config.bits, layer_config.sym)],
layer_config.group_size,
has_zp=not layer_config.sym,
) and check_moe_marlin_supports_layer(layer, layer_config.group_size)
# AutoGPTQMoEMethod selects its fused-MoE backend through the WNA16 oracle
# (Marlin on CUDA, XPUExpertsWNA16 on XPU). Gate only on the layer-shape
# check like compressed-tensors does; the capability-based
# check_marlin_supported is skipped so the XPU path is reachable.
use_marlin = (layer_config.bits, layer_config.sym) in {
(4, True),
(8, True),
} and check_moe_marlin_supports_layer(layer, layer_config.group_size)
if use_marlin:
return AutoGPTQMoEMethod(
@@ -169,21 +164,12 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
MoeWNA16Method,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supported,
check_moe_marlin_supports_layer,
)
awq_type_map = {
4: scalar_types.uint4,
8: scalar_types.uint8,
}
use_marlin = layer_config.bits in awq_type_map
if use_marlin:
use_marlin = check_marlin_supported(
awq_type_map[layer_config.bits],
layer_config.group_size,
not layer_config.sym,
) and check_moe_marlin_supports_layer(layer, layer_config.group_size)
use_marlin = layer_config.bits in (4, 8) and check_moe_marlin_supports_layer(
layer, layer_config.group_size
)
if use_marlin:
return AutoAWQMoEMethod(