[CPU] Support CPU W4A16 INT4 MoE (#43409)

Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
This commit is contained in:
Yuwen Zhou
2026-06-12 07:12:37 +00:00
committed by GitHub
parent a2c72d4388
commit 0cd9b7af25
8 changed files with 685 additions and 22 deletions
@@ -5,7 +5,12 @@
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._custom_ops import CPUQuantMethod, fused_experts_cpu
from vllm._custom_ops import (
CPUQuantAlgo,
CPUQuantMethod,
convert_weight_packed_scale_zp,
fused_experts_cpu,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -17,6 +22,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
kInt4Static,
kMxfp4Static,
)
from vllm.platforms import current_platform
@@ -318,3 +324,209 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
limit,
True, # is_vnni
)
def prepare_int4_moe_layer_for_cpu(
w13_packed: torch.Tensor,
w2_packed: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
quant_algo: CPUQuantAlgo = CPUQuantAlgo.GPTQ,
w13_zeros: torch.Tensor | None = None,
w2_zeros: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
"""Repack INT4 MoE weights via convert_weight_packed_scale_zp for CPU.
Args:
w13_packed: [E, K//8, 2*I] int32 (packed int4)
w2_packed: [E, I//8, K] int32 (packed int4)
w13_scale: [E, num_groups, 2*I] float16/bf16
w2_scale: [E, num_groups, K] float16/bf16
quant_algo: CPUQuantAlgo.GPTQ or CPUQuantAlgo.AWQ
w13_zeros: optional [E, num_groups, N//8] int32 packed zeros.
If None, synthetic zeros are created for symmetric quant.
w2_zeros: optional [E, num_groups, N//8] int32 packed zeros.
If None, synthetic zeros are created for symmetric quant.
Returns:
(blocked_w13, blocked_w2, blocked_s13, blocked_s2,
blocked_z13, blocked_z2)
"""
E = w13_packed.size(0)
# No qzeros are available in compressed-tensors symmetric checkpoints.
# The GPTQ unpack kernel (unpack_4bit_to_32bit_signed) adds +1 to stored zeros,
# so we store 7 per nibble: 0x77777777 → +1 → 8.
if w13_zeros is None:
num_groups_w13 = w13_scale.size(1)
N_w13 = w13_scale.size(2) # 2*I
_zp = 0x77777777
w13_zeros = torch.full(
(E, num_groups_w13, N_w13 // 8),
_zp,
dtype=torch.int32,
)
if w2_zeros is None:
num_groups_w2 = w2_scale.size(1)
N_w2 = w2_scale.size(2) # K
_zp = 0x77777777
w2_zeros = torch.full(
(E, num_groups_w2, N_w2 // 8),
_zp,
dtype=torch.int32,
)
blocked_w13, blocked_z13, blocked_s13 = convert_weight_packed_scale_zp(
w13_packed, w13_zeros, w13_scale, quant_algo
)
blocked_w2, blocked_z2, blocked_s2 = convert_weight_packed_scale_zp(
w2_packed, w2_zeros, w2_scale, quant_algo
)
return (blocked_w13, blocked_w2, blocked_s13, blocked_s2, blocked_z13, blocked_z2)
class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
"""CPU INT4 W4A16 group-quantized monolithic MoE experts.
Weights are int4 (packed), activations are bf16/fp16.
Internally uses int8 compute via fused_experts_cpu with INT4_W4A8.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
)
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation == MoEActivation.SILU
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kInt4Static, None),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
if apply_router_weight_on_input:
raise NotImplementedError(
"CPUExpertsInt4 (W4A16) does not support "
"apply_router_weight_on_input=True. "
)
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
return fused_experts_cpu(
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
False, # inplace
CPUQuantMethod.INT4_W4A8,
self.w1_scale,
self.w2_scale,
self.w1_zp,
self.w2_zp,
None, # block_size
None, # w1_bias
None, # w2_bias
None, # alpha
None, # limit
True, # is_vnni
)
@@ -45,6 +45,7 @@ logger = init_logger(__name__)
class WNA16MoEBackend(Enum):
MARLIN = "MARLIN"
BATCHED_MARLIN = "BATCHED_MARLIN"
CPU = "CPU"
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
XPU = "XPU"
@@ -65,6 +66,12 @@ def backend_to_kernel_cls(
)
return [XPUExpertsWNA16]
elif backend == WNA16MoEBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsInt4,
)
return [CPUExpertsInt4]
else:
raise ValueError(f"Unknown WNA16 MoE backend: {backend.value}")
@@ -73,6 +80,8 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
"""
Get available backends in priority order based on platform and config.
"""
if current_platform.is_cpu():
return [WNA16MoEBackend.CPU]
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
@@ -210,17 +219,21 @@ def make_wna16_moe_kernel(
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsInt4,
)
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsWNA16,
)
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts
# and BatchedMarlinExperts
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
# BatchedMarlinExperts, XPUExpertsWNA16, and CPUExpertsInt4
assert experts_cls in (
MarlinExperts,
BatchedMarlinExperts,
TrtLlmMxint4ExpertsMonolithic,
XPUExpertsWNA16,
CPUExpertsInt4,
)
is_monolithic = experts_cls.is_monolithic()
@@ -683,6 +696,117 @@ def _process_awq_weights_marlin(
)
def _process_weights_cpu(
quant_config: QuantizationConfig | QuantizationArgs | None,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_g_idx: torch.Tensor | None = None,
w2_g_idx: torch.Tensor | None = None,
w13_qzeros: torch.Tensor | None = None,
w2_qzeros: torch.Tensor | None = None,
w13_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, # w13_qweight
torch.Tensor, # w2_qweight
torch.Tensor, # w13_scales
torch.Tensor, # w2_scales
torch.Tensor | None, # w13_g_idx
torch.Tensor | None, # w2_g_idx
torch.Tensor | None, # w13_g_idx_sort_indices
torch.Tensor | None, # w2_g_idx_sort_indices
torch.Tensor | None, # w13_qzeros
torch.Tensor | None, # w2_qzeros
torch.Tensor | None, # w13_input_global_scale
torch.Tensor | None, # w2_input_global_scale
torch.Tensor | None, # w13_bias
torch.Tensor | None, # w2_bias
]:
"""CPU INT4 W4A16 weight post-processing."""
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_int4_moe_layer_for_cpu,
)
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQConfig,
)
from vllm.model_executor.layers.quantization.awq_marlin import (
AWQMarlinConfig,
)
# Detect packing format.
# AWQ: qweight is [E, K, 2*N//8] (packed along output/N dim).
# GPTQ: qweight is [E, K//8, 2*N] (packed along input/K dim).
# compressed-tensors: qweight is [E, K//8, 2*N] (packed along input/K dim).
if isinstance(quant_config, AWQMarlinConfig):
# AWQ: K is stored unpacked in dim 1.
cpu_quant_algo = ops.CPUQuantAlgo.AWQ
elif isinstance(quant_config, (AutoGPTQConfig, QuantizationArgs)):
# GPTQ / compressed-tensors: K//8 is stored packed in dim 1.
if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act:
raise NotImplementedError(
"CPU WNA16 MoE backend does not support GPTQ with "
"desc_act=True. The fused MoE kernel has no g_idx "
"reordering support."
)
cpu_quant_algo = ops.CPUQuantAlgo.GPTQ
else:
raise TypeError(
"CPU WNA16 MoE backend requires AWQMarlinConfig, AutoGPTQConfig "
f"or QuantizationArgs, got {type(quant_config).__name__}."
)
# Determine zero points for repacking.
w13_zeros: torch.Tensor | None = None
w2_zeros: torch.Tensor | None = None
if w13_qzeros is not None:
w13_zeros = (
w13_qzeros.data.view(torch.int32)
if w13_qzeros.dtype != torch.int32
else w13_qzeros.data
)
if w2_qzeros is not None:
w2_zeros = (
w2_qzeros.data.view(torch.int32)
if w2_qzeros.dtype != torch.int32
else w2_qzeros.data
)
(
blocked_w13,
blocked_w2,
blocked_s13,
blocked_s2,
blocked_z13,
blocked_z2,
) = prepare_int4_moe_layer_for_cpu(
w13,
w2,
w13_scale,
w2_scale,
quant_algo=cpu_quant_algo,
w13_zeros=w13_zeros,
w2_zeros=w2_zeros,
)
return (
blocked_w13,
blocked_w2,
blocked_s13,
blocked_s2,
w13_g_idx,
w2_g_idx,
None, # w13_g_idx_sort_indices (unused on CPU)
None, # w2_g_idx_sort_indices (unused on CPU)
blocked_z13,
blocked_z2,
None, # w13_input_global_scale
None, # w2_input_global_scale
w13_bias.to(torch.float32) if w13_bias is not None else None,
w2_bias.to(torch.float32) if w2_bias is not None else None,
)
def _process_weights_xpu(
layer: torch.nn.Module,
quant_config: QuantizationConfig,
@@ -857,6 +981,20 @@ def convert_to_wna16_moe_kernel_format(
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.CPU:
return _process_weights_cpu(
quant_config,
w13,
w2,
w13_scale,
w2_scale,
w13_g_idx,
w2_g_idx,
w13_qzeros,
w2_qzeros,
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM:
return _process_weights_flashinfer(
w13,
@@ -16,6 +16,7 @@ from vllm.model_executor.kernels.linear import (
)
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEExpertsModular,
FusedMoEMethodBase,
FusedMoEQuantConfig,
FusedMoeWeightScaleSupported,
@@ -640,8 +641,11 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
device = layer.w13_qweight.device
layer.workspace = marlin_make_workspace_new(device, 4)
if self.experts_cls is not None and issubclass(
self.experts_cls, FusedMoEExpertsModular
):
device = layer.w13_qweight.device
layer.workspace = marlin_make_workspace_new(device, 4)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
is_a_8bit = self.input_dtype is not None and self.input_dtype.itemsize == 1
@@ -660,8 +664,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2_g_idx,
w13_g_idx_sort_indices,
w2_g_idx_sort_indices,
_w13_qzeros,
_w2_qzeros,
w13_qzeros,
w2_qzeros,
w13_input_global_scale,
w2_input_global_scale,
w13_bias,
@@ -689,6 +693,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
replace_parameter(layer, "w2_g_idx", w2_g_idx)
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
if w13_qzeros is not None:
replace_parameter(layer, "w13_qzeros", w13_qzeros)
if w2_qzeros is not None:
replace_parameter(layer, "w2_qzeros", w2_qzeros)
if w13_input_global_scale is not None:
if hasattr(layer, "w13_input_global_scale"):
replace_parameter(
@@ -735,8 +743,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
is_k_full=self.is_k_full,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_g_idx_sort_indices=layer.w13_g_idx_sort_indices,
w2_g_idx_sort_indices=layer.w2_g_idx_sort_indices,
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
routing_tables=layer._expert_routing_tables(),
)
@@ -750,12 +758,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2_scale=layer.w2_scales,
weight_bits=self.quant_config.weight_bits,
group_size=self.quant_config.group_size,
w1_zp=getattr(layer, "w13_qzeros", None)
if not self.quant_config.is_sym
else None,
w2_zp=getattr(layer, "w2_qzeros", None)
if not self.quant_config.is_sym
else None,
w1_zp=getattr(layer, "w13_qzeros", None),
w2_zp=getattr(layer, "w2_qzeros", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
@@ -794,3 +798,27 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
def apply_monolithic(
self,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
router_logits=router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
@@ -700,8 +700,8 @@ class AWQMarlinMoEMethod(FusedMoEMethodBase):
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
w13_g_idx_sort_indices=layer.w13_g_idx_sort_indices,
w2_g_idx_sort_indices=layer.w2_g_idx_sort_indices,
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
routing_tables=layer._expert_routing_tables(),
)
@@ -757,3 +757,27 @@ class AWQMarlinMoEMethod(FusedMoEMethodBase):
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
def apply_monolithic(
self,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
router_logits=router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
@@ -10,6 +10,7 @@ from compressed_tensors.quantization import (
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEExpertsModular,
RoutedExperts,
SharedExperts,
)
@@ -414,8 +415,9 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
if not self.symmetric:
if w13_qzeros is not None:
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
if w2_qzeros is not None:
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
# Marlin-specific parameters (not needed for Flashinfer)
@@ -437,9 +439,12 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
layer.workspace = marlin_make_workspace_new(
layer.w13_weight_g_idx.device, 4
)
if self.experts_cls is not None and issubclass(
self.experts_cls, FusedMoEExpertsModular
):
layer.workspace = marlin_make_workspace_new(
layer.w13_weight_g_idx.device, 4
)
# Alias packed weights to w13_weight/w2_weight for the modular kernel interface
layer.w13_weight = layer.w13_weight_packed