[CPU] Support cpu compressed-tensor w8a8 int8 moe (#42920)

Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
Signed-off-by: Yuwen Zhou <yuwen.zhou@intel.com>
This commit is contained in:
Yuwen Zhou
2026-06-29 03:04:05 +00:00
committed by GitHub
parent a2abce646f
commit 58d6a6e60a
6 changed files with 410 additions and 9 deletions
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU FP8 W8A16 and MXFP4 W4A16 fused MoE experts."""
"""CPU quantized fused MoE experts."""
import torch
@@ -23,10 +23,16 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
kInt4Static,
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
kMxfp4Static,
)
from vllm.platforms import current_platform
# ===========================================================================
# FP8 W8A16 MoE
# ===========================================================================
def prepare_fp8_moe_layer_for_cpu(
w13: torch.Tensor,
@@ -177,6 +183,11 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
)
# ===========================================================================
# MXFP4 W4A16 MoE
# ===========================================================================
def prepare_mxfp4_moe_layer_for_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
@@ -326,6 +337,11 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
)
# ===========================================================================
# INT4 W4A16 MoE
# ===========================================================================
def prepare_int4_moe_layer_for_cpu(
w13_packed: torch.Tensor,
w2_packed: torch.Tensor,
@@ -529,3 +545,159 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
None, # limit
True, # is_vnni
)
# ===========================================================================
# INT8 W8A8 MoE
# ===========================================================================
def prepare_int8_moe_layer_for_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""VNNI-prepack INT8 MoE weights for CPU kernel."""
packed_w13 = torch.ops._C.convert_weight_packed(w13)
packed_w2 = torch.ops._C.convert_weight_packed(w2)
return packed_w13, packed_w2
class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
monolithic MoE experts."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(
moe_config,
quant_config,
)
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation == MoEActivation.SILU
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kInt8StaticChannelSym, kInt8DynamicTokenSym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""VNNI-prepack INT8 MoE weights for CPU kernel."""
from vllm.model_executor.utils import replace_parameter
w13, w2 = prepare_int8_moe_layer_for_cpu(layer.w13_weight, layer.w2_weight)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
return fused_experts_cpu(
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
False, # inplace
CPUQuantMethod.INT8_W8A8,
self.w1_scale,
self.w2_scale,
None, # w1_zero
None, # w2_zero
None, # block_size (per-channel, no block)
None, # w1_bias
None, # w2_bias
None, # alpha
None, # limit
True, # is_vnni
)
@@ -22,12 +22,14 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
class Int8MoeBackend(Enum):
TRITON = "TRITON"
CPU = "CPU"
def _get_priority_backends(
@@ -36,7 +38,18 @@ def _get_priority_backends(
"""
Get available backends in priority order based on platform and config.
"""
return [Int8MoeBackend.TRITON]
_AVAILABLE_BACKENDS = [
Int8MoeBackend.TRITON,
Int8MoeBackend.CPU,
]
def _move_to_front(backends: list[Int8MoeBackend], backend: Int8MoeBackend) -> None:
backends.insert(0, backends.pop(backends.index(backend)))
if current_platform.is_cpu():
_move_to_front(_AVAILABLE_BACKENDS, Int8MoeBackend.CPU)
return _AVAILABLE_BACKENDS
def backend_to_kernel_cls(
@@ -49,6 +62,13 @@ def backend_to_kernel_cls(
return [TritonExperts]
elif backend == Int8MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsInt8,
)
return [CPUExpertsInt8]
else:
raise ValueError(f"Unknown Int8 MoE backend: {backend.value}")
@@ -176,6 +196,24 @@ def make_int8_moe_quant_config(
)
def convert_to_int8_moe_kernel_format(
int8_backend: Int8MoeBackend,
w13: torch.Tensor,
w2: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Convert INT8 MoE weights to backend-specific kernel format."""
if int8_backend == Int8MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_int8_moe_layer_for_cpu,
)
w13, w2 = prepare_int8_moe_layer_for_cpu(w13, w2)
elif int8_backend != Int8MoeBackend.TRITON:
raise ValueError(f"Unsupported Int8 MoE backend: {int8_backend.value}")
return w13, w2
def make_int8_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from compressed_tensors.quantization import (
QuantizationArgs,
@@ -20,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
convert_to_int8_moe_kernel_format,
make_int8_moe_kernel,
make_int8_moe_quant_config,
select_int8_moe_backend,
@@ -31,7 +31,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
logger = init_logger(__name__)
@@ -142,6 +142,14 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
layer.w2_input_scale = None
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
w13, w2 = convert_to_int8_moe_kernel_format(
int8_backend=self.int8_backend,
w13=layer.w13_weight,
w2=layer.w2_weight,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.experts_cls is not None
self.moe_kernel = make_int8_moe_kernel(
@@ -193,3 +201,27 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
def apply_monolithic(
self,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
x,
layer.w13_weight,
layer.w2_weight,
router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)