From 8f03625b3d14969e59b2adb06097b2f20be5ce8c Mon Sep 17 00:00:00 2001 From: Ganesh R Date: Tue, 1 Sep 2026 15:01:31 +0530 Subject: [PATCH] [CPU][Zen] Route Int8 MoE inference through zentorch on AMD (#44834) Signed-off-by: R Co-authored-by: Cursor Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .buildkite/hardware_tests/cpu.yaml | 3 + tests/kernels/moe/test_zen_cpu_int8_moe.py | 253 ++++++++++++++++++ .../layers/fused_moe/experts/cpu_moe.py | 162 +++++++++++ .../layers/fused_moe/oracle/int8.py | 3 +- .../compressed_tensors_moe_w8a8_int8.py | 71 +++++ vllm/model_executor/models/mixtral.py | 9 +- 6 files changed, 499 insertions(+), 2 deletions(-) create mode 100644 tests/kernels/moe/test_zen_cpu_int8_moe.py diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index 2dad3c5592..c3806a1795 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -18,6 +18,8 @@ steps: - vllm/model_executor/layers/attention/mla_attention.py - tests/kernels/moe/test_cpu_fused_moe.py - tests/kernels/moe/test_cpu_quant_fused_moe.py + - tests/kernels/moe/test_zen_cpu_int8_moe.py + - vllm/model_executor/layers/fused_moe/experts/cpu_moe.py - tests/kernels/test_onednn.py - tests/kernels/test_awq_int4_to_int8.py - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py @@ -36,6 +38,7 @@ steps: pytest -x -v -s tests/kernels/attention/test_amx_mla.py pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py + pytest -x -v -s tests/kernels/moe/test_zen_cpu_int8_moe.py pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py pytest -x -v -s tests/kernels/test_onednn.py pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py diff --git a/tests/kernels/moe/test_zen_cpu_int8_moe.py b/tests/kernels/moe/test_zen_cpu_int8_moe.py new file mode 100644 index 0000000000..a01bc3d1bf --- /dev/null +++ b/tests/kernels/moe/test_zen_cpu_int8_moe.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the Zen CPU INT8 MoE experts (ZenCPUExpertsInt8).""" + +import pytest +import torch + +from tests.kernels.moe.test_cpu_fused_moe import ( + _make_moe_config, + quantize_per_channel, + ref_fused_moe_int8, +) +from vllm.model_executor.kernels.linear.zentorch_utils import ( + _ZENTORCH_MOE_ACTIVATIONS, +) +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEParallelConfig, + int8_w8a8_moe_quant_config, +) +from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + ZenCPUExpertsInt8, + select_experts, +) +from vllm.model_executor.layers.fused_moe.oracle.int8 import ( + Int8MoeBackend, + backend_to_kernel_cls, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt8DynamicTokenSym, + kInt8StaticChannelSym, +) +from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_random_seed + +if not current_platform.is_cpu(): + pytest.skip("skipping CPU-only tests", allow_module_level=True) + +EXPERT_NUM = 8 +HIDDEN_SIZE = 128 +INTERMEDIATE_SIZE = 128 +BATCH_SIZE = 32 +# zentorch names its activations as strings; the experts API is enum-typed. +ZENTORCH_ACT = [act for act in MoEActivation if act.value in _ZENTORCH_MOE_ACTIVATIONS] + + +@pytest.fixture +def mock_zentorch_fused_moe(): + """Reference zentorch_fused_moe, registered when zentorch is absent.""" + if hasattr(torch.ops.zentorch, "zentorch_fused_moe"): + yield None + return + + calls: dict[str, dict] = {} + + def _fused_moe( + output, + input, + w13, + w2, + w13_bias, + w2_bias, + topk_weights, + topk_id, + skip_weighted, + act, + w13_scales=None, + w2_scales=None, + zentorch_op_name="", + ): + calls["fused_moe"] = { + "w13": w13, + "w2": w2, + "w13_bias": w13_bias, + "w2_bias": w2_bias, + "topk_weights": topk_weights, + "topk_id": topk_id, + "skip_weighted": skip_weighted, + "act": act, + "w13_scales": w13_scales, + "w2_scales": w2_scales, + } + output.copy_( + ref_fused_moe_int8( + input, + w13, + w2, + w13_scales.float(), + w2_scales.float(), + None if w13_bias is None else w13_bias.float(), + None if w2_bias is None else w2_bias.float(), + topk_weights, + topk_id, + MoEActivation(act), + ) + ) + + lib_def = torch.library.Library("zentorch", "DEF") + lib_def.define( + "zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, " + "Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, " + "Tensor topk_id, bool skip_weighted, str act, " + "Tensor? w13_scales=None, Tensor? w2_scales=None, *, " + "str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()" + ) + lib_impl = torch.library.Library("zentorch", "IMPL", "CPU") + lib_impl.impl("zentorch_fused_moe", _fused_moe) + + yield calls + + lib_impl._destroy() + lib_def._destroy() + + +def _make_int8_moe_weights(use_bias: bool, dtype: torch.dtype = torch.bfloat16): + up_dim = 2 * INTERMEDIATE_SIZE + w13_fp = torch.randn((EXPERT_NUM, up_dim, HIDDEN_SIZE), dtype=dtype) / ( + 0.5 * HIDDEN_SIZE**0.5 + ) + w2_fp = torch.randn((EXPERT_NUM, HIDDEN_SIZE, INTERMEDIATE_SIZE), dtype=dtype) / ( + 0.5 * INTERMEDIATE_SIZE**0.5 + ) + w13, w13_scale = quantize_per_channel(w13_fp) + w2, w2_scale = quantize_per_channel(w2_fp) + w13_scale = w13_scale.to(torch.bfloat16).float() + w2_scale = w2_scale.to(torch.bfloat16).float() + + w13_bias = w2_bias = None + if use_bias: + w13_bias = torch.randn((EXPERT_NUM, up_dim), dtype=dtype) / (0.5 * up_dim**0.5) + w2_bias = torch.randn((EXPERT_NUM, HIDDEN_SIZE), dtype=dtype) / ( + 0.5 * HIDDEN_SIZE**0.5 + ) + return w13, w2, w13_scale, w2_scale, w13_bias, w2_bias + + +def test_zen_int8_first_in_cpu_backend_priority(): + """The Zen experts are preferred over the generic CPU int8 experts.""" + assert backend_to_kernel_cls(Int8MoeBackend.CPU)[0] is ZenCPUExpertsInt8 + + +def test_zen_int8_support_predicates(mock_zentorch_fused_moe, monkeypatch): + monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: False) + assert not ZenCPUExpertsInt8._supports_current_device() + monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True) + assert ZenCPUExpertsInt8._supports_current_device() + assert ZenCPUExpertsInt8._supports_quant_scheme( + kInt8StaticChannelSym, kInt8DynamicTokenSym + ) + assert not ZenCPUExpertsInt8._supports_quant_scheme(kInt8StaticChannelSym, None) + for act in ZENTORCH_ACT: + assert ZenCPUExpertsInt8._supports_activation(act) + assert not ZenCPUExpertsInt8._supports_activation(MoEActivation.RELU2) + assert not ZenCPUExpertsInt8._supports_activation( + MoEActivation.SWIGLUOAI_UNINTERLEAVE + ) + assert not ZenCPUExpertsInt8._supports_no_act_and_mul() + assert ZenCPUExpertsInt8._supports_parallel_config( + FusedMoEParallelConfig.make_no_parallel() + ) + + +@pytest.mark.parametrize("act", ZENTORCH_ACT) +@pytest.mark.parametrize("use_bias", [False, True]) +def test_zen_int8_dispatch_contract( + default_vllm_config, + mock_zentorch_fused_moe, + act: MoEActivation, + use_bias: bool, +): + """apply() hands zentorch the weights, scales and biases it expects.""" + if mock_zentorch_fused_moe is None: + pytest.skip("real zentorch is installed; the recorded call is unavailable") + set_random_seed(0) + + topk_num = EXPERT_NUM // 2 + w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = _make_int8_moe_weights(use_bias) + moe_config = _make_moe_config( + EXPERT_NUM, + HIDDEN_SIZE, + INTERMEDIATE_SIZE, + topk_num, + torch.bfloat16, + act, + use_bias, + ) + # A layer carries per-channel scales as [E, N, 1]. + quant_config = int8_w8a8_moe_quant_config( + w1_scale=w13_scale.unsqueeze(-1), + w2_scale=w2_scale.unsqueeze(-1), + a1_scale=None, + a2_scale=None, + w1_bias=w13_bias, + w2_bias=w2_bias, + per_act_token_quant=True, + ) + + input = torch.randn((BATCH_SIZE, HIDDEN_SIZE), dtype=torch.bfloat16) / ( + 0.5 * HIDDEN_SIZE**0.5 + ) + router_logits = torch.randn((BATCH_SIZE, EXPERT_NUM), dtype=torch.bfloat16) + + experts = ZenCPUExpertsInt8(moe_config, quant_config) + # zentorch_fused_moe takes no expert_map. + assert not experts.supports_expert_map() + output = experts.apply( + hidden_states=input, + w1=w13, + w2=w2, + router_logits=router_logits, + activation=act, + global_num_experts=EXPERT_NUM, + expert_map=None, + a1q_scale=None, + apply_router_weight_on_input=False, + ) + + call = mock_zentorch_fused_moe["fused_moe"] + assert call["w13_scales"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE) + assert call["w2_scales"].shape == (EXPERT_NUM, HIDDEN_SIZE) + assert call["w13_scales"].dtype == torch.bfloat16 + assert call["w2_scales"].dtype == torch.bfloat16 + assert call["act"] == act.value.lower() + assert call["skip_weighted"] is False + assert call["topk_weights"].dtype == torch.float32 + assert call["topk_id"].dtype == torch.int32 + if use_bias: + assert call["w13_bias"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE) + assert call["w2_bias"].shape == (EXPERT_NUM, HIDDEN_SIZE) + else: + assert call["w13_bias"] is None and call["w2_bias"] is None + + topk_weights, topk_ids = select_experts( + hidden_states=input, + router_logits=router_logits, + use_grouped_topk=False, + top_k=topk_num, + renormalize=False, + scoring_func="softmax", + ) + ref = ref_fused_moe_int8( + input, + w13, + w2, + w13_scale, + w2_scale, + None if w13_bias is None else w13_bias.float(), + None if w2_bias is None else w2_bias.float(), + topk_weights.to(torch.float32), + topk_ids.to(torch.int32), + act, + ) + torch.testing.assert_close(output, ref) diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 3cdf280c2b..00c3dcc17b 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -20,7 +20,10 @@ from vllm._custom_ops import ( cpu_prepack_moe_weight_int8, fused_experts_cpu, ) +from vllm.logger import init_logger from vllm.model_executor.kernels.linear.zentorch_utils import ( + _ZENTORCH_MOE_ACTIVATIONS, + has_zentorch_op, is_zentorch_moe_config_supported, is_zentorch_moe_supported, ) @@ -44,6 +47,7 @@ from vllm.model_executor.utils import replace_parameter from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.math_utils import round_up +logger = init_logger(__name__) # =========================================================================== # Routing # =========================================================================== @@ -1382,3 +1386,161 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic): "neon", skip_weighted=apply_router_weight_on_input, ) + + +class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic): + """AMD Zen INT8 MoE with per-token activation and channelwise weight + quantization, dispatched through zentorch.""" + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__(moe_config, quant_config) + assert self.w1_scale is not None and self.w2_scale is not None, ( + "ZenCPUExpertsInt8 requires per-channel weight scales on the layer." + ) + num_experts = self.w1_scale.shape[0] + self._w1_scale_bf16 = ( + self.w1_scale.detach() + .to(torch.bfloat16) + .reshape(num_experts, -1) + .contiguous() + ) + self._w2_scale_bf16 = ( + self.w2_scale.detach() + .to(torch.bfloat16) + .reshape(num_experts, -1) + .contiguous() + ) + self._w1_bias_bf16 = ( + None + if self.w1_bias is None + else self.w1_bias.detach().to(torch.bfloat16).contiguous() + ) + self._w2_bias_bf16 = ( + None + if self.w2_bias is None + else self.w2_bias.detach().to(torch.bfloat16).contiguous() + ) + logger.info_once( + "[zen_cpu] Using zentorch_fused_moe for W8A8 INT8 MoE (monolithic experts)" + ) + + @property + def expects_unquantized_inputs(self) -> bool: + # zentorch_fused_moe quantizes activations itself. + return True + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + return has_zentorch_op(["zentorch_fused_moe"]) + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return str(activation.value).lower() in _ZENTORCH_MOE_ACTIVATIONS + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + return not moe_parallel_config.use_ep + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return (weight_key, activation_key) == ( + kInt8StaticChannelSym, + kInt8DynamicTokenSym, + ) + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Default, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + def supports_expert_map(self) -> bool: + return False + + def apply( + self, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + apply_router_weight_on_input: bool, + num_expert_group: int | None = None, + e_score_correction_bias: torch.Tensor | None = None, + routed_scaling_factor: float | None = None, + topk_group: int | None = None, + ) -> torch.Tensor: + topk_weights, topk_ids = select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + use_grouped_topk=num_expert_group is not None, + top_k=self.moe_config.experts_per_token, + renormalize=self.moe_config.routing_method + in ( + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ), + topk_group=topk_group, + num_expert_group=num_expert_group, + scoring_func="softmax", + routed_scaling_factor=( + routed_scaling_factor if routed_scaling_factor is not None else 1.0 + ), + e_score_correction_bias=e_score_correction_bias, + ) + + if apply_router_weight_on_input: + assert topk_ids.size(1) == 1, ( + "apply_router_weight_on_input is only implemented for topk=1" + ) + hidden_states = hidden_states.mul(topk_weights.to(hidden_states.dtype)) + + output = torch.empty_like(hidden_states) + torch.ops.zentorch.zentorch_fused_moe( + output, + hidden_states, + w1, + w2, + self._w1_bias_bf16, + self._w2_bias_bf16, + topk_weights.to(torch.float32).contiguous(), + topk_ids.to(torch.int32).contiguous(), + apply_router_weight_on_input, # skip_weighted + str(activation.value).lower(), + self._w1_scale_bf16, + self._w2_scale_bf16, + ) + return output diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py index 2d6e5b1369..cd15dceaf5 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py @@ -73,9 +73,10 @@ def backend_to_kernel_cls( from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( ArmCPUExpertsInt8, CPUExpertsInt8, + ZenCPUExpertsInt8, ) - return [ArmCPUExpertsInt8, CPUExpertsInt8] + return [ZenCPUExpertsInt8, ArmCPUExpertsInt8, CPUExpertsInt8] else: raise ValueError(f"Unknown Int8 MoE backend: {backend.value}") diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py index f27c54c8f4..1f6e87a672 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py @@ -83,6 +83,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): params_dtype: torch.dtype, **extra_weight_attrs, ): + # Capture bias dtype before params_dtype is overridden to int8 below. + bias_dtype = params_dtype params_dtype = torch.int8 w13_num_shards = 2 if self.moe.is_act_and_mul else 1 @@ -111,6 +113,26 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): layer.register_parameter("w2_weight", w2_weight) set_weight_attrs(w2_weight, extra_weight_attrs) + # Per-expert biases (gpt-oss), allocated only when has_bias. + if self.moe.has_bias: + w13_bias = torch.nn.Parameter( + torch.zeros( + num_experts, + w13_num_shards * intermediate_size_per_partition, + dtype=bias_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_bias", w13_bias) + set_weight_attrs(w13_bias, extra_weight_attrs) + + w2_bias = torch.nn.Parameter( + torch.zeros(num_experts, hidden_size, dtype=bias_dtype), + requires_grad=False, + ) + layer.register_parameter("w2_bias", w2_bias) + set_weight_attrs(w2_bias, extra_weight_attrs) + # WEIGHT_SCALES assert self.weight_quant.strategy == QuantizationStrategy.CHANNEL w13_weight_scale = torch.nn.Parameter( @@ -141,6 +163,53 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): layer.w2_input_scale = None def process_weights_after_loading(self, layer: RoutedExperts) -> None: + # ZenDNN's swiglu_oai_mul reads gate/up interleaved (stride 2), not the + # half-split layout _load_w13 leaves; reorder before the conversion below. + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + ZenCPUExpertsInt8, + ) + + _act = getattr(layer, "activation", None) + _act_str = getattr(_act, "value", _act) + if ( + self.experts_cls is ZenCPUExpertsInt8 + and isinstance(_act_str, str) + and _act_str.lower() == "swigluoai" + ): + two_i = layer.w13_weight.size(1) + i = two_i // 2 + device = layer.w13_weight.device + perm = torch.stack( + [ + torch.arange(0, i, device=device), + torch.arange(i, two_i, device=device), + ], + dim=1, + ).flatten() + has_w13_bias = getattr(layer, "w13_bias", None) is not None + logger.info_once( + "[zen_cpu][swigluoai-permute] Reordering w13 from " + "half-split to interleaved for ZenDNN swiglu_oai_mul " + "kernel: E=%d, 2I=%d, I=%d, has_w13_bias=%s", + layer.w13_weight.size(0), + two_i, + i, + has_w13_bias, + ) + layer.w13_weight = torch.nn.Parameter( + layer.w13_weight.data[:, perm, :].contiguous(), + requires_grad=False, + ) + layer.w13_weight_scale = torch.nn.Parameter( + layer.w13_weight_scale.data[:, perm].contiguous(), + requires_grad=False, + ) + if has_w13_bias: + layer.w13_bias = torch.nn.Parameter( + layer.w13_bias.data[:, perm].contiguous(), + requires_grad=False, + ) + w13, w2 = convert_to_int8_moe_kernel_format( int8_backend=self.int8_backend, w13=layer.w13_weight, @@ -169,6 +238,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): w2_scale=layer.w2_weight_scale, a1_scale=layer.w13_input_scale, a2_scale=layer.w2_input_scale, + w1_bias=getattr(layer, "w13_bias", None), + w2_bias=getattr(layer, "w2_bias", None), per_act_token_quant=True, layer=layer, ) diff --git a/vllm/model_executor/models/mixtral.py b/vllm/model_executor/models/mixtral.py index d154208606..717c935751 100644 --- a/vllm/model_executor/models/mixtral.py +++ b/vllm/model_executor/models/mixtral.py @@ -288,7 +288,14 @@ class MixtralModel(nn.Module): ".q_proj": (".qkv_proj", "q"), ".k_proj": (".qkv_proj", "k"), ".v_proj": (".qkv_proj", "v"), - } + }, + orig_to_new_substr={ + # W8A8 compressed-tensors checkpoints name experts gate/up/down; + # map to the w1/w3/w2 the FusedMoE loader expects (no-op if native). + ".gate_proj.": ".w1.", + ".up_proj.": ".w3.", + ".down_proj.": ".w2.", + }, ) def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):