From 5addde2a0afa1e27b6e0c478ba9b6995724f006d Mon Sep 17 00:00:00 2001 From: Ganesh R Date: Thu, 1 Oct 2026 13:21:53 +0530 Subject: [PATCH] [CPU][Zen] Pass f32 weight scales to the zentorch INT8 MoE (#59434) Signed-off-by: Ganesh R Co-authored-by: Cursor Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> --- tests/kernels/moe/test_zen_cpu_int8_moe.py | 5 +++-- .../layers/fused_moe/experts/cpu_moe.py | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/kernels/moe/test_zen_cpu_int8_moe.py b/tests/kernels/moe/test_zen_cpu_int8_moe.py index cdcfe3b165..83e051a1e3 100644 --- a/tests/kernels/moe/test_zen_cpu_int8_moe.py +++ b/tests/kernels/moe/test_zen_cpu_int8_moe.py @@ -224,8 +224,9 @@ def test_zen_int8_dispatch_contract( call = mock_zentorch_fused_moe["fused_moe"] assert call["w13_scales"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE) assert call["w2_scales"].shape == (EXPERT_NUM, HIDDEN_SIZE) - assert call["w13_scales"].dtype == torch.bfloat16 - assert call["w2_scales"].dtype == torch.bfloat16 + # f32 scales keep the layer on zentorch's fast routed MoE executor. + assert call["w13_scales"].dtype == torch.float32 + assert call["w2_scales"].dtype == torch.float32 assert call["act"] == act.value.lower() assert call["skip_weighted"] is False assert call["topk_weights"].dtype == torch.float32 diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 9b66a1f853..14b25ce30e 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -1502,15 +1502,17 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular): "ZenCPUExpertsInt8 requires per-channel weight scales on the layer." ) num_experts = self.w1_scale.shape[0] - self._w1_scale_bf16 = ( + # ZenDNN's fast routed MoE kernel only accepts f32 expert scales; bf16 + # silently drops the layer onto the slower per-expert path. + self._w1_scale_f32 = ( self.w1_scale.detach() - .to(torch.bfloat16) + .to(torch.float32) .reshape(num_experts, -1) .contiguous() ) - self._w2_scale_bf16 = ( + self._w2_scale_f32 = ( self.w2_scale.detach() - .to(torch.bfloat16) + .to(torch.float32) .reshape(num_experts, -1) .contiguous() ) @@ -1637,8 +1639,8 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular): topk_ids.to(torch.int32).contiguous(), apply_router_weight_on_input, # skip_weighted str(activation.value).lower(), - self._w1_scale_bf16, - self._w2_scale_bf16, + self._w1_scale_f32, + self._w2_scale_f32, )