[CPU][Zen] Route BF16 MoE inference through zentorch on AMD (#44201)

Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Priyansh Jain
2026-08-10 23:20:05 -07:00
committed by GitHub
co-authored by Andreas Karatzas
parent b76cf758db
commit 2acb055ed7
4 changed files with 316 additions and 3 deletions
@@ -4,11 +4,32 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
import torch
from vllm.platforms import current_platform
__all__ = ["has_zentorch_op"]
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
__all__ = [
"has_zentorch_op",
"is_zentorch_moe_config_supported",
"is_zentorch_moe_supported",
"_ZENTORCH_MOE_ACTIVATIONS",
]
_ZENTORCH_MOE_ACTIVATIONS = frozenset({"gelu", "gelu_tanh", "silu", "swigluoai"})
def _moe_activation_to_str(activation: object) -> str:
"""Normalize activation to a lowercase string (enum-safe)."""
if hasattr(activation, "value"):
return str(activation.value).lower()
return str(activation).lower()
def has_zentorch_op(op_names: list[str]) -> bool:
@@ -21,3 +42,44 @@ def has_zentorch_op(op_names: list[str]) -> bool:
if ns is None:
return False
return all(hasattr(ns, op_name) for op_name in op_names)
def is_zentorch_moe_config_supported(moe_config: FusedMoEConfig) -> bool:
if not has_zentorch_op(["zentorch_fused_moe"]):
return False
if not moe_config.is_act_and_mul:
return False
act = _moe_activation_to_str(moe_config.activation)
return act in _ZENTORCH_MOE_ACTIVATIONS
def is_zentorch_moe_supported(layer: torch.nn.Module) -> bool:
if not has_zentorch_op(["zentorch_fused_moe"]):
logging.debug(
"Skipping zentorch fused-MoE: not a Zen CPU or "
"zentorch not loaded; using default MoE."
)
return False
moe_config = getattr(layer, "moe_config", None)
if moe_config is not None and not moe_config.is_act_and_mul:
logging.debug(
"Skipping zentorch fused-MoE: layer is not a gated "
"(act-and-mul, e.g. SwiGLU) MLP, the only structure supported."
)
return False
activation = getattr(layer, "activation", None)
if activation is None:
logging.debug(
"Skipping zentorch fused-MoE: layer has no 'activation' "
"attribute, so the activation can't be verified."
)
return False
act = str(getattr(activation, "value", activation)).lower()
if act not in _ZENTORCH_MOE_ACTIVATIONS:
logging.debug(
"Skipping zentorch fused-MoE: activation %r unsupported (supported: %s).",
act,
_ZENTORCH_MOE_ACTIVATIONS,
)
return False
return True
@@ -20,6 +20,10 @@ from vllm._custom_ops import (
cpu_prepack_moe_weight_int8,
fused_experts_cpu,
)
from vllm.model_executor.kernels.linear.zentorch_utils import (
is_zentorch_moe_config_supported,
is_zentorch_moe_supported,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -214,6 +218,7 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
self.renormalize = False
self.scoring_func = "softmax"
self.custom_routing_function: Callable | None = None
self._use_zentorch = False
@property
def expects_unquantized_inputs(self) -> bool:
@@ -240,6 +245,8 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
)
if not supported:
return supported, reason
if is_zentorch_moe_config_supported(moe_config):
return True, None
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
return cpu_cls._supports_grouped_gemm(moe_config)
@@ -287,11 +294,14 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
return True
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self._pad_moe_intermediate(layer)
self.use_grouped_topk = layer.use_grouped_topk
self.renormalize = layer.renormalize
self.scoring_func = layer.scoring_func
self.custom_routing_function = layer.custom_routing_function
self._use_zentorch = is_zentorch_moe_supported(layer)
if self._use_zentorch:
return
self._pad_moe_intermediate(layer)
replace_parameter(
layer, "w13_weight", cpu_prepack_moe_weight(layer.w13_weight, self.isa)
)
@@ -388,6 +398,22 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
)
hidden_states.mul_(topk_weights.to(hidden_states.dtype))
if self._use_zentorch:
output = torch.empty_like(hidden_states)
torch.ops.zentorch.zentorch_fused_moe(
output,
hidden_states,
w1,
w2,
self.w1_bias,
self.w2_bias,
topk_weights,
topk_ids,
apply_router_weight_on_input,
str(activation.value).lower(),
)
return output
return cpu_fused_moe(
hidden_states,
w1,
@@ -430,6 +456,8 @@ class X86CPUUnquantizedExperts(CPUUnquantizedExperts):
)
if not supported:
return supported, reason
if is_zentorch_moe_config_supported(moe_config):
return True, None
if moe_config.in_dtype != torch.bfloat16:
return False, "kernel requires bfloat16 activations"
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
@@ -464,6 +492,8 @@ class ArmCPUUnquantizedExperts(CPUUnquantizedExperts):
)
if not supported:
return supported, reason
if is_zentorch_moe_config_supported(moe_config):
return True, None
if moe_config.in_dtype != torch.bfloat16:
return False, "kernel requires bfloat16 activations"
cpu_cls = cast(type[CPUUnquantizedExperts], cls)