[CPU][Zen] Route BF16 MoE inference through zentorch on AMD (#44201)
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
b76cf758db
commit
2acb055ed7
@@ -4,11 +4,32 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
__all__ = ["has_zentorch_op"]
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
|
||||
__all__ = [
|
||||
"has_zentorch_op",
|
||||
"is_zentorch_moe_config_supported",
|
||||
"is_zentorch_moe_supported",
|
||||
"_ZENTORCH_MOE_ACTIVATIONS",
|
||||
]
|
||||
|
||||
|
||||
_ZENTORCH_MOE_ACTIVATIONS = frozenset({"gelu", "gelu_tanh", "silu", "swigluoai"})
|
||||
|
||||
|
||||
def _moe_activation_to_str(activation: object) -> str:
|
||||
"""Normalize activation to a lowercase string (enum-safe)."""
|
||||
if hasattr(activation, "value"):
|
||||
return str(activation.value).lower()
|
||||
return str(activation).lower()
|
||||
|
||||
|
||||
def has_zentorch_op(op_names: list[str]) -> bool:
|
||||
@@ -21,3 +42,44 @@ def has_zentorch_op(op_names: list[str]) -> bool:
|
||||
if ns is None:
|
||||
return False
|
||||
return all(hasattr(ns, op_name) for op_name in op_names)
|
||||
|
||||
|
||||
def is_zentorch_moe_config_supported(moe_config: FusedMoEConfig) -> bool:
|
||||
if not has_zentorch_op(["zentorch_fused_moe"]):
|
||||
return False
|
||||
if not moe_config.is_act_and_mul:
|
||||
return False
|
||||
act = _moe_activation_to_str(moe_config.activation)
|
||||
return act in _ZENTORCH_MOE_ACTIVATIONS
|
||||
|
||||
|
||||
def is_zentorch_moe_supported(layer: torch.nn.Module) -> bool:
|
||||
if not has_zentorch_op(["zentorch_fused_moe"]):
|
||||
logging.debug(
|
||||
"Skipping zentorch fused-MoE: not a Zen CPU or "
|
||||
"zentorch not loaded; using default MoE."
|
||||
)
|
||||
return False
|
||||
moe_config = getattr(layer, "moe_config", None)
|
||||
if moe_config is not None and not moe_config.is_act_and_mul:
|
||||
logging.debug(
|
||||
"Skipping zentorch fused-MoE: layer is not a gated "
|
||||
"(act-and-mul, e.g. SwiGLU) MLP, the only structure supported."
|
||||
)
|
||||
return False
|
||||
activation = getattr(layer, "activation", None)
|
||||
if activation is None:
|
||||
logging.debug(
|
||||
"Skipping zentorch fused-MoE: layer has no 'activation' "
|
||||
"attribute, so the activation can't be verified."
|
||||
)
|
||||
return False
|
||||
act = str(getattr(activation, "value", activation)).lower()
|
||||
if act not in _ZENTORCH_MOE_ACTIVATIONS:
|
||||
logging.debug(
|
||||
"Skipping zentorch fused-MoE: activation %r unsupported (supported: %s).",
|
||||
act,
|
||||
_ZENTORCH_MOE_ACTIVATIONS,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user