[CPU][Zen] Route Int8 MoE inference through zentorch on AMD (#44834)

Signed-off-by: R <Ganesh.R@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ganesh R
2026-09-01 09:31:31 +00:00
committed by GitHub
co-authored by Cursor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent c35551f892
commit 8f03625b3d
6 changed files with 499 additions and 2 deletions
@@ -20,7 +20,10 @@ from vllm._custom_ops import (
cpu_prepack_moe_weight_int8,
fused_experts_cpu,
)
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear.zentorch_utils import (
_ZENTORCH_MOE_ACTIVATIONS,
has_zentorch_op,
is_zentorch_moe_config_supported,
is_zentorch_moe_supported,
)
@@ -44,6 +47,7 @@ from vllm.model_executor.utils import replace_parameter
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.math_utils import round_up
logger = init_logger(__name__)
# ===========================================================================
# Routing
# ===========================================================================
@@ -1382,3 +1386,161 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
"neon",
skip_weighted=apply_router_weight_on_input,
)
class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
"""AMD Zen INT8 MoE with per-token activation and channelwise weight
quantization, dispatched through zentorch."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
assert self.w1_scale is not None and self.w2_scale is not None, (
"ZenCPUExpertsInt8 requires per-channel weight scales on the layer."
)
num_experts = self.w1_scale.shape[0]
self._w1_scale_bf16 = (
self.w1_scale.detach()
.to(torch.bfloat16)
.reshape(num_experts, -1)
.contiguous()
)
self._w2_scale_bf16 = (
self.w2_scale.detach()
.to(torch.bfloat16)
.reshape(num_experts, -1)
.contiguous()
)
self._w1_bias_bf16 = (
None
if self.w1_bias is None
else self.w1_bias.detach().to(torch.bfloat16).contiguous()
)
self._w2_bias_bf16 = (
None
if self.w2_bias is None
else self.w2_bias.detach().to(torch.bfloat16).contiguous()
)
logger.info_once(
"[zen_cpu] Using zentorch_fused_moe for W8A8 INT8 MoE (monolithic experts)"
)
@property
def expects_unquantized_inputs(self) -> bool:
# zentorch_fused_moe quantizes activations itself.
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return has_zentorch_op(["zentorch_fused_moe"])
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return str(activation.value).lower() in _ZENTORCH_MOE_ACTIVATIONS
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return not moe_parallel_config.use_ep
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) == (
kInt8StaticChannelSym,
kInt8DynamicTokenSym,
)
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
if apply_router_weight_on_input:
assert topk_ids.size(1) == 1, (
"apply_router_weight_on_input is only implemented for topk=1"
)
hidden_states = hidden_states.mul(topk_weights.to(hidden_states.dtype))
output = torch.empty_like(hidden_states)
torch.ops.zentorch.zentorch_fused_moe(
output,
hidden_states,
w1,
w2,
self._w1_bias_bf16,
self._w2_bias_bf16,
topk_weights.to(torch.float32).contiguous(),
topk_ids.to(torch.int32).contiguous(),
apply_router_weight_on_input, # skip_weighted
str(activation.value).lower(),
self._w1_scale_bf16,
self._w2_scale_bf16,
)
return output
@@ -73,9 +73,10 @@ def backend_to_kernel_cls(
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
ArmCPUExpertsInt8,
CPUExpertsInt8,
ZenCPUExpertsInt8,
)
return [ArmCPUExpertsInt8, CPUExpertsInt8]
return [ZenCPUExpertsInt8, ArmCPUExpertsInt8, CPUExpertsInt8]
else:
raise ValueError(f"Unknown Int8 MoE backend: {backend.value}")
@@ -83,6 +83,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
# Capture bias dtype before params_dtype is overridden to int8 below.
bias_dtype = params_dtype
params_dtype = torch.int8
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
@@ -111,6 +113,26 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
layer.register_parameter("w2_weight", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
# Per-expert biases (gpt-oss), allocated only when has_bias.
if self.moe.has_bias:
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
w13_num_shards * intermediate_size_per_partition,
dtype=bias_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs)
w2_bias = torch.nn.Parameter(
torch.zeros(num_experts, hidden_size, dtype=bias_dtype),
requires_grad=False,
)
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
# WEIGHT_SCALES
assert self.weight_quant.strategy == QuantizationStrategy.CHANNEL
w13_weight_scale = torch.nn.Parameter(
@@ -141,6 +163,53 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
layer.w2_input_scale = None
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
# ZenDNN's swiglu_oai_mul reads gate/up interleaved (stride 2), not the
# half-split layout _load_w13 leaves; reorder before the conversion below.
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
ZenCPUExpertsInt8,
)
_act = getattr(layer, "activation", None)
_act_str = getattr(_act, "value", _act)
if (
self.experts_cls is ZenCPUExpertsInt8
and isinstance(_act_str, str)
and _act_str.lower() == "swigluoai"
):
two_i = layer.w13_weight.size(1)
i = two_i // 2
device = layer.w13_weight.device
perm = torch.stack(
[
torch.arange(0, i, device=device),
torch.arange(i, two_i, device=device),
],
dim=1,
).flatten()
has_w13_bias = getattr(layer, "w13_bias", None) is not None
logger.info_once(
"[zen_cpu][swigluoai-permute] Reordering w13 from "
"half-split to interleaved for ZenDNN swiglu_oai_mul "
"kernel: E=%d, 2I=%d, I=%d, has_w13_bias=%s",
layer.w13_weight.size(0),
two_i,
i,
has_w13_bias,
)
layer.w13_weight = torch.nn.Parameter(
layer.w13_weight.data[:, perm, :].contiguous(),
requires_grad=False,
)
layer.w13_weight_scale = torch.nn.Parameter(
layer.w13_weight_scale.data[:, perm].contiguous(),
requires_grad=False,
)
if has_w13_bias:
layer.w13_bias = torch.nn.Parameter(
layer.w13_bias.data[:, perm].contiguous(),
requires_grad=False,
)
w13, w2 = convert_to_int8_moe_kernel_format(
int8_backend=self.int8_backend,
w13=layer.w13_weight,
@@ -169,6 +238,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
w2_scale=layer.w2_weight_scale,
a1_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
per_act_token_quant=True,
layer=layer,
)
+8 -1
View File
@@ -288,7 +288,14 @@ class MixtralModel(nn.Module):
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
},
orig_to_new_substr={
# W8A8 compressed-tensors checkpoints name experts gate/up/down;
# map to the w1/w3/w2 the FusedMoE loader expects (no-op if native).
".gate_proj.": ".w1.",
".up_proj.": ".w3.",
".down_proj.": ".w2.",
},
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):