[CPU][Zen] Add DA8W4 (W4A8) int4 support for dense and MoE layers (#54024)

Signed-off-by: R <Ganesh.R@amd.com>
Signed-off-by: Ganesh R <Ganesh.R@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ganesh R
2026-10-01 15:18:25 +08:00
committed by GitHub
co-authored by Cursor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4d6874d137
commit e12291d733
9 changed files with 1184 additions and 15 deletions
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Zentorch W4A16 GPTQ weight-only-quantized linear kernel for AMD Zen CPUs.
"""Zentorch int4 weight-quantized linear kernels for AMD Zen CPUs.
Selected by ``choose_mp_linear_kernel`` ahead of the generic oneDNN-backed
``CPUWNA16LinearKernel``. When ``can_implement`` rejects a layer, the selector
@@ -9,12 +9,13 @@ falls through to the next kernel in ``_POSSIBLE_KERNELS[PlatformEnum.CPU]``.
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from .cpu import CPUWNA16LinearKernel
from .cpu import _CPUWNA16_SUPPORTED_QUANT_TYPES, CPUWNA16LinearKernel
from .MPLinearKernel import MPLinearLayerConfig
logger = init_logger(__name__)
@@ -34,17 +35,35 @@ def _import_unpack_from_int32():
class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
"""W4A16 GPTQ kernel backed by ``torch.ops.zentorch.zentorch_woq_linear``."""
"""Int4 kernel backed by ``zentorch_dynamic_qlinear`` (DA8W4) or
``zentorch_woq_linear`` (W4A16)."""
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
ok, reason = super().can_implement(c)
if not ok:
return ok, reason
if not current_platform.is_zen_cpu():
return False, "ZentorchWNA16 requires an AMD Zen CPU."
# Not super(): its N/K multiple-of-32 rule is a oneDNN packing
# constraint the zentorch repack does not share.
if c.weight_type not in _CPUWNA16_SUPPORTED_QUANT_TYPES:
return (
False,
f"Quant type ({c.weight_type}) not supported by "
"CPUWNA16, supported types are: "
f"{_CPUWNA16_SUPPORTED_QUANT_TYPES}",
)
# -1 is per-channel, one group spanning K. Anything else has to divide
# K, which is stricter than the parent's multiple-of-2 rule.
if c.group_size != -1:
in_features = c.partition_weight_shape[0]
if c.group_size <= 0 or in_features % c.group_size != 0:
return (
False,
f"Group size ({c.group_size}) must divide input size "
f"({in_features})",
)
if not has_zentorch_op(["zentorch_woq_repack_weight", "zentorch_woq_linear"]):
return (
False,
@@ -89,8 +108,81 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
num_groups = weight_scale.shape[1]
return num_groups > 0 and in_features % num_groups == 0
def _maybe_process_da8w4_weights(self, layer: torch.nn.Module) -> bool:
"""Repack ``layer`` into the DA8W4 layout, or leave it untouched.
Returns False without mutating the layer when it is not eligible, so
the caller falls through to the W4A16 paths.
"""
if not envs.VLLM_CPU_INT4_W4A8:
return False
if not has_zentorch_op(["zentorch_dynamic_qlinear"]):
return False
# DA8W4 is symmetric-only, and the kernel rejects f32 activations.
if self.config.zero_points or self.config.weight_type == scalar_types.uint4:
return False
if self.config.act_type != torch.bfloat16:
return False
if not self._zentorch_woq_eligible(layer):
return False
weight_q = getattr(layer, self.w_q_name)
weight_s = getattr(layer, self.w_s_name)
weight_packed = weight_q.data if hasattr(weight_q, "data") else weight_q
weight_scale = weight_s.data if hasattr(weight_s, "data") else weight_s
bits = self.config.weight_type.mantissa
pack_factor = torch.iinfo(weight_packed.dtype).bits // bits
out_features, num_groups = weight_scale.shape[0], weight_scale.shape[1]
in_features = weight_packed.shape[1] * pack_factor
# AOCL sym_quant requires K/G to be a multiple of 4; K must be even to
# pack two s4 values per byte.
if (in_features // num_groups) % 4 != 0 or in_features % 2 != 0:
return False
if self.w_zp_name is not None:
setattr(layer, self.w_zp_name, None)
unpack_from_int32 = _import_unpack_from_int32()
weight_unpacked = unpack_from_int32(
weight_packed,
bits,
torch.Size([out_features, in_features]),
packed_dim=weight_q.packed_dim,
)
# The WOQ repack packs 8 int4 per int32, which on little-endian is the
# same byte stream as s4 [N, K/2]; the kernel takes either view.
layer._zentorch_da8w4_packed = (
torch.ops.zentorch.zentorch_woq_repack_weight.default(
weight_unpacked.to(torch.int8).contiguous()
).view(torch.int8)
)
# CT stores scales as [N, G]; DA8W4 wants per-group {G, N}.
layer._zentorch_da8w4_scale = weight_scale.t().to(torch.bfloat16).contiguous()
for param_name in (self.w_q_name, self.w_s_name):
param = getattr(layer, param_name, None)
if param is not None and hasattr(param, "data"):
param.data = torch.empty(0)
layer._zentorch_kind = "compressed_tensors_w4a8_da8w4"
layer._zentorch_da8w4 = True
layer._zentorch_processed_weights = True
logger.info_once(
"[zen_cpu] Using zentorch_dynamic_qlinear for DA8W4 (W4A8) "
"(weight_type=%s, group_size=%d)",
self.config.weight_type,
in_features // num_groups,
)
return True
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Repack CT GPTQ weights into the zentorch WOQ layout.
"""Repack CT GPTQ weights into the zentorch DA8W4 or WOQ layout.
Falls back to ``CPUWNA16LinearKernel.process_weights_after_loading``
via ``super()`` when the layer doesn't satisfy
@@ -101,6 +193,9 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
if getattr(layer, "_zentorch_processed_weights", False):
return
if self._maybe_process_da8w4_weights(layer):
return
if not self._zentorch_woq_eligible(layer):
logger.info_once(
"[zen_cpu] ZentorchWNA16 fast path not eligible for this "
@@ -186,6 +281,16 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if getattr(layer, "_zentorch_da8w4", False):
# The kernel reads bias through a raw pointer, so it must be
# contiguous; the weight and scales already are.
return torch.ops.zentorch.zentorch_dynamic_qlinear.default(
x if x.dtype == torch.bfloat16 else x.to(torch.bfloat16),
layer._zentorch_da8w4_packed,
layer._zentorch_da8w4_scale,
bias.contiguous() if bias is not None else None,
)
if getattr(layer, "_zentorch_processed_weights", False):
return torch.ops.zentorch.zentorch_woq_linear.default(
x,