[CPU][Zen] Add DA8W4 (W4A8) int4 support for dense and MoE layers (#54024)
Signed-off-by: R <Ganesh.R@amd.com> Signed-off-by: Ganesh R <Ganesh.R@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4d6874d137
commit
e12291d733
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Zentorch W4A16 GPTQ weight-only-quantized linear kernel for AMD Zen CPUs.
|
||||
"""Zentorch int4 weight-quantized linear kernels for AMD Zen CPUs.
|
||||
|
||||
Selected by ``choose_mp_linear_kernel`` ahead of the generic oneDNN-backed
|
||||
``CPUWNA16LinearKernel``. When ``can_implement`` rejects a layer, the selector
|
||||
@@ -9,12 +9,13 @@ falls through to the next kernel in ``_POSSIBLE_KERNELS[PlatformEnum.CPU]``.
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
from .cpu import CPUWNA16LinearKernel
|
||||
from .cpu import _CPUWNA16_SUPPORTED_QUANT_TYPES, CPUWNA16LinearKernel
|
||||
from .MPLinearKernel import MPLinearLayerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -34,17 +35,35 @@ def _import_unpack_from_int32():
|
||||
|
||||
|
||||
class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
"""W4A16 GPTQ kernel backed by ``torch.ops.zentorch.zentorch_woq_linear``."""
|
||||
"""Int4 kernel backed by ``zentorch_dynamic_qlinear`` (DA8W4) or
|
||||
``zentorch_woq_linear`` (W4A16)."""
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
ok, reason = super().can_implement(c)
|
||||
if not ok:
|
||||
return ok, reason
|
||||
|
||||
if not current_platform.is_zen_cpu():
|
||||
return False, "ZentorchWNA16 requires an AMD Zen CPU."
|
||||
|
||||
# Not super(): its N/K multiple-of-32 rule is a oneDNN packing
|
||||
# constraint the zentorch repack does not share.
|
||||
if c.weight_type not in _CPUWNA16_SUPPORTED_QUANT_TYPES:
|
||||
return (
|
||||
False,
|
||||
f"Quant type ({c.weight_type}) not supported by "
|
||||
"CPUWNA16, supported types are: "
|
||||
f"{_CPUWNA16_SUPPORTED_QUANT_TYPES}",
|
||||
)
|
||||
|
||||
# -1 is per-channel, one group spanning K. Anything else has to divide
|
||||
# K, which is stricter than the parent's multiple-of-2 rule.
|
||||
if c.group_size != -1:
|
||||
in_features = c.partition_weight_shape[0]
|
||||
if c.group_size <= 0 or in_features % c.group_size != 0:
|
||||
return (
|
||||
False,
|
||||
f"Group size ({c.group_size}) must divide input size "
|
||||
f"({in_features})",
|
||||
)
|
||||
|
||||
if not has_zentorch_op(["zentorch_woq_repack_weight", "zentorch_woq_linear"]):
|
||||
return (
|
||||
False,
|
||||
@@ -89,8 +108,81 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
num_groups = weight_scale.shape[1]
|
||||
return num_groups > 0 and in_features % num_groups == 0
|
||||
|
||||
def _maybe_process_da8w4_weights(self, layer: torch.nn.Module) -> bool:
|
||||
"""Repack ``layer`` into the DA8W4 layout, or leave it untouched.
|
||||
|
||||
Returns False without mutating the layer when it is not eligible, so
|
||||
the caller falls through to the W4A16 paths.
|
||||
"""
|
||||
if not envs.VLLM_CPU_INT4_W4A8:
|
||||
return False
|
||||
|
||||
if not has_zentorch_op(["zentorch_dynamic_qlinear"]):
|
||||
return False
|
||||
|
||||
# DA8W4 is symmetric-only, and the kernel rejects f32 activations.
|
||||
if self.config.zero_points or self.config.weight_type == scalar_types.uint4:
|
||||
return False
|
||||
if self.config.act_type != torch.bfloat16:
|
||||
return False
|
||||
|
||||
if not self._zentorch_woq_eligible(layer):
|
||||
return False
|
||||
|
||||
weight_q = getattr(layer, self.w_q_name)
|
||||
weight_s = getattr(layer, self.w_s_name)
|
||||
weight_packed = weight_q.data if hasattr(weight_q, "data") else weight_q
|
||||
weight_scale = weight_s.data if hasattr(weight_s, "data") else weight_s
|
||||
|
||||
bits = self.config.weight_type.mantissa
|
||||
pack_factor = torch.iinfo(weight_packed.dtype).bits // bits
|
||||
out_features, num_groups = weight_scale.shape[0], weight_scale.shape[1]
|
||||
in_features = weight_packed.shape[1] * pack_factor
|
||||
|
||||
# AOCL sym_quant requires K/G to be a multiple of 4; K must be even to
|
||||
# pack two s4 values per byte.
|
||||
if (in_features // num_groups) % 4 != 0 or in_features % 2 != 0:
|
||||
return False
|
||||
|
||||
if self.w_zp_name is not None:
|
||||
setattr(layer, self.w_zp_name, None)
|
||||
|
||||
unpack_from_int32 = _import_unpack_from_int32()
|
||||
|
||||
weight_unpacked = unpack_from_int32(
|
||||
weight_packed,
|
||||
bits,
|
||||
torch.Size([out_features, in_features]),
|
||||
packed_dim=weight_q.packed_dim,
|
||||
)
|
||||
# The WOQ repack packs 8 int4 per int32, which on little-endian is the
|
||||
# same byte stream as s4 [N, K/2]; the kernel takes either view.
|
||||
layer._zentorch_da8w4_packed = (
|
||||
torch.ops.zentorch.zentorch_woq_repack_weight.default(
|
||||
weight_unpacked.to(torch.int8).contiguous()
|
||||
).view(torch.int8)
|
||||
)
|
||||
# CT stores scales as [N, G]; DA8W4 wants per-group {G, N}.
|
||||
layer._zentorch_da8w4_scale = weight_scale.t().to(torch.bfloat16).contiguous()
|
||||
|
||||
for param_name in (self.w_q_name, self.w_s_name):
|
||||
param = getattr(layer, param_name, None)
|
||||
if param is not None and hasattr(param, "data"):
|
||||
param.data = torch.empty(0)
|
||||
|
||||
layer._zentorch_kind = "compressed_tensors_w4a8_da8w4"
|
||||
layer._zentorch_da8w4 = True
|
||||
layer._zentorch_processed_weights = True
|
||||
logger.info_once(
|
||||
"[zen_cpu] Using zentorch_dynamic_qlinear for DA8W4 (W4A8) "
|
||||
"(weight_type=%s, group_size=%d)",
|
||||
self.config.weight_type,
|
||||
in_features // num_groups,
|
||||
)
|
||||
return True
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Repack CT GPTQ weights into the zentorch WOQ layout.
|
||||
"""Repack CT GPTQ weights into the zentorch DA8W4 or WOQ layout.
|
||||
|
||||
Falls back to ``CPUWNA16LinearKernel.process_weights_after_loading``
|
||||
via ``super()`` when the layer doesn't satisfy
|
||||
@@ -101,6 +193,9 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
if getattr(layer, "_zentorch_processed_weights", False):
|
||||
return
|
||||
|
||||
if self._maybe_process_da8w4_weights(layer):
|
||||
return
|
||||
|
||||
if not self._zentorch_woq_eligible(layer):
|
||||
logger.info_once(
|
||||
"[zen_cpu] ZentorchWNA16 fast path not eligible for this "
|
||||
@@ -186,6 +281,16 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if getattr(layer, "_zentorch_da8w4", False):
|
||||
# The kernel reads bias through a raw pointer, so it must be
|
||||
# contiguous; the weight and scales already are.
|
||||
return torch.ops.zentorch.zentorch_dynamic_qlinear.default(
|
||||
x if x.dtype == torch.bfloat16 else x.to(torch.bfloat16),
|
||||
layer._zentorch_da8w4_packed,
|
||||
layer._zentorch_da8w4_scale,
|
||||
bias.contiguous() if bias is not None else None,
|
||||
)
|
||||
|
||||
if getattr(layer, "_zentorch_processed_weights", False):
|
||||
return torch.ops.zentorch.zentorch_woq_linear.default(
|
||||
x,
|
||||
|
||||
Reference in New Issue
Block a user