[CPU] Migrate cpu_awq into awq_marlin (#43841)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-05-28 22:36:31 +08:00
committed by GitHub
parent 552eb81918
commit 20d69d100a
7 changed files with 112 additions and 401 deletions
@@ -4,6 +4,7 @@
import torch
from vllm import _custom_ops as ops
from vllm import envs
from vllm.model_executor.layers.quantization.utils.quant_utils import (
pack_quantized_values_into_int32,
unpack_quantized_values_into_int32,
@@ -58,18 +59,11 @@ class CPUWNA16LinearKernel(MPLinearKernel):
return True, None
# note assumes that
# `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0} (marlin)
# or: {input_dim = 1, output_dim = 0, packed_dim = 1} (CT)
# `weight_scale` is: {input_dim = 0, output_dim = 1} (marlin)
# or: {input_dim = 1, output_dim = 0} (CT)
# `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1} (marlin)
# or: {input_dim = 1, output_dim = 0, packed_dim = 0} (CT)
def _process_gptq_weights(self, layer: torch.nn.Module):
# `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
# `weight_scale` is: {input_dim = 0, output_dim = 1}
# `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1}
def _process_gptq_weights_w4a16(self, layer: torch.nn.Module):
packed_weight = getattr(layer, self.w_q_name)
assert packed_weight.input_dim == packed_weight.packed_dim
is_ct_format = packed_weight.input_dim == 1
if is_ct_format:
packed_weight = packed_weight.t()
bits = self.config.weight_type.mantissa
pack_factor = 32 // bits
p_w_k, _ = packed_weight.size()
@@ -92,14 +86,59 @@ class CPUWNA16LinearKernel(MPLinearKernel):
)
getattr(layer, self.w_q_name).data = weight
# transpose scale, zp for CT format
if is_ct_format:
scales = getattr(layer, self.w_s_name)
scales.data = scales.t().contiguous()
if self.config.zero_points:
assert self.w_zp_name
zp = getattr(layer, self.w_zp_name)
zp.data = zp.t().contiguous()
# note assumes that
# `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
# `weight_scale` is: {input_dim = 0, output_dim = 1}
# `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1}
def _process_gptq_weights_w4a8(self, layer: torch.nn.Module):
packed_weight = getattr(layer, self.w_q_name)
scales = getattr(layer, self.w_s_name)
group_num = scales.data.size(0)
zp_output_size = scales.data.size(1) // 8
if self.config.zero_points:
assert self.w_zp_name
packed_zp = getattr(layer, self.w_zp_name)
else:
# w4a8 kernel always requires zp, allocate a fake zp
assert self.w_zp_name
packed_zp = torch.nn.Parameter(
torch.ones(group_num, zp_output_size, dtype=torch.int32) * -2004318072,
requires_grad=False,
)
setattr(layer, self.w_zp_name, packed_zp)
# FIXME: some bugs in convert_weight_packed_scale_zp with GPTQ format,
# repack to AWQ weight
weight = unpack_quantized_values_into_int32(
packed_weight, self.config.weight_type, 0
)
input_size, output_size = weight.size()
weight = weight.view(input_size, output_size // 8, 8)
weight = weight[:, :, (0, 2, 4, 6, 1, 3, 5, 7)].reshape(input_size, output_size)
weight = pack_quantized_values_into_int32(
weight, self.config.weight_type, 1
).contiguous()
zp = unpack_quantized_values_into_int32(packed_zp, self.config.weight_type, 1)
zp = zp.view(group_num, output_size // 8, 8)
zp = zp[:, :, (0, 2, 4, 6, 1, 3, 5, 7)].reshape(group_num, output_size)
zp = pack_quantized_values_into_int32(
zp, self.config.weight_type, 1
).contiguous()
blocked_w, blocked_zp, blocked_s = ops.convert_weight_packed_scale_zp(
weight,
zp,
scales.data,
ops.CPUQuantAlgo.AWQ,
)
if layer.bias is not None:
layer.bias.data = layer.bias.float()
packed_weight.data = blocked_w
scales.data = blocked_s
packed_zp.data = blocked_zp
def process_weights_after_loading(self, layer: torch.nn.Module):
if (not self.config.zero_points) and (self.w_zp_name is not None):
@@ -108,16 +147,39 @@ class CPUWNA16LinearKernel(MPLinearKernel):
if (not self.config.has_g_idx) and (self.w_gidx_name is not None):
setattr(layer, self.w_gidx_name, None)
w_input_dim = getattr(layer, self.w_q_name).input_dim
w_pack_dim = getattr(layer, self.w_q_name).packed_dim
quant_method = "gptq" if w_pack_dim == w_input_dim else "awq"
weights = getattr(layer, self.w_q_name)
# Require GPTQ pack format
assert weights.input_dim == weights.packed_dim
if quant_method == "gptq":
# GPTQ
self._process_gptq_weights(layer)
# Weights in CT format is [output_size, input_size]
if weights.input_dim == 1:
weights.data = weights.t()
# Scales in CT format is [output_size, group_num]
scales = getattr(layer, self.w_s_name)
if scales.output_dim == 0:
scales.data = scales.t().contiguous()
# Zero points in CT format is [output_size, group_num]
# Zero points in awq_marlin format is [output_size, group_num]
if self.config.zero_points:
assert self.w_zp_name
zp = getattr(layer, self.w_zp_name)
if zp.output_dim == 0:
zp.data = zp.t().contiguous()
layer.use_w4a8 = (
envs.VLLM_CPU_INT4_W4A8
and not self.config.has_g_idx
and self.config.act_type == torch.bfloat16
and torch.cpu._is_amx_tile_supported()
)
# layer.use_w4a8 = False
# AWQ format will be converted to GPTQ format in `AWQMarlinLinearMethod`
if layer.use_w4a8:
self._process_gptq_weights_w4a8(layer)
else:
# AWQ
raise NotImplementedError("AWQ is not supported in CPUWNA16LinearKernel")
self._process_gptq_weights_w4a16(layer)
def apply_weights(
self,
@@ -126,16 +188,25 @@ class CPUWNA16LinearKernel(MPLinearKernel):
bias: torch.Tensor | None = None,
) -> torch.Tensor:
w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
x = ops.cpu_gemm_wna16(
input=x,
q_weight=w_q,
scales=w_s,
zeros=w_zp,
g_idx=w_gidx,
bias=bias,
pack_factor=8, # 32 // 4
isa_hint=layer.isa_hint,
)
if layer.use_w4a8:
x = ops.int4_scaled_mm_cpu(
x=x,
w=w_q,
w_zeros=w_zp,
w_scales=w_s,
bias=bias,
)
else:
x = ops.cpu_gemm_wna16(
input=x,
q_weight=w_q,
scales=w_s,
zeros=w_zp,
g_idx=w_gidx,
bias=bias,
pack_factor=8, # 32 // 4
isa_hint=layer.isa_hint,
)
return x