[CPU] Support int8 compute mode in CPU AWQ (#35697)

Signed-off-by: Yintong Lu <yintong.lu@intel.com>
This commit is contained in:
Yintong Lu
2026-03-31 15:27:37 +08:00
committed by GitHub
parent 42318c840b
commit f09daea261
10 changed files with 1197 additions and 11 deletions
@@ -7,9 +7,8 @@ import torch
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
from transformers import PretrainedConfig
from vllm._custom_ops import (
cpu_gemm_wna16,
)
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import (
LinearBase,
@@ -230,7 +229,14 @@ class CPUAWQLinearMethod(LinearMethodBase):
layer.register_parameter("scales", scales)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
torch.set_printoptions(profile="full", linewidth=5000, sci_mode=False)
layer.use_w4a8 = envs.VLLM_CPU_INT4_W4A8 and torch.cpu._is_amx_tile_supported()
if layer.use_w4a8:
self._process_weights_sglang_int4(layer)
else:
self._process_weights_woq(layer)
def _process_weights_woq(self, layer: torch.nn.Module) -> None:
"""Original WOQ int4 repack path."""
packed_weight = layer.qweight.data
packed_zeros = layer.qzeros.data
group_num = packed_zeros.size(0)
@@ -266,8 +272,6 @@ class CPUAWQLinearMethod(LinearMethodBase):
)
zeros = pack_cols(zeros, bits, group_num, output_size).contiguous()
# make 16 output channel as a block and transpose to
# the make the block contiguous
weight = pack_cols(weight, bits, input_size, output_size)
weight = (
weight.view(input_size, -1, 16 // pack_factor)
@@ -278,13 +282,40 @@ class CPUAWQLinearMethod(LinearMethodBase):
layer.qweight.data = weight
layer.qzeros.data = zeros
def _process_weights_sglang_int4(self, layer: torch.nn.Module) -> None:
"""SGLang INT4 W4A8 path: pack int4 weights with VNNI reordering."""
packed_weight = layer.qweight.data
packed_zeros = layer.qzeros.data
scales = layer.scales.data
blocked_w, blocked_zp, blocked_s = torch.ops._C.convert_weight_packed_scale_zp(
packed_weight, packed_zeros, scales
)
layer.packed_weight = blocked_w
layer.packed_qzeros = blocked_zp
layer.packed_scales = blocked_s
layer.qweight = None
layer.qzeros = None
layer.scales = None
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
x = cpu_gemm_wna16(
if layer.use_w4a8:
return self._apply_sglang_int4(layer, x, bias)
return self._apply_woq(layer, x, bias)
def _apply_woq(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""Original WOQ int4 GEMM path."""
x = ops.cpu_gemm_wna16(
input=x,
q_weight=layer.qweight,
scales=layer.scales,
@@ -296,6 +327,26 @@ class CPUAWQLinearMethod(LinearMethodBase):
)
return x
def _apply_sglang_int4(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""SGLang INT4 W4A8 GEMM path."""
x_shape = x.shape
x_2d = x.reshape(-1, x_shape[-1]) if len(x_shape) > 2 else x
out = torch.ops._C.int4_scaled_mm_cpu(
x_2d,
layer.packed_weight,
layer.packed_qzeros,
layer.packed_scales,
bias,
)
out = out.reshape(x_shape[:-1] + (out.size(-1),)) if len(x_shape) > 2 else out
return out
def _get_isa_hint(dtype: torch.dtype) -> str:
supports_amx = torch.cpu._is_amx_tile_supported()