From 20d69d100a967d51788bd852169afb516ba18434 Mon Sep 17 00:00:00 2001 From: "Li, Jiang" Date: Thu, 28 May 2026 22:36:31 +0800 Subject: [PATCH] [CPU] Migrate cpu_awq into awq_marlin (#43841) Signed-off-by: jiang1.li --- .buildkite/hardware_tests/cpu.yaml | 7 +- vllm/config/model.py | 1 - .../kernels/linear/mixed_precision/cpu.py | 145 +++++-- .../layers/quantization/__init__.py | 3 - .../layers/quantization/awq_marlin.py | 2 +- .../layers/quantization/cpu_wna16.py | 354 ------------------ vllm/v1/metrics/perf.py | 1 - 7 files changed, 112 insertions(+), 401 deletions(-) delete mode 100644 vllm/model_executor/layers/quantization/cpu_wna16.py diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index 19716bab6d..73503664c4 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -64,7 +64,7 @@ steps: - vllm/v1/worker/gpu/ commands: - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m " + bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m " uv pip install git+https://github.com/triton-lang/triton-cpu.git@270e696d VLLM_USE_V2_MODEL_RUNNER=1 pytest -x -v -s tests/models/language/generation/test_granite.py -m cpu_model" @@ -74,11 +74,10 @@ steps: no_plugin: true source_file_dependencies: - csrc/cpu/ - - vllm/model_executor/layers/quantization/cpu_wna16.py - vllm/model_executor/layers/quantization/auto_gptq.py - vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py - - vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py - - vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py + - vllm/model_executor/kernels/linear/mixed_precision/cpu.py + - vllm/model_executor/kernels/linear/scaled_mm/cpu.py - vllm/model_executor/layers/fused_moe/experts/cpu_moe.py - tests/quantization/test_compressed_tensors.py - tests/quantization/test_cpu_wna16.py diff --git a/vllm/config/model.py b/vllm/config/model.py index 491bd1a415..b41ab189c1 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -1028,7 +1028,6 @@ class ModelConfig: "mxfp4", "gpt_oss_mxfp4", "deepseek_v4_fp8", - "cpu_awq", "humming", "gguf", ] diff --git a/vllm/model_executor/kernels/linear/mixed_precision/cpu.py b/vllm/model_executor/kernels/linear/mixed_precision/cpu.py index 72e5620123..b364d1ad96 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/cpu.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/cpu.py @@ -4,6 +4,7 @@ import torch from vllm import _custom_ops as ops +from vllm import envs from vllm.model_executor.layers.quantization.utils.quant_utils import ( pack_quantized_values_into_int32, unpack_quantized_values_into_int32, @@ -58,18 +59,11 @@ class CPUWNA16LinearKernel(MPLinearKernel): return True, None # note assumes that - # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0} (marlin) - # or: {input_dim = 1, output_dim = 0, packed_dim = 1} (CT) - # `weight_scale` is: {input_dim = 0, output_dim = 1} (marlin) - # or: {input_dim = 1, output_dim = 0} (CT) - # `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1} (marlin) - # or: {input_dim = 1, output_dim = 0, packed_dim = 0} (CT) - def _process_gptq_weights(self, layer: torch.nn.Module): + # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0} + # `weight_scale` is: {input_dim = 0, output_dim = 1} + # `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1} + def _process_gptq_weights_w4a16(self, layer: torch.nn.Module): packed_weight = getattr(layer, self.w_q_name) - assert packed_weight.input_dim == packed_weight.packed_dim - is_ct_format = packed_weight.input_dim == 1 - if is_ct_format: - packed_weight = packed_weight.t() bits = self.config.weight_type.mantissa pack_factor = 32 // bits p_w_k, _ = packed_weight.size() @@ -92,14 +86,59 @@ class CPUWNA16LinearKernel(MPLinearKernel): ) getattr(layer, self.w_q_name).data = weight - # transpose scale, zp for CT format - if is_ct_format: - scales = getattr(layer, self.w_s_name) - scales.data = scales.t().contiguous() - if self.config.zero_points: - assert self.w_zp_name - zp = getattr(layer, self.w_zp_name) - zp.data = zp.t().contiguous() + # note assumes that + # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0} + # `weight_scale` is: {input_dim = 0, output_dim = 1} + # `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1} + def _process_gptq_weights_w4a8(self, layer: torch.nn.Module): + packed_weight = getattr(layer, self.w_q_name) + scales = getattr(layer, self.w_s_name) + group_num = scales.data.size(0) + zp_output_size = scales.data.size(1) // 8 + if self.config.zero_points: + assert self.w_zp_name + packed_zp = getattr(layer, self.w_zp_name) + else: + # w4a8 kernel always requires zp, allocate a fake zp + assert self.w_zp_name + packed_zp = torch.nn.Parameter( + torch.ones(group_num, zp_output_size, dtype=torch.int32) * -2004318072, + requires_grad=False, + ) + setattr(layer, self.w_zp_name, packed_zp) + + # FIXME: some bugs in convert_weight_packed_scale_zp with GPTQ format, + # repack to AWQ weight + weight = unpack_quantized_values_into_int32( + packed_weight, self.config.weight_type, 0 + ) + input_size, output_size = weight.size() + weight = weight.view(input_size, output_size // 8, 8) + weight = weight[:, :, (0, 2, 4, 6, 1, 3, 5, 7)].reshape(input_size, output_size) + weight = pack_quantized_values_into_int32( + weight, self.config.weight_type, 1 + ).contiguous() + + zp = unpack_quantized_values_into_int32(packed_zp, self.config.weight_type, 1) + zp = zp.view(group_num, output_size // 8, 8) + zp = zp[:, :, (0, 2, 4, 6, 1, 3, 5, 7)].reshape(group_num, output_size) + zp = pack_quantized_values_into_int32( + zp, self.config.weight_type, 1 + ).contiguous() + + blocked_w, blocked_zp, blocked_s = ops.convert_weight_packed_scale_zp( + weight, + zp, + scales.data, + ops.CPUQuantAlgo.AWQ, + ) + + if layer.bias is not None: + layer.bias.data = layer.bias.float() + + packed_weight.data = blocked_w + scales.data = blocked_s + packed_zp.data = blocked_zp def process_weights_after_loading(self, layer: torch.nn.Module): if (not self.config.zero_points) and (self.w_zp_name is not None): @@ -108,16 +147,39 @@ class CPUWNA16LinearKernel(MPLinearKernel): if (not self.config.has_g_idx) and (self.w_gidx_name is not None): setattr(layer, self.w_gidx_name, None) - w_input_dim = getattr(layer, self.w_q_name).input_dim - w_pack_dim = getattr(layer, self.w_q_name).packed_dim - quant_method = "gptq" if w_pack_dim == w_input_dim else "awq" + weights = getattr(layer, self.w_q_name) + # Require GPTQ pack format + assert weights.input_dim == weights.packed_dim - if quant_method == "gptq": - # GPTQ - self._process_gptq_weights(layer) + # Weights in CT format is [output_size, input_size] + if weights.input_dim == 1: + weights.data = weights.t() + + # Scales in CT format is [output_size, group_num] + scales = getattr(layer, self.w_s_name) + if scales.output_dim == 0: + scales.data = scales.t().contiguous() + + # Zero points in CT format is [output_size, group_num] + # Zero points in awq_marlin format is [output_size, group_num] + if self.config.zero_points: + assert self.w_zp_name + zp = getattr(layer, self.w_zp_name) + if zp.output_dim == 0: + zp.data = zp.t().contiguous() + + layer.use_w4a8 = ( + envs.VLLM_CPU_INT4_W4A8 + and not self.config.has_g_idx + and self.config.act_type == torch.bfloat16 + and torch.cpu._is_amx_tile_supported() + ) + # layer.use_w4a8 = False + # AWQ format will be converted to GPTQ format in `AWQMarlinLinearMethod` + if layer.use_w4a8: + self._process_gptq_weights_w4a8(layer) else: - # AWQ - raise NotImplementedError("AWQ is not supported in CPUWNA16LinearKernel") + self._process_gptq_weights_w4a16(layer) def apply_weights( self, @@ -126,16 +188,25 @@ class CPUWNA16LinearKernel(MPLinearKernel): bias: torch.Tensor | None = None, ) -> torch.Tensor: w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer) - x = ops.cpu_gemm_wna16( - input=x, - q_weight=w_q, - scales=w_s, - zeros=w_zp, - g_idx=w_gidx, - bias=bias, - pack_factor=8, # 32 // 4 - isa_hint=layer.isa_hint, - ) + if layer.use_w4a8: + x = ops.int4_scaled_mm_cpu( + x=x, + w=w_q, + w_zeros=w_zp, + w_scales=w_s, + bias=bias, + ) + else: + x = ops.cpu_gemm_wna16( + input=x, + q_weight=w_q, + scales=w_s, + zeros=w_zp, + g_idx=w_gidx, + bias=bias, + pack_factor=8, # 32 // 4 + isa_hint=layer.isa_hint, + ) return x diff --git a/vllm/model_executor/layers/quantization/__init__.py b/vllm/model_executor/layers/quantization/__init__.py index 5fb3c930f8..2506c62d39 100644 --- a/vllm/model_executor/layers/quantization/__init__.py +++ b/vllm/model_executor/layers/quantization/__init__.py @@ -34,7 +34,6 @@ QuantizationMethods = Literal[ "mxfp4", "gpt_oss_mxfp4", "deepseek_v4_fp8", - "cpu_awq", "online", # Below are online quant shorthand names (see vllm.config.quantization). # Listed here as strings to avoid a circular import; kept in sync with @@ -122,7 +121,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: from .compressed_tensors.compressed_tensors import ( CompressedTensorsConfig, ) - from .cpu_wna16 import CPUAWQConfig from .experts_int8 import ExpertsInt8Config from .fbgemm_fp8 import FBGEMMFp8Config from .fp8 import Fp8Config @@ -166,7 +164,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: "mxfp4": Mxfp4Config, "gpt_oss_mxfp4": GptOssMxfp4Config, "deepseek_v4_fp8": DeepseekV4FP8Config, - "cpu_awq": CPUAWQConfig, "humming": HummingConfig, "online": OnlineQuantizationConfig, } diff --git a/vllm/model_executor/layers/quantization/awq_marlin.py b/vllm/model_executor/layers/quantization/awq_marlin.py index 0692d3b84c..14ebd920fd 100644 --- a/vllm/model_executor/layers/quantization/awq_marlin.py +++ b/vllm/model_executor/layers/quantization/awq_marlin.py @@ -332,7 +332,7 @@ class AWQMarlinConfig(QuantizationConfig): group_size = quant_config.get("group_size") zero_point = quant_config.get("zero_point") - if not current_platform.is_cuda_alike(): + if not (current_platform.is_cuda_alike() or current_platform.is_cpu()): return False if quant_method != "awq": diff --git a/vllm/model_executor/layers/quantization/cpu_wna16.py b/vllm/model_executor/layers/quantization/cpu_wna16.py deleted file mode 100644 index 6853013af8..0000000000 --- a/vllm/model_executor/layers/quantization/cpu_wna16.py +++ /dev/null @@ -1,354 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import Any - -import torch -from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE -from transformers import PretrainedConfig - -import vllm.envs as envs -from vllm import _custom_ops as ops -from vllm.logger import init_logger -from vllm.model_executor.layers.linear import ( - LinearBase, - LinearMethodBase, - UnquantizedLinearMethod, -) -from vllm.model_executor.layers.quantization import QuantizationMethods -from vllm.model_executor.layers.quantization.base_config import ( - QuantizationConfig, - QuantizeMethodBase, -) -from vllm.model_executor.layers.quantization.utils.quant_utils import ( - is_layer_skipped, - pack_cols, - unpack_cols, -) -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead -from vllm.model_executor.models.utils import WeightsMapper -from vllm.model_executor.parameter import ( - GroupQuantScaleParameter, - PackedvLLMParameter, -) -from vllm.platforms import current_platform -from vllm.transformers_utils.config import get_safetensors_params_metadata - -logger = init_logger(__name__) - - -class CPUAWQConfig(QuantizationConfig): - """Config class for CPU AWQ""" - - def __init__( - self, - weight_bits: int, - group_size: int, - zero_point: bool, - lm_head_quantized: bool, - modules_to_not_convert: list[str] | None, - full_config: dict[str, Any], - ) -> None: - super().__init__() - assert weight_bits == 4 - self.pack_factor = 32 // weight_bits # packed into int32 - self.group_size = group_size - self.zero_point = zero_point - self.lm_head_quantized = lm_head_quantized - self.weight_bits = weight_bits - self.modules_to_not_convert = modules_to_not_convert or [] - self.full_config = full_config - - def __repr__(self) -> str: - return ( - f"AWQMarlinConfig(" - f"group_size={self.group_size}, " - f"zero_point={self.zero_point}, " - f"lm_head_quantized={self.lm_head_quantized}, " - f"modules_to_not_convert={self.modules_to_not_convert})" - ) - - @classmethod - def get_name(cls) -> "QuantizationMethods": - return "cpu_awq" - - @classmethod - def get_supported_act_dtypes(cls) -> list[torch.dtype]: - return [torch.half, torch.bfloat16] - - @classmethod - def get_min_capability(cls) -> int: - return -1 - - @classmethod - def get_config_filenames(cls) -> list[str]: - return ["quantize_config.json"] - - @classmethod - def from_config(cls, config: dict[str, Any]) -> "CPUAWQConfig": - weight_bits = cls.get_from_keys(config, ["bits"]) - group_size = cls.get_from_keys(config, ["group_size"]) - zero_point = cls.get_from_keys(config, ["zero_point"]) - lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False) - modules_to_not_convert = cls.get_from_keys_or( - config, ["modules_to_not_convert"], None - ) - return cls( - weight_bits, - group_size, - zero_point, - lm_head_quantized, - modules_to_not_convert, - config, - ) - - @classmethod - def override_quantization_method( - cls, hf_quant_cfg, user_quant, hf_config=None - ) -> "QuantizationMethods | None": - quant_method = hf_quant_cfg.get("quant_method", "").lower() - if current_platform.is_cpu() and (quant_method == "awq"): - return cls.get_name() - return None - - def get_quant_method( - self, layer: torch.nn.Module, prefix: str - ) -> "QuantizeMethodBase | None": - if isinstance(layer, LinearBase) or ( - isinstance(layer, ParallelLMHead) and self.lm_head_quantized - ): - if is_layer_skipped( - prefix, - self.modules_to_not_convert, - self.packed_modules_mapping, - skip_with_substr=True, - ): - return UnquantizedLinearMethod() - return CPUAWQLinearMethod(self) - return None - - def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"): - if self.modules_to_not_convert: - self.modules_to_not_convert = hf_to_vllm_mapper.apply_list( - self.modules_to_not_convert - ) - - def maybe_update_config( - self, - model_name: str, - hf_config: PretrainedConfig | None = None, - revision: str | None = None, - ): - if self.modules_to_not_convert: - return - - unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32] - metadata = get_safetensors_params_metadata(model_name, revision=revision) - layers = {param_name.rsplit(".", 1)[0] for param_name in metadata} - quant_layers: set[str] = { - param_name.rsplit(".", 1)[0] - for param_name, info in metadata.items() - if (dtype := info.get("dtype", None)) - and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes - } - self.modules_to_not_convert = list(layers - quant_layers) - - -class CPUAWQLinearMethod(LinearMethodBase): - """Linear method for CPU AWQ. - - Args: - quant_config: The CPU AWQ quantization config. - """ - - def __init__(self, quant_config: CPUAWQConfig) -> None: - self.quant_config = quant_config - assert self.quant_config.zero_point - - def create_weights( - self, - layer: torch.nn.Module, - input_size_per_partition: int, - output_partition_sizes: list[int], - input_size: int, - output_size: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ) -> None: - del output_size - output_size_per_partition = sum(output_partition_sizes) - weight_loader = extra_weight_attrs.get("weight_loader") - - # Normalize group_size - if self.quant_config.group_size != -1: - group_size = self.quant_config.group_size - else: - group_size = input_size - - qweight = PackedvLLMParameter( - data=torch.empty( - input_size_per_partition, - output_size_per_partition // self.quant_config.pack_factor, - dtype=torch.int32, - ), - input_dim=0, - output_dim=1, - packed_dim=1, - packed_factor=self.quant_config.pack_factor, - weight_loader=weight_loader, - ) - - num_groups = input_size_per_partition // group_size - - qzeros = PackedvLLMParameter( - data=torch.empty( - num_groups, - output_size_per_partition // self.quant_config.pack_factor, - dtype=torch.int32, - ), - input_dim=0, - output_dim=1, - packed_dim=1, - packed_factor=self.quant_config.pack_factor, - weight_loader=weight_loader, - ) - - scales = GroupQuantScaleParameter( - data=torch.empty( - num_groups, - output_size_per_partition, - dtype=params_dtype, - ), - input_dim=0, - output_dim=1, - weight_loader=weight_loader, - ) - - layer.register_parameter("qweight", qweight) - layer.register_parameter("qzeros", qzeros) - layer.register_parameter("scales", scales) - - def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - layer.use_w4a8 = envs.VLLM_CPU_INT4_W4A8 and torch.cpu._is_amx_tile_supported() - if layer.use_w4a8: - self._process_weights_sglang_int4(layer) - else: - self._process_weights_woq(layer) - - def _process_weights_woq(self, layer: torch.nn.Module) -> None: - """Original WOQ int4 repack path.""" - packed_weight = layer.qweight.data - packed_zeros = layer.qzeros.data - group_num = packed_zeros.size(0) - bits = self.quant_config.weight_bits - pack_factor = int(self.quant_config.pack_factor) - input_size, packed_output_size = packed_weight.size() - output_size = packed_output_size * pack_factor - isa_hint = _get_isa_hint(layer.scales.dtype) - layer.isa_hint = isa_hint - - interleave_map = (0, 4, 1, 5, 2, 6, 3, 7) - weight = unpack_cols( - packed_weight, - bits, - input_size, - output_size, - ) - zeros = unpack_cols( - packed_zeros, - bits, - group_num, - output_size, - ) - weight = ( - weight.view(input_size, -1, pack_factor)[:, :, interleave_map] - .reshape(input_size, output_size) - .contiguous() - ) - zeros = ( - zeros.view(group_num, -1, pack_factor)[:, :, interleave_map] - .reshape(group_num, output_size) - .contiguous() - ) - - zeros = pack_cols(zeros, bits, group_num, output_size).contiguous() - weight = pack_cols(weight, bits, input_size, output_size) - weight = ( - weight.view(input_size, -1, 16 // pack_factor) - .permute(1, 0, 2) - .reshape(-1, input_size * 16 // pack_factor) - .contiguous() - ) - layer.qweight.data = weight - layer.qzeros.data = zeros - - def _process_weights_sglang_int4(self, layer: torch.nn.Module) -> None: - """SGLang INT4 W4A8 path: pack int4 weights with VNNI reordering.""" - packed_weight = layer.qweight.data - packed_zeros = layer.qzeros.data - scales = layer.scales.data - blocked_w, blocked_zp, blocked_s = ops.convert_weight_packed_scale_zp( - packed_weight, - packed_zeros, - scales, - ops.CPUQuantAlgo.AWQ, - ) - - layer.packed_weight = blocked_w - layer.packed_qzeros = blocked_zp - layer.packed_scales = blocked_s - layer.qweight = None - layer.qzeros = None - layer.scales = None - - def apply( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: torch.Tensor | None = None, - ) -> torch.Tensor: - if layer.use_w4a8: - return self._apply_sglang_int4(layer, x, bias) - return self._apply_woq(layer, x, bias) - - def _apply_woq( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: torch.Tensor | None = None, - ) -> torch.Tensor: - """Original WOQ int4 GEMM path.""" - x = ops.cpu_gemm_wna16( - input=x, - q_weight=layer.qweight, - scales=layer.scales, - zeros=layer.qzeros, - g_idx=None, - bias=bias, - pack_factor=8, - isa_hint=layer.isa_hint, - ) - return x - - def _apply_sglang_int4( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: torch.Tensor | None = None, - ) -> torch.Tensor: - """SGLang INT4 W4A8 GEMM path.""" - return ops.int4_scaled_mm_cpu( - x, - layer.packed_weight, - layer.packed_qzeros, - layer.packed_scales, - bias, - ) - - -def _get_isa_hint(dtype: torch.dtype) -> str: - supports_amx = torch.cpu._is_amx_tile_supported() - if supports_amx and dtype in (torch.bfloat16,): - return "amx" - else: - return "vec" diff --git a/vllm/v1/metrics/perf.py b/vllm/v1/metrics/perf.py index 91629cb578..38135b9b15 100644 --- a/vllm/v1/metrics/perf.py +++ b/vllm/v1/metrics/perf.py @@ -72,7 +72,6 @@ _QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = { "quark": 0.5, "moe_wna16": 0.5, "inc": 0.5, - "cpu_awq": 0.5, "experts_int8": 1, }