[CPU] Migrate cpu_awq into awq_marlin (#43841)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -64,7 +64,7 @@ steps:
|
||||
- vllm/v1/worker/gpu/
|
||||
commands:
|
||||
- |
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
|
||||
uv pip install git+https://github.com/triton-lang/triton-cpu.git@270e696d
|
||||
VLLM_USE_V2_MODEL_RUNNER=1 pytest -x -v -s tests/models/language/generation/test_granite.py -m cpu_model"
|
||||
|
||||
@@ -74,11 +74,10 @@ steps:
|
||||
no_plugin: true
|
||||
source_file_dependencies:
|
||||
- csrc/cpu/
|
||||
- vllm/model_executor/layers/quantization/cpu_wna16.py
|
||||
- vllm/model_executor/layers/quantization/auto_gptq.py
|
||||
- vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
|
||||
- vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py
|
||||
- vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py
|
||||
- vllm/model_executor/kernels/linear/mixed_precision/cpu.py
|
||||
- vllm/model_executor/kernels/linear/scaled_mm/cpu.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
|
||||
- tests/quantization/test_compressed_tensors.py
|
||||
- tests/quantization/test_cpu_wna16.py
|
||||
|
||||
@@ -1028,7 +1028,6 @@ class ModelConfig:
|
||||
"mxfp4",
|
||||
"gpt_oss_mxfp4",
|
||||
"deepseek_v4_fp8",
|
||||
"cpu_awq",
|
||||
"humming",
|
||||
"gguf",
|
||||
]
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
pack_quantized_values_into_int32,
|
||||
unpack_quantized_values_into_int32,
|
||||
@@ -58,18 +59,11 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
return True, None
|
||||
|
||||
# note assumes that
|
||||
# `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0} (marlin)
|
||||
# or: {input_dim = 1, output_dim = 0, packed_dim = 1} (CT)
|
||||
# `weight_scale` is: {input_dim = 0, output_dim = 1} (marlin)
|
||||
# or: {input_dim = 1, output_dim = 0} (CT)
|
||||
# `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1} (marlin)
|
||||
# or: {input_dim = 1, output_dim = 0, packed_dim = 0} (CT)
|
||||
def _process_gptq_weights(self, layer: torch.nn.Module):
|
||||
# `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
|
||||
# `weight_scale` is: {input_dim = 0, output_dim = 1}
|
||||
# `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1}
|
||||
def _process_gptq_weights_w4a16(self, layer: torch.nn.Module):
|
||||
packed_weight = getattr(layer, self.w_q_name)
|
||||
assert packed_weight.input_dim == packed_weight.packed_dim
|
||||
is_ct_format = packed_weight.input_dim == 1
|
||||
if is_ct_format:
|
||||
packed_weight = packed_weight.t()
|
||||
bits = self.config.weight_type.mantissa
|
||||
pack_factor = 32 // bits
|
||||
p_w_k, _ = packed_weight.size()
|
||||
@@ -92,14 +86,59 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
)
|
||||
getattr(layer, self.w_q_name).data = weight
|
||||
|
||||
# transpose scale, zp for CT format
|
||||
if is_ct_format:
|
||||
scales = getattr(layer, self.w_s_name)
|
||||
scales.data = scales.t().contiguous()
|
||||
if self.config.zero_points:
|
||||
assert self.w_zp_name
|
||||
zp = getattr(layer, self.w_zp_name)
|
||||
zp.data = zp.t().contiguous()
|
||||
# note assumes that
|
||||
# `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}
|
||||
# `weight_scale` is: {input_dim = 0, output_dim = 1}
|
||||
# `weight_zp` is: {input_dim = 0, output_dim = 1, packed_dim = 1}
|
||||
def _process_gptq_weights_w4a8(self, layer: torch.nn.Module):
|
||||
packed_weight = getattr(layer, self.w_q_name)
|
||||
scales = getattr(layer, self.w_s_name)
|
||||
group_num = scales.data.size(0)
|
||||
zp_output_size = scales.data.size(1) // 8
|
||||
if self.config.zero_points:
|
||||
assert self.w_zp_name
|
||||
packed_zp = getattr(layer, self.w_zp_name)
|
||||
else:
|
||||
# w4a8 kernel always requires zp, allocate a fake zp
|
||||
assert self.w_zp_name
|
||||
packed_zp = torch.nn.Parameter(
|
||||
torch.ones(group_num, zp_output_size, dtype=torch.int32) * -2004318072,
|
||||
requires_grad=False,
|
||||
)
|
||||
setattr(layer, self.w_zp_name, packed_zp)
|
||||
|
||||
# FIXME: some bugs in convert_weight_packed_scale_zp with GPTQ format,
|
||||
# repack to AWQ weight
|
||||
weight = unpack_quantized_values_into_int32(
|
||||
packed_weight, self.config.weight_type, 0
|
||||
)
|
||||
input_size, output_size = weight.size()
|
||||
weight = weight.view(input_size, output_size // 8, 8)
|
||||
weight = weight[:, :, (0, 2, 4, 6, 1, 3, 5, 7)].reshape(input_size, output_size)
|
||||
weight = pack_quantized_values_into_int32(
|
||||
weight, self.config.weight_type, 1
|
||||
).contiguous()
|
||||
|
||||
zp = unpack_quantized_values_into_int32(packed_zp, self.config.weight_type, 1)
|
||||
zp = zp.view(group_num, output_size // 8, 8)
|
||||
zp = zp[:, :, (0, 2, 4, 6, 1, 3, 5, 7)].reshape(group_num, output_size)
|
||||
zp = pack_quantized_values_into_int32(
|
||||
zp, self.config.weight_type, 1
|
||||
).contiguous()
|
||||
|
||||
blocked_w, blocked_zp, blocked_s = ops.convert_weight_packed_scale_zp(
|
||||
weight,
|
||||
zp,
|
||||
scales.data,
|
||||
ops.CPUQuantAlgo.AWQ,
|
||||
)
|
||||
|
||||
if layer.bias is not None:
|
||||
layer.bias.data = layer.bias.float()
|
||||
|
||||
packed_weight.data = blocked_w
|
||||
scales.data = blocked_s
|
||||
packed_zp.data = blocked_zp
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
if (not self.config.zero_points) and (self.w_zp_name is not None):
|
||||
@@ -108,16 +147,39 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
if (not self.config.has_g_idx) and (self.w_gidx_name is not None):
|
||||
setattr(layer, self.w_gidx_name, None)
|
||||
|
||||
w_input_dim = getattr(layer, self.w_q_name).input_dim
|
||||
w_pack_dim = getattr(layer, self.w_q_name).packed_dim
|
||||
quant_method = "gptq" if w_pack_dim == w_input_dim else "awq"
|
||||
weights = getattr(layer, self.w_q_name)
|
||||
# Require GPTQ pack format
|
||||
assert weights.input_dim == weights.packed_dim
|
||||
|
||||
if quant_method == "gptq":
|
||||
# GPTQ
|
||||
self._process_gptq_weights(layer)
|
||||
# Weights in CT format is [output_size, input_size]
|
||||
if weights.input_dim == 1:
|
||||
weights.data = weights.t()
|
||||
|
||||
# Scales in CT format is [output_size, group_num]
|
||||
scales = getattr(layer, self.w_s_name)
|
||||
if scales.output_dim == 0:
|
||||
scales.data = scales.t().contiguous()
|
||||
|
||||
# Zero points in CT format is [output_size, group_num]
|
||||
# Zero points in awq_marlin format is [output_size, group_num]
|
||||
if self.config.zero_points:
|
||||
assert self.w_zp_name
|
||||
zp = getattr(layer, self.w_zp_name)
|
||||
if zp.output_dim == 0:
|
||||
zp.data = zp.t().contiguous()
|
||||
|
||||
layer.use_w4a8 = (
|
||||
envs.VLLM_CPU_INT4_W4A8
|
||||
and not self.config.has_g_idx
|
||||
and self.config.act_type == torch.bfloat16
|
||||
and torch.cpu._is_amx_tile_supported()
|
||||
)
|
||||
# layer.use_w4a8 = False
|
||||
# AWQ format will be converted to GPTQ format in `AWQMarlinLinearMethod`
|
||||
if layer.use_w4a8:
|
||||
self._process_gptq_weights_w4a8(layer)
|
||||
else:
|
||||
# AWQ
|
||||
raise NotImplementedError("AWQ is not supported in CPUWNA16LinearKernel")
|
||||
self._process_gptq_weights_w4a16(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -126,16 +188,25 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
w_q, w_s, w_zp, w_gidx = self._get_weight_params(layer)
|
||||
x = ops.cpu_gemm_wna16(
|
||||
input=x,
|
||||
q_weight=w_q,
|
||||
scales=w_s,
|
||||
zeros=w_zp,
|
||||
g_idx=w_gidx,
|
||||
bias=bias,
|
||||
pack_factor=8, # 32 // 4
|
||||
isa_hint=layer.isa_hint,
|
||||
)
|
||||
if layer.use_w4a8:
|
||||
x = ops.int4_scaled_mm_cpu(
|
||||
x=x,
|
||||
w=w_q,
|
||||
w_zeros=w_zp,
|
||||
w_scales=w_s,
|
||||
bias=bias,
|
||||
)
|
||||
else:
|
||||
x = ops.cpu_gemm_wna16(
|
||||
input=x,
|
||||
q_weight=w_q,
|
||||
scales=w_s,
|
||||
zeros=w_zp,
|
||||
g_idx=w_gidx,
|
||||
bias=bias,
|
||||
pack_factor=8, # 32 // 4
|
||||
isa_hint=layer.isa_hint,
|
||||
)
|
||||
return x
|
||||
|
||||
|
||||
|
||||
@@ -34,7 +34,6 @@ QuantizationMethods = Literal[
|
||||
"mxfp4",
|
||||
"gpt_oss_mxfp4",
|
||||
"deepseek_v4_fp8",
|
||||
"cpu_awq",
|
||||
"online",
|
||||
# Below are online quant shorthand names (see vllm.config.quantization).
|
||||
# Listed here as strings to avoid a circular import; kept in sync with
|
||||
@@ -122,7 +121,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
from .compressed_tensors.compressed_tensors import (
|
||||
CompressedTensorsConfig,
|
||||
)
|
||||
from .cpu_wna16 import CPUAWQConfig
|
||||
from .experts_int8 import ExpertsInt8Config
|
||||
from .fbgemm_fp8 import FBGEMMFp8Config
|
||||
from .fp8 import Fp8Config
|
||||
@@ -166,7 +164,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"mxfp4": Mxfp4Config,
|
||||
"gpt_oss_mxfp4": GptOssMxfp4Config,
|
||||
"deepseek_v4_fp8": DeepseekV4FP8Config,
|
||||
"cpu_awq": CPUAWQConfig,
|
||||
"humming": HummingConfig,
|
||||
"online": OnlineQuantizationConfig,
|
||||
}
|
||||
|
||||
@@ -332,7 +332,7 @@ class AWQMarlinConfig(QuantizationConfig):
|
||||
group_size = quant_config.get("group_size")
|
||||
zero_point = quant_config.get("zero_point")
|
||||
|
||||
if not current_platform.is_cuda_alike():
|
||||
if not (current_platform.is_cuda_alike() or current_platform.is_cpu()):
|
||||
return False
|
||||
|
||||
if quant_method != "awq":
|
||||
|
||||
@@ -1,354 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
LinearMethodBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
pack_cols,
|
||||
unpack_cols,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.config import get_safetensors_params_metadata
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CPUAWQConfig(QuantizationConfig):
|
||||
"""Config class for CPU AWQ"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_bits: int,
|
||||
group_size: int,
|
||||
zero_point: bool,
|
||||
lm_head_quantized: bool,
|
||||
modules_to_not_convert: list[str] | None,
|
||||
full_config: dict[str, Any],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert weight_bits == 4
|
||||
self.pack_factor = 32 // weight_bits # packed into int32
|
||||
self.group_size = group_size
|
||||
self.zero_point = zero_point
|
||||
self.lm_head_quantized = lm_head_quantized
|
||||
self.weight_bits = weight_bits
|
||||
self.modules_to_not_convert = modules_to_not_convert or []
|
||||
self.full_config = full_config
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"AWQMarlinConfig("
|
||||
f"group_size={self.group_size}, "
|
||||
f"zero_point={self.zero_point}, "
|
||||
f"lm_head_quantized={self.lm_head_quantized}, "
|
||||
f"modules_to_not_convert={self.modules_to_not_convert})"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> "QuantizationMethods":
|
||||
return "cpu_awq"
|
||||
|
||||
@classmethod
|
||||
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
|
||||
return [torch.half, torch.bfloat16]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return -1
|
||||
|
||||
@classmethod
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return ["quantize_config.json"]
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "CPUAWQConfig":
|
||||
weight_bits = cls.get_from_keys(config, ["bits"])
|
||||
group_size = cls.get_from_keys(config, ["group_size"])
|
||||
zero_point = cls.get_from_keys(config, ["zero_point"])
|
||||
lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False)
|
||||
modules_to_not_convert = cls.get_from_keys_or(
|
||||
config, ["modules_to_not_convert"], None
|
||||
)
|
||||
return cls(
|
||||
weight_bits,
|
||||
group_size,
|
||||
zero_point,
|
||||
lm_head_quantized,
|
||||
modules_to_not_convert,
|
||||
config,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> "QuantizationMethods | None":
|
||||
quant_method = hf_quant_cfg.get("quant_method", "").lower()
|
||||
if current_platform.is_cpu() and (quant_method == "awq"):
|
||||
return cls.get_name()
|
||||
return None
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
if isinstance(layer, LinearBase) or (
|
||||
isinstance(layer, ParallelLMHead) and self.lm_head_quantized
|
||||
):
|
||||
if is_layer_skipped(
|
||||
prefix,
|
||||
self.modules_to_not_convert,
|
||||
self.packed_modules_mapping,
|
||||
skip_with_substr=True,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
return CPUAWQLinearMethod(self)
|
||||
return None
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
|
||||
if self.modules_to_not_convert:
|
||||
self.modules_to_not_convert = hf_to_vllm_mapper.apply_list(
|
||||
self.modules_to_not_convert
|
||||
)
|
||||
|
||||
def maybe_update_config(
|
||||
self,
|
||||
model_name: str,
|
||||
hf_config: PretrainedConfig | None = None,
|
||||
revision: str | None = None,
|
||||
):
|
||||
if self.modules_to_not_convert:
|
||||
return
|
||||
|
||||
unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]
|
||||
metadata = get_safetensors_params_metadata(model_name, revision=revision)
|
||||
layers = {param_name.rsplit(".", 1)[0] for param_name in metadata}
|
||||
quant_layers: set[str] = {
|
||||
param_name.rsplit(".", 1)[0]
|
||||
for param_name, info in metadata.items()
|
||||
if (dtype := info.get("dtype", None))
|
||||
and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes
|
||||
}
|
||||
self.modules_to_not_convert = list(layers - quant_layers)
|
||||
|
||||
|
||||
class CPUAWQLinearMethod(LinearMethodBase):
|
||||
"""Linear method for CPU AWQ.
|
||||
|
||||
Args:
|
||||
quant_config: The CPU AWQ quantization config.
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: CPUAWQConfig) -> None:
|
||||
self.quant_config = quant_config
|
||||
assert self.quant_config.zero_point
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
del output_size
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
|
||||
# Normalize group_size
|
||||
if self.quant_config.group_size != -1:
|
||||
group_size = self.quant_config.group_size
|
||||
else:
|
||||
group_size = input_size
|
||||
|
||||
qweight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition,
|
||||
output_size_per_partition // self.quant_config.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.quant_config.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
num_groups = input_size_per_partition // group_size
|
||||
|
||||
qzeros = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
num_groups,
|
||||
output_size_per_partition // self.quant_config.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.quant_config.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
scales = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
num_groups,
|
||||
output_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
layer.register_parameter("qweight", qweight)
|
||||
layer.register_parameter("qzeros", qzeros)
|
||||
layer.register_parameter("scales", scales)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.use_w4a8 = envs.VLLM_CPU_INT4_W4A8 and torch.cpu._is_amx_tile_supported()
|
||||
if layer.use_w4a8:
|
||||
self._process_weights_sglang_int4(layer)
|
||||
else:
|
||||
self._process_weights_woq(layer)
|
||||
|
||||
def _process_weights_woq(self, layer: torch.nn.Module) -> None:
|
||||
"""Original WOQ int4 repack path."""
|
||||
packed_weight = layer.qweight.data
|
||||
packed_zeros = layer.qzeros.data
|
||||
group_num = packed_zeros.size(0)
|
||||
bits = self.quant_config.weight_bits
|
||||
pack_factor = int(self.quant_config.pack_factor)
|
||||
input_size, packed_output_size = packed_weight.size()
|
||||
output_size = packed_output_size * pack_factor
|
||||
isa_hint = _get_isa_hint(layer.scales.dtype)
|
||||
layer.isa_hint = isa_hint
|
||||
|
||||
interleave_map = (0, 4, 1, 5, 2, 6, 3, 7)
|
||||
weight = unpack_cols(
|
||||
packed_weight,
|
||||
bits,
|
||||
input_size,
|
||||
output_size,
|
||||
)
|
||||
zeros = unpack_cols(
|
||||
packed_zeros,
|
||||
bits,
|
||||
group_num,
|
||||
output_size,
|
||||
)
|
||||
weight = (
|
||||
weight.view(input_size, -1, pack_factor)[:, :, interleave_map]
|
||||
.reshape(input_size, output_size)
|
||||
.contiguous()
|
||||
)
|
||||
zeros = (
|
||||
zeros.view(group_num, -1, pack_factor)[:, :, interleave_map]
|
||||
.reshape(group_num, output_size)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
zeros = pack_cols(zeros, bits, group_num, output_size).contiguous()
|
||||
weight = pack_cols(weight, bits, input_size, output_size)
|
||||
weight = (
|
||||
weight.view(input_size, -1, 16 // pack_factor)
|
||||
.permute(1, 0, 2)
|
||||
.reshape(-1, input_size * 16 // pack_factor)
|
||||
.contiguous()
|
||||
)
|
||||
layer.qweight.data = weight
|
||||
layer.qzeros.data = zeros
|
||||
|
||||
def _process_weights_sglang_int4(self, layer: torch.nn.Module) -> None:
|
||||
"""SGLang INT4 W4A8 path: pack int4 weights with VNNI reordering."""
|
||||
packed_weight = layer.qweight.data
|
||||
packed_zeros = layer.qzeros.data
|
||||
scales = layer.scales.data
|
||||
blocked_w, blocked_zp, blocked_s = ops.convert_weight_packed_scale_zp(
|
||||
packed_weight,
|
||||
packed_zeros,
|
||||
scales,
|
||||
ops.CPUQuantAlgo.AWQ,
|
||||
)
|
||||
|
||||
layer.packed_weight = blocked_w
|
||||
layer.packed_qzeros = blocked_zp
|
||||
layer.packed_scales = blocked_s
|
||||
layer.qweight = None
|
||||
layer.qzeros = None
|
||||
layer.scales = None
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if layer.use_w4a8:
|
||||
return self._apply_sglang_int4(layer, x, bias)
|
||||
return self._apply_woq(layer, x, bias)
|
||||
|
||||
def _apply_woq(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Original WOQ int4 GEMM path."""
|
||||
x = ops.cpu_gemm_wna16(
|
||||
input=x,
|
||||
q_weight=layer.qweight,
|
||||
scales=layer.scales,
|
||||
zeros=layer.qzeros,
|
||||
g_idx=None,
|
||||
bias=bias,
|
||||
pack_factor=8,
|
||||
isa_hint=layer.isa_hint,
|
||||
)
|
||||
return x
|
||||
|
||||
def _apply_sglang_int4(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""SGLang INT4 W4A8 GEMM path."""
|
||||
return ops.int4_scaled_mm_cpu(
|
||||
x,
|
||||
layer.packed_weight,
|
||||
layer.packed_qzeros,
|
||||
layer.packed_scales,
|
||||
bias,
|
||||
)
|
||||
|
||||
|
||||
def _get_isa_hint(dtype: torch.dtype) -> str:
|
||||
supports_amx = torch.cpu._is_amx_tile_supported()
|
||||
if supports_amx and dtype in (torch.bfloat16,):
|
||||
return "amx"
|
||||
else:
|
||||
return "vec"
|
||||
@@ -72,7 +72,6 @@ _QUANT_WEIGHT_BYTE_SIZE: dict[str, float] = {
|
||||
"quark": 0.5,
|
||||
"moe_wna16": 0.5,
|
||||
"inc": 0.5,
|
||||
"cpu_awq": 0.5,
|
||||
"experts_int8": 1,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user