[quant][autoround]Refactor INC quantization into package with INCScheme orchestrator (#40601)
Signed-off-by: yiliu30 <yi4.liu@intel.com> Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com> Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com> Co-authored-by: n1ck-guo <heng.guo@intel.com> Co-authored-by: Zhenzhong1 <zhenzhong.xu@intel.com>
This commit is contained in:
co-authored by
n1ck-guo
Zhenzhong1
parent
06e1e0885c
commit
bcb518ad7a
@@ -1,794 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fractions import Fraction
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
LinearMethodBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
QuantizationMethods,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
RowvLLMParameter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class INCConfig(QuantizationConfig):
|
||||
"""Config class for Intel Neural Compressor (INC).
|
||||
Repo: https://github.com/intel/neural-compressor
|
||||
"""
|
||||
|
||||
SUPPORTED_BITS = {2, 3, 4, 8}
|
||||
SUPPORTED_DTYPES = {"int"}
|
||||
SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq"}
|
||||
SUPPORTED_BACKENDS = {
|
||||
"auto",
|
||||
"gptq",
|
||||
"gptq:marlin",
|
||||
"awq",
|
||||
"awq:marlin",
|
||||
"marlin",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_bits: int,
|
||||
group_size: int,
|
||||
sym: bool = True,
|
||||
packing_format: str = "auto_round:auto_gptq",
|
||||
block_name_to_quantize: str | list[str] | None = None,
|
||||
extra_config: dict[str, Any] | None = None,
|
||||
data_type: str = "int",
|
||||
backend: str = "auto",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if weight_bits not in self.SUPPORTED_BITS:
|
||||
raise ValueError(
|
||||
f"Unsupported weight_bits: {weight_bits}, "
|
||||
f"currently only support {self.SUPPORTED_BITS}."
|
||||
)
|
||||
if data_type not in self.SUPPORTED_DTYPES:
|
||||
raise ValueError(
|
||||
f"Unsupported data_type: {data_type},"
|
||||
f" currently only support {self.SUPPORTED_DTYPES}."
|
||||
)
|
||||
if packing_format not in self.SUPPORTED_FORMATS:
|
||||
raise ValueError(
|
||||
f"Unsupported packing_format: {packing_format}, "
|
||||
f"currently only support {self.SUPPORTED_FORMATS}."
|
||||
)
|
||||
if backend not in self.SUPPORTED_BACKENDS:
|
||||
raise ValueError(
|
||||
f"Unsupported backend: {backend}, "
|
||||
f"currently only support {self.SUPPORTED_BACKENDS}."
|
||||
)
|
||||
|
||||
self.weight_bits = weight_bits
|
||||
self.group_size = group_size
|
||||
self.sym = sym
|
||||
self.packing_format = packing_format
|
||||
self.block_name_to_quantize = (
|
||||
block_name_to_quantize.split(",")
|
||||
if isinstance(block_name_to_quantize, str)
|
||||
else block_name_to_quantize
|
||||
)
|
||||
self.extra_config = extra_config
|
||||
self.data_type = data_type
|
||||
self.backend = backend
|
||||
self.pack_factor = Fraction(32, weight_bits)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"INCConfig(weight_bits={self.weight_bits}, "
|
||||
f"group_size={self.group_size}, sym={self.sym})"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "inc"
|
||||
|
||||
@classmethod
|
||||
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
|
||||
return [torch.half, torch.bfloat16]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 60
|
||||
|
||||
@classmethod
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return ["quantization_config.json"]
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
|
||||
return cls(
|
||||
weight_bits=cls.get_from_keys(config, ["bits"]),
|
||||
group_size=cls.get_from_keys(config, ["group_size"]),
|
||||
sym=cls.get_from_keys(config, ["sym"]),
|
||||
packing_format=cls.get_from_keys_or(
|
||||
config, ["packing_format"], "auto_round:auto_gptq"
|
||||
),
|
||||
block_name_to_quantize=cls.get_from_keys_or(
|
||||
config, ["block_name_to_quantize", "to_quant_block_names"], None
|
||||
),
|
||||
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
|
||||
data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
|
||||
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
|
||||
)
|
||||
|
||||
def get_layer_config(self, layer, layer_name: str):
|
||||
def get_config(name: str, quantized: bool = True):
|
||||
if not self.extra_config:
|
||||
return (
|
||||
self.weight_bits if quantized else 16,
|
||||
self.group_size if quantized else -1,
|
||||
self.sym if quantized else True,
|
||||
)
|
||||
|
||||
# exact match first
|
||||
if name in self.extra_config:
|
||||
cfg = self.extra_config[name]
|
||||
return (
|
||||
cfg.get("bits", self.weight_bits if quantized else 16),
|
||||
cfg.get("group_size", self.group_size if quantized else -1),
|
||||
cfg.get("sym", self.sym if quantized else True),
|
||||
)
|
||||
|
||||
REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
|
||||
for pattern, cfg in self.extra_config.items():
|
||||
if not isinstance(pattern, str) or not any(
|
||||
c in REGEX_SPECIAL_CHARS for c in pattern
|
||||
):
|
||||
continue
|
||||
|
||||
try:
|
||||
if re.search(re.compile(pattern), name) is not None:
|
||||
return (
|
||||
cfg.get("bits", self.weight_bits if quantized else 16),
|
||||
cfg.get("group_size", self.group_size if quantized else -1),
|
||||
cfg.get("sym", self.sym if quantized else True),
|
||||
)
|
||||
except re.error:
|
||||
# Invalid regex, ignore.
|
||||
continue
|
||||
|
||||
return (
|
||||
self.weight_bits if quantized else 16,
|
||||
self.group_size if quantized else -1,
|
||||
self.sym if quantized else True,
|
||||
)
|
||||
|
||||
# 1. Exact match from config
|
||||
if self.extra_config and layer_name in self.extra_config:
|
||||
return get_config(layer_name)
|
||||
|
||||
# 2. Determine whether layer should be quantized
|
||||
quantized = not isinstance(layer, ParallelLMHead)
|
||||
if self.block_name_to_quantize:
|
||||
quantized = any(
|
||||
layer_name.startswith(name) for name in self.block_name_to_quantize
|
||||
)
|
||||
|
||||
# 3. Handle fused MoE
|
||||
if self.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
|
||||
moe_configs = [
|
||||
get_config(name, quantized)
|
||||
for name in self.extra_config
|
||||
if name.startswith(layer_name)
|
||||
]
|
||||
if moe_configs:
|
||||
if len(set(moe_configs)) == 1:
|
||||
return moe_configs[0]
|
||||
raise ValueError(
|
||||
f"Fused MoE layer '{layer_name}' requires "
|
||||
f"consistent quant config for all sub-layers"
|
||||
)
|
||||
|
||||
# 4. Handle fused QKV or other patterns
|
||||
if self.extra_config:
|
||||
for fusion_key, sub_keys in self.packed_modules_mapping.items():
|
||||
if fusion_key in layer_name and layer_name.count(fusion_key) == 1:
|
||||
sub_names = [
|
||||
layer_name.replace(fusion_key, sub_key) for sub_key in sub_keys
|
||||
]
|
||||
sub_configs = [get_config(name, quantized) for name in sub_names]
|
||||
if len(set(sub_configs)) == 1:
|
||||
return sub_configs[0]
|
||||
raise ValueError(
|
||||
f"Fused module '{layer_name}' requires "
|
||||
f"consistent quant config for {sub_names}"
|
||||
)
|
||||
|
||||
# 5. Fallback or try a regular expression match
|
||||
return get_config(layer_name, quantized)
|
||||
|
||||
def check_quantized(self, weight_bits: int) -> bool:
|
||||
return weight_bits < 16
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
|
||||
if self.block_name_to_quantize is not None:
|
||||
self.block_name_to_quantize = hf_to_vllm_mapper.apply_list(
|
||||
self.block_name_to_quantize
|
||||
)
|
||||
if self.extra_config is not None:
|
||||
self.extra_config = hf_to_vllm_mapper.apply_dict(self.extra_config)
|
||||
|
||||
def apply_awq_quant_layer(self, layer, prefix: str, backend: str = "auto"):
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_marlin_supported,
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
|
||||
if not self.check_quantized(weight_bits):
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
return UnquantizedLinearMethod()
|
||||
else:
|
||||
return None
|
||||
|
||||
logger.debug(
|
||||
"[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
|
||||
prefix,
|
||||
layer.__class__.__name__,
|
||||
weight_bits,
|
||||
group_size,
|
||||
sym,
|
||||
)
|
||||
if backend == "auto" or "marlin" in backend:
|
||||
AWQ_TYPE_MAP = {
|
||||
4: scalar_types.uint4,
|
||||
8: scalar_types.uint8,
|
||||
}
|
||||
use_marlin = (weight_bits in AWQ_TYPE_MAP) and check_marlin_supported(
|
||||
AWQ_TYPE_MAP[weight_bits], group_size, not sym
|
||||
)
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
use_marlin = use_marlin and check_moe_marlin_supports_layer(
|
||||
layer, group_size
|
||||
)
|
||||
|
||||
else:
|
||||
use_marlin = False
|
||||
if use_marlin:
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import (
|
||||
AWQMarlinConfig,
|
||||
AWQMarlinLinearMethod,
|
||||
AWQMarlinMoEMethod,
|
||||
)
|
||||
|
||||
quant_args_marlin = AWQMarlinConfig(
|
||||
weight_bits=weight_bits,
|
||||
group_size=group_size,
|
||||
zero_point=not sym,
|
||||
lm_head_quantized=False,
|
||||
full_config={},
|
||||
modules_to_not_convert=[],
|
||||
)
|
||||
else:
|
||||
from vllm.model_executor.layers.quantization.awq import (
|
||||
AWQConfig,
|
||||
AWQLinearMethod,
|
||||
)
|
||||
|
||||
quant_args = AWQConfig(
|
||||
weight_bits=weight_bits,
|
||||
group_size=group_size,
|
||||
zero_point=not sym,
|
||||
)
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
if use_marlin:
|
||||
return AWQMarlinMoEMethod(quant_args_marlin, layer.moe_config)
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
||||
|
||||
config = {
|
||||
"quant_method": "awq",
|
||||
"bits": weight_bits,
|
||||
"group_size": group_size,
|
||||
"zero_point": not sym,
|
||||
"lm_head": False,
|
||||
}
|
||||
return MoeWNA16Config.from_config(config).get_quant_method(layer, prefix)
|
||||
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
if use_marlin:
|
||||
return AWQMarlinLinearMethod(quant_args_marlin)
|
||||
else:
|
||||
return AWQLinearMethod(quant_args)
|
||||
return None
|
||||
|
||||
def apply_gptq_quant_layer(self, layer, prefix: str, backend: str = "auto"):
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_marlin_supported,
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
|
||||
if not self.check_quantized(weight_bits):
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
return UnquantizedLinearMethod()
|
||||
else:
|
||||
return None
|
||||
|
||||
logger.debug(
|
||||
"[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
|
||||
prefix,
|
||||
layer.__class__.__name__,
|
||||
weight_bits,
|
||||
group_size,
|
||||
sym,
|
||||
)
|
||||
if backend == "auto" or "marlin" in backend:
|
||||
GPTQ_TYPE_MAP = {
|
||||
(4, True): scalar_types.uint4b8,
|
||||
(8, True): scalar_types.uint8b128,
|
||||
}
|
||||
use_marlin = (weight_bits, sym) in GPTQ_TYPE_MAP and check_marlin_supported(
|
||||
GPTQ_TYPE_MAP[(weight_bits, sym)], group_size, has_zp=not sym
|
||||
)
|
||||
if isinstance(layer, RoutedExperts):
|
||||
use_marlin = use_marlin and check_moe_marlin_supports_layer(
|
||||
layer, group_size
|
||||
)
|
||||
else:
|
||||
use_marlin = False
|
||||
if use_marlin:
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQConfig,
|
||||
AutoGPTQLinearMethod,
|
||||
AutoGPTQMoEMethod,
|
||||
)
|
||||
|
||||
quant_args_marlin = AutoGPTQConfig(
|
||||
weight_bits=weight_bits,
|
||||
group_size=group_size,
|
||||
is_sym=sym,
|
||||
lm_head_quantized=False,
|
||||
desc_act=False,
|
||||
dynamic={},
|
||||
full_config={},
|
||||
)
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
if use_marlin:
|
||||
return AutoGPTQMoEMethod(quant_args_marlin, layer.moe_config)
|
||||
else:
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import (
|
||||
MoeWNA16Config,
|
||||
)
|
||||
|
||||
config = {
|
||||
"quant_method": "gptq",
|
||||
"bits": weight_bits,
|
||||
"group_size": group_size,
|
||||
"sym": sym,
|
||||
"lm_head": False,
|
||||
}
|
||||
return MoeWNA16Config.from_config(config).get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
if use_marlin:
|
||||
return AutoGPTQLinearMethod(quant_args_marlin)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"INC quantization with bits={weight_bits}, sym={sym} "
|
||||
"is not supported. Only 4-bit and 8-bit symmetric "
|
||||
"quantization is supported with Marlin kernels."
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def apply_xpu_w4a16_quant_layer(self, layer, prefix: str):
|
||||
weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
|
||||
|
||||
if not self.check_quantized(weight_bits):
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
return UnquantizedLinearMethod()
|
||||
else:
|
||||
return None
|
||||
|
||||
if weight_bits != 4:
|
||||
raise NotImplementedError(
|
||||
f"INC on XPU only supports 4-bit quantization, "
|
||||
f"got weight_bits={weight_bits}."
|
||||
)
|
||||
if not sym:
|
||||
raise NotImplementedError(
|
||||
"INC W4A16 on XPU only supports symmetric quantization for now."
|
||||
)
|
||||
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
is_ark_available, ark_error, _, _ = get_ark_state()
|
||||
if is_ark_available:
|
||||
return INCARKLinearMethod(
|
||||
weight_bits=weight_bits,
|
||||
group_size=group_size,
|
||||
sym=sym,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"ARK backend is unavailable for layer %s; "
|
||||
"falling back to the default XPU INC path. Error: %s",
|
||||
prefix,
|
||||
ark_error or "unknown error",
|
||||
)
|
||||
|
||||
return INCXPULinearMethod(
|
||||
weight_bits=weight_bits,
|
||||
group_size=group_size,
|
||||
sym=sym,
|
||||
)
|
||||
return None
|
||||
|
||||
def apply_cpu_w4a16_quant_layer(self, layer, prefix: str):
|
||||
weight_bits, group_size, sym = self.get_layer_config(layer, prefix)
|
||||
if not self.check_quantized(weight_bits):
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
return UnquantizedLinearMethod()
|
||||
else:
|
||||
return None
|
||||
|
||||
if weight_bits != 4:
|
||||
raise NotImplementedError(
|
||||
f"INC on CPU only supports 4-bit quantization, "
|
||||
f"got weight_bits={weight_bits}."
|
||||
)
|
||||
if not sym:
|
||||
raise NotImplementedError(
|
||||
"INC W4A16 on CPU only supports symmetric quantization for now."
|
||||
)
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
is_ark_available, ark_error, _, _ = get_ark_state()
|
||||
if is_ark_available:
|
||||
return INCARKLinearMethod(
|
||||
weight_bits=weight_bits,
|
||||
group_size=group_size,
|
||||
sym=sym,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"ARK backend is unavailable for layer %s; "
|
||||
"falling back to the default CPU INC path. Error: %s",
|
||||
prefix,
|
||||
ark_error or "unknown error",
|
||||
)
|
||||
|
||||
return self.apply_gptq_quant_layer(layer, prefix)
|
||||
return None
|
||||
|
||||
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
|
||||
if prefix and self.extra_config:
|
||||
for layer_name in self.extra_config:
|
||||
if (
|
||||
layer_name == prefix or layer_name == f"model.{prefix}"
|
||||
) and self.extra_config[layer_name].get("bits", 16) >= 16:
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
if current_platform.is_xpu():
|
||||
return self.apply_xpu_w4a16_quant_layer(layer, prefix)
|
||||
is_gptq = "gptq" in self.packing_format or "gptq" in self.backend
|
||||
if current_platform.is_cpu() and is_gptq:
|
||||
return self.apply_cpu_w4a16_quant_layer(layer, prefix)
|
||||
if is_gptq:
|
||||
return self.apply_gptq_quant_layer(layer, prefix)
|
||||
if "awq" in self.packing_format or "awq" in self.backend:
|
||||
return self.apply_awq_quant_layer(layer, prefix)
|
||||
|
||||
raise NotImplementedError(
|
||||
f"Unsupported quantization configuration for layer '{prefix}'. "
|
||||
f"Platform: CPU={current_platform.is_cpu()}. "
|
||||
f"Platform: XPU={current_platform.is_xpu()}. "
|
||||
f"Format: {self.packing_format}, Backend: {self.backend}."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> "QuantizationMethods | None":
|
||||
"""Override the `auto-round` method to `inc`."""
|
||||
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
|
||||
if is_auto_round_format:
|
||||
return cls.get_name()
|
||||
return None
|
||||
|
||||
|
||||
class INCXPULinearBase(LinearMethodBase):
|
||||
def __init__(self, weight_bits: int, group_size: int, sym: bool):
|
||||
self.weight_bits = weight_bits
|
||||
self.group_size = group_size
|
||||
self.sym = sym
|
||||
self.pack_factor = 32 // weight_bits
|
||||
|
||||
def _create_inc_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Any,
|
||||
group_size: int,
|
||||
pack_factor: int,
|
||||
) -> None:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
scales_and_zp_size = input_size_per_partition // group_size
|
||||
|
||||
qweight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition // pack_factor,
|
||||
output_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=0,
|
||||
packed_factor=pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
scales = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
scales_and_zp_size,
|
||||
output_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
qzeros = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
scales_and_zp_size,
|
||||
output_size_per_partition // pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
layer.register_parameter("qweight", qweight)
|
||||
layer.register_parameter("scales", scales)
|
||||
layer.register_parameter("qzeros", qzeros)
|
||||
|
||||
g_idx = RowvLLMParameter(
|
||||
data=torch.tensor(
|
||||
[i // group_size for i in range(input_size_per_partition)],
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("g_idx", g_idx)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
self._create_inc_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
params_dtype=params_dtype,
|
||||
weight_loader=extra_weight_attrs.get("weight_loader"),
|
||||
group_size=self.group_size,
|
||||
pack_factor=self.pack_factor,
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_ark_state() -> tuple[bool, str | None, Any | None, Any | None]:
|
||||
"""Return ARK availability, error details, cached instance, and QuantLinear."""
|
||||
try:
|
||||
import auto_round_kernel
|
||||
from auto_round_kernel.qlinear import QuantLinear
|
||||
|
||||
logger.info("Successfully imported auto_round_kernel.")
|
||||
except ImportError as error:
|
||||
return False, str(error), None, None
|
||||
|
||||
ark_loader = getattr(auto_round_kernel, "_ark_instance", None)
|
||||
if not callable(ark_loader):
|
||||
return False, "auto_round_kernel does not expose _ark_instance().", None, None
|
||||
|
||||
try:
|
||||
ark_instance = ark_loader()
|
||||
except Exception as error:
|
||||
return False, str(error), None, None
|
||||
|
||||
if ark_instance is None:
|
||||
return False, "auto_round_kernel._ark_instance() returned None.", None, None
|
||||
|
||||
return True, None, ark_instance, QuantLinear
|
||||
|
||||
|
||||
class INCXPULinearMethod(INCXPULinearBase):
|
||||
"""XPU linear method for INC w4a16 GPTQ quantization (symmetric only).
|
||||
|
||||
Repacks GPTQ weights from [in_packed, out] to oneDNN [out, in_packed]
|
||||
layout and calls torch.ops._xpu_C.int4_gemm_w4a16.
|
||||
|
||||
GPTQ format: qweight [in_packed, out] with sequential nibble order.
|
||||
|
||||
Note: Asymmetric quantization (sym=false) is not for now.
|
||||
|
||||
FIXME(yiliu30): Refine the implementation to reuse XPUwNa16LinearKernel.
|
||||
"""
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Repack GPTQ weights into kernel-ready NT layout."""
|
||||
device = layer.qweight.data.device
|
||||
|
||||
# oneDNN int4 kernel requires strides[0]==1 ("NT format"), but GPTQ
|
||||
# checkpoint is [K_packed, N] contiguous with strides (N, 1).
|
||||
# Two transposes are needed — neither alone can achieve this:
|
||||
# 1. .t().contiguous() → [N, K_packed] contiguous in memory
|
||||
# 2. .t() → [K_packed, N] view with strides (1, K_packed)
|
||||
# The result has the same logical shape but strides[0]==1 as required.
|
||||
qweight_ct = layer.qweight.data.t().contiguous()
|
||||
layer.qweight = Parameter(qweight_ct.t(), requires_grad=False)
|
||||
|
||||
# Scales: [num_groups, out] — no change needed
|
||||
layer.scales = Parameter(layer.scales.data, requires_grad=False)
|
||||
|
||||
# Symmetric: GPTQ v1 stores qzeros=7, effective zp = 7+1 = 8
|
||||
# Kernel expects int8 scalar = 8
|
||||
layer.qzeros = Parameter(
|
||||
torch.tensor([8], dtype=torch.int8, device=device),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# qweight is already in NT layout [K_packed, N] (strides (1, K_packed))
|
||||
# from process_weights_after_loading — pass directly to kernel.
|
||||
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a16(
|
||||
reshaped_x,
|
||||
layer.qweight,
|
||||
bias,
|
||||
layer.scales,
|
||||
layer.qzeros,
|
||||
self.group_size,
|
||||
None, # g_idx not needed: desc_act is always False for INC models
|
||||
)
|
||||
return out.reshape(out_shape)
|
||||
|
||||
|
||||
class INCARKLinearMethod(INCXPULinearBase):
|
||||
"""XPU & CPU w4a16 linear method for INC quantization utilizing the ARK backend.
|
||||
|
||||
See: https://github.com/intel/auto-round/blob/main/auto_round_extension/ark/README.md
|
||||
|
||||
Repacks GPTQ/INC weights into ARK's layout.
|
||||
"""
|
||||
|
||||
def __init__(self, weight_bits: int, group_size: int, sym: bool):
|
||||
super().__init__(weight_bits=weight_bits, group_size=group_size, sym=sym)
|
||||
|
||||
is_available, error_str, _, quant_linear_cls = get_ark_state()
|
||||
if not is_available or quant_linear_cls is None:
|
||||
reason = error_str or "unknown error"
|
||||
raise ImportError(f"Failed to import auto_round_kernel. {reason}")
|
||||
|
||||
self.QuantLinear = quant_linear_cls
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
super().create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
layer.in_features = input_size_per_partition
|
||||
layer.out_features = sum(output_partition_sizes)
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if hasattr(layer, "input_size_per_partition"):
|
||||
in_features = layer.input_size_per_partition
|
||||
elif hasattr(layer, "input_size"):
|
||||
in_features = layer.input_size
|
||||
else:
|
||||
raise AttributeError("Cannot determine in_features for layer.")
|
||||
|
||||
if hasattr(layer, "output_partition_sizes"):
|
||||
out_features = sum(layer.output_partition_sizes)
|
||||
elif hasattr(layer, "output_size_per_partition"):
|
||||
out_features = layer.output_size_per_partition
|
||||
elif hasattr(layer, "output_size"):
|
||||
out_features = layer.output_size
|
||||
else:
|
||||
out_features = layer.scales.shape[-1]
|
||||
|
||||
ark_linear = self.QuantLinear(
|
||||
bits=self.weight_bits,
|
||||
group_size=self.group_size,
|
||||
sym=self.sym,
|
||||
in_features=in_features,
|
||||
out_features=out_features,
|
||||
bias=layer.bias is not None,
|
||||
weight_dtype=layer.params_dtype,
|
||||
)
|
||||
|
||||
ark_linear.to(layer.qweight.device)
|
||||
|
||||
with torch.no_grad():
|
||||
ark_linear.qweight.copy_(layer.qweight.detach())
|
||||
|
||||
if hasattr(layer, "qzeros") and layer.qzeros is not None:
|
||||
ark_linear.qzeros.copy_(layer.qzeros.detach())
|
||||
else:
|
||||
ark_linear.qzeros = None
|
||||
|
||||
ark_linear.scales.copy_(layer.scales.detach())
|
||||
|
||||
if hasattr(layer, "bias") and layer.bias is not None:
|
||||
ark_linear.bias.copy_(layer.bias.detach())
|
||||
|
||||
ark_linear.post_init()
|
||||
|
||||
layer.ark_linear = ark_linear
|
||||
|
||||
del layer.qweight
|
||||
if hasattr(layer, "qzeros"):
|
||||
del layer.qzeros
|
||||
del layer.scales
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return layer.ark_linear.forward(x)
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .inc import INCConfig
|
||||
|
||||
__all__ = ["INCConfig"]
|
||||
@@ -0,0 +1,188 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from .inc import INCConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class INCLayerConfig:
|
||||
bits: int
|
||||
group_size: int
|
||||
sym: bool
|
||||
packing_format: str
|
||||
backend: str
|
||||
data_type: str
|
||||
quantized: bool
|
||||
|
||||
@property
|
||||
def is_gptq(self) -> bool:
|
||||
return "gptq" in self.packing_format or "gptq" in self.backend
|
||||
|
||||
@property
|
||||
def is_awq(self) -> bool:
|
||||
return "awq" in self.packing_format or "awq" in self.backend
|
||||
|
||||
@property
|
||||
def is_wna16_int(self) -> bool:
|
||||
return self.data_type == "int" and self.quantized
|
||||
|
||||
@property
|
||||
def is_mxfp4(self) -> bool:
|
||||
return self.data_type == "mx_fp" and self.bits == 4
|
||||
|
||||
@property
|
||||
def is_mxfp8(self) -> bool:
|
||||
return self.data_type == "mx_fp" and self.bits == 8
|
||||
|
||||
|
||||
class INCConfigParser:
|
||||
def __init__(self, config: "INCConfig") -> None:
|
||||
self._config = config
|
||||
|
||||
def resolve(self, layer: "torch.nn.Module", layer_name: str) -> INCLayerConfig:
|
||||
bits, group_size, sym = self._resolve_raw(layer, layer_name)
|
||||
return INCLayerConfig(
|
||||
bits=bits,
|
||||
group_size=group_size,
|
||||
sym=sym,
|
||||
packing_format=self._config.packing_format,
|
||||
backend=self._config.backend,
|
||||
data_type=self._config.data_type,
|
||||
quantized=bits < 16,
|
||||
)
|
||||
|
||||
def get_layer_config(
|
||||
self, layer: "torch.nn.Module", layer_name: str
|
||||
) -> tuple[int, int, bool]:
|
||||
layer_config = self.resolve(layer, layer_name)
|
||||
return layer_config.bits, layer_config.group_size, layer_config.sym
|
||||
|
||||
def _resolve_raw(
|
||||
self, layer: "torch.nn.Module", layer_name: str
|
||||
) -> tuple[int, int, bool]:
|
||||
REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
|
||||
|
||||
def is_explicitly_configured(name: str) -> bool:
|
||||
"""Return True if *name* has an explicit entry in extra_config,
|
||||
either via exact key match or via a regex pattern key."""
|
||||
if not self._config.extra_config:
|
||||
return False
|
||||
if name in self._config.extra_config:
|
||||
return True
|
||||
for pattern in self._config.extra_config:
|
||||
if not isinstance(pattern, str) or not any(
|
||||
c in REGEX_SPECIAL_CHARS for c in pattern
|
||||
):
|
||||
continue
|
||||
try:
|
||||
if re.search(re.compile(pattern), name) is not None:
|
||||
return True
|
||||
except re.error:
|
||||
continue
|
||||
return False
|
||||
|
||||
def get_config(name: str, quantized: bool = True) -> tuple[int, int, bool]:
|
||||
if not self._config.extra_config:
|
||||
return (
|
||||
self._config.weight_bits if quantized else 16,
|
||||
self._config.group_size if quantized else -1,
|
||||
self._config.sym if quantized else True,
|
||||
)
|
||||
|
||||
if name in self._config.extra_config:
|
||||
cfg = self._config.extra_config[name]
|
||||
return (
|
||||
cfg.get("bits", self._config.weight_bits if quantized else 16),
|
||||
cfg.get(
|
||||
"group_size",
|
||||
self._config.group_size if quantized else -1,
|
||||
),
|
||||
cfg.get("sym", self._config.sym if quantized else True),
|
||||
)
|
||||
|
||||
regex_special_chars = set(r"*+?^$()[]{}|\\")
|
||||
for pattern, cfg in self._config.extra_config.items():
|
||||
if not isinstance(pattern, str) or not any(
|
||||
c in regex_special_chars for c in pattern
|
||||
):
|
||||
continue
|
||||
|
||||
try:
|
||||
if re.search(re.compile(pattern), name) is not None:
|
||||
return (
|
||||
cfg.get(
|
||||
"bits",
|
||||
self._config.weight_bits if quantized else 16,
|
||||
),
|
||||
cfg.get(
|
||||
"group_size",
|
||||
self._config.group_size if quantized else -1,
|
||||
),
|
||||
cfg.get("sym", self._config.sym if quantized else True),
|
||||
)
|
||||
except re.error:
|
||||
continue
|
||||
|
||||
return (
|
||||
self._config.weight_bits if quantized else 16,
|
||||
self._config.group_size if quantized else -1,
|
||||
self._config.sym if quantized else True,
|
||||
)
|
||||
|
||||
if self._config.extra_config and layer_name in self._config.extra_config:
|
||||
return get_config(layer_name)
|
||||
|
||||
quantized = not isinstance(layer, ParallelLMHead)
|
||||
if self._config.block_name_to_quantize:
|
||||
quantized = any(
|
||||
layer_name.startswith(name)
|
||||
for name in self._config.block_name_to_quantize
|
||||
)
|
||||
|
||||
if self._config.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
|
||||
moe_configs = [
|
||||
get_config(name, quantized)
|
||||
for name in self._config.extra_config
|
||||
if name.startswith(layer_name)
|
||||
]
|
||||
if moe_configs:
|
||||
if len(set(moe_configs)) == 1:
|
||||
return moe_configs[0]
|
||||
raise ValueError(
|
||||
f"Fused MoE layer '{layer_name}' requires "
|
||||
f"consistent quant config for all sub-layers"
|
||||
)
|
||||
|
||||
if self._config.extra_config:
|
||||
for fusion_key, sub_keys in self._config.packed_modules_mapping.items():
|
||||
if fusion_key in layer_name and layer_name.count(fusion_key) == 1:
|
||||
sub_names = [
|
||||
layer_name.replace(fusion_key, sub_key) for sub_key in sub_keys
|
||||
]
|
||||
# Only trigger if at least one sub_name is explicitly
|
||||
# configured in extra_config (via exact match or regex).
|
||||
# This prevents false matches when a short fusion_key
|
||||
# (e.g. "qkv") is merely a substring of a longer layer
|
||||
# name (e.g. "in_proj_qkvz") and none of the generated
|
||||
# sub_names are actually configured.
|
||||
if not any(is_explicitly_configured(n) for n in sub_names):
|
||||
continue
|
||||
sub_configs = [get_config(name, quantized) for name in sub_names]
|
||||
if len(set(sub_configs)) == 1:
|
||||
return sub_configs[0]
|
||||
raise ValueError(
|
||||
f"Fused module '{layer_name}' requires "
|
||||
f"consistent quant config for {sub_names}"
|
||||
)
|
||||
|
||||
return get_config(layer_name, quantized)
|
||||
@@ -0,0 +1,192 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fractions import Fraction
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
RoutedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import (
|
||||
QuantizationConfig,
|
||||
QuantizationMethods,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
from .config_parser import INCConfigParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class INCConfig(QuantizationConfig):
|
||||
"""Config class for Intel Neural Compressor (INC).
|
||||
Repo: https://github.com/intel/neural-compressor
|
||||
"""
|
||||
|
||||
SUPPORTED_BITS = {2, 3, 4, 8}
|
||||
SUPPORTED_DTYPES = {"int"}
|
||||
SUPPORTED_FORMATS = {"auto_round:auto_gptq", "auto_round:auto_awq"}
|
||||
SUPPORTED_BACKENDS = {
|
||||
"auto",
|
||||
"gptq",
|
||||
"gptq:marlin",
|
||||
"awq",
|
||||
"awq:marlin",
|
||||
"marlin",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_bits: int,
|
||||
group_size: int,
|
||||
sym: bool = True,
|
||||
packing_format: str = "auto_round:auto_gptq",
|
||||
block_name_to_quantize: str | list[str] | None = None,
|
||||
extra_config: dict[str, Any] | None = None,
|
||||
data_type: str = "int",
|
||||
backend: str = "auto",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if weight_bits not in self.SUPPORTED_BITS:
|
||||
raise ValueError(
|
||||
f"Unsupported weight_bits: {weight_bits}, "
|
||||
f"currently only support {self.SUPPORTED_BITS}."
|
||||
)
|
||||
if data_type not in self.SUPPORTED_DTYPES:
|
||||
raise ValueError(
|
||||
f"Unsupported data_type: {data_type},"
|
||||
f" currently only support {self.SUPPORTED_DTYPES}."
|
||||
)
|
||||
if packing_format not in self.SUPPORTED_FORMATS:
|
||||
raise ValueError(
|
||||
f"Unsupported packing_format: {packing_format}, "
|
||||
f"currently only support {self.SUPPORTED_FORMATS}."
|
||||
)
|
||||
if backend not in self.SUPPORTED_BACKENDS:
|
||||
raise ValueError(
|
||||
f"Unsupported backend: {backend}, "
|
||||
f"currently only support {self.SUPPORTED_BACKENDS}."
|
||||
)
|
||||
|
||||
self.weight_bits = weight_bits
|
||||
self.group_size = group_size
|
||||
self.sym = sym
|
||||
self.packing_format = packing_format
|
||||
self.block_name_to_quantize = (
|
||||
block_name_to_quantize.split(",")
|
||||
if isinstance(block_name_to_quantize, str)
|
||||
else block_name_to_quantize
|
||||
)
|
||||
self.extra_config = extra_config
|
||||
self.data_type = data_type
|
||||
self.backend = backend
|
||||
self.pack_factor = Fraction(32, weight_bits)
|
||||
self.config_parser = INCConfigParser(self)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"INCConfig(weight_bits={self.weight_bits}, "
|
||||
f"group_size={self.group_size}, sym={self.sym})"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "inc"
|
||||
|
||||
@classmethod
|
||||
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
|
||||
return [torch.half, torch.bfloat16]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 60
|
||||
|
||||
@classmethod
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return ["quantization_config.json"]
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
|
||||
return cls(
|
||||
weight_bits=cls.get_from_keys(config, ["bits"]),
|
||||
group_size=cls.get_from_keys(config, ["group_size"]),
|
||||
sym=cls.get_from_keys(config, ["sym"]),
|
||||
packing_format=cls.get_from_keys_or(
|
||||
config, ["packing_format"], "auto_round:auto_gptq"
|
||||
),
|
||||
block_name_to_quantize=cls.get_from_keys_or(
|
||||
config, ["block_name_to_quantize", "to_quant_block_names"], None
|
||||
),
|
||||
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
|
||||
data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
|
||||
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
|
||||
)
|
||||
|
||||
def get_layer_config(self, layer, layer_name: str):
|
||||
return self.config_parser.get_layer_config(layer, layer_name)
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
|
||||
if self.block_name_to_quantize is not None:
|
||||
self.block_name_to_quantize = hf_to_vllm_mapper.apply_list(
|
||||
self.block_name_to_quantize
|
||||
)
|
||||
if self.extra_config is not None:
|
||||
self.extra_config = hf_to_vllm_mapper.apply_dict(self.extra_config)
|
||||
|
||||
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
|
||||
from .schemes.factory import resolve_scheme
|
||||
|
||||
# Match original: check model.-prefixed names for unquantized layers
|
||||
if prefix and self.extra_config:
|
||||
for layer_name in self.extra_config:
|
||||
if (
|
||||
layer_name == prefix or layer_name == f"model.{prefix}"
|
||||
) and self.extra_config[layer_name].get("bits", 16) >= 16:
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
layer_config = self.config_parser.resolve(layer, prefix)
|
||||
if not layer_config.quantized:
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
return UnquantizedLinearMethod()
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
return None
|
||||
|
||||
logger.debug(
|
||||
"[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
|
||||
prefix,
|
||||
layer.__class__.__name__,
|
||||
layer_config.bits,
|
||||
layer_config.group_size,
|
||||
layer_config.sym,
|
||||
)
|
||||
|
||||
scheme = resolve_scheme(layer_config)
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
return scheme.get_linear_method(self, layer, prefix, layer_config)
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return scheme.get_moe_method(self, layer, prefix, layer_config)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> "QuantizationMethods | None":
|
||||
"""Override the `auto-round` method to `inc`."""
|
||||
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
|
||||
if is_auto_round_format:
|
||||
return cls.get_name()
|
||||
return None
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.linear import LinearMethodBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .schemes.inc_scheme import INCLinearScheme
|
||||
|
||||
|
||||
class INCLinearMethod(LinearMethodBase):
|
||||
def __init__(self, scheme: "INCLinearScheme") -> None:
|
||||
self.scheme = scheme
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
return self.scheme.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
return self.scheme.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.scheme.apply_weights(layer, x, bias)
|
||||
@@ -0,0 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .factory import resolve_scheme
|
||||
from .inc_scheme import INCLinearScheme, INCScheme
|
||||
from .inc_wna16_scheme import INCWna16Scheme
|
||||
|
||||
__all__ = [
|
||||
"INCScheme",
|
||||
"INCLinearScheme",
|
||||
"INCWna16Scheme",
|
||||
"resolve_scheme",
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..config_parser import INCLayerConfig
|
||||
from .inc_scheme import INCScheme
|
||||
|
||||
|
||||
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
|
||||
from .inc_wna16_scheme import INCWna16Scheme
|
||||
|
||||
scheme_list: list[type[INCScheme]] = [
|
||||
INCWna16Scheme,
|
||||
]
|
||||
|
||||
for scheme_cls in scheme_list:
|
||||
if scheme_cls.can_handle(layer_config):
|
||||
return scheme_cls()
|
||||
|
||||
raise NotImplementedError(f"No INC scheme found for layer config: {layer_config}")
|
||||
@@ -0,0 +1,104 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
|
||||
from vllm.model_executor.layers.linear import LinearMethodBase
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
|
||||
from ..config_parser import INCLayerConfig
|
||||
from ..inc import INCConfig
|
||||
|
||||
|
||||
class INCScheme(ABC):
|
||||
"""One class per quant type. Single registration point for the factory.
|
||||
|
||||
Each subclass defines:
|
||||
- can_handle(): when does this scheme apply?
|
||||
- get_linear_method(): required — how to quantize Linear layers
|
||||
- get_moe_method(): optional — how to quantize MoE layers
|
||||
- get_kvcache_method(): optional — how to quantize KV cache
|
||||
|
||||
Schemes that don't support MoE/KVCache inherit the default raise.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def can_handle(layer_config: "INCLayerConfig") -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_linear_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
) -> "LinearMethodBase":
|
||||
raise NotImplementedError
|
||||
|
||||
def get_moe_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
) -> "FusedMoEMethodBase | None":
|
||||
"""Optional. Override if this scheme supports MoE.
|
||||
Default raises NotImplementedError."""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support MoE layers. "
|
||||
f"Layer config: {layer_config}"
|
||||
)
|
||||
|
||||
def get_kvcache_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
) -> "QuantizationMethods":
|
||||
"""Optional. Override if this scheme supports KV cache quantization.
|
||||
Default raises NotImplementedError."""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support KV cache quantization. "
|
||||
f"Layer config: {layer_config}"
|
||||
)
|
||||
|
||||
|
||||
class INCLinearScheme(ABC):
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def create_weights(
|
||||
self,
|
||||
layer: "torch.nn.Module",
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: "torch.dtype",
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: "torch.nn.Module",
|
||||
x: "torch.Tensor",
|
||||
bias: "torch.Tensor | None" = None,
|
||||
) -> "torch.Tensor":
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,399 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_marlin_supported,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
RowvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
from .inc_scheme import INCLinearScheme
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..config_parser import INCLayerConfig
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_ark_state() -> tuple[bool, str | None, Any | None, Any | None]:
|
||||
"""Return ARK availability, error details, cached module, and QuantLinear."""
|
||||
try:
|
||||
import auto_round_kernel as ark
|
||||
from auto_round_kernel.qlinear import QuantLinear
|
||||
|
||||
logger.info("Successfully imported auto_round_kernel.")
|
||||
except ImportError as error:
|
||||
return False, str(error), None, None
|
||||
|
||||
if getattr(ark, "cpu_lib", None) is None and getattr(ark, "xpu_lib", None) is None:
|
||||
return (
|
||||
False,
|
||||
"No ARK backend library is available.",
|
||||
None,
|
||||
None,
|
||||
)
|
||||
logger.info("Successfully loaded auto_round_kernel backend library.")
|
||||
|
||||
return True, None, ark, QuantLinear
|
||||
|
||||
|
||||
class INCWNA16LinearScheme(INCLinearScheme):
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
self.layer_config = layer_config
|
||||
self.inner_method = self._build_inner_method()
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 60
|
||||
|
||||
def _build_inner_method(self):
|
||||
if self.layer_config.is_gptq:
|
||||
return self._build_gptq_method()
|
||||
if self.layer_config.is_awq:
|
||||
return self._build_awq_method()
|
||||
raise NotImplementedError(
|
||||
f"WNA16 linear scheme does not support {self.layer_config}"
|
||||
)
|
||||
|
||||
def _build_gptq_method(self):
|
||||
gptq_type_map = {
|
||||
(4, True): scalar_types.uint4b8,
|
||||
(8, True): scalar_types.uint8b128,
|
||||
}
|
||||
use_marlin = (
|
||||
self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
|
||||
) and (self.layer_config.bits, self.layer_config.sym) in gptq_type_map
|
||||
if use_marlin:
|
||||
use_marlin = check_marlin_supported(
|
||||
gptq_type_map[(self.layer_config.bits, self.layer_config.sym)],
|
||||
self.layer_config.group_size,
|
||||
has_zp=not self.layer_config.sym,
|
||||
)
|
||||
|
||||
if use_marlin:
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQLinearMethod,
|
||||
)
|
||||
|
||||
return AutoGPTQLinearMethod(
|
||||
AutoGPTQConfig(
|
||||
weight_bits=self.layer_config.bits,
|
||||
group_size=self.layer_config.group_size,
|
||||
desc_act=False,
|
||||
is_sym=self.layer_config.sym,
|
||||
lm_head_quantized=False,
|
||||
dynamic={},
|
||||
full_config={},
|
||||
)
|
||||
)
|
||||
|
||||
raise NotImplementedError(
|
||||
f"INC quantization with bits={self.layer_config.bits}, "
|
||||
f"sym={self.layer_config.sym} is not supported. "
|
||||
"Only 4-bit and 8-bit symmetric quantization is supported "
|
||||
"with Marlin kernels."
|
||||
)
|
||||
|
||||
def _build_awq_method(self):
|
||||
awq_type_map = {
|
||||
4: scalar_types.uint4,
|
||||
8: scalar_types.uint8,
|
||||
}
|
||||
use_marlin = (
|
||||
self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
|
||||
) and self.layer_config.bits in awq_type_map
|
||||
if use_marlin:
|
||||
use_marlin = check_marlin_supported(
|
||||
awq_type_map[self.layer_config.bits],
|
||||
self.layer_config.group_size,
|
||||
not self.layer_config.sym,
|
||||
)
|
||||
|
||||
if use_marlin:
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import (
|
||||
AWQMarlinLinearMethod,
|
||||
)
|
||||
|
||||
return AWQMarlinLinearMethod(
|
||||
AWQMarlinConfig(
|
||||
weight_bits=self.layer_config.bits,
|
||||
group_size=self.layer_config.group_size,
|
||||
zero_point=not self.layer_config.sym,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=[],
|
||||
full_config={},
|
||||
)
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.awq import AWQLinearMethod
|
||||
|
||||
return AWQLinearMethod(
|
||||
AWQConfig(
|
||||
weight_bits=self.layer_config.bits,
|
||||
group_size=self.layer_config.group_size,
|
||||
zero_point=not self.layer_config.sym,
|
||||
)
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: "torch.nn.Module",
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: "torch.dtype",
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
return self.inner_method.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
|
||||
return self.inner_method.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: "torch.nn.Module",
|
||||
x: "torch.Tensor",
|
||||
bias: "torch.Tensor | None" = None,
|
||||
) -> "torch.Tensor":
|
||||
return self.inner_method.apply(layer, x, bias)
|
||||
|
||||
|
||||
class INCXPULinearBase(INCLinearScheme):
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
self.weight_bits = layer_config.bits
|
||||
self.group_size = layer_config.group_size
|
||||
self.sym = layer_config.sym
|
||||
self.pack_factor = 32 // self.weight_bits
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 0
|
||||
|
||||
def _create_inc_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Any,
|
||||
) -> None:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
scales_and_zp_size = input_size_per_partition // self.group_size
|
||||
|
||||
qweight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition // self.pack_factor,
|
||||
output_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=0,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
scales = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
scales_and_zp_size,
|
||||
output_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
qzeros = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
scales_and_zp_size,
|
||||
output_size_per_partition // self.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
layer.register_parameter("qweight", qweight)
|
||||
layer.register_parameter("scales", scales)
|
||||
layer.register_parameter("qzeros", qzeros)
|
||||
|
||||
g_idx = RowvLLMParameter(
|
||||
data=torch.tensor(
|
||||
[i // self.group_size for i in range(input_size_per_partition)],
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("g_idx", g_idx)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
del input_size, output_size
|
||||
self._create_inc_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
params_dtype=params_dtype,
|
||||
weight_loader=extra_weight_attrs.get("weight_loader"),
|
||||
)
|
||||
|
||||
|
||||
class INCXPULinearMethod(INCXPULinearBase):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
device = layer.qweight.data.device
|
||||
|
||||
qweight_ct = layer.qweight.data.t().contiguous()
|
||||
layer.qweight = Parameter(qweight_ct.t(), requires_grad=False)
|
||||
layer.scales = Parameter(layer.scales.data, requires_grad=False)
|
||||
layer.qzeros = Parameter(
|
||||
torch.tensor([8], dtype=torch.int8, device=device),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a16(
|
||||
reshaped_x,
|
||||
layer.qweight,
|
||||
bias,
|
||||
layer.scales,
|
||||
layer.qzeros,
|
||||
self.group_size,
|
||||
None,
|
||||
)
|
||||
return out.reshape(out_shape)
|
||||
|
||||
|
||||
class INCARKLinearMethod(INCXPULinearBase):
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
super().__init__(layer_config)
|
||||
|
||||
is_available, error_str, _, quant_linear_cls = get_ark_state()
|
||||
if not is_available or quant_linear_cls is None:
|
||||
reason = error_str or "unknown error"
|
||||
raise ImportError(f"Failed to import auto_round_kernel. {reason}")
|
||||
|
||||
self.quant_linear_cls = quant_linear_cls
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
super().create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
layer.in_features = input_size_per_partition
|
||||
layer.out_features = sum(output_partition_sizes)
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if hasattr(layer, "input_size_per_partition"):
|
||||
in_features = layer.input_size_per_partition
|
||||
elif hasattr(layer, "input_size"):
|
||||
in_features = layer.input_size
|
||||
else:
|
||||
raise AttributeError("Cannot determine in_features for layer.")
|
||||
|
||||
if hasattr(layer, "output_partition_sizes"):
|
||||
out_features = sum(layer.output_partition_sizes)
|
||||
elif hasattr(layer, "output_size_per_partition"):
|
||||
out_features = layer.output_size_per_partition
|
||||
elif hasattr(layer, "output_size"):
|
||||
out_features = layer.output_size
|
||||
else:
|
||||
out_features = layer.scales.shape[-1]
|
||||
|
||||
ark_linear = self.quant_linear_cls(
|
||||
bits=self.weight_bits,
|
||||
group_size=self.group_size,
|
||||
sym=self.sym,
|
||||
in_features=in_features,
|
||||
out_features=out_features,
|
||||
bias=layer.bias is not None,
|
||||
weight_dtype=layer.params_dtype,
|
||||
)
|
||||
ark_linear.to(layer.qweight.device)
|
||||
|
||||
with torch.no_grad():
|
||||
ark_linear.qweight.copy_(layer.qweight.detach())
|
||||
if hasattr(layer, "qzeros") and layer.qzeros is not None:
|
||||
ark_linear.qzeros.copy_(layer.qzeros.detach())
|
||||
else:
|
||||
ark_linear.qzeros = None
|
||||
ark_linear.scales.copy_(layer.scales.detach())
|
||||
if hasattr(layer, "bias") and layer.bias is not None:
|
||||
ark_linear.bias.copy_(layer.bias.detach())
|
||||
|
||||
ark_linear.post_init()
|
||||
layer.ark_linear = ark_linear
|
||||
|
||||
del layer.qweight
|
||||
if hasattr(layer, "qzeros"):
|
||||
del layer.qzeros
|
||||
del layer.scales
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
del bias
|
||||
return layer.ark_linear.forward(x)
|
||||
|
||||
|
||||
class INCXPUW4A16LinearScheme(INCXPULinearMethod):
|
||||
pass
|
||||
@@ -0,0 +1,201 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
from ..inc_linear import INCLinearMethod
|
||||
from .inc_scheme import INCScheme
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from ..config_parser import INCLayerConfig
|
||||
from ..inc import INCConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class INCWna16Scheme(INCScheme):
|
||||
@staticmethod
|
||||
def can_handle(layer_config: "INCLayerConfig") -> bool:
|
||||
return layer_config.is_wna16_int
|
||||
|
||||
def get_linear_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, layer
|
||||
if current_platform.is_xpu():
|
||||
if layer_config.bits == 4 and layer_config.sym:
|
||||
from .inc_wna16_linear import (
|
||||
INCARKLinearMethod,
|
||||
INCXPULinearMethod,
|
||||
get_ark_state,
|
||||
)
|
||||
|
||||
is_ark_available, ark_error, _, _ = get_ark_state()
|
||||
if is_ark_available:
|
||||
return INCLinearMethod(INCARKLinearMethod(layer_config))
|
||||
|
||||
logger.debug(
|
||||
"ARK backend is unavailable for layer %s; "
|
||||
"falling back to the default XPU INC path. Error: %s",
|
||||
prefix,
|
||||
ark_error or "unknown error",
|
||||
)
|
||||
return INCLinearMethod(INCXPULinearMethod(layer_config))
|
||||
raise NotImplementedError(f"INC on XPU: unsupported config {layer_config}")
|
||||
|
||||
if current_platform.is_cpu() and layer_config.is_gptq:
|
||||
if layer_config.bits == 4 and layer_config.sym:
|
||||
from .inc_wna16_linear import (
|
||||
INCARKLinearMethod,
|
||||
INCWNA16LinearScheme,
|
||||
get_ark_state,
|
||||
)
|
||||
|
||||
is_ark_available, ark_error, _, _ = get_ark_state()
|
||||
if is_ark_available:
|
||||
return INCLinearMethod(INCARKLinearMethod(layer_config))
|
||||
|
||||
logger.debug(
|
||||
"ARK backend is unavailable for layer %s; "
|
||||
"falling back to the default CPU INC path. Error: %s",
|
||||
prefix,
|
||||
ark_error or "unknown error",
|
||||
)
|
||||
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
|
||||
raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
|
||||
|
||||
from .inc_wna16_linear import INCWNA16LinearScheme
|
||||
|
||||
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
|
||||
|
||||
def get_moe_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, prefix
|
||||
# XPU and CPU do not support MoE quantization yet
|
||||
if current_platform.is_xpu() or current_platform.is_cpu():
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if layer_config.is_gptq:
|
||||
return _resolve_gptq_moe(layer, layer_config)
|
||||
if layer_config.is_awq:
|
||||
return _resolve_awq_moe(layer, layer_config)
|
||||
raise NotImplementedError(f"WNA16 MoE does not support config {layer_config}")
|
||||
|
||||
|
||||
def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import (
|
||||
MoeWNA16Config,
|
||||
MoeWNA16Method,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_marlin_supported,
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
gptq_type_map = {
|
||||
(4, True): scalar_types.uint4b8,
|
||||
(8, True): scalar_types.uint8b128,
|
||||
}
|
||||
use_marlin = (layer_config.bits, layer_config.sym) in gptq_type_map
|
||||
if use_marlin:
|
||||
use_marlin = check_marlin_supported(
|
||||
gptq_type_map[(layer_config.bits, layer_config.sym)],
|
||||
layer_config.group_size,
|
||||
has_zp=not layer_config.sym,
|
||||
) and check_moe_marlin_supports_layer(layer, layer_config.group_size)
|
||||
|
||||
if use_marlin:
|
||||
return AutoGPTQMoEMethod(
|
||||
AutoGPTQConfig(
|
||||
weight_bits=layer_config.bits,
|
||||
group_size=layer_config.group_size,
|
||||
desc_act=False,
|
||||
is_sym=layer_config.sym,
|
||||
lm_head_quantized=False,
|
||||
dynamic={},
|
||||
full_config={},
|
||||
),
|
||||
layer.moe_config,
|
||||
)
|
||||
|
||||
moe_config = MoeWNA16Config.from_config(
|
||||
{
|
||||
"quant_method": "gptq",
|
||||
"bits": layer_config.bits,
|
||||
"group_size": layer_config.group_size,
|
||||
"sym": layer_config.sym,
|
||||
"lm_head": False,
|
||||
}
|
||||
)
|
||||
return MoeWNA16Method(moe_config, layer.moe_config)
|
||||
|
||||
|
||||
def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinMoEMethod
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import (
|
||||
MoeWNA16Config,
|
||||
MoeWNA16Method,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_marlin_supported,
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
awq_type_map = {
|
||||
4: scalar_types.uint4,
|
||||
8: scalar_types.uint8,
|
||||
}
|
||||
use_marlin = layer_config.bits in awq_type_map
|
||||
if use_marlin:
|
||||
use_marlin = check_marlin_supported(
|
||||
awq_type_map[layer_config.bits],
|
||||
layer_config.group_size,
|
||||
not layer_config.sym,
|
||||
) and check_moe_marlin_supports_layer(layer, layer_config.group_size)
|
||||
|
||||
if use_marlin:
|
||||
return AWQMarlinMoEMethod(
|
||||
AWQMarlinConfig(
|
||||
weight_bits=layer_config.bits,
|
||||
group_size=layer_config.group_size,
|
||||
zero_point=not layer_config.sym,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=[],
|
||||
full_config={},
|
||||
),
|
||||
layer.moe_config,
|
||||
)
|
||||
|
||||
moe_config = MoeWNA16Config.from_config(
|
||||
{
|
||||
"quant_method": "awq",
|
||||
"bits": layer_config.bits,
|
||||
"group_size": layer_config.group_size,
|
||||
"zero_point": not layer_config.sym,
|
||||
"lm_head": False,
|
||||
}
|
||||
)
|
||||
return MoeWNA16Method(moe_config, layer.moe_config)
|
||||
Reference in New Issue
Block a user