fix(quantization): Fix AWQ dequantize on Intel XPU and refactor AutoAWQ config (#42727)

Signed-off-by: Alex <alex.tech.lab@outlook.com>
Signed-off-by: AlexHuang <jihuihuang@tencent.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
AlexHuang
2026-06-18 20:12:28 +08:00
committed by GitHub
co-authored by Claude Kunshang Ji mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 351c72d6e5
commit 8d4f54966c
20 changed files with 580 additions and 429 deletions
+2 -2
View File
@@ -49,7 +49,7 @@ To run an AWQ model with vLLM, you can use [TheBloke/Llama-2-7b-Chat-AWQ](https:
```bash
python examples/deployment/llm_engine_example.py \
--model TheBloke/Llama-2-7b-Chat-AWQ \
--quantization awq
--quantization auto_awq
```
AWQ models are also supported directly through the LLM entrypoint:
@@ -70,7 +70,7 @@ AWQ models are also supported directly through the LLM entrypoint:
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
# Create an LLM.
llm = LLM(model="TheBloke/Llama-2-7b-Chat-AWQ", quantization="AWQ")
llm = LLM(model="TheBloke/Llama-2-7b-Chat-AWQ", quantization="auto_awq")
# Generate texts from the prompts. The output is a list of RequestOutput objects
# that contain the prompt, generated text, and other information.
outputs = llm.generate(prompts, sampling_params)
+231
View File
@@ -0,0 +1,231 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for AutoAWQConfig behavior after unification.
These tests verify the bug fixes for:
1. CPU platform override conflict (auto_awq should not override on CPU)
2. MoE fallback compatibility (full_config["quant_method"] should be "awq")
3. Config attribute consistency
4. End-to-end quantization method loading (auto_awq loads and runs correctly)
Note: Tests that require importing the full auto_awq module (which has GPU-dependent
imports) should use subprocess or be run in a GPU environment.
"""
from __future__ import annotations
import pytest
import torch
from tests.quantization.utils import is_quant_method_supported
def _get_auto_awq_config_source() -> str:
"""Read the AutoAWQConfig class source code for isolated testing."""
import inspect
import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module
return inspect.getsource(auto_awq_module.AutoAWQConfig)
class TestAutoAWQConfigFromConfig:
"""Tests for AutoAWQConfig.from_config behavior.
These tests require GPU environment to import the full module.
They are skipped on non-GPU platforms.
"""
def test_full_config_quant_method_is_awq_for_moe_fallback(self):
"""full_config should have quant_method='awq' for MoE fallback compatibility.
MoeWNA16Config only accepts 'gptq' or 'awq' as linear_quant_method.
If full_config has 'auto_awq', the MoE fallback will fail.
"""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
config = {
"w_bit": 4,
"q_group_size": 128,
"zero_point": True,
"lm_head": False,
}
awq_config = AutoAWQConfig.from_config(config)
# Verify quant_method is 'awq' for MoE fallback
assert awq_config.full_config["quant_method"] == "awq", (
f"Expected quant_method='awq', got {awq_config.full_config['quant_method']}"
)
def test_full_config_preserves_other_fields(self):
"""full_config should preserve all original config fields."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
config = {
"w_bit": 4,
"q_group_size": 128,
"zero_point": True,
"lm_head": False,
"custom_field": "custom_value",
}
awq_config = AutoAWQConfig.from_config(config)
assert awq_config.full_config["w_bit"] == 4
assert awq_config.full_config["q_group_size"] == 128
assert awq_config.full_config["zero_point"] is True
assert awq_config.full_config["lm_head"] is False
assert awq_config.full_config["custom_field"] == "custom_value"
def test_full_config_is_copy_not_original(self):
"""full_config should be a copy, not the original dict."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
config = {
"w_bit": 4,
"q_group_size": 128,
"zero_point": True,
"lm_head": False,
}
original_quant_method = config.get("quant_method")
AutoAWQConfig.from_config(config)
# Original config should not be modified
assert config.get("quant_method") == original_quant_method
class TestAutoAWQConfigAttributes:
"""Tests for AutoAWQConfig attribute consistency.
These tests require GPU environment to import the full module.
They are skipped on non-GPU platforms.
"""
def test_config_attributes_match_input(self):
"""Config attributes should match input values."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
awq_config = AutoAWQConfig(
weight_bits=4,
group_size=128,
zero_point=True,
lm_head_quantized=False,
modules_to_not_convert=["lm_head"],
)
assert awq_config.weight_bits == 4
assert awq_config.group_size == 128
assert awq_config.zero_point is True
assert awq_config.lm_head_quantized is False
assert awq_config.modules_to_not_convert == ["lm_head"]
def test_pack_factor_for_4bit(self):
"""Pack factor should be 8 for 4-bit quantization."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
awq_config = AutoAWQConfig(
weight_bits=4,
group_size=128,
zero_point=True,
lm_head_quantized=False,
)
assert awq_config.pack_factor == 8 # 32 // 4
class TestAutoAWQConfigOverrideLogic:
"""Tests for override logic by parsing source code (no GPU import required)."""
def _get_auto_awq_source(self) -> str:
"""Read the auto_awq.py source file."""
import inspect
import pathlib
import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module
source_path = inspect.getfile(auto_awq_module)
return pathlib.Path(source_path).read_text()
def test_cpu_check_in_override_method(self):
"""override_quantization_method should check current_platform.is_cpu()."""
source = self._get_auto_awq_source()
# Verify the CPU check exists in override method
assert "current_platform.is_cpu()" in source, (
"override_quantization_method should check is_cpu()"
)
assert "return None" in source, (
"override_quantization_method should return None on CPU"
)
def test_quant_method_normalization_in_from_config(self):
"""from_config should normalize quant_method to 'awq' for MoE fallback."""
source = self._get_auto_awq_source()
# Verify the normalization exists
assert (
'"quant_method"] = "awq"' in source or "'quant_method'] = 'awq'" in source
), "from_config should set quant_method='awq' in full_config"
# =============================================================================
# End-to-end integration tests (require GPU environment)
# =============================================================================
PROMPT = "On the surface of Mars, we found"
# Small AWQ model for testing - using Qwen2 1.5B which has official AWQ checkpoint
AWQ_MODELS = [
"Qwen/Qwen2-1.5B-Instruct-AWQ",
]
@pytest.mark.skipif(
not is_quant_method_supported("auto_awq"),
reason="auto_awq is not supported on this GPU type.",
)
@pytest.mark.parametrize("model_id", AWQ_MODELS)
def test_auto_awq_quantization_method(vllm_runner, model_id: str, monkeypatch):
"""Test that quantization='auto_awq' loads and runs correctly."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
with vllm_runner(
model_id,
dtype=torch.float16,
quantization="auto_awq",
max_model_len=2048,
enforce_eager=True,
) as llm:
def check_model(model):
from vllm.model_executor.layers.quantization.auto_awq import (
AutoAWQLinearMethod,
AutoAWQMarlinLinearMethod,
)
for name, submodule in model.named_modules():
if name == "model.layers.0.self_attn.qkv_proj":
# Should use either AutoAWQLinearMethod (Triton) or
# AutoAWQMarlinLinearMethod (Marlin) depending on hardware
assert isinstance(
submodule.quant_method,
(AutoAWQLinearMethod, AutoAWQMarlinLinearMethod),
), (
f"Expected AutoAWQLinearMethod or AutoAWQMarlinLinearMethod "
f"for {name}, got {type(submodule.quant_method)}"
)
break
llm.apply_model(check_model)
outputs = llm.generate_greedy([PROMPT], max_tokens=8)
assert outputs
assert len(outputs[0][1]) > 0
def test_auto_awq_config_get_name():
"""Test that AutoAWQConfig.get_name() returns 'auto_awq'."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
assert AutoAWQConfig.get_name() == "auto_awq"
+2 -2
View File
@@ -645,11 +645,11 @@ def test_resolve_awq_moe_uses_marlin_when_supported(monkeypatch) -> None:
lambda *args, **kwargs: True,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.awq_marlin.verify_marlin_supported",
"vllm.model_executor.layers.quantization.auto_awq.verify_marlin_supported",
lambda *args, **kwargs: None,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.awq_marlin.AWQMarlinMoEMethod",
"vllm.model_executor.layers.quantization.auto_awq.AutoAWQMoEMethod",
DummyMethod,
)
+5 -3
View File
@@ -43,16 +43,18 @@ MODEL_ARG_EXPTYPES = [
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "auto_gptq"),
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
# AUTOAWQ
# AutoAWQConfig.override_quantization_method() returns "auto_awq" for AWQ models
# when user_quant is None, "awq", "awq_marlin", "marlin", or "auto_awq"
(
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
None,
"awq_marlin" if current_platform.is_cuda_alike() else "awq",
"auto_awq",
),
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "awq"),
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "auto_awq"),
(
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
"marlin",
"awq_marlin" if current_platform.is_cuda_alike() else "ERROR",
"auto_awq" if current_platform.is_cuda_alike() else "ERROR",
),
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
]
+2
View File
@@ -984,6 +984,8 @@ class ModelConfig:
"auto_gptq",
"gptq",
"gptq_marlin",
"auto_awq",
"awq",
"awq_marlin",
"inc",
"moe_wna16",
@@ -175,7 +175,7 @@ class CPUWNA16LinearKernel(MPLinearKernel):
and torch.cpu._is_amx_tile_supported()
)
# layer.use_w4a8 = False
# AWQ format will be converted to GPTQ format in `AWQMarlinLinearMethod`
# AWQ format will be converted to GPTQ format in `AutoAWQMarlinLinearMethod`
if layer.use_w4a8:
self._process_gptq_weights_w4a8(layer)
else:
@@ -728,18 +728,18 @@ def _process_weights_cpu(
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_int4_moe_layer_for_cpu,
)
from vllm.model_executor.layers.quantization.auto_awq import (
AutoAWQConfig,
)
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQConfig,
)
from vllm.model_executor.layers.quantization.awq_marlin import (
AWQMarlinConfig,
)
# Detect packing format.
# AWQ: qweight is [E, K, 2*N//8] (packed along output/N dim).
# GPTQ: qweight is [E, K//8, 2*N] (packed along input/K dim).
# compressed-tensors: qweight is [E, K//8, 2*N] (packed along input/K dim).
if isinstance(quant_config, AWQMarlinConfig):
if isinstance(quant_config, AutoAWQConfig):
# AWQ: K is stored unpacked in dim 1.
cpu_quant_algo = ops.CPUQuantAlgo.AWQ
elif isinstance(quant_config, (AutoGPTQConfig, QuantizationArgs)):
@@ -753,7 +753,7 @@ def _process_weights_cpu(
cpu_quant_algo = ops.CPUQuantAlgo.GPTQ
else:
raise TypeError(
"CPU WNA16 MoE backend requires AWQMarlinConfig, AutoGPTQConfig "
"CPU WNA16 MoE backend requires AutoAWQConfig, AutoGPTQConfig "
f"or QuantizationArgs, got {type(quant_config).__name__}."
)
@@ -916,14 +916,14 @@ def convert_to_wna16_moe_kernel_format(
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
):
from vllm.model_executor.layers.quantization.auto_awq import (
AutoAWQConfig,
)
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQConfig,
)
from vllm.model_executor.layers.quantization.awq_marlin import (
AWQMarlinConfig,
)
if isinstance(quant_config, AWQMarlinConfig):
if isinstance(quant_config, AutoAWQConfig):
if w13_qzeros is None or w2_qzeros is None:
raise ValueError("AWQ Marlin MoE requires zero-point tensors.")
@@ -958,7 +958,7 @@ def convert_to_wna16_moe_kernel_format(
actorder = quant_config.actorder
else:
raise TypeError(
"Marlin WNA16 MoE backend requires AutoGPTQConfig, AWQMarlinConfig or "
"Marlin WNA16 MoE backend requires AutoAWQConfig, AutoGPTQConfig or "
f"QuantizationArgs, got {type(quant_config).__name__}."
)
if w13_g_idx is None or w2_g_idx is None:
+2 -2
View File
@@ -46,8 +46,8 @@ WEIGHT_LOADER_V2_SUPPORTED = [
"UnquantizedLinearMethod",
"CompressedTensorsLinearMethod",
"CompressedTensorsLinearTransformMethod",
"AWQMarlinLinearMethod",
"AWQLinearMethod",
"AutoAWQMarlinLinearMethod",
"AutoAWQLinearMethod",
"AutoGPTQLinearMethod",
"Fp8LinearMethod",
"FBGEMMFp8LinearMethod",
@@ -48,8 +48,8 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_update,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
from vllm.model_executor.layers.quantization.inc import INCConfig
from vllm.model_executor.model_loader.weight_utils import (
sharded_weight_loader,
@@ -628,7 +628,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
return (
current_platform.is_cuda()
and not self.gqa_interleaved_layout
and isinstance(quant_config, (AWQMarlinConfig, AutoGPTQConfig, INCConfig))
and isinstance(quant_config, (AutoAWQConfig, AutoGPTQConfig, INCConfig))
)
def split_ba(self, ba: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
@@ -11,6 +11,7 @@ logger = init_logger(__name__)
QuantizationMethods = Literal[
"awq",
"auto_awq",
"fp8",
"fbgemm_fp8",
"fp_quant",
@@ -113,9 +114,8 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.models.deepseek_v4 import DeepseekV4FP8Config
from .auto_awq import AutoAWQConfig
from .auto_gptq import AutoGPTQConfig
from .awq import AWQConfig
from .awq_marlin import AWQMarlinConfig
from .bitsandbytes import BitsAndBytesConfig
from .compressed_tensors.compressed_tensors import (
CompressedTensorsConfig,
@@ -138,7 +138,9 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
from .torchao import TorchAOConfig
method_to_config: dict[str, type[QuantizationConfig]] = {
"awq": AWQConfig,
"awq": AutoAWQConfig,
"awq_marlin": AutoAWQConfig,
"auto_awq": AutoAWQConfig,
"fp8": Fp8Config,
"fbgemm_fp8": FBGEMMFp8Config,
"fp_quant": FPQuantConfig,
@@ -149,7 +151,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
"auto_gptq": AutoGPTQConfig,
"gptq": AutoGPTQConfig,
"gptq_marlin": AutoGPTQConfig,
"awq_marlin": AWQMarlinConfig,
"compressed-tensors": CompressedTensorsConfig,
"bitsandbytes": BitsAndBytesConfig,
"experts_int8": ExpertsInt8Config,
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Union
import torch
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
@@ -9,6 +9,7 @@ from torch.nn import Parameter
from transformers import PretrainedConfig
import vllm.model_executor.layers.fused_moe # noqa
from vllm import _custom_ops as ops
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
@@ -36,7 +37,6 @@ from vllm.model_executor.layers.linear import (
UnquantizedLinearMethod,
set_weight_attrs,
)
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
@@ -55,7 +55,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4Static,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.parameter import GroupQuantScaleParameter, PackedvLLMParameter
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.transformers_utils.config import get_safetensors_params_metadata
@@ -164,8 +167,12 @@ def _convert_awq_to_standard_format(
setattr(layer, w_zp_name, new_zp_param)
class AWQMarlinConfig(QuantizationConfig):
"""Config class for AWQ Marlin"""
class AutoAWQConfig(QuantizationConfig):
"""Config class for AutoAWQ quantization.
Unified config that supports multiple backends: Triton, Marlin, and XPU.
Reference: https://arxiv.org/abs/2306.00978
"""
# num_bits -> type
TYPE_MAP = {
@@ -178,8 +185,8 @@ class AWQMarlinConfig(QuantizationConfig):
group_size: int,
zero_point: bool,
lm_head_quantized: bool,
modules_to_not_convert: list[str] | None,
full_config: dict[str, Any],
modules_to_not_convert: list[str] | None = None,
full_config: dict[str, Any] | None = None,
) -> None:
super().__init__()
self.pack_factor = 32 // weight_bits # packed into int32
@@ -188,23 +195,22 @@ class AWQMarlinConfig(QuantizationConfig):
self.lm_head_quantized = lm_head_quantized
self.weight_bits = weight_bits
self.modules_to_not_convert = modules_to_not_convert or []
self.full_config = full_config
self.full_config = full_config or {}
if self.weight_bits not in self.TYPE_MAP:
supported = ", ".join(str(k) for k in self.TYPE_MAP)
raise ValueError(
f"Unsupported num_bits = {self.weight_bits}. "
f"Supported num_bits = {self.TYPE_MAP.keys()}"
f"Supported: {supported}. "
f"For 8-bit AWQ, use Marlin backend by setting "
f"backend='awq:marlin' or backend='marlin'."
)
self.quant_type = self.TYPE_MAP[self.weight_bits]
verify_marlin_supported(
self.quant_type, group_size=self.group_size, has_zp=self.zero_point
)
def __repr__(self) -> str:
return (
f"AWQMarlinConfig(quant_type={self.quant_type}, "
f"AutoAWQConfig(quant_type={self.quant_type}, "
f"group_size={self.group_size}, "
f"zero_point={self.zero_point}, "
f"lm_head_quantized={self.lm_head_quantized}, "
@@ -213,7 +219,7 @@ class AWQMarlinConfig(QuantizationConfig):
@classmethod
def get_name(cls) -> "QuantizationMethods":
return "awq_marlin"
return "auto_awq"
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
@@ -225,60 +231,59 @@ class AWQMarlinConfig(QuantizationConfig):
@classmethod
def get_config_filenames(cls) -> list[str]:
return ["quantize_config.json"]
return ["quantize_config.json", "quant_config.json"]
@classmethod
def from_config(cls, config: dict[str, Any]) -> "AWQMarlinConfig":
weight_bits = cls.get_from_keys(config, ["bits"])
group_size = cls.get_from_keys(config, ["group_size"])
def from_config(cls, config: dict[str, Any]) -> "AutoAWQConfig":
weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
zero_point = cls.get_from_keys(config, ["zero_point"])
lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False)
modules_to_not_convert = cls.get_from_keys_or(
config, ["modules_to_not_convert"], None
)
# Ensure full_config uses "awq" as quant_method for MoE fallback compatibility.
# MoeWNA16Config only accepts "gptq" or "awq", so we normalize here.
full_config = config.copy()
full_config["quant_method"] = "awq"
return cls(
weight_bits,
group_size,
zero_point,
lm_head_quantized,
modules_to_not_convert,
config,
full_config,
)
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant, hf_config=None
) -> "QuantizationMethods | None":
# Skip override to marlin kernels, as they are not
# batch invariant
if envs.VLLM_BATCH_INVARIANT:
"""Override to use AutoAWQ for compatible AWQ models."""
# Don't override on CPU - let cpu_awq handle it
if current_platform.is_cpu():
return None
can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)
is_valid_user_quant = (
user_quant is None or user_quant == "marlin" or user_quant == "awq_marlin"
quant_method = hf_quant_cfg.get("quant_method", "").lower()
if quant_method != "awq":
return None
is_valid_user_quant = user_quant is None or user_quant in (
"awq",
"awq_marlin",
"auto_awq",
"marlin",
)
if can_convert and is_valid_user_quant:
msg = (
"The model is convertible to {} during runtime."
" Using {} kernel.".format(cls.get_name(), cls.get_name())
)
logger.info(msg)
if is_valid_user_quant:
return cls.get_name()
if can_convert and user_quant == "awq":
logger.info(
"Detected that the model can run with awq_marlin"
", however you specified quantization=awq explicitly,"
" so forcing awq. Use quantization=awq_marlin for"
" faster inference"
)
return None
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
) -> Union["LinearMethodBase", "QuantizeMethodBase"] | None:
if isinstance(layer, LinearBase) or (
isinstance(layer, ParallelLMHead) and self.lm_head_quantized
):
@@ -289,41 +294,66 @@ class AWQMarlinConfig(QuantizationConfig):
skip_with_substr=True,
):
return UnquantizedLinearMethod()
# Check if the layer is supported by AWQMarlin; tile-misaligned
# shapes are fixed by padding at weight prep.
if not check_marlin_supports_layer(
layer, self.group_size, allow_tile_padding=True
):
logger.warning_once(
"Layer '%s' is not supported by AWQMarlin. Falling back to unoptimized AWQ kernels.", # noqa: E501
prefix,
)
return AWQConfig.from_config(self.full_config).get_quant_method(
layer, prefix
)
quant_method = AWQMarlinLinearMethod(self)
quant_method.input_dtype = get_marlin_input_dtype(prefix)
return quant_method
elif isinstance(layer, RoutedExperts):
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
# Check if XPU - use XPU-specific linear method
if current_platform.is_xpu():
return AutoAWQXPULinearMethod(self)
# On CPU, use Marlin linear method which uses choose_mp_linear_kernel
# to select the best available kernel (CPUWNA16LinearKernel on CPU)
if current_platform.is_cpu():
return AutoAWQMarlinLinearMethod(self)
# Check if Marlin is supported and not using batch invariant mode
# (Marlin kernels are not batch invariant)
use_marlin = (
not envs.VLLM_BATCH_INVARIANT
and current_platform.is_cuda()
and check_marlin_supported(
self.quant_type, self.group_size, self.zero_point
)
)
if use_marlin:
# tile-misaligned shapes are fixed by padding at weight prep
if not check_marlin_supports_layer(
layer, self.group_size, allow_tile_padding=True
):
logger.warning_once(
"Layer '%s' is not supported by AutoAWQMarlin. "
"Falling back to unoptimized AWQ kernels.",
prefix,
)
return AutoAWQLinearMethod(self)
quant_method = AutoAWQMarlinLinearMethod(self)
quant_method.input_dtype = get_marlin_input_dtype(prefix)
return quant_method
return AutoAWQLinearMethod(self)
elif isinstance(layer, RoutedExperts):
if is_layer_skipped(
prefix,
getattr(self, "modules_to_not_convert", []),
skip_with_substr=True,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
if not check_moe_marlin_supports_layer(layer, self.group_size):
logger.warning_once(
f"Layer '{prefix}' is not supported by AWQMoeMarlin. "
f"Layer '{prefix}' is not supported by AutoAWQMoEMarlin. "
"Falling back to Moe WNA16 kernels."
)
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
)
return MoeWNA16Config.from_config(self.full_config).get_quant_method(
layer, prefix
)
moe_quant_method = AWQMarlinMoEMethod(self, layer.moe_config)
moe_quant_method.input_dtype = get_marlin_input_dtype(prefix)
return moe_quant_method
return AutoAWQMoEMethod(self, layer.moe_config)
return None
@classmethod
@@ -378,7 +408,7 @@ class AWQMarlinConfig(QuantizationConfig):
self.modules_to_not_convert = list(layers - quant_layers)
class AWQMarlinLinearMethod(LinearMethodBase):
class AutoAWQMarlinLinearMethod(LinearMethodBase):
"""Linear method for AWQ Marlin.
Uses choose_mp_linear_kernel to select the best available kernel
@@ -390,16 +420,18 @@ class AWQMarlinLinearMethod(LinearMethodBase):
_kernel_backends_being_used: set[str] = set()
def __init__(self, quant_config: AWQMarlinConfig) -> None:
def __init__(self, quant_config: AutoAWQConfig) -> None:
self.quant_config = quant_config
self.quant_type = scalar_types.uint4
self.input_dtype = None
verify_marlin_supported(
quant_type=self.quant_config.quant_type,
group_size=self.quant_config.group_size,
has_zp=self.quant_config.zero_point,
)
# Skip Marlin verification on CPU - it will use CPUWNA16LinearKernel
if not current_platform.is_cpu():
verify_marlin_supported(
quant_type=self.quant_config.quant_type,
group_size=self.quant_config.group_size,
has_zp=self.quant_config.zero_point,
)
def create_weights(
self,
@@ -435,7 +467,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
if kernel_type.__name__ not in self._kernel_backends_being_used:
logger.info("Using %s for AWQMarlinLinearMethod", kernel_type.__name__)
logger.info("Using %s for AutoAWQMarlinLinearMethod", kernel_type.__name__)
self._kernel_backends_being_used.add(kernel_type.__name__)
# Weights are loaded in AWQ checkpoint format (packed along output dim).
@@ -509,16 +541,16 @@ class AWQMarlinLinearMethod(LinearMethodBase):
return self.kernel.apply_weights(layer, x, bias)
class AWQMarlinMoEMethod(FusedMoEMethodBase):
class AutoAWQMoEMethod(FusedMoEMethodBase):
def __init__(
self,
quant_config: AWQMarlinConfig,
quant_config: AutoAWQConfig,
moe: FusedMoEConfig,
):
super().__init__(moe)
self.quant_config = quant_config
if self.quant_config.weight_bits != 4:
raise ValueError("AWQMarlinMoEMethod only supports 4bit now.")
raise ValueError("AutoAWQMoEMethod only supports 4bit now.")
self.quant_type = scalar_types.uint4
self.input_dtype = None
self.use_marlin = True
@@ -784,3 +816,185 @@ class AWQMarlinMoEMethod(FusedMoEMethodBase):
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
class BaseAWQLinearMethod(LinearMethodBase):
"""Base class for AWQ linear methods with shared weight creation logic."""
def __init__(self, quant_config: AutoAWQConfig):
self.quant_config = quant_config
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
# Normalize group_size
if self.quant_config.group_size != -1:
group_size = self.quant_config.group_size
else:
group_size = input_size
if input_size_per_partition % group_size != 0:
raise ValueError(
"The input size is not aligned with the quantized "
"weight shape. This can be caused by too large "
"tensor parallel size."
)
output_size_per_partition = sum(output_partition_sizes)
if output_size_per_partition % self.quant_config.pack_factor != 0:
raise ValueError(
"The output size is not aligned with the quantized "
"weight shape. This can be caused by too large "
"tensor parallel size."
)
weight_loader = extra_weight_attrs.get("weight_loader")
qweight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition,
output_size_per_partition // self.quant_config.pack_factor,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.quant_config.pack_factor,
weight_loader=weight_loader,
)
num_groups = input_size_per_partition // group_size
qzeros = PackedvLLMParameter(
data=torch.empty(
num_groups,
output_size_per_partition // self.quant_config.pack_factor,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.quant_config.pack_factor,
weight_loader=weight_loader,
)
scales = GroupQuantScaleParameter(
data=torch.empty(
num_groups,
output_size_per_partition,
dtype=params_dtype,
),
input_dim=0,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("qweight", qweight)
layer.register_parameter("qzeros", qzeros)
layer.register_parameter("scales", scales)
class AutoAWQLinearMethod(BaseAWQLinearMethod):
"""Linear method for AWQ using Triton kernels.
Args:
quant_config: The AWQ quantization config.
"""
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False)
layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False)
layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False)
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
qweight = layer.qweight
scales = layer.scales
qzeros = layer.qzeros
pack_factor = self.quant_config.pack_factor
out_shape = x.shape[:-1] + (qweight.shape[-1] * pack_factor,)
reshaped_x = x.reshape(-1, x.shape[-1])
# num_tokens >= threshold
FP16_MATMUL_HEURISTIC_CONDITION = x.shape[:-1].numel() >= 256
# Batch invariant mode requires torch.matmul path
# for Triton override
if FP16_MATMUL_HEURISTIC_CONDITION or envs.VLLM_BATCH_INVARIANT:
out = ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0)
out = torch.matmul(reshaped_x, out)
else:
out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros, pack_factor)
if bias is not None:
out.add_(bias)
return out.reshape(out_shape)
class AutoAWQXPULinearMethod(BaseAWQLinearMethod):
"""Linear method for AWQ on XPU using int4 GEMM kernel.
Args:
quant_config: The AWQ quantization config.
"""
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False)
layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False)
layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False)
try:
from vllm_xpu_kernels.quantization._quantize_convert import (
AWQUtils,
transpose_onednn_woq_format,
)
except ImportError as e:
raise ImportError(
"XPU AWQ requires vllm-xpu-kernels. "
"Please install it with: pip install vllm-xpu-kernels"
) from e
layer.xpu_output_size = layer.qweight.size(1) * self.quant_config.pack_factor
qweight_new, qzeros_new = AWQUtils.repack(layer.qweight, layer.qzeros)
if qweight_new.shape != layer.qweight.data.shape:
layer.qweight.data = layer.qweight.data.view_as(qweight_new)
if qzeros_new.shape != layer.qzeros.data.shape:
layer.qzeros.data = layer.qzeros.data.view_as(qzeros_new)
layer.qweight.data.copy_(qweight_new)
layer.qzeros.data.copy_(qzeros_new)
transpose_onednn_woq_format(layer, "awq", False)
def _get_group_size(self, layer: torch.nn.Module) -> int:
"""Get the effective group size for kernel computation."""
if self.quant_config.group_size != -1:
return self.quant_config.group_size
return layer.qweight.shape[0] # input_size_per_partition
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1])
group_size = self._get_group_size(layer)
out = torch.ops._xpu_C.int4_gemm_w4a16(
reshaped_x,
layer.qweight,
bias,
layer.scales,
layer.qzeros,
group_size,
None,
)
out_shape = x.shape[:-1] + (layer.xpu_output_size,)
return out.reshape(out_shape)
@@ -1,286 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any, Union
import torch
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
from transformers import PretrainedConfig
from vllm import _custom_ops as ops
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.linear import (
LinearBase,
LinearMethodBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
from vllm.model_executor.parameter import GroupQuantScaleParameter, PackedvLLMParameter
from vllm.transformers_utils.config import get_safetensors_params_metadata
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.models.utils import WeightsMapper
logger = init_logger(__name__)
class AWQConfig(QuantizationConfig):
"""Config class for AWQ.
Reference: https://arxiv.org/abs/2306.00978
"""
def __init__(
self,
weight_bits: int,
group_size: int,
zero_point: bool,
modules_to_not_convert: list[str] | None = None,
) -> None:
super().__init__()
self.weight_bits = weight_bits
self.group_size = group_size
self.zero_point = zero_point
self.modules_to_not_convert = modules_to_not_convert or []
if self.weight_bits != 4:
raise ValueError(
"Currently, only 4-bit weight quantization is supported for "
f"AWQ, but got {self.weight_bits} bits."
)
self.pack_factor = 32 // self.weight_bits
def __repr__(self) -> str:
return (
f"AWQConfig(weight_bits={self.weight_bits}, "
f"group_size={self.group_size}, "
f"zero_point={self.zero_point}, "
f"modules_to_not_convert={self.modules_to_not_convert})"
)
def get_name(self) -> "QuantizationMethods":
return "awq"
def get_supported_act_dtypes(self) -> list[torch.dtype]:
return [torch.half]
@classmethod
def get_min_capability(cls) -> int:
# The AWQ kernel only supports Turing or newer GPUs.
return 75
@staticmethod
def get_config_filenames() -> list[str]:
return [
"quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq
# E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq
"quantize_config.json",
]
@classmethod
def from_config(cls, config: dict[str, Any]) -> "AWQConfig":
weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
zero_point = cls.get_from_keys(config, ["zero_point"])
modules_to_not_convert = cls.get_from_keys_or(
config, ["modules_to_not_convert"], None
)
return cls(weight_bits, group_size, zero_point, modules_to_not_convert)
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> Union["LinearMethodBase", "QuantizeMethodBase"] | None:
if isinstance(layer, LinearBase):
if is_layer_skipped(
prefix,
self.modules_to_not_convert,
self.packed_modules_mapping,
skip_with_substr=True,
):
return UnquantizedLinearMethod()
return AWQLinearMethod(self)
elif isinstance(layer, RoutedExperts):
# Lazy import to avoid circular import.
from .awq_marlin import AWQMarlinConfig
from .moe_wna16 import MoeWNA16Config
from .utils.marlin_utils import check_moe_marlin_supports_layer
if not check_moe_marlin_supports_layer(layer, self.group_size):
logger.warning_once(
f"Layer '{prefix}' is not supported by AWQMoeMarlin. "
"Falling back to Moe WNA16 kernels."
)
config = {
"quant_method": "awq",
"bits": self.weight_bits,
"group_size": self.group_size,
"zero_point": self.zero_point,
"lm_head": False,
"modules_to_not_convert": self.modules_to_not_convert,
}
return MoeWNA16Config.from_config(config).get_quant_method(
layer, prefix
)
marlin_compatible_config_dict = {
"quant_method": "awq",
"bits": self.weight_bits,
"group_size": self.group_size,
"zero_point": self.zero_point,
"lm_head": False,
"modules_to_not_convert": self.modules_to_not_convert,
}
awq_marlin_config = AWQMarlinConfig.from_config(
marlin_compatible_config_dict
)
return awq_marlin_config.get_quant_method(layer, prefix)
return None
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
if self.modules_to_not_convert:
self.modules_to_not_convert = hf_to_vllm_mapper.apply_list(
self.modules_to_not_convert
)
def maybe_update_config(
self,
model_name: str,
hf_config: PretrainedConfig | None = None,
revision: str | None = None,
):
if self.modules_to_not_convert:
return
unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]
metadata = get_safetensors_params_metadata(model_name, revision=revision)
layers = {param_name.rsplit(".", 1)[0] for param_name in metadata}
quant_layers: set[str] = {
param_name.rsplit(".", 1)[0]
for param_name, info in metadata.items()
if (dtype := info.get("dtype", None))
and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes
}
self.modules_to_not_convert = list(layers - quant_layers)
class AWQLinearMethod(LinearMethodBase):
"""Linear method for AWQ.
Args:
quant_config: The AWQ quantization config.
"""
def __init__(self, quant_config: AWQConfig):
self.quant_config = quant_config
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
# Normalize group_size
if self.quant_config.group_size != -1:
group_size = self.quant_config.group_size
else:
group_size = input_size
if input_size_per_partition % group_size != 0:
raise ValueError(
"The input size is not aligned with the quantized "
"weight shape. This can be caused by too large "
"tensor parallel size."
)
output_size_per_partition = sum(output_partition_sizes)
if output_size_per_partition % self.quant_config.pack_factor != 0:
raise ValueError(
"The output size is not aligned with the quantized "
"weight shape. This can be caused by too large "
"tensor parallel size."
)
weight_loader = extra_weight_attrs.get("weight_loader")
qweight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition,
output_size_per_partition // self.quant_config.pack_factor,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.quant_config.pack_factor,
weight_loader=weight_loader,
)
num_groups = input_size_per_partition // group_size
qzeros = PackedvLLMParameter(
data=torch.empty(
num_groups,
output_size_per_partition // self.quant_config.pack_factor,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.quant_config.pack_factor,
weight_loader=weight_loader,
)
scales = GroupQuantScaleParameter(
data=torch.empty(
num_groups,
output_size_per_partition,
dtype=params_dtype,
),
input_dim=0,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("qweight", qweight)
layer.register_parameter("qzeros", qzeros)
layer.register_parameter("scales", scales)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False)
layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False)
layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False)
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
qweight = layer.qweight
scales = layer.scales
qzeros = layer.qzeros
pack_factor = self.quant_config.pack_factor
out_shape = x.shape[:-1] + (qweight.shape[-1] * pack_factor,)
reshaped_x = x.reshape(-1, x.shape[-1])
# num_tokens >= threshold
FP16_MATMUL_HEURISTIC_CONDITION = x.shape[:-1].numel() >= 256
# Batch invariant mode requires torch.matmul path
# for Triton override
if FP16_MATMUL_HEURISTIC_CONDITION or envs.VLLM_BATCH_INVARIANT:
out = ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0)
out = torch.matmul(reshaped_x, out)
else:
out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros, pack_factor)
if bias is not None:
out.add_(bias)
return out.reshape(out_shape)
@@ -8,9 +8,8 @@ import torch
from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supported,
)
@@ -125,12 +124,12 @@ class INCWNA16LinearScheme(INCLinearScheme):
)
if use_marlin:
from vllm.model_executor.layers.quantization.awq_marlin import (
AWQMarlinLinearMethod,
from vllm.model_executor.layers.quantization.auto_awq import (
AutoAWQMarlinLinearMethod,
)
return AWQMarlinLinearMethod(
AWQMarlinConfig(
return AutoAWQMarlinLinearMethod(
AutoAWQConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
zero_point=not self.layer_config.sym,
@@ -140,13 +139,16 @@ class INCWNA16LinearScheme(INCLinearScheme):
)
)
from vllm.model_executor.layers.quantization.awq import AWQLinearMethod
from vllm.model_executor.layers.quantization.auto_awq import (
AutoAWQLinearMethod,
)
return AWQLinearMethod(
AWQConfig(
return AutoAWQLinearMethod(
AutoAWQConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
zero_point=not self.layer_config.sym,
lm_head_quantized=False,
)
)
@@ -4,8 +4,8 @@
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
@@ -154,7 +154,7 @@ def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinMoEMethod
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQMoEMethod
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
MoeWNA16Method,
@@ -177,8 +177,8 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
) and check_moe_marlin_supports_layer(layer, layer_config.group_size)
if use_marlin:
return AWQMarlinMoEMethod(
AWQMarlinConfig(
return AutoAWQMoEMethod(
AutoAWQConfig(
weight_bits=layer_config.bits,
group_size=layer_config.group_size,
zero_point=not layer_config.sym,
@@ -27,9 +27,6 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supports_layer,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
@@ -55,10 +52,8 @@ class MoeWNA16Config(QuantizationConfig):
self.lm_head_quantized = lm_head_quantized
self.linear_quant_method = linear_quant_method
self.full_config = full_config
self.use_marlin = False
# Avoid circular import
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
if self.linear_quant_method == "gptq":
pass
@@ -67,7 +62,7 @@ class MoeWNA16Config(QuantizationConfig):
device_capability = (
-1 if capability_tuple is None else capability_tuple.to_int()
)
awq_min_capability = AWQConfig.get_min_capability()
awq_min_capability = AutoAWQConfig.get_min_capability()
if device_capability < awq_min_capability:
raise ValueError(
"The quantization method moe_wna16 + awq is not supported "
@@ -75,7 +70,6 @@ class MoeWNA16Config(QuantizationConfig):
f"Minimum capability: {awq_min_capability}. "
f"Current capability: {device_capability}."
)
self.use_marlin = AWQMarlinConfig.is_awq_marlin_compatible(full_config)
else:
raise ValueError("moe_wna16 only support gptq and awq.")
@@ -148,9 +142,9 @@ class MoeWNA16Config(QuantizationConfig):
-1 if capability_tuple is None else capability_tuple.to_int()
)
# Avoid circular import
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
awq_min_capability = AWQConfig.get_min_capability()
awq_min_capability = AutoAWQConfig.get_min_capability()
gptq_compatible = quant_method == "gptq" and not desc_act and num_bits in [4, 8]
awq_compatible = (
@@ -170,29 +164,19 @@ class MoeWNA16Config(QuantizationConfig):
return UnquantizedLinearMethod()
elif isinstance(layer, LinearBase):
# Avoid circular import
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQConfig,
)
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.awq_marlin import (
AWQMarlinConfig,
)
if self.linear_quant_method == "gptq":
return AutoGPTQConfig.from_config(self.full_config).get_quant_method(
layer, prefix
)
elif self.linear_quant_method in ("awq", "awq_marlin"):
if self.use_marlin and check_marlin_supports_layer(
layer, self.group_size
):
return AWQMarlinConfig.from_config(
self.full_config
).get_quant_method(layer, prefix)
else:
return AWQConfig.from_config(self.full_config).get_quant_method(
layer, prefix
)
return AutoAWQConfig.from_config(self.full_config).get_quant_method(
layer, prefix
)
else:
raise ValueError("moe_wna16 only support gptq and awq.")
elif isinstance(layer, RoutedExperts):
+2 -2
View File
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
@@ -420,7 +420,7 @@ class Cohere2VisionForConditionalGeneration(
):
# the awq models from OpenGVLab missing `modules_to_not_convert`
# patch the quant_config to add `modules_to_not_convert` back
if isinstance(quant_config, AWQConfig):
if isinstance(quant_config, AutoAWQConfig):
text_config = config.text_config
llm_quant_config = getattr(text_config, "quantization_config", None)
if (not quant_config.modules_to_not_convert) and (
+2 -2
View File
@@ -20,7 +20,7 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.models.intern_vit import (
InternVisionModel,
)
@@ -608,7 +608,7 @@ class InternVLChatModel(
):
# the awq models from OpenGVLab missing `modules_to_not_convert`
# patch the quant_config to add `modules_to_not_convert` back
if isinstance(quant_config, AWQConfig):
if isinstance(quant_config, AutoAWQConfig):
text_config = config.text_config
llm_quant_config = getattr(text_config, "quantization_config", None)
if (not quant_config.modules_to_not_convert) and (
+2 -2
View File
@@ -11,7 +11,7 @@ from vllm.config import VllmConfig
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.models.internvl import (
BaseInternVLDummyInputsBuilder,
BaseInternVLMultiModalProcessor,
@@ -144,7 +144,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor
):
# the awq models from OpenGVLab missing `modules_to_not_convert`
# patch the quant_config to add `modules_to_not_convert` back
if isinstance(quant_config, AWQConfig):
if isinstance(quant_config, AutoAWQConfig):
text_config = config.get_text_config()
llm_quant_config = getattr(text_config, "quantization_config", None)
if (not quant_config.modules_to_not_convert) and (
+2 -2
View File
@@ -19,7 +19,7 @@ from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.awq import AWQConfig
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.models.intern_vit import (
InternVisionModel,
)
@@ -205,7 +205,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
):
# the awq models from OpenGVLab missing `modules_to_not_convert`
# patch the quant_config to add `modules_to_not_convert` back
if isinstance(quant_config, AWQConfig):
if isinstance(quant_config, AutoAWQConfig):
text_config = config.text_config
llm_quant_config = getattr(text_config, "quantization_config", None)
if (not quant_config.modules_to_not_convert) and (
+1
View File
@@ -440,6 +440,7 @@ class RocmPlatform(Platform):
supported_quantization: list[str] = [
"awq",
"auto_awq",
"awq_marlin", # will be overwritten with awq
"gptq",
"gptq_marlin",