fix(quantization): Fix AWQ dequantize on Intel XPU and refactor AutoAWQ config (#42727)
Signed-off-by: Alex <alex.tech.lab@outlook.com> Signed-off-by: AlexHuang <jihuihuang@tencent.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Kunshang Ji
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
351c72d6e5
commit
8d4f54966c
@@ -49,7 +49,7 @@ To run an AWQ model with vLLM, you can use [TheBloke/Llama-2-7b-Chat-AWQ](https:
|
||||
```bash
|
||||
python examples/deployment/llm_engine_example.py \
|
||||
--model TheBloke/Llama-2-7b-Chat-AWQ \
|
||||
--quantization awq
|
||||
--quantization auto_awq
|
||||
```
|
||||
|
||||
AWQ models are also supported directly through the LLM entrypoint:
|
||||
@@ -70,7 +70,7 @@ AWQ models are also supported directly through the LLM entrypoint:
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
|
||||
|
||||
# Create an LLM.
|
||||
llm = LLM(model="TheBloke/Llama-2-7b-Chat-AWQ", quantization="AWQ")
|
||||
llm = LLM(model="TheBloke/Llama-2-7b-Chat-AWQ", quantization="auto_awq")
|
||||
# Generate texts from the prompts. The output is a list of RequestOutput objects
|
||||
# that contain the prompt, generated text, and other information.
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for AutoAWQConfig behavior after unification.
|
||||
|
||||
These tests verify the bug fixes for:
|
||||
1. CPU platform override conflict (auto_awq should not override on CPU)
|
||||
2. MoE fallback compatibility (full_config["quant_method"] should be "awq")
|
||||
3. Config attribute consistency
|
||||
4. End-to-end quantization method loading (auto_awq loads and runs correctly)
|
||||
|
||||
Note: Tests that require importing the full auto_awq module (which has GPU-dependent
|
||||
imports) should use subprocess or be run in a GPU environment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
|
||||
|
||||
def _get_auto_awq_config_source() -> str:
|
||||
"""Read the AutoAWQConfig class source code for isolated testing."""
|
||||
import inspect
|
||||
|
||||
import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module
|
||||
|
||||
return inspect.getsource(auto_awq_module.AutoAWQConfig)
|
||||
|
||||
|
||||
class TestAutoAWQConfigFromConfig:
|
||||
"""Tests for AutoAWQConfig.from_config behavior.
|
||||
|
||||
These tests require GPU environment to import the full module.
|
||||
They are skipped on non-GPU platforms.
|
||||
"""
|
||||
|
||||
def test_full_config_quant_method_is_awq_for_moe_fallback(self):
|
||||
"""full_config should have quant_method='awq' for MoE fallback compatibility.
|
||||
|
||||
MoeWNA16Config only accepts 'gptq' or 'awq' as linear_quant_method.
|
||||
If full_config has 'auto_awq', the MoE fallback will fail.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
config = {
|
||||
"w_bit": 4,
|
||||
"q_group_size": 128,
|
||||
"zero_point": True,
|
||||
"lm_head": False,
|
||||
}
|
||||
awq_config = AutoAWQConfig.from_config(config)
|
||||
|
||||
# Verify quant_method is 'awq' for MoE fallback
|
||||
assert awq_config.full_config["quant_method"] == "awq", (
|
||||
f"Expected quant_method='awq', got {awq_config.full_config['quant_method']}"
|
||||
)
|
||||
|
||||
def test_full_config_preserves_other_fields(self):
|
||||
"""full_config should preserve all original config fields."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
config = {
|
||||
"w_bit": 4,
|
||||
"q_group_size": 128,
|
||||
"zero_point": True,
|
||||
"lm_head": False,
|
||||
"custom_field": "custom_value",
|
||||
}
|
||||
awq_config = AutoAWQConfig.from_config(config)
|
||||
|
||||
assert awq_config.full_config["w_bit"] == 4
|
||||
assert awq_config.full_config["q_group_size"] == 128
|
||||
assert awq_config.full_config["zero_point"] is True
|
||||
assert awq_config.full_config["lm_head"] is False
|
||||
assert awq_config.full_config["custom_field"] == "custom_value"
|
||||
|
||||
def test_full_config_is_copy_not_original(self):
|
||||
"""full_config should be a copy, not the original dict."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
config = {
|
||||
"w_bit": 4,
|
||||
"q_group_size": 128,
|
||||
"zero_point": True,
|
||||
"lm_head": False,
|
||||
}
|
||||
original_quant_method = config.get("quant_method")
|
||||
|
||||
AutoAWQConfig.from_config(config)
|
||||
|
||||
# Original config should not be modified
|
||||
assert config.get("quant_method") == original_quant_method
|
||||
|
||||
|
||||
class TestAutoAWQConfigAttributes:
|
||||
"""Tests for AutoAWQConfig attribute consistency.
|
||||
|
||||
These tests require GPU environment to import the full module.
|
||||
They are skipped on non-GPU platforms.
|
||||
"""
|
||||
|
||||
def test_config_attributes_match_input(self):
|
||||
"""Config attributes should match input values."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
awq_config = AutoAWQConfig(
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
zero_point=True,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=["lm_head"],
|
||||
)
|
||||
|
||||
assert awq_config.weight_bits == 4
|
||||
assert awq_config.group_size == 128
|
||||
assert awq_config.zero_point is True
|
||||
assert awq_config.lm_head_quantized is False
|
||||
assert awq_config.modules_to_not_convert == ["lm_head"]
|
||||
|
||||
def test_pack_factor_for_4bit(self):
|
||||
"""Pack factor should be 8 for 4-bit quantization."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
awq_config = AutoAWQConfig(
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
zero_point=True,
|
||||
lm_head_quantized=False,
|
||||
)
|
||||
|
||||
assert awq_config.pack_factor == 8 # 32 // 4
|
||||
|
||||
|
||||
class TestAutoAWQConfigOverrideLogic:
|
||||
"""Tests for override logic by parsing source code (no GPU import required)."""
|
||||
|
||||
def _get_auto_awq_source(self) -> str:
|
||||
"""Read the auto_awq.py source file."""
|
||||
import inspect
|
||||
import pathlib
|
||||
|
||||
import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module
|
||||
|
||||
source_path = inspect.getfile(auto_awq_module)
|
||||
return pathlib.Path(source_path).read_text()
|
||||
|
||||
def test_cpu_check_in_override_method(self):
|
||||
"""override_quantization_method should check current_platform.is_cpu()."""
|
||||
source = self._get_auto_awq_source()
|
||||
|
||||
# Verify the CPU check exists in override method
|
||||
assert "current_platform.is_cpu()" in source, (
|
||||
"override_quantization_method should check is_cpu()"
|
||||
)
|
||||
assert "return None" in source, (
|
||||
"override_quantization_method should return None on CPU"
|
||||
)
|
||||
|
||||
def test_quant_method_normalization_in_from_config(self):
|
||||
"""from_config should normalize quant_method to 'awq' for MoE fallback."""
|
||||
source = self._get_auto_awq_source()
|
||||
|
||||
# Verify the normalization exists
|
||||
assert (
|
||||
'"quant_method"] = "awq"' in source or "'quant_method'] = 'awq'" in source
|
||||
), "from_config should set quant_method='awq' in full_config"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# End-to-end integration tests (require GPU environment)
|
||||
# =============================================================================
|
||||
|
||||
PROMPT = "On the surface of Mars, we found"
|
||||
|
||||
# Small AWQ model for testing - using Qwen2 1.5B which has official AWQ checkpoint
|
||||
AWQ_MODELS = [
|
||||
"Qwen/Qwen2-1.5B-Instruct-AWQ",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("auto_awq"),
|
||||
reason="auto_awq is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", AWQ_MODELS)
|
||||
def test_auto_awq_quantization_method(vllm_runner, model_id: str, monkeypatch):
|
||||
"""Test that quantization='auto_awq' loads and runs correctly."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
with vllm_runner(
|
||||
model_id,
|
||||
dtype=torch.float16,
|
||||
quantization="auto_awq",
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
) as llm:
|
||||
|
||||
def check_model(model):
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
AutoAWQLinearMethod,
|
||||
AutoAWQMarlinLinearMethod,
|
||||
)
|
||||
|
||||
for name, submodule in model.named_modules():
|
||||
if name == "model.layers.0.self_attn.qkv_proj":
|
||||
# Should use either AutoAWQLinearMethod (Triton) or
|
||||
# AutoAWQMarlinLinearMethod (Marlin) depending on hardware
|
||||
assert isinstance(
|
||||
submodule.quant_method,
|
||||
(AutoAWQLinearMethod, AutoAWQMarlinLinearMethod),
|
||||
), (
|
||||
f"Expected AutoAWQLinearMethod or AutoAWQMarlinLinearMethod "
|
||||
f"for {name}, got {type(submodule.quant_method)}"
|
||||
)
|
||||
break
|
||||
|
||||
llm.apply_model(check_model)
|
||||
|
||||
outputs = llm.generate_greedy([PROMPT], max_tokens=8)
|
||||
assert outputs
|
||||
assert len(outputs[0][1]) > 0
|
||||
|
||||
|
||||
def test_auto_awq_config_get_name():
|
||||
"""Test that AutoAWQConfig.get_name() returns 'auto_awq'."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
assert AutoAWQConfig.get_name() == "auto_awq"
|
||||
@@ -645,11 +645,11 @@ def test_resolve_awq_moe_uses_marlin_when_supported(monkeypatch) -> None:
|
||||
lambda *args, **kwargs: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.awq_marlin.verify_marlin_supported",
|
||||
"vllm.model_executor.layers.quantization.auto_awq.verify_marlin_supported",
|
||||
lambda *args, **kwargs: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.awq_marlin.AWQMarlinMoEMethod",
|
||||
"vllm.model_executor.layers.quantization.auto_awq.AutoAWQMoEMethod",
|
||||
DummyMethod,
|
||||
)
|
||||
|
||||
|
||||
@@ -43,16 +43,18 @@ MODEL_ARG_EXPTYPES = [
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "auto_gptq"),
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
|
||||
# AUTOAWQ
|
||||
# AutoAWQConfig.override_quantization_method() returns "auto_awq" for AWQ models
|
||||
# when user_quant is None, "awq", "awq_marlin", "marlin", or "auto_awq"
|
||||
(
|
||||
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
|
||||
None,
|
||||
"awq_marlin" if current_platform.is_cuda_alike() else "awq",
|
||||
"auto_awq",
|
||||
),
|
||||
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "awq"),
|
||||
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "auto_awq"),
|
||||
(
|
||||
"TheBloke/OpenHermes-2.5-Mistral-7B-AWQ",
|
||||
"marlin",
|
||||
"awq_marlin" if current_platform.is_cuda_alike() else "ERROR",
|
||||
"auto_awq" if current_platform.is_cuda_alike() else "ERROR",
|
||||
),
|
||||
("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"),
|
||||
]
|
||||
|
||||
@@ -984,6 +984,8 @@ class ModelConfig:
|
||||
"auto_gptq",
|
||||
"gptq",
|
||||
"gptq_marlin",
|
||||
"auto_awq",
|
||||
"awq",
|
||||
"awq_marlin",
|
||||
"inc",
|
||||
"moe_wna16",
|
||||
|
||||
@@ -175,7 +175,7 @@ class CPUWNA16LinearKernel(MPLinearKernel):
|
||||
and torch.cpu._is_amx_tile_supported()
|
||||
)
|
||||
# layer.use_w4a8 = False
|
||||
# AWQ format will be converted to GPTQ format in `AWQMarlinLinearMethod`
|
||||
# AWQ format will be converted to GPTQ format in `AutoAWQMarlinLinearMethod`
|
||||
if layer.use_w4a8:
|
||||
self._process_gptq_weights_w4a8(layer)
|
||||
else:
|
||||
|
||||
@@ -728,18 +728,18 @@ def _process_weights_cpu(
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
prepare_int4_moe_layer_for_cpu,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
AutoAWQConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import (
|
||||
AWQMarlinConfig,
|
||||
)
|
||||
|
||||
# Detect packing format.
|
||||
# AWQ: qweight is [E, K, 2*N//8] (packed along output/N dim).
|
||||
# GPTQ: qweight is [E, K//8, 2*N] (packed along input/K dim).
|
||||
# compressed-tensors: qweight is [E, K//8, 2*N] (packed along input/K dim).
|
||||
if isinstance(quant_config, AWQMarlinConfig):
|
||||
if isinstance(quant_config, AutoAWQConfig):
|
||||
# AWQ: K is stored unpacked in dim 1.
|
||||
cpu_quant_algo = ops.CPUQuantAlgo.AWQ
|
||||
elif isinstance(quant_config, (AutoGPTQConfig, QuantizationArgs)):
|
||||
@@ -753,7 +753,7 @@ def _process_weights_cpu(
|
||||
cpu_quant_algo = ops.CPUQuantAlgo.GPTQ
|
||||
else:
|
||||
raise TypeError(
|
||||
"CPU WNA16 MoE backend requires AWQMarlinConfig, AutoGPTQConfig "
|
||||
"CPU WNA16 MoE backend requires AutoAWQConfig, AutoGPTQConfig "
|
||||
f"or QuantizationArgs, got {type(quant_config).__name__}."
|
||||
)
|
||||
|
||||
@@ -916,14 +916,14 @@ def convert_to_wna16_moe_kernel_format(
|
||||
WNA16MoEBackend.MARLIN,
|
||||
WNA16MoEBackend.BATCHED_MARLIN,
|
||||
):
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
AutoAWQConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import (
|
||||
AWQMarlinConfig,
|
||||
)
|
||||
|
||||
if isinstance(quant_config, AWQMarlinConfig):
|
||||
if isinstance(quant_config, AutoAWQConfig):
|
||||
if w13_qzeros is None or w2_qzeros is None:
|
||||
raise ValueError("AWQ Marlin MoE requires zero-point tensors.")
|
||||
|
||||
@@ -958,7 +958,7 @@ def convert_to_wna16_moe_kernel_format(
|
||||
actorder = quant_config.actorder
|
||||
else:
|
||||
raise TypeError(
|
||||
"Marlin WNA16 MoE backend requires AutoGPTQConfig, AWQMarlinConfig or "
|
||||
"Marlin WNA16 MoE backend requires AutoAWQConfig, AutoGPTQConfig or "
|
||||
f"QuantizationArgs, got {type(quant_config).__name__}."
|
||||
)
|
||||
if w13_g_idx is None or w2_g_idx is None:
|
||||
|
||||
@@ -46,8 +46,8 @@ WEIGHT_LOADER_V2_SUPPORTED = [
|
||||
"UnquantizedLinearMethod",
|
||||
"CompressedTensorsLinearMethod",
|
||||
"CompressedTensorsLinearTransformMethod",
|
||||
"AWQMarlinLinearMethod",
|
||||
"AWQLinearMethod",
|
||||
"AutoAWQMarlinLinearMethod",
|
||||
"AutoAWQLinearMethod",
|
||||
"AutoGPTQLinearMethod",
|
||||
"Fp8LinearMethod",
|
||||
"FBGEMMFp8LinearMethod",
|
||||
|
||||
@@ -48,8 +48,8 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
sharded_weight_loader,
|
||||
@@ -628,7 +628,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
return (
|
||||
current_platform.is_cuda()
|
||||
and not self.gqa_interleaved_layout
|
||||
and isinstance(quant_config, (AWQMarlinConfig, AutoGPTQConfig, INCConfig))
|
||||
and isinstance(quant_config, (AutoAWQConfig, AutoGPTQConfig, INCConfig))
|
||||
)
|
||||
|
||||
def split_ba(self, ba: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -11,6 +11,7 @@ logger = init_logger(__name__)
|
||||
|
||||
QuantizationMethods = Literal[
|
||||
"awq",
|
||||
"auto_awq",
|
||||
"fp8",
|
||||
"fbgemm_fp8",
|
||||
"fp_quant",
|
||||
@@ -113,9 +114,8 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.models.deepseek_v4 import DeepseekV4FP8Config
|
||||
|
||||
from .auto_awq import AutoAWQConfig
|
||||
from .auto_gptq import AutoGPTQConfig
|
||||
from .awq import AWQConfig
|
||||
from .awq_marlin import AWQMarlinConfig
|
||||
from .bitsandbytes import BitsAndBytesConfig
|
||||
from .compressed_tensors.compressed_tensors import (
|
||||
CompressedTensorsConfig,
|
||||
@@ -138,7 +138,9 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
from .torchao import TorchAOConfig
|
||||
|
||||
method_to_config: dict[str, type[QuantizationConfig]] = {
|
||||
"awq": AWQConfig,
|
||||
"awq": AutoAWQConfig,
|
||||
"awq_marlin": AutoAWQConfig,
|
||||
"auto_awq": AutoAWQConfig,
|
||||
"fp8": Fp8Config,
|
||||
"fbgemm_fp8": FBGEMMFp8Config,
|
||||
"fp_quant": FPQuantConfig,
|
||||
@@ -149,7 +151,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
|
||||
"auto_gptq": AutoGPTQConfig,
|
||||
"gptq": AutoGPTQConfig,
|
||||
"gptq_marlin": AutoGPTQConfig,
|
||||
"awq_marlin": AWQMarlinConfig,
|
||||
"compressed-tensors": CompressedTensorsConfig,
|
||||
"bitsandbytes": BitsAndBytesConfig,
|
||||
"experts_int8": ExpertsInt8Config,
|
||||
|
||||
+286
-72
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, Union
|
||||
|
||||
import torch
|
||||
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
||||
@@ -9,6 +9,7 @@ from torch.nn import Parameter
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
import vllm.model_executor.layers.fused_moe # noqa
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
@@ -36,7 +37,6 @@ from vllm.model_executor.layers.linear import (
|
||||
UnquantizedLinearMethod,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
@@ -55,7 +55,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4Static,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.parameter import GroupQuantScaleParameter, PackedvLLMParameter
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.transformers_utils.config import get_safetensors_params_metadata
|
||||
@@ -164,8 +167,12 @@ def _convert_awq_to_standard_format(
|
||||
setattr(layer, w_zp_name, new_zp_param)
|
||||
|
||||
|
||||
class AWQMarlinConfig(QuantizationConfig):
|
||||
"""Config class for AWQ Marlin"""
|
||||
class AutoAWQConfig(QuantizationConfig):
|
||||
"""Config class for AutoAWQ quantization.
|
||||
|
||||
Unified config that supports multiple backends: Triton, Marlin, and XPU.
|
||||
Reference: https://arxiv.org/abs/2306.00978
|
||||
"""
|
||||
|
||||
# num_bits -> type
|
||||
TYPE_MAP = {
|
||||
@@ -178,8 +185,8 @@ class AWQMarlinConfig(QuantizationConfig):
|
||||
group_size: int,
|
||||
zero_point: bool,
|
||||
lm_head_quantized: bool,
|
||||
modules_to_not_convert: list[str] | None,
|
||||
full_config: dict[str, Any],
|
||||
modules_to_not_convert: list[str] | None = None,
|
||||
full_config: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.pack_factor = 32 // weight_bits # packed into int32
|
||||
@@ -188,23 +195,22 @@ class AWQMarlinConfig(QuantizationConfig):
|
||||
self.lm_head_quantized = lm_head_quantized
|
||||
self.weight_bits = weight_bits
|
||||
self.modules_to_not_convert = modules_to_not_convert or []
|
||||
self.full_config = full_config
|
||||
self.full_config = full_config or {}
|
||||
|
||||
if self.weight_bits not in self.TYPE_MAP:
|
||||
supported = ", ".join(str(k) for k in self.TYPE_MAP)
|
||||
raise ValueError(
|
||||
f"Unsupported num_bits = {self.weight_bits}. "
|
||||
f"Supported num_bits = {self.TYPE_MAP.keys()}"
|
||||
f"Supported: {supported}. "
|
||||
f"For 8-bit AWQ, use Marlin backend by setting "
|
||||
f"backend='awq:marlin' or backend='marlin'."
|
||||
)
|
||||
|
||||
self.quant_type = self.TYPE_MAP[self.weight_bits]
|
||||
|
||||
verify_marlin_supported(
|
||||
self.quant_type, group_size=self.group_size, has_zp=self.zero_point
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"AWQMarlinConfig(quant_type={self.quant_type}, "
|
||||
f"AutoAWQConfig(quant_type={self.quant_type}, "
|
||||
f"group_size={self.group_size}, "
|
||||
f"zero_point={self.zero_point}, "
|
||||
f"lm_head_quantized={self.lm_head_quantized}, "
|
||||
@@ -213,7 +219,7 @@ class AWQMarlinConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def get_name(cls) -> "QuantizationMethods":
|
||||
return "awq_marlin"
|
||||
return "auto_awq"
|
||||
|
||||
@classmethod
|
||||
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
|
||||
@@ -225,60 +231,59 @@ class AWQMarlinConfig(QuantizationConfig):
|
||||
|
||||
@classmethod
|
||||
def get_config_filenames(cls) -> list[str]:
|
||||
return ["quantize_config.json"]
|
||||
return ["quantize_config.json", "quant_config.json"]
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "AWQMarlinConfig":
|
||||
weight_bits = cls.get_from_keys(config, ["bits"])
|
||||
group_size = cls.get_from_keys(config, ["group_size"])
|
||||
def from_config(cls, config: dict[str, Any]) -> "AutoAWQConfig":
|
||||
weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
|
||||
group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
|
||||
zero_point = cls.get_from_keys(config, ["zero_point"])
|
||||
lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False)
|
||||
modules_to_not_convert = cls.get_from_keys_or(
|
||||
config, ["modules_to_not_convert"], None
|
||||
)
|
||||
# Ensure full_config uses "awq" as quant_method for MoE fallback compatibility.
|
||||
# MoeWNA16Config only accepts "gptq" or "awq", so we normalize here.
|
||||
full_config = config.copy()
|
||||
full_config["quant_method"] = "awq"
|
||||
return cls(
|
||||
weight_bits,
|
||||
group_size,
|
||||
zero_point,
|
||||
lm_head_quantized,
|
||||
modules_to_not_convert,
|
||||
config,
|
||||
full_config,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> "QuantizationMethods | None":
|
||||
# Skip override to marlin kernels, as they are not
|
||||
# batch invariant
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
"""Override to use AutoAWQ for compatible AWQ models."""
|
||||
# Don't override on CPU - let cpu_awq handle it
|
||||
if current_platform.is_cpu():
|
||||
return None
|
||||
|
||||
can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)
|
||||
is_valid_user_quant = (
|
||||
user_quant is None or user_quant == "marlin" or user_quant == "awq_marlin"
|
||||
quant_method = hf_quant_cfg.get("quant_method", "").lower()
|
||||
|
||||
if quant_method != "awq":
|
||||
return None
|
||||
|
||||
is_valid_user_quant = user_quant is None or user_quant in (
|
||||
"awq",
|
||||
"awq_marlin",
|
||||
"auto_awq",
|
||||
"marlin",
|
||||
)
|
||||
|
||||
if can_convert and is_valid_user_quant:
|
||||
msg = (
|
||||
"The model is convertible to {} during runtime."
|
||||
" Using {} kernel.".format(cls.get_name(), cls.get_name())
|
||||
)
|
||||
logger.info(msg)
|
||||
if is_valid_user_quant:
|
||||
return cls.get_name()
|
||||
|
||||
if can_convert and user_quant == "awq":
|
||||
logger.info(
|
||||
"Detected that the model can run with awq_marlin"
|
||||
", however you specified quantization=awq explicitly,"
|
||||
" so forcing awq. Use quantization=awq_marlin for"
|
||||
" faster inference"
|
||||
)
|
||||
return None
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
) -> Union["LinearMethodBase", "QuantizeMethodBase"] | None:
|
||||
if isinstance(layer, LinearBase) or (
|
||||
isinstance(layer, ParallelLMHead) and self.lm_head_quantized
|
||||
):
|
||||
@@ -289,41 +294,66 @@ class AWQMarlinConfig(QuantizationConfig):
|
||||
skip_with_substr=True,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
# Check if the layer is supported by AWQMarlin; tile-misaligned
|
||||
# shapes are fixed by padding at weight prep.
|
||||
if not check_marlin_supports_layer(
|
||||
layer, self.group_size, allow_tile_padding=True
|
||||
):
|
||||
logger.warning_once(
|
||||
"Layer '%s' is not supported by AWQMarlin. Falling back to unoptimized AWQ kernels.", # noqa: E501
|
||||
prefix,
|
||||
)
|
||||
return AWQConfig.from_config(self.full_config).get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
quant_method = AWQMarlinLinearMethod(self)
|
||||
quant_method.input_dtype = get_marlin_input_dtype(prefix)
|
||||
return quant_method
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
||||
|
||||
# Check if XPU - use XPU-specific linear method
|
||||
if current_platform.is_xpu():
|
||||
return AutoAWQXPULinearMethod(self)
|
||||
|
||||
# On CPU, use Marlin linear method which uses choose_mp_linear_kernel
|
||||
# to select the best available kernel (CPUWNA16LinearKernel on CPU)
|
||||
if current_platform.is_cpu():
|
||||
return AutoAWQMarlinLinearMethod(self)
|
||||
|
||||
# Check if Marlin is supported and not using batch invariant mode
|
||||
# (Marlin kernels are not batch invariant)
|
||||
use_marlin = (
|
||||
not envs.VLLM_BATCH_INVARIANT
|
||||
and current_platform.is_cuda()
|
||||
and check_marlin_supported(
|
||||
self.quant_type, self.group_size, self.zero_point
|
||||
)
|
||||
)
|
||||
|
||||
if use_marlin:
|
||||
# tile-misaligned shapes are fixed by padding at weight prep
|
||||
if not check_marlin_supports_layer(
|
||||
layer, self.group_size, allow_tile_padding=True
|
||||
):
|
||||
logger.warning_once(
|
||||
"Layer '%s' is not supported by AutoAWQMarlin. "
|
||||
"Falling back to unoptimized AWQ kernels.",
|
||||
prefix,
|
||||
)
|
||||
return AutoAWQLinearMethod(self)
|
||||
quant_method = AutoAWQMarlinLinearMethod(self)
|
||||
quant_method.input_dtype = get_marlin_input_dtype(prefix)
|
||||
return quant_method
|
||||
|
||||
return AutoAWQLinearMethod(self)
|
||||
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
if is_layer_skipped(
|
||||
prefix,
|
||||
getattr(self, "modules_to_not_convert", []),
|
||||
skip_with_substr=True,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
|
||||
if not check_moe_marlin_supports_layer(layer, self.group_size):
|
||||
logger.warning_once(
|
||||
f"Layer '{prefix}' is not supported by AWQMoeMarlin. "
|
||||
f"Layer '{prefix}' is not supported by AutoAWQMoEMarlin. "
|
||||
"Falling back to Moe WNA16 kernels."
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import (
|
||||
MoeWNA16Config,
|
||||
)
|
||||
|
||||
return MoeWNA16Config.from_config(self.full_config).get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
moe_quant_method = AWQMarlinMoEMethod(self, layer.moe_config)
|
||||
moe_quant_method.input_dtype = get_marlin_input_dtype(prefix)
|
||||
return moe_quant_method
|
||||
|
||||
return AutoAWQMoEMethod(self, layer.moe_config)
|
||||
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
@@ -378,7 +408,7 @@ class AWQMarlinConfig(QuantizationConfig):
|
||||
self.modules_to_not_convert = list(layers - quant_layers)
|
||||
|
||||
|
||||
class AWQMarlinLinearMethod(LinearMethodBase):
|
||||
class AutoAWQMarlinLinearMethod(LinearMethodBase):
|
||||
"""Linear method for AWQ Marlin.
|
||||
|
||||
Uses choose_mp_linear_kernel to select the best available kernel
|
||||
@@ -390,16 +420,18 @@ class AWQMarlinLinearMethod(LinearMethodBase):
|
||||
|
||||
_kernel_backends_being_used: set[str] = set()
|
||||
|
||||
def __init__(self, quant_config: AWQMarlinConfig) -> None:
|
||||
def __init__(self, quant_config: AutoAWQConfig) -> None:
|
||||
self.quant_config = quant_config
|
||||
self.quant_type = scalar_types.uint4
|
||||
self.input_dtype = None
|
||||
|
||||
verify_marlin_supported(
|
||||
quant_type=self.quant_config.quant_type,
|
||||
group_size=self.quant_config.group_size,
|
||||
has_zp=self.quant_config.zero_point,
|
||||
)
|
||||
# Skip Marlin verification on CPU - it will use CPUWNA16LinearKernel
|
||||
if not current_platform.is_cpu():
|
||||
verify_marlin_supported(
|
||||
quant_type=self.quant_config.quant_type,
|
||||
group_size=self.quant_config.group_size,
|
||||
has_zp=self.quant_config.zero_point,
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -435,7 +467,7 @@ class AWQMarlinLinearMethod(LinearMethodBase):
|
||||
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
|
||||
|
||||
if kernel_type.__name__ not in self._kernel_backends_being_used:
|
||||
logger.info("Using %s for AWQMarlinLinearMethod", kernel_type.__name__)
|
||||
logger.info("Using %s for AutoAWQMarlinLinearMethod", kernel_type.__name__)
|
||||
self._kernel_backends_being_used.add(kernel_type.__name__)
|
||||
|
||||
# Weights are loaded in AWQ checkpoint format (packed along output dim).
|
||||
@@ -509,16 +541,16 @@ class AWQMarlinLinearMethod(LinearMethodBase):
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
|
||||
|
||||
class AWQMarlinMoEMethod(FusedMoEMethodBase):
|
||||
class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
def __init__(
|
||||
self,
|
||||
quant_config: AWQMarlinConfig,
|
||||
quant_config: AutoAWQConfig,
|
||||
moe: FusedMoEConfig,
|
||||
):
|
||||
super().__init__(moe)
|
||||
self.quant_config = quant_config
|
||||
if self.quant_config.weight_bits != 4:
|
||||
raise ValueError("AWQMarlinMoEMethod only supports 4bit now.")
|
||||
raise ValueError("AutoAWQMoEMethod only supports 4bit now.")
|
||||
self.quant_type = scalar_types.uint4
|
||||
self.input_dtype = None
|
||||
self.use_marlin = True
|
||||
@@ -784,3 +816,185 @@ class AWQMarlinMoEMethod(FusedMoEMethodBase):
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
)
|
||||
|
||||
|
||||
class BaseAWQLinearMethod(LinearMethodBase):
|
||||
"""Base class for AWQ linear methods with shared weight creation logic."""
|
||||
|
||||
def __init__(self, quant_config: AutoAWQConfig):
|
||||
self.quant_config = quant_config
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
# Normalize group_size
|
||||
if self.quant_config.group_size != -1:
|
||||
group_size = self.quant_config.group_size
|
||||
else:
|
||||
group_size = input_size
|
||||
|
||||
if input_size_per_partition % group_size != 0:
|
||||
raise ValueError(
|
||||
"The input size is not aligned with the quantized "
|
||||
"weight shape. This can be caused by too large "
|
||||
"tensor parallel size."
|
||||
)
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
if output_size_per_partition % self.quant_config.pack_factor != 0:
|
||||
raise ValueError(
|
||||
"The output size is not aligned with the quantized "
|
||||
"weight shape. This can be caused by too large "
|
||||
"tensor parallel size."
|
||||
)
|
||||
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
qweight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition,
|
||||
output_size_per_partition // self.quant_config.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.quant_config.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
num_groups = input_size_per_partition // group_size
|
||||
|
||||
qzeros = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
num_groups,
|
||||
output_size_per_partition // self.quant_config.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.quant_config.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
scales = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
num_groups,
|
||||
output_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
layer.register_parameter("qweight", qweight)
|
||||
layer.register_parameter("qzeros", qzeros)
|
||||
layer.register_parameter("scales", scales)
|
||||
|
||||
|
||||
class AutoAWQLinearMethod(BaseAWQLinearMethod):
|
||||
"""Linear method for AWQ using Triton kernels.
|
||||
|
||||
Args:
|
||||
quant_config: The AWQ quantization config.
|
||||
"""
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False)
|
||||
layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False)
|
||||
layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
qweight = layer.qweight
|
||||
scales = layer.scales
|
||||
qzeros = layer.qzeros
|
||||
pack_factor = self.quant_config.pack_factor
|
||||
out_shape = x.shape[:-1] + (qweight.shape[-1] * pack_factor,)
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
|
||||
# num_tokens >= threshold
|
||||
FP16_MATMUL_HEURISTIC_CONDITION = x.shape[:-1].numel() >= 256
|
||||
# Batch invariant mode requires torch.matmul path
|
||||
# for Triton override
|
||||
if FP16_MATMUL_HEURISTIC_CONDITION or envs.VLLM_BATCH_INVARIANT:
|
||||
out = ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0)
|
||||
out = torch.matmul(reshaped_x, out)
|
||||
else:
|
||||
out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros, pack_factor)
|
||||
if bias is not None:
|
||||
out.add_(bias)
|
||||
return out.reshape(out_shape)
|
||||
|
||||
|
||||
class AutoAWQXPULinearMethod(BaseAWQLinearMethod):
|
||||
"""Linear method for AWQ on XPU using int4 GEMM kernel.
|
||||
|
||||
Args:
|
||||
quant_config: The AWQ quantization config.
|
||||
"""
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False)
|
||||
layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False)
|
||||
layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False)
|
||||
|
||||
try:
|
||||
from vllm_xpu_kernels.quantization._quantize_convert import (
|
||||
AWQUtils,
|
||||
transpose_onednn_woq_format,
|
||||
)
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"XPU AWQ requires vllm-xpu-kernels. "
|
||||
"Please install it with: pip install vllm-xpu-kernels"
|
||||
) from e
|
||||
|
||||
layer.xpu_output_size = layer.qweight.size(1) * self.quant_config.pack_factor
|
||||
qweight_new, qzeros_new = AWQUtils.repack(layer.qweight, layer.qzeros)
|
||||
if qweight_new.shape != layer.qweight.data.shape:
|
||||
layer.qweight.data = layer.qweight.data.view_as(qweight_new)
|
||||
if qzeros_new.shape != layer.qzeros.data.shape:
|
||||
layer.qzeros.data = layer.qzeros.data.view_as(qzeros_new)
|
||||
layer.qweight.data.copy_(qweight_new)
|
||||
layer.qzeros.data.copy_(qzeros_new)
|
||||
transpose_onednn_woq_format(layer, "awq", False)
|
||||
|
||||
def _get_group_size(self, layer: torch.nn.Module) -> int:
|
||||
"""Get the effective group size for kernel computation."""
|
||||
if self.quant_config.group_size != -1:
|
||||
return self.quant_config.group_size
|
||||
return layer.qweight.shape[0] # input_size_per_partition
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
group_size = self._get_group_size(layer)
|
||||
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a16(
|
||||
reshaped_x,
|
||||
layer.qweight,
|
||||
bias,
|
||||
layer.scales,
|
||||
layer.qzeros,
|
||||
group_size,
|
||||
None,
|
||||
)
|
||||
out_shape = x.shape[:-1] + (layer.xpu_output_size,)
|
||||
return out.reshape(out_shape)
|
||||
@@ -1,286 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Union
|
||||
|
||||
import torch
|
||||
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
LinearMethodBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
|
||||
from vllm.model_executor.parameter import GroupQuantScaleParameter, PackedvLLMParameter
|
||||
from vllm.transformers_utils.config import get_safetensors_params_metadata
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AWQConfig(QuantizationConfig):
|
||||
"""Config class for AWQ.
|
||||
|
||||
Reference: https://arxiv.org/abs/2306.00978
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_bits: int,
|
||||
group_size: int,
|
||||
zero_point: bool,
|
||||
modules_to_not_convert: list[str] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.weight_bits = weight_bits
|
||||
self.group_size = group_size
|
||||
self.zero_point = zero_point
|
||||
self.modules_to_not_convert = modules_to_not_convert or []
|
||||
|
||||
if self.weight_bits != 4:
|
||||
raise ValueError(
|
||||
"Currently, only 4-bit weight quantization is supported for "
|
||||
f"AWQ, but got {self.weight_bits} bits."
|
||||
)
|
||||
self.pack_factor = 32 // self.weight_bits
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"AWQConfig(weight_bits={self.weight_bits}, "
|
||||
f"group_size={self.group_size}, "
|
||||
f"zero_point={self.zero_point}, "
|
||||
f"modules_to_not_convert={self.modules_to_not_convert})"
|
||||
)
|
||||
|
||||
def get_name(self) -> "QuantizationMethods":
|
||||
return "awq"
|
||||
|
||||
def get_supported_act_dtypes(self) -> list[torch.dtype]:
|
||||
return [torch.half]
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
# The AWQ kernel only supports Turing or newer GPUs.
|
||||
return 75
|
||||
|
||||
@staticmethod
|
||||
def get_config_filenames() -> list[str]:
|
||||
return [
|
||||
"quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq
|
||||
# E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq
|
||||
"quantize_config.json",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "AWQConfig":
|
||||
weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
|
||||
group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
|
||||
zero_point = cls.get_from_keys(config, ["zero_point"])
|
||||
modules_to_not_convert = cls.get_from_keys_or(
|
||||
config, ["modules_to_not_convert"], None
|
||||
)
|
||||
return cls(weight_bits, group_size, zero_point, modules_to_not_convert)
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> Union["LinearMethodBase", "QuantizeMethodBase"] | None:
|
||||
if isinstance(layer, LinearBase):
|
||||
if is_layer_skipped(
|
||||
prefix,
|
||||
self.modules_to_not_convert,
|
||||
self.packed_modules_mapping,
|
||||
skip_with_substr=True,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
return AWQLinearMethod(self)
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
# Lazy import to avoid circular import.
|
||||
from .awq_marlin import AWQMarlinConfig
|
||||
from .moe_wna16 import MoeWNA16Config
|
||||
from .utils.marlin_utils import check_moe_marlin_supports_layer
|
||||
|
||||
if not check_moe_marlin_supports_layer(layer, self.group_size):
|
||||
logger.warning_once(
|
||||
f"Layer '{prefix}' is not supported by AWQMoeMarlin. "
|
||||
"Falling back to Moe WNA16 kernels."
|
||||
)
|
||||
config = {
|
||||
"quant_method": "awq",
|
||||
"bits": self.weight_bits,
|
||||
"group_size": self.group_size,
|
||||
"zero_point": self.zero_point,
|
||||
"lm_head": False,
|
||||
"modules_to_not_convert": self.modules_to_not_convert,
|
||||
}
|
||||
return MoeWNA16Config.from_config(config).get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
marlin_compatible_config_dict = {
|
||||
"quant_method": "awq",
|
||||
"bits": self.weight_bits,
|
||||
"group_size": self.group_size,
|
||||
"zero_point": self.zero_point,
|
||||
"lm_head": False,
|
||||
"modules_to_not_convert": self.modules_to_not_convert,
|
||||
}
|
||||
awq_marlin_config = AWQMarlinConfig.from_config(
|
||||
marlin_compatible_config_dict
|
||||
)
|
||||
return awq_marlin_config.get_quant_method(layer, prefix)
|
||||
return None
|
||||
|
||||
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
|
||||
if self.modules_to_not_convert:
|
||||
self.modules_to_not_convert = hf_to_vllm_mapper.apply_list(
|
||||
self.modules_to_not_convert
|
||||
)
|
||||
|
||||
def maybe_update_config(
|
||||
self,
|
||||
model_name: str,
|
||||
hf_config: PretrainedConfig | None = None,
|
||||
revision: str | None = None,
|
||||
):
|
||||
if self.modules_to_not_convert:
|
||||
return
|
||||
|
||||
unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]
|
||||
metadata = get_safetensors_params_metadata(model_name, revision=revision)
|
||||
layers = {param_name.rsplit(".", 1)[0] for param_name in metadata}
|
||||
quant_layers: set[str] = {
|
||||
param_name.rsplit(".", 1)[0]
|
||||
for param_name, info in metadata.items()
|
||||
if (dtype := info.get("dtype", None))
|
||||
and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes
|
||||
}
|
||||
self.modules_to_not_convert = list(layers - quant_layers)
|
||||
|
||||
|
||||
class AWQLinearMethod(LinearMethodBase):
|
||||
"""Linear method for AWQ.
|
||||
|
||||
Args:
|
||||
quant_config: The AWQ quantization config.
|
||||
"""
|
||||
|
||||
def __init__(self, quant_config: AWQConfig):
|
||||
self.quant_config = quant_config
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
# Normalize group_size
|
||||
if self.quant_config.group_size != -1:
|
||||
group_size = self.quant_config.group_size
|
||||
else:
|
||||
group_size = input_size
|
||||
|
||||
if input_size_per_partition % group_size != 0:
|
||||
raise ValueError(
|
||||
"The input size is not aligned with the quantized "
|
||||
"weight shape. This can be caused by too large "
|
||||
"tensor parallel size."
|
||||
)
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
if output_size_per_partition % self.quant_config.pack_factor != 0:
|
||||
raise ValueError(
|
||||
"The output size is not aligned with the quantized "
|
||||
"weight shape. This can be caused by too large "
|
||||
"tensor parallel size."
|
||||
)
|
||||
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
qweight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition,
|
||||
output_size_per_partition // self.quant_config.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.quant_config.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
num_groups = input_size_per_partition // group_size
|
||||
|
||||
qzeros = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
num_groups,
|
||||
output_size_per_partition // self.quant_config.pack_factor,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.quant_config.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
scales = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
num_groups,
|
||||
output_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
layer.register_parameter("qweight", qweight)
|
||||
layer.register_parameter("qzeros", qzeros)
|
||||
layer.register_parameter("scales", scales)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False)
|
||||
layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False)
|
||||
layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
qweight = layer.qweight
|
||||
scales = layer.scales
|
||||
qzeros = layer.qzeros
|
||||
pack_factor = self.quant_config.pack_factor
|
||||
out_shape = x.shape[:-1] + (qweight.shape[-1] * pack_factor,)
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
|
||||
# num_tokens >= threshold
|
||||
FP16_MATMUL_HEURISTIC_CONDITION = x.shape[:-1].numel() >= 256
|
||||
# Batch invariant mode requires torch.matmul path
|
||||
# for Triton override
|
||||
if FP16_MATMUL_HEURISTIC_CONDITION or envs.VLLM_BATCH_INVARIANT:
|
||||
out = ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0)
|
||||
out = torch.matmul(reshaped_x, out)
|
||||
else:
|
||||
out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros, pack_factor)
|
||||
if bias is not None:
|
||||
out.add_(bias)
|
||||
return out.reshape(out_shape)
|
||||
@@ -8,9 +8,8 @@ import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_marlin_supported,
|
||||
)
|
||||
@@ -125,12 +124,12 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
)
|
||||
|
||||
if use_marlin:
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import (
|
||||
AWQMarlinLinearMethod,
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
AutoAWQMarlinLinearMethod,
|
||||
)
|
||||
|
||||
return AWQMarlinLinearMethod(
|
||||
AWQMarlinConfig(
|
||||
return AutoAWQMarlinLinearMethod(
|
||||
AutoAWQConfig(
|
||||
weight_bits=self.layer_config.bits,
|
||||
group_size=self.layer_config.group_size,
|
||||
zero_point=not self.layer_config.sym,
|
||||
@@ -140,13 +139,16 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
)
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.awq import AWQLinearMethod
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
AutoAWQLinearMethod,
|
||||
)
|
||||
|
||||
return AWQLinearMethod(
|
||||
AWQConfig(
|
||||
return AutoAWQLinearMethod(
|
||||
AutoAWQConfig(
|
||||
weight_bits=self.layer_config.bits,
|
||||
group_size=self.layer_config.group_size,
|
||||
zero_point=not self.layer_config.sym,
|
||||
lm_head_quantized=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
@@ -154,7 +154,7 @@ def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
|
||||
|
||||
def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinMoEMethod
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQMoEMethod
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import (
|
||||
MoeWNA16Config,
|
||||
MoeWNA16Method,
|
||||
@@ -177,8 +177,8 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
) and check_moe_marlin_supports_layer(layer, layer_config.group_size)
|
||||
|
||||
if use_marlin:
|
||||
return AWQMarlinMoEMethod(
|
||||
AWQMarlinConfig(
|
||||
return AutoAWQMoEMethod(
|
||||
AutoAWQConfig(
|
||||
weight_bits=layer_config.bits,
|
||||
group_size=layer_config.group_size,
|
||||
zero_point=not layer_config.sym,
|
||||
|
||||
@@ -27,9 +27,6 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_marlin_supports_layer,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -55,10 +52,8 @@ class MoeWNA16Config(QuantizationConfig):
|
||||
self.lm_head_quantized = lm_head_quantized
|
||||
self.linear_quant_method = linear_quant_method
|
||||
self.full_config = full_config
|
||||
self.use_marlin = False
|
||||
# Avoid circular import
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
if self.linear_quant_method == "gptq":
|
||||
pass
|
||||
@@ -67,7 +62,7 @@ class MoeWNA16Config(QuantizationConfig):
|
||||
device_capability = (
|
||||
-1 if capability_tuple is None else capability_tuple.to_int()
|
||||
)
|
||||
awq_min_capability = AWQConfig.get_min_capability()
|
||||
awq_min_capability = AutoAWQConfig.get_min_capability()
|
||||
if device_capability < awq_min_capability:
|
||||
raise ValueError(
|
||||
"The quantization method moe_wna16 + awq is not supported "
|
||||
@@ -75,7 +70,6 @@ class MoeWNA16Config(QuantizationConfig):
|
||||
f"Minimum capability: {awq_min_capability}. "
|
||||
f"Current capability: {device_capability}."
|
||||
)
|
||||
self.use_marlin = AWQMarlinConfig.is_awq_marlin_compatible(full_config)
|
||||
else:
|
||||
raise ValueError("moe_wna16 only support gptq and awq.")
|
||||
|
||||
@@ -148,9 +142,9 @@ class MoeWNA16Config(QuantizationConfig):
|
||||
-1 if capability_tuple is None else capability_tuple.to_int()
|
||||
)
|
||||
# Avoid circular import
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
awq_min_capability = AWQConfig.get_min_capability()
|
||||
awq_min_capability = AutoAWQConfig.get_min_capability()
|
||||
|
||||
gptq_compatible = quant_method == "gptq" and not desc_act and num_bits in [4, 8]
|
||||
awq_compatible = (
|
||||
@@ -170,29 +164,19 @@ class MoeWNA16Config(QuantizationConfig):
|
||||
return UnquantizedLinearMethod()
|
||||
elif isinstance(layer, LinearBase):
|
||||
# Avoid circular import
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import (
|
||||
AWQMarlinConfig,
|
||||
)
|
||||
|
||||
if self.linear_quant_method == "gptq":
|
||||
return AutoGPTQConfig.from_config(self.full_config).get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
elif self.linear_quant_method in ("awq", "awq_marlin"):
|
||||
if self.use_marlin and check_marlin_supports_layer(
|
||||
layer, self.group_size
|
||||
):
|
||||
return AWQMarlinConfig.from_config(
|
||||
self.full_config
|
||||
).get_quant_method(layer, prefix)
|
||||
else:
|
||||
return AWQConfig.from_config(self.full_config).get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
return AutoAWQConfig.from_config(self.full_config).get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
else:
|
||||
raise ValueError("moe_wna16 only support gptq and awq.")
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
|
||||
@@ -26,7 +26,7 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
@@ -420,7 +420,7 @@ class Cohere2VisionForConditionalGeneration(
|
||||
):
|
||||
# the awq models from OpenGVLab missing `modules_to_not_convert`
|
||||
# patch the quant_config to add `modules_to_not_convert` back
|
||||
if isinstance(quant_config, AWQConfig):
|
||||
if isinstance(quant_config, AutoAWQConfig):
|
||||
text_config = config.text_config
|
||||
llm_quant_config = getattr(text_config, "quantization_config", None)
|
||||
if (not quant_config.modules_to_not_convert) and (
|
||||
|
||||
@@ -20,7 +20,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.models.intern_vit import (
|
||||
InternVisionModel,
|
||||
)
|
||||
@@ -608,7 +608,7 @@ class InternVLChatModel(
|
||||
):
|
||||
# the awq models from OpenGVLab missing `modules_to_not_convert`
|
||||
# patch the quant_config to add `modules_to_not_convert` back
|
||||
if isinstance(quant_config, AWQConfig):
|
||||
if isinstance(quant_config, AutoAWQConfig):
|
||||
text_config = config.text_config
|
||||
llm_quant_config = getattr(text_config, "quantization_config", None)
|
||||
if (not quant_config.modules_to_not_convert) and (
|
||||
|
||||
@@ -11,7 +11,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.pooler import DispatchPooler
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.models.internvl import (
|
||||
BaseInternVLDummyInputsBuilder,
|
||||
BaseInternVLMultiModalProcessor,
|
||||
@@ -144,7 +144,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor
|
||||
):
|
||||
# the awq models from OpenGVLab missing `modules_to_not_convert`
|
||||
# patch the quant_config to add `modules_to_not_convert` back
|
||||
if isinstance(quant_config, AWQConfig):
|
||||
if isinstance(quant_config, AutoAWQConfig):
|
||||
text_config = config.get_text_config()
|
||||
llm_quant_config = getattr(text_config, "quantization_config", None)
|
||||
if (not quant_config.modules_to_not_convert) and (
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.awq import AWQConfig
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
from vllm.model_executor.models.intern_vit import (
|
||||
InternVisionModel,
|
||||
)
|
||||
@@ -205,7 +205,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
):
|
||||
# the awq models from OpenGVLab missing `modules_to_not_convert`
|
||||
# patch the quant_config to add `modules_to_not_convert` back
|
||||
if isinstance(quant_config, AWQConfig):
|
||||
if isinstance(quant_config, AutoAWQConfig):
|
||||
text_config = config.text_config
|
||||
llm_quant_config = getattr(text_config, "quantization_config", None)
|
||||
if (not quant_config.modules_to_not_convert) and (
|
||||
|
||||
@@ -440,6 +440,7 @@ class RocmPlatform(Platform):
|
||||
|
||||
supported_quantization: list[str] = [
|
||||
"awq",
|
||||
"auto_awq",
|
||||
"awq_marlin", # will be overwritten with awq
|
||||
"gptq",
|
||||
"gptq_marlin",
|
||||
|
||||
Reference in New Issue
Block a user