diff --git a/docs/features/quantization/auto_awq.md b/docs/features/quantization/auto_awq.md index e93005f263..39dfd6fec1 100644 --- a/docs/features/quantization/auto_awq.md +++ b/docs/features/quantization/auto_awq.md @@ -49,7 +49,7 @@ To run an AWQ model with vLLM, you can use [TheBloke/Llama-2-7b-Chat-AWQ](https: ```bash python examples/deployment/llm_engine_example.py \ --model TheBloke/Llama-2-7b-Chat-AWQ \ - --quantization awq + --quantization auto_awq ``` AWQ models are also supported directly through the LLM entrypoint: @@ -70,7 +70,7 @@ AWQ models are also supported directly through the LLM entrypoint: sampling_params = SamplingParams(temperature=0.8, top_p=0.95) # Create an LLM. - llm = LLM(model="TheBloke/Llama-2-7b-Chat-AWQ", quantization="AWQ") + llm = LLM(model="TheBloke/Llama-2-7b-Chat-AWQ", quantization="auto_awq") # Generate texts from the prompts. The output is a list of RequestOutput objects # that contain the prompt, generated text, and other information. outputs = llm.generate(prompts, sampling_params) diff --git a/tests/quantization/test_auto_awq.py b/tests/quantization/test_auto_awq.py new file mode 100644 index 0000000000..dcb2b11c8f --- /dev/null +++ b/tests/quantization/test_auto_awq.py @@ -0,0 +1,231 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Tests for AutoAWQConfig behavior after unification. + +These tests verify the bug fixes for: +1. CPU platform override conflict (auto_awq should not override on CPU) +2. MoE fallback compatibility (full_config["quant_method"] should be "awq") +3. Config attribute consistency +4. End-to-end quantization method loading (auto_awq loads and runs correctly) + +Note: Tests that require importing the full auto_awq module (which has GPU-dependent +imports) should use subprocess or be run in a GPU environment. +""" + +from __future__ import annotations + +import pytest +import torch + +from tests.quantization.utils import is_quant_method_supported + + +def _get_auto_awq_config_source() -> str: + """Read the AutoAWQConfig class source code for isolated testing.""" + import inspect + + import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module + + return inspect.getsource(auto_awq_module.AutoAWQConfig) + + +class TestAutoAWQConfigFromConfig: + """Tests for AutoAWQConfig.from_config behavior. + + These tests require GPU environment to import the full module. + They are skipped on non-GPU platforms. + """ + + def test_full_config_quant_method_is_awq_for_moe_fallback(self): + """full_config should have quant_method='awq' for MoE fallback compatibility. + + MoeWNA16Config only accepts 'gptq' or 'awq' as linear_quant_method. + If full_config has 'auto_awq', the MoE fallback will fail. + """ + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig + + config = { + "w_bit": 4, + "q_group_size": 128, + "zero_point": True, + "lm_head": False, + } + awq_config = AutoAWQConfig.from_config(config) + + # Verify quant_method is 'awq' for MoE fallback + assert awq_config.full_config["quant_method"] == "awq", ( + f"Expected quant_method='awq', got {awq_config.full_config['quant_method']}" + ) + + def test_full_config_preserves_other_fields(self): + """full_config should preserve all original config fields.""" + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig + + config = { + "w_bit": 4, + "q_group_size": 128, + "zero_point": True, + "lm_head": False, + "custom_field": "custom_value", + } + awq_config = AutoAWQConfig.from_config(config) + + assert awq_config.full_config["w_bit"] == 4 + assert awq_config.full_config["q_group_size"] == 128 + assert awq_config.full_config["zero_point"] is True + assert awq_config.full_config["lm_head"] is False + assert awq_config.full_config["custom_field"] == "custom_value" + + def test_full_config_is_copy_not_original(self): + """full_config should be a copy, not the original dict.""" + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig + + config = { + "w_bit": 4, + "q_group_size": 128, + "zero_point": True, + "lm_head": False, + } + original_quant_method = config.get("quant_method") + + AutoAWQConfig.from_config(config) + + # Original config should not be modified + assert config.get("quant_method") == original_quant_method + + +class TestAutoAWQConfigAttributes: + """Tests for AutoAWQConfig attribute consistency. + + These tests require GPU environment to import the full module. + They are skipped on non-GPU platforms. + """ + + def test_config_attributes_match_input(self): + """Config attributes should match input values.""" + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig + + awq_config = AutoAWQConfig( + weight_bits=4, + group_size=128, + zero_point=True, + lm_head_quantized=False, + modules_to_not_convert=["lm_head"], + ) + + assert awq_config.weight_bits == 4 + assert awq_config.group_size == 128 + assert awq_config.zero_point is True + assert awq_config.lm_head_quantized is False + assert awq_config.modules_to_not_convert == ["lm_head"] + + def test_pack_factor_for_4bit(self): + """Pack factor should be 8 for 4-bit quantization.""" + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig + + awq_config = AutoAWQConfig( + weight_bits=4, + group_size=128, + zero_point=True, + lm_head_quantized=False, + ) + + assert awq_config.pack_factor == 8 # 32 // 4 + + +class TestAutoAWQConfigOverrideLogic: + """Tests for override logic by parsing source code (no GPU import required).""" + + def _get_auto_awq_source(self) -> str: + """Read the auto_awq.py source file.""" + import inspect + import pathlib + + import vllm.model_executor.layers.quantization.auto_awq as auto_awq_module + + source_path = inspect.getfile(auto_awq_module) + return pathlib.Path(source_path).read_text() + + def test_cpu_check_in_override_method(self): + """override_quantization_method should check current_platform.is_cpu().""" + source = self._get_auto_awq_source() + + # Verify the CPU check exists in override method + assert "current_platform.is_cpu()" in source, ( + "override_quantization_method should check is_cpu()" + ) + assert "return None" in source, ( + "override_quantization_method should return None on CPU" + ) + + def test_quant_method_normalization_in_from_config(self): + """from_config should normalize quant_method to 'awq' for MoE fallback.""" + source = self._get_auto_awq_source() + + # Verify the normalization exists + assert ( + '"quant_method"] = "awq"' in source or "'quant_method'] = 'awq'" in source + ), "from_config should set quant_method='awq' in full_config" + + +# ============================================================================= +# End-to-end integration tests (require GPU environment) +# ============================================================================= + +PROMPT = "On the surface of Mars, we found" + +# Small AWQ model for testing - using Qwen2 1.5B which has official AWQ checkpoint +AWQ_MODELS = [ + "Qwen/Qwen2-1.5B-Instruct-AWQ", +] + + +@pytest.mark.skipif( + not is_quant_method_supported("auto_awq"), + reason="auto_awq is not supported on this GPU type.", +) +@pytest.mark.parametrize("model_id", AWQ_MODELS) +def test_auto_awq_quantization_method(vllm_runner, model_id: str, monkeypatch): + """Test that quantization='auto_awq' loads and runs correctly.""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + + with vllm_runner( + model_id, + dtype=torch.float16, + quantization="auto_awq", + max_model_len=2048, + enforce_eager=True, + ) as llm: + + def check_model(model): + from vllm.model_executor.layers.quantization.auto_awq import ( + AutoAWQLinearMethod, + AutoAWQMarlinLinearMethod, + ) + + for name, submodule in model.named_modules(): + if name == "model.layers.0.self_attn.qkv_proj": + # Should use either AutoAWQLinearMethod (Triton) or + # AutoAWQMarlinLinearMethod (Marlin) depending on hardware + assert isinstance( + submodule.quant_method, + (AutoAWQLinearMethod, AutoAWQMarlinLinearMethod), + ), ( + f"Expected AutoAWQLinearMethod or AutoAWQMarlinLinearMethod " + f"for {name}, got {type(submodule.quant_method)}" + ) + break + + llm.apply_model(check_model) + + outputs = llm.generate_greedy([PROMPT], max_tokens=8) + assert outputs + assert len(outputs[0][1]) > 0 + + +def test_auto_awq_config_get_name(): + """Test that AutoAWQConfig.get_name() returns 'auto_awq'.""" + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig + + assert AutoAWQConfig.get_name() == "auto_awq" diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index 5cd599f721..a826bba955 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -645,11 +645,11 @@ def test_resolve_awq_moe_uses_marlin_when_supported(monkeypatch) -> None: lambda *args, **kwargs: True, ) monkeypatch.setattr( - "vllm.model_executor.layers.quantization.awq_marlin.verify_marlin_supported", + "vllm.model_executor.layers.quantization.auto_awq.verify_marlin_supported", lambda *args, **kwargs: None, ) monkeypatch.setattr( - "vllm.model_executor.layers.quantization.awq_marlin.AWQMarlinMoEMethod", + "vllm.model_executor.layers.quantization.auto_awq.AutoAWQMoEMethod", DummyMethod, ) diff --git a/tests/quantization/test_configs.py b/tests/quantization/test_configs.py index fe5f8735d6..85b67da433 100644 --- a/tests/quantization/test_configs.py +++ b/tests/quantization/test_configs.py @@ -43,16 +43,18 @@ MODEL_ARG_EXPTYPES = [ ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "auto_gptq"), ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"), # AUTOAWQ + # AutoAWQConfig.override_quantization_method() returns "auto_awq" for AWQ models + # when user_quant is None, "awq", "awq_marlin", "marlin", or "auto_awq" ( "TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", None, - "awq_marlin" if current_platform.is_cuda_alike() else "awq", + "auto_awq", ), - ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "awq"), + ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "awq", "auto_awq"), ( "TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "marlin", - "awq_marlin" if current_platform.is_cuda_alike() else "ERROR", + "auto_awq" if current_platform.is_cuda_alike() else "ERROR", ), ("TheBloke/OpenHermes-2.5-Mistral-7B-AWQ", "gptq", "ERROR"), ] diff --git a/vllm/config/model.py b/vllm/config/model.py index 87c0eec1bf..37549e188e 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -984,6 +984,8 @@ class ModelConfig: "auto_gptq", "gptq", "gptq_marlin", + "auto_awq", + "awq", "awq_marlin", "inc", "moe_wna16", diff --git a/vllm/model_executor/kernels/linear/mixed_precision/cpu.py b/vllm/model_executor/kernels/linear/mixed_precision/cpu.py index b364d1ad96..928fa97a4f 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/cpu.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/cpu.py @@ -175,7 +175,7 @@ class CPUWNA16LinearKernel(MPLinearKernel): and torch.cpu._is_amx_tile_supported() ) # layer.use_w4a8 = False - # AWQ format will be converted to GPTQ format in `AWQMarlinLinearMethod` + # AWQ format will be converted to GPTQ format in `AutoAWQMarlinLinearMethod` if layer.use_w4a8: self._process_gptq_weights_w4a8(layer) else: diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index 8de6269e2e..cbd12b3e60 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -728,18 +728,18 @@ def _process_weights_cpu( from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( prepare_int4_moe_layer_for_cpu, ) + from vllm.model_executor.layers.quantization.auto_awq import ( + AutoAWQConfig, + ) from vllm.model_executor.layers.quantization.auto_gptq import ( AutoGPTQConfig, ) - from vllm.model_executor.layers.quantization.awq_marlin import ( - AWQMarlinConfig, - ) # Detect packing format. # AWQ: qweight is [E, K, 2*N//8] (packed along output/N dim). # GPTQ: qweight is [E, K//8, 2*N] (packed along input/K dim). # compressed-tensors: qweight is [E, K//8, 2*N] (packed along input/K dim). - if isinstance(quant_config, AWQMarlinConfig): + if isinstance(quant_config, AutoAWQConfig): # AWQ: K is stored unpacked in dim 1. cpu_quant_algo = ops.CPUQuantAlgo.AWQ elif isinstance(quant_config, (AutoGPTQConfig, QuantizationArgs)): @@ -753,7 +753,7 @@ def _process_weights_cpu( cpu_quant_algo = ops.CPUQuantAlgo.GPTQ else: raise TypeError( - "CPU WNA16 MoE backend requires AWQMarlinConfig, AutoGPTQConfig " + "CPU WNA16 MoE backend requires AutoAWQConfig, AutoGPTQConfig " f"or QuantizationArgs, got {type(quant_config).__name__}." ) @@ -916,14 +916,14 @@ def convert_to_wna16_moe_kernel_format( WNA16MoEBackend.MARLIN, WNA16MoEBackend.BATCHED_MARLIN, ): + from vllm.model_executor.layers.quantization.auto_awq import ( + AutoAWQConfig, + ) from vllm.model_executor.layers.quantization.auto_gptq import ( AutoGPTQConfig, ) - from vllm.model_executor.layers.quantization.awq_marlin import ( - AWQMarlinConfig, - ) - if isinstance(quant_config, AWQMarlinConfig): + if isinstance(quant_config, AutoAWQConfig): if w13_qzeros is None or w2_qzeros is None: raise ValueError("AWQ Marlin MoE requires zero-point tensors.") @@ -958,7 +958,7 @@ def convert_to_wna16_moe_kernel_format( actorder = quant_config.actorder else: raise TypeError( - "Marlin WNA16 MoE backend requires AutoGPTQConfig, AWQMarlinConfig or " + "Marlin WNA16 MoE backend requires AutoAWQConfig, AutoGPTQConfig or " f"QuantizationArgs, got {type(quant_config).__name__}." ) if w13_g_idx is None or w2_g_idx is None: diff --git a/vllm/model_executor/layers/linear.py b/vllm/model_executor/layers/linear.py index 9ee3a231b9..48c1902e29 100644 --- a/vllm/model_executor/layers/linear.py +++ b/vllm/model_executor/layers/linear.py @@ -46,8 +46,8 @@ WEIGHT_LOADER_V2_SUPPORTED = [ "UnquantizedLinearMethod", "CompressedTensorsLinearMethod", "CompressedTensorsLinearTransformMethod", - "AWQMarlinLinearMethod", - "AWQLinearMethod", + "AutoAWQMarlinLinearMethod", + "AutoAWQLinearMethod", "AutoGPTQLinearMethod", "Fp8LinearMethod", "FBGEMMFp8LinearMethod", diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index a7a609eba9..06bfe5c5de 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -48,8 +48,8 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( causal_conv1d_update, ) from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig -from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig from vllm.model_executor.layers.quantization.inc import INCConfig from vllm.model_executor.model_loader.weight_utils import ( sharded_weight_loader, @@ -628,7 +628,7 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): return ( current_platform.is_cuda() and not self.gqa_interleaved_layout - and isinstance(quant_config, (AWQMarlinConfig, AutoGPTQConfig, INCConfig)) + and isinstance(quant_config, (AutoAWQConfig, AutoGPTQConfig, INCConfig)) ) def split_ba(self, ba: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: diff --git a/vllm/model_executor/layers/quantization/__init__.py b/vllm/model_executor/layers/quantization/__init__.py index 53f4e7d2a8..866bc30a15 100644 --- a/vllm/model_executor/layers/quantization/__init__.py +++ b/vllm/model_executor/layers/quantization/__init__.py @@ -11,6 +11,7 @@ logger = init_logger(__name__) QuantizationMethods = Literal[ "awq", + "auto_awq", "fp8", "fbgemm_fp8", "fp_quant", @@ -113,9 +114,8 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig from vllm.models.deepseek_v4 import DeepseekV4FP8Config + from .auto_awq import AutoAWQConfig from .auto_gptq import AutoGPTQConfig - from .awq import AWQConfig - from .awq_marlin import AWQMarlinConfig from .bitsandbytes import BitsAndBytesConfig from .compressed_tensors.compressed_tensors import ( CompressedTensorsConfig, @@ -138,7 +138,9 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: from .torchao import TorchAOConfig method_to_config: dict[str, type[QuantizationConfig]] = { - "awq": AWQConfig, + "awq": AutoAWQConfig, + "awq_marlin": AutoAWQConfig, + "auto_awq": AutoAWQConfig, "fp8": Fp8Config, "fbgemm_fp8": FBGEMMFp8Config, "fp_quant": FPQuantConfig, @@ -149,7 +151,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: "auto_gptq": AutoGPTQConfig, "gptq": AutoGPTQConfig, "gptq_marlin": AutoGPTQConfig, - "awq_marlin": AWQMarlinConfig, "compressed-tensors": CompressedTensorsConfig, "bitsandbytes": BitsAndBytesConfig, "experts_int8": ExpertsInt8Config, diff --git a/vllm/model_executor/layers/quantization/awq_marlin.py b/vllm/model_executor/layers/quantization/auto_awq.py similarity index 69% rename from vllm/model_executor/layers/quantization/awq_marlin.py rename to vllm/model_executor/layers/quantization/auto_awq.py index b8fe2f272a..a524c8c193 100644 --- a/vllm/model_executor/layers/quantization/awq_marlin.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Union import torch from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE @@ -9,6 +9,7 @@ from torch.nn import Parameter from transformers import PretrainedConfig import vllm.model_executor.layers.fused_moe # noqa +from vllm import _custom_ops as ops from vllm import envs from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( @@ -36,7 +37,6 @@ from vllm.model_executor.layers.linear import ( UnquantizedLinearMethod, set_weight_attrs, ) -from vllm.model_executor.layers.quantization.awq import AWQConfig from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, @@ -55,7 +55,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kInt4Static, ) from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead -from vllm.model_executor.parameter import GroupQuantScaleParameter, PackedvLLMParameter +from vllm.model_executor.parameter import ( + GroupQuantScaleParameter, + PackedvLLMParameter, +) from vllm.platforms import current_platform from vllm.scalar_type import scalar_types from vllm.transformers_utils.config import get_safetensors_params_metadata @@ -164,8 +167,12 @@ def _convert_awq_to_standard_format( setattr(layer, w_zp_name, new_zp_param) -class AWQMarlinConfig(QuantizationConfig): - """Config class for AWQ Marlin""" +class AutoAWQConfig(QuantizationConfig): + """Config class for AutoAWQ quantization. + + Unified config that supports multiple backends: Triton, Marlin, and XPU. + Reference: https://arxiv.org/abs/2306.00978 + """ # num_bits -> type TYPE_MAP = { @@ -178,8 +185,8 @@ class AWQMarlinConfig(QuantizationConfig): group_size: int, zero_point: bool, lm_head_quantized: bool, - modules_to_not_convert: list[str] | None, - full_config: dict[str, Any], + modules_to_not_convert: list[str] | None = None, + full_config: dict[str, Any] | None = None, ) -> None: super().__init__() self.pack_factor = 32 // weight_bits # packed into int32 @@ -188,23 +195,22 @@ class AWQMarlinConfig(QuantizationConfig): self.lm_head_quantized = lm_head_quantized self.weight_bits = weight_bits self.modules_to_not_convert = modules_to_not_convert or [] - self.full_config = full_config + self.full_config = full_config or {} if self.weight_bits not in self.TYPE_MAP: + supported = ", ".join(str(k) for k in self.TYPE_MAP) raise ValueError( f"Unsupported num_bits = {self.weight_bits}. " - f"Supported num_bits = {self.TYPE_MAP.keys()}" + f"Supported: {supported}. " + f"For 8-bit AWQ, use Marlin backend by setting " + f"backend='awq:marlin' or backend='marlin'." ) self.quant_type = self.TYPE_MAP[self.weight_bits] - verify_marlin_supported( - self.quant_type, group_size=self.group_size, has_zp=self.zero_point - ) - def __repr__(self) -> str: return ( - f"AWQMarlinConfig(quant_type={self.quant_type}, " + f"AutoAWQConfig(quant_type={self.quant_type}, " f"group_size={self.group_size}, " f"zero_point={self.zero_point}, " f"lm_head_quantized={self.lm_head_quantized}, " @@ -213,7 +219,7 @@ class AWQMarlinConfig(QuantizationConfig): @classmethod def get_name(cls) -> "QuantizationMethods": - return "awq_marlin" + return "auto_awq" @classmethod def get_supported_act_dtypes(cls) -> list[torch.dtype]: @@ -225,60 +231,59 @@ class AWQMarlinConfig(QuantizationConfig): @classmethod def get_config_filenames(cls) -> list[str]: - return ["quantize_config.json"] + return ["quantize_config.json", "quant_config.json"] @classmethod - def from_config(cls, config: dict[str, Any]) -> "AWQMarlinConfig": - weight_bits = cls.get_from_keys(config, ["bits"]) - group_size = cls.get_from_keys(config, ["group_size"]) + def from_config(cls, config: dict[str, Any]) -> "AutoAWQConfig": + weight_bits = cls.get_from_keys(config, ["w_bit", "bits"]) + group_size = cls.get_from_keys(config, ["q_group_size", "group_size"]) zero_point = cls.get_from_keys(config, ["zero_point"]) lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False) modules_to_not_convert = cls.get_from_keys_or( config, ["modules_to_not_convert"], None ) + # Ensure full_config uses "awq" as quant_method for MoE fallback compatibility. + # MoeWNA16Config only accepts "gptq" or "awq", so we normalize here. + full_config = config.copy() + full_config["quant_method"] = "awq" return cls( weight_bits, group_size, zero_point, lm_head_quantized, modules_to_not_convert, - config, + full_config, ) @classmethod def override_quantization_method( cls, hf_quant_cfg, user_quant, hf_config=None ) -> "QuantizationMethods | None": - # Skip override to marlin kernels, as they are not - # batch invariant - if envs.VLLM_BATCH_INVARIANT: + """Override to use AutoAWQ for compatible AWQ models.""" + # Don't override on CPU - let cpu_awq handle it + if current_platform.is_cpu(): return None - can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg) - is_valid_user_quant = ( - user_quant is None or user_quant == "marlin" or user_quant == "awq_marlin" + quant_method = hf_quant_cfg.get("quant_method", "").lower() + + if quant_method != "awq": + return None + + is_valid_user_quant = user_quant is None or user_quant in ( + "awq", + "awq_marlin", + "auto_awq", + "marlin", ) - if can_convert and is_valid_user_quant: - msg = ( - "The model is convertible to {} during runtime." - " Using {} kernel.".format(cls.get_name(), cls.get_name()) - ) - logger.info(msg) + if is_valid_user_quant: return cls.get_name() - if can_convert and user_quant == "awq": - logger.info( - "Detected that the model can run with awq_marlin" - ", however you specified quantization=awq explicitly," - " so forcing awq. Use quantization=awq_marlin for" - " faster inference" - ) return None def get_quant_method( self, layer: torch.nn.Module, prefix: str - ) -> "QuantizeMethodBase | None": + ) -> Union["LinearMethodBase", "QuantizeMethodBase"] | None: if isinstance(layer, LinearBase) or ( isinstance(layer, ParallelLMHead) and self.lm_head_quantized ): @@ -289,41 +294,66 @@ class AWQMarlinConfig(QuantizationConfig): skip_with_substr=True, ): return UnquantizedLinearMethod() - # Check if the layer is supported by AWQMarlin; tile-misaligned - # shapes are fixed by padding at weight prep. - if not check_marlin_supports_layer( - layer, self.group_size, allow_tile_padding=True - ): - logger.warning_once( - "Layer '%s' is not supported by AWQMarlin. Falling back to unoptimized AWQ kernels.", # noqa: E501 - prefix, - ) - return AWQConfig.from_config(self.full_config).get_quant_method( - layer, prefix - ) - quant_method = AWQMarlinLinearMethod(self) - quant_method.input_dtype = get_marlin_input_dtype(prefix) - return quant_method - elif isinstance(layer, RoutedExperts): - from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config + # Check if XPU - use XPU-specific linear method + if current_platform.is_xpu(): + return AutoAWQXPULinearMethod(self) + + # On CPU, use Marlin linear method which uses choose_mp_linear_kernel + # to select the best available kernel (CPUWNA16LinearKernel on CPU) + if current_platform.is_cpu(): + return AutoAWQMarlinLinearMethod(self) + + # Check if Marlin is supported and not using batch invariant mode + # (Marlin kernels are not batch invariant) + use_marlin = ( + not envs.VLLM_BATCH_INVARIANT + and current_platform.is_cuda() + and check_marlin_supported( + self.quant_type, self.group_size, self.zero_point + ) + ) + + if use_marlin: + # tile-misaligned shapes are fixed by padding at weight prep + if not check_marlin_supports_layer( + layer, self.group_size, allow_tile_padding=True + ): + logger.warning_once( + "Layer '%s' is not supported by AutoAWQMarlin. " + "Falling back to unoptimized AWQ kernels.", + prefix, + ) + return AutoAWQLinearMethod(self) + quant_method = AutoAWQMarlinLinearMethod(self) + quant_method.input_dtype = get_marlin_input_dtype(prefix) + return quant_method + + return AutoAWQLinearMethod(self) + + elif isinstance(layer, RoutedExperts): if is_layer_skipped( prefix, getattr(self, "modules_to_not_convert", []), skip_with_substr=True, ): return UnquantizedFusedMoEMethod(layer.moe_config) + if not check_moe_marlin_supports_layer(layer, self.group_size): logger.warning_once( - f"Layer '{prefix}' is not supported by AWQMoeMarlin. " + f"Layer '{prefix}' is not supported by AutoAWQMoEMarlin. " "Falling back to Moe WNA16 kernels." ) + from vllm.model_executor.layers.quantization.moe_wna16 import ( + MoeWNA16Config, + ) + return MoeWNA16Config.from_config(self.full_config).get_quant_method( layer, prefix ) - moe_quant_method = AWQMarlinMoEMethod(self, layer.moe_config) - moe_quant_method.input_dtype = get_marlin_input_dtype(prefix) - return moe_quant_method + + return AutoAWQMoEMethod(self, layer.moe_config) + return None @classmethod @@ -378,7 +408,7 @@ class AWQMarlinConfig(QuantizationConfig): self.modules_to_not_convert = list(layers - quant_layers) -class AWQMarlinLinearMethod(LinearMethodBase): +class AutoAWQMarlinLinearMethod(LinearMethodBase): """Linear method for AWQ Marlin. Uses choose_mp_linear_kernel to select the best available kernel @@ -390,16 +420,18 @@ class AWQMarlinLinearMethod(LinearMethodBase): _kernel_backends_being_used: set[str] = set() - def __init__(self, quant_config: AWQMarlinConfig) -> None: + def __init__(self, quant_config: AutoAWQConfig) -> None: self.quant_config = quant_config self.quant_type = scalar_types.uint4 self.input_dtype = None - verify_marlin_supported( - quant_type=self.quant_config.quant_type, - group_size=self.quant_config.group_size, - has_zp=self.quant_config.zero_point, - ) + # Skip Marlin verification on CPU - it will use CPUWNA16LinearKernel + if not current_platform.is_cpu(): + verify_marlin_supported( + quant_type=self.quant_config.quant_type, + group_size=self.quant_config.group_size, + has_zp=self.quant_config.zero_point, + ) def create_weights( self, @@ -435,7 +467,7 @@ class AWQMarlinLinearMethod(LinearMethodBase): kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config) if kernel_type.__name__ not in self._kernel_backends_being_used: - logger.info("Using %s for AWQMarlinLinearMethod", kernel_type.__name__) + logger.info("Using %s for AutoAWQMarlinLinearMethod", kernel_type.__name__) self._kernel_backends_being_used.add(kernel_type.__name__) # Weights are loaded in AWQ checkpoint format (packed along output dim). @@ -509,16 +541,16 @@ class AWQMarlinLinearMethod(LinearMethodBase): return self.kernel.apply_weights(layer, x, bias) -class AWQMarlinMoEMethod(FusedMoEMethodBase): +class AutoAWQMoEMethod(FusedMoEMethodBase): def __init__( self, - quant_config: AWQMarlinConfig, + quant_config: AutoAWQConfig, moe: FusedMoEConfig, ): super().__init__(moe) self.quant_config = quant_config if self.quant_config.weight_bits != 4: - raise ValueError("AWQMarlinMoEMethod only supports 4bit now.") + raise ValueError("AutoAWQMoEMethod only supports 4bit now.") self.quant_type = scalar_types.uint4 self.input_dtype = None self.use_marlin = True @@ -784,3 +816,185 @@ class AWQMarlinMoEMethod(FusedMoEMethodBase): e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, ) + + +class BaseAWQLinearMethod(LinearMethodBase): + """Base class for AWQ linear methods with shared weight creation logic.""" + + def __init__(self, quant_config: AutoAWQConfig): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + # Normalize group_size + if self.quant_config.group_size != -1: + group_size = self.quant_config.group_size + else: + group_size = input_size + + if input_size_per_partition % group_size != 0: + raise ValueError( + "The input size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size." + ) + + output_size_per_partition = sum(output_partition_sizes) + if output_size_per_partition % self.quant_config.pack_factor != 0: + raise ValueError( + "The output size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size." + ) + + weight_loader = extra_weight_attrs.get("weight_loader") + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader, + ) + + num_groups = input_size_per_partition // group_size + + qzeros = PackedvLLMParameter( + data=torch.empty( + num_groups, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader, + ) + + scales = GroupQuantScaleParameter( + data=torch.empty( + num_groups, + output_size_per_partition, + dtype=params_dtype, + ), + input_dim=0, + output_dim=1, + weight_loader=weight_loader, + ) + + layer.register_parameter("qweight", qweight) + layer.register_parameter("qzeros", qzeros) + layer.register_parameter("scales", scales) + + +class AutoAWQLinearMethod(BaseAWQLinearMethod): + """Linear method for AWQ using Triton kernels. + + Args: + quant_config: The AWQ quantization config. + """ + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False) + layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False) + layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + qweight = layer.qweight + scales = layer.scales + qzeros = layer.qzeros + pack_factor = self.quant_config.pack_factor + out_shape = x.shape[:-1] + (qweight.shape[-1] * pack_factor,) + reshaped_x = x.reshape(-1, x.shape[-1]) + + # num_tokens >= threshold + FP16_MATMUL_HEURISTIC_CONDITION = x.shape[:-1].numel() >= 256 + # Batch invariant mode requires torch.matmul path + # for Triton override + if FP16_MATMUL_HEURISTIC_CONDITION or envs.VLLM_BATCH_INVARIANT: + out = ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0) + out = torch.matmul(reshaped_x, out) + else: + out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros, pack_factor) + if bias is not None: + out.add_(bias) + return out.reshape(out_shape) + + +class AutoAWQXPULinearMethod(BaseAWQLinearMethod): + """Linear method for AWQ on XPU using int4 GEMM kernel. + + Args: + quant_config: The AWQ quantization config. + """ + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False) + layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False) + layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False) + + try: + from vllm_xpu_kernels.quantization._quantize_convert import ( + AWQUtils, + transpose_onednn_woq_format, + ) + except ImportError as e: + raise ImportError( + "XPU AWQ requires vllm-xpu-kernels. " + "Please install it with: pip install vllm-xpu-kernels" + ) from e + + layer.xpu_output_size = layer.qweight.size(1) * self.quant_config.pack_factor + qweight_new, qzeros_new = AWQUtils.repack(layer.qweight, layer.qzeros) + if qweight_new.shape != layer.qweight.data.shape: + layer.qweight.data = layer.qweight.data.view_as(qweight_new) + if qzeros_new.shape != layer.qzeros.data.shape: + layer.qzeros.data = layer.qzeros.data.view_as(qzeros_new) + layer.qweight.data.copy_(qweight_new) + layer.qzeros.data.copy_(qzeros_new) + transpose_onednn_woq_format(layer, "awq", False) + + def _get_group_size(self, layer: torch.nn.Module) -> int: + """Get the effective group size for kernel computation.""" + if self.quant_config.group_size != -1: + return self.quant_config.group_size + return layer.qweight.shape[0] # input_size_per_partition + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + reshaped_x = x.reshape(-1, x.shape[-1]) + group_size = self._get_group_size(layer) + + out = torch.ops._xpu_C.int4_gemm_w4a16( + reshaped_x, + layer.qweight, + bias, + layer.scales, + layer.qzeros, + group_size, + None, + ) + out_shape = x.shape[:-1] + (layer.xpu_output_size,) + return out.reshape(out_shape) diff --git a/vllm/model_executor/layers/quantization/awq.py b/vllm/model_executor/layers/quantization/awq.py deleted file mode 100644 index edacfc7633..0000000000 --- a/vllm/model_executor/layers/quantization/awq.py +++ /dev/null @@ -1,286 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import TYPE_CHECKING, Any, Union - -import torch -from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE -from transformers import PretrainedConfig - -from vllm import _custom_ops as ops -from vllm import envs -from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import RoutedExperts -from vllm.model_executor.layers.linear import ( - LinearBase, - LinearMethodBase, - UnquantizedLinearMethod, -) -from vllm.model_executor.layers.quantization.base_config import ( - QuantizationConfig, - QuantizeMethodBase, -) -from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped -from vllm.model_executor.parameter import GroupQuantScaleParameter, PackedvLLMParameter -from vllm.transformers_utils.config import get_safetensors_params_metadata - -if TYPE_CHECKING: - from vllm.model_executor.layers.quantization import QuantizationMethods - from vllm.model_executor.models.utils import WeightsMapper - -logger = init_logger(__name__) - - -class AWQConfig(QuantizationConfig): - """Config class for AWQ. - - Reference: https://arxiv.org/abs/2306.00978 - """ - - def __init__( - self, - weight_bits: int, - group_size: int, - zero_point: bool, - modules_to_not_convert: list[str] | None = None, - ) -> None: - super().__init__() - self.weight_bits = weight_bits - self.group_size = group_size - self.zero_point = zero_point - self.modules_to_not_convert = modules_to_not_convert or [] - - if self.weight_bits != 4: - raise ValueError( - "Currently, only 4-bit weight quantization is supported for " - f"AWQ, but got {self.weight_bits} bits." - ) - self.pack_factor = 32 // self.weight_bits - - def __repr__(self) -> str: - return ( - f"AWQConfig(weight_bits={self.weight_bits}, " - f"group_size={self.group_size}, " - f"zero_point={self.zero_point}, " - f"modules_to_not_convert={self.modules_to_not_convert})" - ) - - def get_name(self) -> "QuantizationMethods": - return "awq" - - def get_supported_act_dtypes(self) -> list[torch.dtype]: - return [torch.half] - - @classmethod - def get_min_capability(cls) -> int: - # The AWQ kernel only supports Turing or newer GPUs. - return 75 - - @staticmethod - def get_config_filenames() -> list[str]: - return [ - "quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq - # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq - "quantize_config.json", - ] - - @classmethod - def from_config(cls, config: dict[str, Any]) -> "AWQConfig": - weight_bits = cls.get_from_keys(config, ["w_bit", "bits"]) - group_size = cls.get_from_keys(config, ["q_group_size", "group_size"]) - zero_point = cls.get_from_keys(config, ["zero_point"]) - modules_to_not_convert = cls.get_from_keys_or( - config, ["modules_to_not_convert"], None - ) - return cls(weight_bits, group_size, zero_point, modules_to_not_convert) - - def get_quant_method( - self, layer: torch.nn.Module, prefix: str - ) -> Union["LinearMethodBase", "QuantizeMethodBase"] | None: - if isinstance(layer, LinearBase): - if is_layer_skipped( - prefix, - self.modules_to_not_convert, - self.packed_modules_mapping, - skip_with_substr=True, - ): - return UnquantizedLinearMethod() - return AWQLinearMethod(self) - elif isinstance(layer, RoutedExperts): - # Lazy import to avoid circular import. - from .awq_marlin import AWQMarlinConfig - from .moe_wna16 import MoeWNA16Config - from .utils.marlin_utils import check_moe_marlin_supports_layer - - if not check_moe_marlin_supports_layer(layer, self.group_size): - logger.warning_once( - f"Layer '{prefix}' is not supported by AWQMoeMarlin. " - "Falling back to Moe WNA16 kernels." - ) - config = { - "quant_method": "awq", - "bits": self.weight_bits, - "group_size": self.group_size, - "zero_point": self.zero_point, - "lm_head": False, - "modules_to_not_convert": self.modules_to_not_convert, - } - return MoeWNA16Config.from_config(config).get_quant_method( - layer, prefix - ) - marlin_compatible_config_dict = { - "quant_method": "awq", - "bits": self.weight_bits, - "group_size": self.group_size, - "zero_point": self.zero_point, - "lm_head": False, - "modules_to_not_convert": self.modules_to_not_convert, - } - awq_marlin_config = AWQMarlinConfig.from_config( - marlin_compatible_config_dict - ) - return awq_marlin_config.get_quant_method(layer, prefix) - return None - - def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"): - if self.modules_to_not_convert: - self.modules_to_not_convert = hf_to_vllm_mapper.apply_list( - self.modules_to_not_convert - ) - - def maybe_update_config( - self, - model_name: str, - hf_config: PretrainedConfig | None = None, - revision: str | None = None, - ): - if self.modules_to_not_convert: - return - - unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32] - metadata = get_safetensors_params_metadata(model_name, revision=revision) - layers = {param_name.rsplit(".", 1)[0] for param_name in metadata} - quant_layers: set[str] = { - param_name.rsplit(".", 1)[0] - for param_name, info in metadata.items() - if (dtype := info.get("dtype", None)) - and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes - } - self.modules_to_not_convert = list(layers - quant_layers) - - -class AWQLinearMethod(LinearMethodBase): - """Linear method for AWQ. - - Args: - quant_config: The AWQ quantization config. - """ - - def __init__(self, quant_config: AWQConfig): - self.quant_config = quant_config - - def create_weights( - self, - layer: torch.nn.Module, - input_size_per_partition: int, - output_partition_sizes: list[int], - input_size: int, - output_size: int, - params_dtype: torch.dtype, - **extra_weight_attrs, - ): - # Normalize group_size - if self.quant_config.group_size != -1: - group_size = self.quant_config.group_size - else: - group_size = input_size - - if input_size_per_partition % group_size != 0: - raise ValueError( - "The input size is not aligned with the quantized " - "weight shape. This can be caused by too large " - "tensor parallel size." - ) - - output_size_per_partition = sum(output_partition_sizes) - if output_size_per_partition % self.quant_config.pack_factor != 0: - raise ValueError( - "The output size is not aligned with the quantized " - "weight shape. This can be caused by too large " - "tensor parallel size." - ) - - weight_loader = extra_weight_attrs.get("weight_loader") - qweight = PackedvLLMParameter( - data=torch.empty( - input_size_per_partition, - output_size_per_partition // self.quant_config.pack_factor, - dtype=torch.int32, - ), - input_dim=0, - output_dim=1, - packed_dim=1, - packed_factor=self.quant_config.pack_factor, - weight_loader=weight_loader, - ) - - num_groups = input_size_per_partition // group_size - - qzeros = PackedvLLMParameter( - data=torch.empty( - num_groups, - output_size_per_partition // self.quant_config.pack_factor, - dtype=torch.int32, - ), - input_dim=0, - output_dim=1, - packed_dim=1, - packed_factor=self.quant_config.pack_factor, - weight_loader=weight_loader, - ) - - scales = GroupQuantScaleParameter( - data=torch.empty( - num_groups, - output_size_per_partition, - dtype=params_dtype, - ), - input_dim=0, - output_dim=1, - weight_loader=weight_loader, - ) - - layer.register_parameter("qweight", qweight) - layer.register_parameter("qzeros", qzeros) - layer.register_parameter("scales", scales) - - def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False) - layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False) - layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False) - - def apply( - self, - layer: torch.nn.Module, - x: torch.Tensor, - bias: torch.Tensor | None = None, - ) -> torch.Tensor: - qweight = layer.qweight - scales = layer.scales - qzeros = layer.qzeros - pack_factor = self.quant_config.pack_factor - out_shape = x.shape[:-1] + (qweight.shape[-1] * pack_factor,) - reshaped_x = x.reshape(-1, x.shape[-1]) - - # num_tokens >= threshold - FP16_MATMUL_HEURISTIC_CONDITION = x.shape[:-1].numel() >= 256 - # Batch invariant mode requires torch.matmul path - # for Triton override - if FP16_MATMUL_HEURISTIC_CONDITION or envs.VLLM_BATCH_INVARIANT: - out = ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0) - out = torch.matmul(reshaped_x, out) - else: - out = ops.awq_gemm(reshaped_x, qweight, scales, qzeros, pack_factor) - if bias is not None: - out.add_(bias) - return out.reshape(out_shape) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py index e0ffc6ac28..646865bbfc 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py @@ -8,9 +8,8 @@ import torch from torch.nn.parameter import Parameter from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig -from vllm.model_executor.layers.quantization.awq import AWQConfig -from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig from vllm.model_executor.layers.quantization.utils.marlin_utils import ( check_marlin_supported, ) @@ -125,12 +124,12 @@ class INCWNA16LinearScheme(INCLinearScheme): ) if use_marlin: - from vllm.model_executor.layers.quantization.awq_marlin import ( - AWQMarlinLinearMethod, + from vllm.model_executor.layers.quantization.auto_awq import ( + AutoAWQMarlinLinearMethod, ) - return AWQMarlinLinearMethod( - AWQMarlinConfig( + return AutoAWQMarlinLinearMethod( + AutoAWQConfig( weight_bits=self.layer_config.bits, group_size=self.layer_config.group_size, zero_point=not self.layer_config.sym, @@ -140,13 +139,16 @@ class INCWNA16LinearScheme(INCLinearScheme): ) ) - from vllm.model_executor.layers.quantization.awq import AWQLinearMethod + from vllm.model_executor.layers.quantization.auto_awq import ( + AutoAWQLinearMethod, + ) - return AWQLinearMethod( - AWQConfig( + return AutoAWQLinearMethod( + AutoAWQConfig( weight_bits=self.layer_config.bits, group_size=self.layer_config.group_size, zero_point=not self.layer_config.sym, + lm_head_quantized=False, ) ) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py index 7b6c10de2a..e994b03494 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py @@ -4,8 +4,8 @@ from typing import TYPE_CHECKING from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig -from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig from vllm.platforms import current_platform from vllm.scalar_type import scalar_types @@ -154,7 +154,7 @@ def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"): def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"): - from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinMoEMethod + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQMoEMethod from vllm.model_executor.layers.quantization.moe_wna16 import ( MoeWNA16Config, MoeWNA16Method, @@ -177,8 +177,8 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"): ) and check_moe_marlin_supports_layer(layer, layer_config.group_size) if use_marlin: - return AWQMarlinMoEMethod( - AWQMarlinConfig( + return AutoAWQMoEMethod( + AutoAWQConfig( weight_bits=layer_config.bits, group_size=layer_config.group_size, zero_point=not layer_config.sym, diff --git a/vllm/model_executor/layers/quantization/moe_wna16.py b/vllm/model_executor/layers/quantization/moe_wna16.py index ee4b455ddc..2dabfd436f 100644 --- a/vllm/model_executor/layers/quantization/moe_wna16.py +++ b/vllm/model_executor/layers/quantization/moe_wna16.py @@ -27,9 +27,6 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, ) -from vllm.model_executor.layers.quantization.utils.marlin_utils import ( - check_marlin_supports_layer, -) from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform @@ -55,10 +52,8 @@ class MoeWNA16Config(QuantizationConfig): self.lm_head_quantized = lm_head_quantized self.linear_quant_method = linear_quant_method self.full_config = full_config - self.use_marlin = False # Avoid circular import - from vllm.model_executor.layers.quantization.awq import AWQConfig - from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig if self.linear_quant_method == "gptq": pass @@ -67,7 +62,7 @@ class MoeWNA16Config(QuantizationConfig): device_capability = ( -1 if capability_tuple is None else capability_tuple.to_int() ) - awq_min_capability = AWQConfig.get_min_capability() + awq_min_capability = AutoAWQConfig.get_min_capability() if device_capability < awq_min_capability: raise ValueError( "The quantization method moe_wna16 + awq is not supported " @@ -75,7 +70,6 @@ class MoeWNA16Config(QuantizationConfig): f"Minimum capability: {awq_min_capability}. " f"Current capability: {device_capability}." ) - self.use_marlin = AWQMarlinConfig.is_awq_marlin_compatible(full_config) else: raise ValueError("moe_wna16 only support gptq and awq.") @@ -148,9 +142,9 @@ class MoeWNA16Config(QuantizationConfig): -1 if capability_tuple is None else capability_tuple.to_int() ) # Avoid circular import - from vllm.model_executor.layers.quantization.awq import AWQConfig + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig - awq_min_capability = AWQConfig.get_min_capability() + awq_min_capability = AutoAWQConfig.get_min_capability() gptq_compatible = quant_method == "gptq" and not desc_act and num_bits in [4, 8] awq_compatible = ( @@ -170,29 +164,19 @@ class MoeWNA16Config(QuantizationConfig): return UnquantizedLinearMethod() elif isinstance(layer, LinearBase): # Avoid circular import + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.model_executor.layers.quantization.auto_gptq import ( AutoGPTQConfig, ) - from vllm.model_executor.layers.quantization.awq import AWQConfig - from vllm.model_executor.layers.quantization.awq_marlin import ( - AWQMarlinConfig, - ) if self.linear_quant_method == "gptq": return AutoGPTQConfig.from_config(self.full_config).get_quant_method( layer, prefix ) elif self.linear_quant_method in ("awq", "awq_marlin"): - if self.use_marlin and check_marlin_supports_layer( - layer, self.group_size - ): - return AWQMarlinConfig.from_config( - self.full_config - ).get_quant_method(layer, prefix) - else: - return AWQConfig.from_config(self.full_config).get_quant_method( - layer, prefix - ) + return AutoAWQConfig.from_config(self.full_config).get_quant_method( + layer, prefix + ) else: raise ValueError("moe_wna16 only support gptq and awq.") elif isinstance(layer, RoutedExperts): diff --git a/vllm/model_executor/models/cohere2_vision.py b/vllm/model_executor/models/cohere2_vision.py index c800c21492..302619a8db 100644 --- a/vllm/model_executor/models/cohere2_vision.py +++ b/vllm/model_executor/models/cohere2_vision.py @@ -26,7 +26,7 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.quantization.awq import AWQConfig +from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, @@ -420,7 +420,7 @@ class Cohere2VisionForConditionalGeneration( ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back - if isinstance(quant_config, AWQConfig): + if isinstance(quant_config, AutoAWQConfig): text_config = config.text_config llm_quant_config = getattr(text_config, "quantization_config", None) if (not quant_config.modules_to_not_convert) and ( diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index b75b9c4f20..eae9e66fb7 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -20,7 +20,7 @@ from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.quantization.awq import AWQConfig +from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.model_executor.models.intern_vit import ( InternVisionModel, ) @@ -608,7 +608,7 @@ class InternVLChatModel( ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back - if isinstance(quant_config, AWQConfig): + if isinstance(quant_config, AutoAWQConfig): text_config = config.text_config llm_quant_config = getattr(text_config, "quantization_config", None) if (not quant_config.modules_to_not_convert) and ( diff --git a/vllm/model_executor/models/nemotron_vl.py b/vllm/model_executor/models/nemotron_vl.py index 5b22a607a2..734968819b 100644 --- a/vllm/model_executor/models/nemotron_vl.py +++ b/vllm/model_executor/models/nemotron_vl.py @@ -11,7 +11,7 @@ from vllm.config import VllmConfig from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.pooler import DispatchPooler from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.quantization.awq import AWQConfig +from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.model_executor.models.internvl import ( BaseInternVLDummyInputsBuilder, BaseInternVLMultiModalProcessor, @@ -144,7 +144,7 @@ class LlamaNemotronVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, Suppor ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back - if isinstance(quant_config, AWQConfig): + if isinstance(quant_config, AutoAWQConfig): text_config = config.get_text_config() llm_quant_config = getattr(text_config, "quantization_config", None) if (not quant_config.modules_to_not_convert) and ( diff --git a/vllm/model_executor/models/skyworkr1v.py b/vllm/model_executor/models/skyworkr1v.py index 685b980c3f..d57da08598 100644 --- a/vllm/model_executor/models/skyworkr1v.py +++ b/vllm/model_executor/models/skyworkr1v.py @@ -19,7 +19,7 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.quantization.awq import AWQConfig +from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig from vllm.model_executor.models.intern_vit import ( InternVisionModel, ) @@ -205,7 +205,7 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP): ): # the awq models from OpenGVLab missing `modules_to_not_convert` # patch the quant_config to add `modules_to_not_convert` back - if isinstance(quant_config, AWQConfig): + if isinstance(quant_config, AutoAWQConfig): text_config = config.text_config llm_quant_config = getattr(text_config, "quantization_config", None) if (not quant_config.modules_to_not_convert) and ( diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 13695a142e..9662037b01 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -440,6 +440,7 @@ class RocmPlatform(Platform): supported_quantization: list[str] = [ "awq", + "auto_awq", "awq_marlin", # will be overwritten with awq "gptq", "gptq_marlin",