diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index ced034cb1a..39481456fd 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -58,12 +58,17 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501 from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501 QuarkMoEMethod, QuarkW4A8Fp8MoEMethod, + QuarkW4A16Int4MoEMethod, QuarkW8A8Fp8MoEMethod, QuarkW8A8Int8MoEMethod, ) -from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme +from vllm.model_executor.layers.quantization.quark.schemes import ( + QuarkScheme, + QuarkW4A16Int4, +) from vllm.model_executor.layers.quantization.quark.utils import ( QuarkQTensorHint, + canonicalize_quark_packed_int4, should_ignore_layer, ) from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( @@ -97,6 +102,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kNvfp4Static, ) from vllm.model_executor.models.llama import LlamaForCausalLM +from vllm.model_executor.models.utils import WeightsMapper from vllm.platforms import current_platform from vllm.transformers_utils.repo_utils import hf_api @@ -1876,3 +1882,435 @@ def test_glm5next_attn_loader_accepts_deepseek_weight_scale_inv(): ) assert target.loaded_shard is None # o_proj is a direct (non-fused) proj assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128)) + + +_REVERSE_AWQ_PACK_ORDER = [0, 4, 1, 5, 2, 6, 3, 7] + + +def _quark_int4_config( + *, + pack_method: str = "reorder", + symmetric: bool = True, + exclude: list[str] | None = None, +) -> dict: + return { + "quant_method": "quark", + "export": {"pack_method": pack_method, "kv_cache_group": []}, + "global_quant_config": { + "weight": { + "dtype": "int4", + "group_size": 128, + "symmetric": symmetric, + } + }, + "exclude": exclude or [], + } + + +def _sign_extend_int4_nibbles(t: torch.Tensor) -> torch.Tensor: + mask = (t & 0x8).bool() + t = t.clone() + t[mask] = t[mask] | 0xF0 + return t + + +def _dequantize_quark_signed_awq_torch( + qweight: torch.Tensor, + scales: torch.Tensor, + qzeros: torch.Tensor, + group_size: int, + *, + pack_reorder: bool = True, +) -> torch.Tensor: + bits = 4 + shifts = torch.arange(0, 32, bits, device=qweight.device) + iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + iweights = iweights.view(qweight.shape[0], -1) + zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + zeros = zeros.view(qzeros.shape[0], -1) + + if pack_reorder: + order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device) + else: + order = torch.arange(8, device=qweight.device) + iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape( + qweight.shape[0], -1 + ) + zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1) + iweights = _sign_extend_int4_nibbles(iweights & 0xF) + zeros = _sign_extend_int4_nibbles(zeros & 0xF) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +def _dequantize_awq_unsigned_torch( + qweight: torch.Tensor, + scales: torch.Tensor, + qzeros: torch.Tensor, + group_size: int, + *, + pack_reorder: bool = True, +) -> torch.Tensor: + bits = 4 + shifts = torch.arange(0, 32, bits, device=qweight.device) + iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + iweights = iweights.view(qweight.shape[0], -1) + zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8) + zeros = zeros.view(qzeros.shape[0], -1) + + if pack_reorder: + order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device) + else: + order = torch.arange(8, device=qweight.device) + iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape( + qweight.shape[0], -1 + ) + zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +def _pack_int4_nibbles(nibbles: torch.Tensor, *, pack_reorder: bool) -> torch.Tensor: + pack_order = ( + torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=nibbles.device) + if pack_reorder + else torch.arange(8, device=nibbles.device) + ) + shifts = pack_order * 4 + return ((nibbles.to(torch.int64) & 0xF) << shifts).sum(dim=-1).to(torch.int32) + + +class TestQuarkInt4Format: + """Tests for Quark INT4 export format compatibility.""" + + def test_quark_order_pack_method_uses_native_int4_scheme(self): + quant_config = QuarkConfig.from_config(_quark_int4_config(pack_method="order")) + weight_config = quant_config.quant_config["global_quant_config"] + weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config( + weight_config + ) + assert scheme_cls is QuarkW4A16Int4 + + scheme = quant_config.init_scheme( + scheme_cls, + weight_quant_key=weight_key, + activation_quant_key=act_key, + weight_config=weight_config["weight"], + ) + assert isinstance(scheme, QuarkW4A16Int4) + assert not scheme.pack_reorder + + def test_quark_int4_scheme_supports_asymmetric_weights(self): + quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False)) + weight_config = quant_config.quant_config["global_quant_config"] + weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config( + weight_config + ) + assert scheme_cls is QuarkW4A16Int4 + + scheme = quant_config.init_scheme( + scheme_cls, + weight_quant_key=weight_key, + activation_quant_key=act_key, + weight_config=weight_config["weight"], + ) + assert isinstance(scheme, QuarkW4A16Int4) + assert not scheme.is_symmetric + + @pytest.mark.parametrize("missing_field", ["group_size", "symmetric"]) + def test_quark_int4_scheme_requires_weight_config_fields(self, missing_field): + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4Static, + ) + + weight_config = { + "dtype": "int4", + "group_size": 128, + "symmetric": True, + } + weight_config.pop(missing_field) + quant_config = QuarkConfig.from_config( + { + "quant_method": "quark", + "export": {"pack_method": "reorder", "kv_cache_group": []}, + "global_quant_config": {"weight": weight_config}, + "exclude": [], + } + ) + + with pytest.raises(ValueError, match=missing_field): + quant_config.init_scheme( + QuarkW4A16Int4, + weight_quant_key=kInt4Static, + activation_quant_key=None, + weight_config=weight_config, + ) + + def test_quark_int4_moe_uses_native_moe_method(self): + from unittest.mock import MagicMock, patch + + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4StaticAsym, + ) + + quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False)) + moe_config = type("MoeConfig", (), {})() + moe_config.has_bias = False + + mock_backend = MagicMock() + mock_experts_cls = MagicMock() + with patch( + "vllm.model_executor.layers.quantization.quark.quark_moe" + ".select_wna16_moe_backend", + return_value=(mock_backend, mock_experts_cls), + ): + method = QuarkW4A16Int4MoEMethod( + kInt4StaticAsym, + None, + quant_config.quant_config["global_quant_config"]["weight"], + quant_config.pack_method, + moe_config, + ) + + assert method.group_size == 128 + assert method.pack_reorder + assert method.use_wna16_backend + + def test_triton_wna16_experts_supports_asymmetric_int4(self): + from vllm.model_executor.layers.fused_moe.experts.triton_moe import ( + TritonWNA16Experts, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, + ) + + for key in [kInt4Static, kInt4Static32, kInt4StaticAsym, kInt4Static32Asym]: + assert TritonWNA16Experts._supports_quant_scheme(key, None), ( + f"{key} should be supported by TritonWNA16Experts" + ) + + def test_quark_moe_loader_shards_w2_zero_point_across_tp_ranks(self): + """w2 zero-points are sharded along the intermediate dim per TP rank. + + Loading the same Quark checkpoint tensor at tensor_parallel_size=2 for + both ranks must jointly reconstruct the tensor_parallel_size=1 (full) + result, with no overlap or gap. This guards the custom loader's TP + split for the packed zero-point layout. + """ + + class _FakeMoEConfig: + has_bias = False + + def __init__(self, tp_size, tp_rank): + self.tp_size = tp_size + self.tp_rank = tp_rank + + class _FakeLayer: + def __init__(self, moe_config): + self.moe_config = moe_config + self.intermediate_size_per_partition = 64 + self.group_size_div_factor = 1 + + def load_w2_zero_point(raw_zp, tp_size, tp_rank): + from unittest.mock import MagicMock, patch + + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4StaticAsym, + ) + + quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False)) + moe_config = _FakeMoEConfig(tp_size, tp_rank) + with patch( + "vllm.model_executor.layers.quantization.quark.quark_moe" + ".select_wna16_moe_backend", + return_value=(MagicMock(), MagicMock()), + ): + method = QuarkW4A16Int4MoEMethod( + kInt4StaticAsym, + None, + quant_config.quant_config["global_quant_config"]["weight"], + quant_config.pack_method, + moe_config, + ) + + layer = _FakeLayer(moe_config) + # The loader converts the packed zero-point, then writes the + # per-rank slice into param.data[expert_id]. A single expert dim is + # enough; the last dim shrinks by tp_size after sharding. + param = torch.nn.Parameter( + torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8), + requires_grad=False, + ) + loader = method.get_weight_loader(layer, weight_loader=None) + loader( + param, + raw_zp.clone(), + weight_name="w2_weight_zero_point", + shard_id="w2", + expert_id=0, + ) + return param.data[0] + + # Quark-packed zero-point for one expert: rows = hidden // pack_factor, + # cols carry the (packed) intermediate dim, split evenly across ranks. + raw_zp = torch.randint(0, 256, (8, 16), dtype=torch.uint8) + + full = load_w2_zero_point(raw_zp, tp_size=1, tp_rank=0) + shard0 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=0) + shard1 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=1) + + expected = full.view(full.size(0), 2, -1) + torch.testing.assert_close(shard0, expected[:, 0]) + torch.testing.assert_close(shard1, expected[:, 1]) + + def test_quark_shared_expert_gate_keeps_quantized_tensors(self): + quant_config = QuarkConfig.from_config(_quark_int4_config()) + mapper = quant_config.get_cache_scale_mapper() + + output_names = { + name + for name, _ in mapper.apply( + [ + ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight", + torch.zeros(1), + ), + ( + "model.language_model.layers.0.mlp.shared_expert_gate" + ".weight_scale", + torch.zeros(1), + ), + ( + "model.language_model.layers.0.mlp.shared_expert_gate" + ".weight_zero_point", + torch.zeros(1), + ), + ] + ) + } + + assert ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight" + in output_names + ) + assert ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight_scale" + in output_names + ) + assert ( + "model.language_model.layers.0.mlp.shared_expert_gate.weight_zero_point" + in output_names + ) + + def test_quark_apply_mapper_updates_exclude_and_layer_quant_config(self): + quant_config = QuarkConfig.from_config( + { + **_quark_int4_config(), + "exclude": ["lm_head"], + "layer_quant_config": { + "model.language_model.layers.0.mlp.gate": { + "weight": {"dtype": "float16"}, + }, + }, + } + ) + quant_config.apply_vllm_mapper( + WeightsMapper( + orig_to_new_prefix={ + "lm_head": "language_model.lm_head", + "model.language_model.": "language_model.model.", + }, + ) + ) + + layer_quant_config = quant_config.quant_config["layer_quant_config"] + assert "language_model.model.layers.0.mlp.gate" in layer_quant_config + assert "model.language_model.layers.0.mlp.gate" not in layer_quant_config + assert quant_config.quant_config["exclude"] == [ + "language_model.lm_head", + ] + + def test_quark_apply_mapper_ignores_non_string_list_entries(self): + quant_config = QuarkConfig.from_config( + { + **_quark_int4_config(), + "algo_config": [ + {"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]} + ], + } + ) + quant_config.apply_vllm_mapper(WeightsMapper()) + + assert quant_config.quant_config["algo_config"] == [ + {"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]} + ] + + def test_quark_exclude_matches_via_mapper_prefix(self): + quant_config = QuarkConfig.from_config(_quark_int4_config(exclude=["lm_head"])) + quant_config.apply_vllm_mapper( + WeightsMapper(orig_to_new_prefix={"lm_head": "language_model.lm_head"}) + ) + + exclude_layers = quant_config.quant_config["exclude"] + assert should_ignore_layer("language_model.lm_head", ignore=exclude_layers) + assert not should_ignore_layer( + "language_model.model.layers.0.mlp.gate", ignore=exclude_layers + ) + + +@pytest.mark.parametrize("symmetric", [False, True]) +@pytest.mark.parametrize("pack_method", ["order", "reorder"]) +def test_quark_int4_canonicalizes_pack_for_kernel_layout(pack_method, symmetric): + """Quark pack order is normalized to the layout expected by awq_* ops. + + This reuses the existing AWQ dequant/gemm kernels for compute only; loading + still goes through the native Quark quantization path, not AutoAWQ. + """ + pack_reorder = pack_method == "reorder" + group_size = 2 + packed_values = torch.tensor( + [ + [0, 1, 7, 8, 9, 15, 2, 14], + [15, 8, 0, 3, 12, 7, 1, 9], + ], + dtype=torch.int32, + ) + packed_zeros = torch.zeros((1, 8), dtype=torch.int32) + qweight = _pack_int4_nibbles(packed_values, pack_reorder=pack_reorder).view(2, 1) + qzeros = _pack_int4_nibbles(packed_zeros, pack_reorder=pack_reorder).view(1, 1) + scales = torch.ones((1, 8), dtype=torch.float16) + + dequantize = ( + _dequantize_quark_signed_awq_torch + if symmetric + else _dequantize_awq_unsigned_torch + ) + expected = dequantize( + qweight, + scales, + qzeros, + group_size, + pack_reorder=pack_reorder, + ) + canonical_weight = canonicalize_quark_packed_int4( + qweight, + pack_reorder=pack_reorder, + is_symmetric=symmetric, + ) + canonical_zero = canonicalize_quark_packed_int4( + qzeros, + pack_reorder=pack_reorder, + is_symmetric=symmetric, + ) + actual = _dequantize_awq_unsigned_torch( + canonical_weight, scales, canonical_zero, group_size + ) + + assert torch.equal(actual, expected) diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 3c9c616d0c..8c57059e57 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -864,6 +864,7 @@ def choose_mp_linear_kernel( can_implement, failure_reason = kernel.can_implement(config) if can_implement: + logger.info_once("Using %s for mixed-precision linear", kernel.__name__) return kernel else: failure_reasons.append( diff --git a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py index 9f982644b5..da9666a66b 100644 --- a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py @@ -596,9 +596,9 @@ class TritonWNA16Experts(TritonExperts): ) -> bool: SUPPORTED_W = [ kInt4Static, + kInt4StaticAsym, kInt8Static, kInt4Static32, - kInt4StaticAsym, kInt4Static32Asym, # other group sizes? ] diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index 2c123b8768..71b858e919 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -93,7 +93,7 @@ def _replace_or_register_parameter( def _convert_awq_to_standard_format( layer: torch.nn.Module, w_q_name: str, - w_zp_name: str, + w_zp_name: str | None, size_bits: int, ) -> None: """Convert AWQ weight and zero-point tensors to standard GPTQ-like format. @@ -101,6 +101,7 @@ def _convert_awq_to_standard_format( AWQ packs qweight along the output dim with a non-standard bit order. This converts to standard bit order and repacks qweight along the input dim, matching the format expected by the MPLinearKernel framework. + If w_zp_name is None (symmetric quantization), only the weight is converted. """ pack_factor = 32 // size_bits mask = (1 << size_bits) - 1 @@ -139,6 +140,9 @@ def _convert_awq_to_standard_format( ) setattr(layer, w_q_name, new_param) + if w_zp_name is None: + return + # --- Convert qzeros: fix AWQ bit ordering and repack # AWQ qzeros: (G, N // pack) packed along dim 1, AWQ bit order # Target: (N // pack, G) packed along dim 0, standard bit order diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index 88544fbe99..a3a3a36f00 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import ( QuarkOCP_MX, QuarkScheme, QuarkW4A8_MXFP4_FP8, + QuarkW4A16Int4, QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock, QuarkW8A8Int8, @@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import ( from vllm.model_executor.layers.quantization.quark.utils import ( QuarkQTensorHint, deep_compare, + parse_w4a16_int4_weight_config, should_ignore_layer, ) from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( @@ -54,6 +56,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, kInt4W4A8StaticChannelSym, kInt8DynamicTensorAsym, kInt8DynamicTensorSym, @@ -172,8 +178,10 @@ class QuarkConfig(QuantizationConfig): quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {} for k, v in self.quant_config.items(): - if isinstance(v, list): + if isinstance(v, list) and all(isinstance(item, str) for item in v): quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_list(v) + elif isinstance(v, list): + quant_config_with_hf_to_vllm_mapper[k] = v elif isinstance(v, dict): quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_dict(v) else: @@ -215,6 +223,9 @@ class QuarkConfig(QuantizationConfig): weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key, dynamic_mxfp4_quant=dynamic_mxfp4_quant, + weight_config=self._find_matched_config(prefix, type(layer)).get( + "weight" + ), ) layer.scheme = scheme return QuarkLinearMethod(self) @@ -227,6 +238,7 @@ class QuarkConfig(QuantizationConfig): method_cls=method_cls, weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key, + layer_name=prefix, ) return None @@ -572,6 +584,26 @@ class QuarkConfig(QuantizationConfig): ) return QuantKeyMatch(True, activation_quant_key, weight_quant_key) + def _is_w4a16_int4( + self, + weight_quant: dict[str, Any] | None, + input_quant: dict[str, Any] | None, + ) -> QuantKeyMatch: + if weight_quant is None or input_quant is not None: + return QuantKeyMatch(False, None, None) + + is_int4 = weight_quant.get("dtype") in ("int4", "uint4") + is_packed = self.pack_method in ("order", "reorder") + if not (is_int4 and is_packed): + return QuantKeyMatch(False, None, None) + + group_size, is_symmetric = parse_w4a16_int4_weight_config(weight_quant) + if is_symmetric: + weight_key = kInt4Static32 if group_size == 32 else kInt4Static + else: + weight_key = kInt4Static32Asym if group_size == 32 else kInt4StaticAsym + return QuantKeyMatch(True, None, weight_key) + def _is_w4a8_mxfp4_fp8( self, weight_quant: dict[str, Any] | None, @@ -890,6 +922,12 @@ class QuarkConfig(QuantizationConfig): match.activation_quant_key, QuarkW8A8Int8, ) + elif match := self._is_w4a16_int4(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW4A16Int4, + ) elif match := self._is_w4a8_mxfp4_fp8(weight_config, input_config): return ( match.weight_quant_key, @@ -919,6 +957,7 @@ class QuarkConfig(QuantizationConfig): weight_quant_key: QuantKey | None, activation_quant_key: QuantKey | None, dynamic_mxfp4_quant: bool = False, + weight_config: dict[str, Any] | None = None, ) -> "QuarkScheme": """Construct a Quark scheme selected by get_scheme_cls.""" if scheme_cls not in ( @@ -928,9 +967,24 @@ class QuarkConfig(QuantizationConfig): QuarkOCP_MX, QuarkNVFP4, QuarkW4A8_MXFP4_FP8, + QuarkW4A16Int4, ): raise AssertionError(f"Unsupported Quark scheme class: {scheme_cls}") + scheme: QuarkScheme + if scheme_cls is QuarkW4A16Int4: + normalized_weight_config = self._unwrap_single_quant_config(weight_config) + assert normalized_weight_config is not None + assert weight_quant_key is not None + scheme = QuarkW4A16Int4( + weight_quant_key=weight_quant_key, + activation_quant_key=activation_quant_key, + pack_method=self.pack_method, + weight_config=normalized_weight_config, + ) + self._check_scheme_supported(scheme.get_min_capability()) + return scheme + kwargs: dict[str, Any] = {"activation_quant_key": activation_quant_key} # These classes handle different possible `weight_quant_key`. diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index d2472524a5..3a9f6d6366 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import torch @@ -40,6 +40,12 @@ from vllm.model_executor.layers.fused_moe.oracle.int8 import ( make_int8_moe_quant_config, select_int8_moe_backend, ) +from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( + convert_to_wna16_moe_kernel_format, + make_wna16_moe_kernel, + make_wna16_moe_quant_config, + select_wna16_moe_backend, +) from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( TRITON_BACKENDS, Mxfp4MoeBackend, @@ -56,7 +62,10 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( make_nvfp4_moe_quant_config, select_nvfp4_moe_backend, ) -from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint +from vllm.model_executor.layers.quantization.quark.utils import ( + QuarkQTensorHint, + parse_w4a16_int4_weight_config, +) from vllm.model_executor.layers.quantization.utils.fp8_utils import ( validate_fp8_block_shape_moe, ) @@ -75,6 +84,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, kInt4W4A8StaticChannelSym, kInt8DynamicTensorAsym, kInt8DynamicTensorSym, @@ -102,6 +115,7 @@ logger = init_logger(__name__) __all__ = [ "QuarkMoEMethod", + "QuarkW4A16Int4MoEMethod", "QuarkW8A8Fp8MoEMethod", "QuarkOCP_MX_MoEMethod", "QuarkNvfp4MoEMethod", @@ -179,6 +193,12 @@ class QuarkMoEMethod(FusedMoEMethodBase): match.activation_quant_key, QuarkW8A8Fp8MoEMethod, ) + if match := quant_config._is_w4a16_int4(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW4A16Int4MoEMethod, + ) if match := quant_config._is_w_ocp_mx_a_x( weight_config, input_config, allow_static_fp8=True ): @@ -202,6 +222,7 @@ class QuarkMoEMethod(FusedMoEMethodBase): method_cls: type["QuarkMoEMethod"], weight_quant_key: QuantKey | None, activation_quant_key: QuantKey | None, + layer_name: str | None = None, ) -> "QuarkMoEMethod": if weight_quant_key is None: raise RuntimeError("Unsupported FusedMoe scheme") @@ -226,10 +247,339 @@ class QuarkMoEMethod(FusedMoEMethodBase): return QuarkW8A8Int8MoEMethod( module.moe_config, weight_quant_key, activation_quant_key ) + elif method_cls is QuarkW4A16Int4MoEMethod: + assert layer_name is not None + layer_quant_config = quant_config._find_matched_config( + layer_name, type(module) + ) + weight_config = quant_config._unwrap_single_quant_config( + layer_quant_config.get("weight") + ) + if weight_config is None: + raise RuntimeError( + f"Missing weight quantization config for layer {layer_name}" + ) + return QuarkW4A16Int4MoEMethod( + weight_quant_key, + activation_quant_key, + weight_config, + quant_config.pack_method, + module.moe_config, + quant_config=quant_config, + ) else: raise RuntimeError("Unsupported FusedMoe scheme") +class QuarkW4A16Int4MoEMethod(QuarkMoEMethod): + """Quark packed INT4 weight-only MoE method.""" + + supported_activation_quant_keys: list[QuantKey | None] = [None] + supported_weight_quant_keys: list[QuantKey] = [ + kInt4Static, + kInt4Static32, + kInt4StaticAsym, + kInt4Static32Asym, + ] + + def __init__( + self, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, + weight_config: dict[str, Any], + pack_method: str, + moe: FusedMoEConfig, + quant_config: "QuarkConfig | None" = None, # type: ignore # noqa F821 + ): + super().__init__(moe, weight_quant_key, activation_quant_key) + + self.quant_config = quant_config + # group_size is read from the Quark config rather than weight_quant_key: + # the int4 QuantKeys only encode group_size 32 or -1, but Quark int4 + # checkpoints commonly use group_size 128, which the QuantKey cannot + # represent. Symmetry is unambiguous, so it comes from the key. + self.group_size = parse_w4a16_int4_weight_config(weight_config)[0] + self.is_symmetric = weight_quant_key.symmetric + self.bit8_pack_factor = 2 + self.pack_reorder = pack_method == "reorder" + + self.use_wna16_backend = True + self.wna16_backend, self.experts_cls = select_wna16_moe_backend( + config=self.moe, + weight_key=weight_quant_key, + quant_config=self.quant_config, + may_have_zp=not self.is_symmetric, + may_have_bias=False, + ) + + def create_weights( + self, + layer: RoutedExperts, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + group_size = self.group_size + group_size_div_factor = 1 + + while intermediate_size_per_partition % group_size or hidden_size % group_size: + group_size = group_size // 2 + group_size_div_factor *= 2 + assert group_size >= 32 + layer.group_size = group_size + layer.group_size_div_factor = group_size_div_factor + + strategy = FusedMoeWeightScaleSupported.GROUP.value + extra_weight_attrs.update({"quant_method": strategy, "is_transposed": False}) + + assert "weight_loader" in extra_weight_attrs + weight_loader = extra_weight_attrs["weight_loader"] + extra_weight_attrs["weight_loader"] = self.get_weight_loader( + layer, weight_loader + ) + + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // self.bit8_pack_factor, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition // self.bit8_pack_factor, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + w13_weight_scale = torch.nn.Parameter( + torch.zeros( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // group_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + + w2_weight_scale = torch.nn.Parameter( + torch.zeros( + num_experts, + hidden_size, + intermediate_size_per_partition // group_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + w13_weight_zero_point = torch.nn.Parameter( + torch.zeros( + num_experts, + 2 * intermediate_size_per_partition // self.bit8_pack_factor, + hidden_size // group_size, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_zero_point", w13_weight_zero_point) + set_weight_attrs(w13_weight_zero_point, extra_weight_attrs) + + w2_weight_zero_point = torch.nn.Parameter( + torch.zeros( + num_experts, + hidden_size // self.bit8_pack_factor, + intermediate_size_per_partition // group_size, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_zero_point", w2_weight_zero_point) + set_weight_attrs(w2_weight_zero_point, extra_weight_attrs) + + def process_weights_after_loading(self, layer: RoutedExperts) -> None: + w13_qzeros = layer.w13_weight_zero_point if not self.is_symmetric else None + w2_qzeros = layer.w2_weight_zero_point if not self.is_symmetric else None + + converted = convert_to_wna16_moe_kernel_format( + backend=self.wna16_backend, + layer=layer, + quant_config=self.quant_config, + input_dtype=None, + w13=layer.w13_weight, + w2=layer.w2_weight, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + w13_qzeros=w13_qzeros, + w2_qzeros=w2_qzeros, + ) + + if converted is None: + # Backend rewrote the layer's params in place (e.g. Humming). + self._setup_kernel(layer) + return + + ( + w13_qweight, + w2_qweight, + w13_scales, + w2_scales, + _, # w13_qzeros + _, # w2_qzeros + _, # w13_input_global_scale + _, # w2_input_global_scale + _, # w13_bias + _, # w2_bias + ) = converted + + replace_parameter(layer, "w13_weight", w13_qweight) + replace_parameter(layer, "w2_weight", w2_qweight) + replace_parameter(layer, "w13_weight_scale", w13_scales) + replace_parameter(layer, "w2_weight_scale", w2_scales) + + self._setup_kernel(layer) + + def _setup_kernel(self, layer: RoutedExperts) -> None: + assert self.experts_cls is not None + self.moe_quant_config = self.get_fused_moe_quant_config(layer) + assert self.moe_quant_config is not None + self.moe_kernel = make_wna16_moe_kernel( + moe_quant_config=self.moe_quant_config, + moe_config=self.moe, + experts_cls=self.experts_cls, + backend=self.wna16_backend, + routing_tables=layer._expert_routing_tables(), + ) + + def get_fused_moe_quant_config( + self, layer: RoutedExperts + ) -> FusedMoEQuantConfig | None: + w1_zp = layer.w13_weight_zero_point if not self.is_symmetric else None + w2_zp = layer.w2_weight_zero_point if not self.is_symmetric else None + return make_wna16_moe_quant_config( + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + w1_zp=w1_zp, + w2_zp=w2_zp, + group_size=layer.group_size, + num_bits=4, + ) + + def apply( + self, + layer: RoutedExperts, + x: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + shared_experts: SharedExperts | None, + shared_experts_input: torch.Tensor | None, + ) -> torch.Tensor: + assert self.moe_kernel is not None + return self.moe_kernel.apply( + hidden_states=x, + w1=layer.w13_weight, + w2=layer.w2_weight, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=layer.activation, + global_num_experts=layer.global_num_experts, + apply_router_weight_on_input=layer.apply_router_weight_on_input, + expert_map=layer.expert_map, + shared_experts=shared_experts, + shared_experts_input=shared_experts_input, + ) + + def get_weight_loader(self, layer: RoutedExperts, weight_loader): + def convert_quark_tensor(tensor: torch.Tensor, tensor_type: str): + size0 = tensor.size(0) + tensor = tensor.view(torch.uint8) + + shifter = torch.tensor([0, 4], dtype=torch.uint8, device=tensor.device) + tensor = (tensor[:, :, None] >> shifter) & 0xF + + if self.pack_reorder: + reverse_awq_pack_order = [0, 4, 1, 5, 2, 6, 3, 7] + tensor = tensor.view(-1, 8)[:, reverse_awq_pack_order] + else: + tensor = tensor.view(-1, 8) + if self.is_symmetric: + tensor = tensor ^ 0x8 + tensor = tensor.view(size0, -1) + + tensor = tensor.T.contiguous() + if tensor_type == "weight": + tensor = tensor[:, 1::2] * 16 + tensor[:, ::2] + elif tensor_type == "zero_point": + tensor = tensor[1::2, :] * 16 + tensor[::2, :] + return tensor + + def quark_int4_weight_loader( + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + return_success: bool = False, + ): + loaded_weight = loaded_weight.to(param.data.device) + shard_size = layer.intermediate_size_per_partition + + if weight_name.endswith("_weight"): + loaded_weight = convert_quark_tensor(loaded_weight, "weight") + elif weight_name.endswith("_weight_zero_point"): + loaded_weight = convert_quark_tensor(loaded_weight, "zero_point") + elif weight_name.endswith("_weight_scale"): + loaded_weight = loaded_weight.T + + if layer.group_size_div_factor > 1 and ( + "weight_zero_point" in weight_name or "weight_scale" in weight_name + ): + loaded_weight = loaded_weight.repeat_interleave( + layer.group_size_div_factor, 1 + ) + + if "w13_weight_zero_point" in weight_name: + tensor = loaded_weight.view( + layer.moe_config.tp_size, -1, loaded_weight.size(1) + )[layer.moe_config.tp_rank] + if shard_id == "w1": + param.data[expert_id, : shard_size // 2] = tensor + else: + param.data[expert_id, shard_size // 2 :] = tensor + return True if return_success else None + elif "w2_weight_zero_point" in weight_name: + param.data[expert_id] = loaded_weight.view( + loaded_weight.size(0), layer.moe_config.tp_size, -1 + )[:, layer.moe_config.tp_rank] + return True if return_success else None + + return weight_loader( + param, + loaded_weight, + weight_name, + shard_id, + expert_id, + return_success=return_success, + ) + + return quark_int4_weight_loader + + class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): supported_activation_quant_keys = [ kFp8StaticTensorSym, diff --git a/vllm/model_executor/layers/quantization/quark/schemes/__init__.py b/vllm/model_executor/layers/quantization/quark/schemes/__init__.py index 4db1ba3845..a5857a776f 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/__init__.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/__init__.py @@ -5,6 +5,7 @@ from .quark_nvfp4 import QuarkNVFP4 from .quark_ocp_mx import QuarkOCP_MX from .quark_scheme import QuarkScheme from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8 +from .quark_w4a16_int4 import QuarkW4A16Int4 from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock from .quark_w8a8_int8 import QuarkW8A8Int8 @@ -14,6 +15,7 @@ __all__ = [ "QuarkW8A8Fp8PerBlock", "QuarkW8A8Int8", "QuarkOCP_MX", + "QuarkW4A16Int4", "QuarkW4A8_MXFP4_FP8", "QuarkNVFP4", ] diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a16_int4.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a16_int4.py new file mode 100644 index 0000000000..cded0f80a9 --- /dev/null +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a16_int4.py @@ -0,0 +1,212 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import math +from collections.abc import Callable +from typing import Any + +import torch + +from vllm.model_executor.kernels.linear import ( + MPLinearKernel, + MPLinearLayerConfig, + choose_mp_linear_kernel, +) +from vllm.model_executor.layers.quantization.auto_awq import ( + _convert_awq_to_standard_format, +) +from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import ( + QuarkScheme, +) +from vllm.model_executor.layers.quantization.quark.utils import ( + canonicalize_quark_packed_int4, + parse_w4a16_int4_weight_config, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kInt4Static, + kInt4Static32, + kInt4Static32Asym, + kInt4StaticAsym, +) +from vllm.model_executor.parameter import ( + GroupQuantScaleParameter, + PackedvLLMParameter, +) +from vllm.scalar_type import scalar_types + + +class QuarkW4A16Int4(QuarkScheme): + """Quark packed INT4 weight-only linear scheme via MPLinearKernel.""" + + supported_activation_quant_keys: list[QuantKey | None] = [None] + supported_weight_quant_keys: list[QuantKey] = [ + kInt4Static, + kInt4Static32, + kInt4StaticAsym, + kInt4Static32Asym, + ] + + def __init__( + self, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, + pack_method: str, + weight_config: dict[str, Any], + ): + super().__init__(weight_quant_key, activation_quant_key) + + # group_size is read from the Quark config rather than weight_quant_key: + # the int4 QuantKeys only encode group_size 32 or -1, but Quark int4 + # checkpoints commonly use group_size 128, which the QuantKey cannot + # represent. Symmetry is unambiguous, so it comes from the key. + self.group_size = parse_w4a16_int4_weight_config(weight_config)[0] + self.pack_factor = 8 + self.pack_reorder = pack_method == "reorder" + self.is_symmetric = weight_quant_key.symmetric + self.quant_type = ( + scalar_types.uint4b8 if self.is_symmetric else scalar_types.uint4 + ) + self.kernel: MPLinearKernel | None = None + + @classmethod + def get_min_capability(cls) -> int: + return 70 + + def create_weights( + self, + layer: torch.nn.Module, + output_partition_sizes: list[int], + input_size_per_partition: int, + params_dtype: torch.dtype, + weight_loader: Callable, + **kwargs, + ): + input_size = kwargs["input_size"] + output_size = kwargs["output_size"] + group_size = ( + self.group_size if self.group_size != -1 else input_size_per_partition + ) + if input_size_per_partition % group_size != 0: + raise ValueError( + "The input size is not aligned with the quantized weight shape. " + "This can be caused by too large tensor parallel size. " + f"input_size_per_partition={input_size_per_partition}, " + f"group_size={group_size}." + ) + + output_size_per_partition = sum(output_partition_sizes) + packed_output_size_per_partition = math.ceil( + output_size_per_partition / self.pack_factor + ) + layer.output_size_per_partition = output_size_per_partition + layer.packed_output_size_per_partition = packed_output_size_per_partition + + mp_linear_kernel_config = MPLinearLayerConfig( + full_weight_shape=(input_size, output_size), + partition_weight_shape=( + input_size_per_partition, + output_size_per_partition, + ), + weight_type=self.quant_type, + act_type=params_dtype, + group_size=group_size, + zero_points=not self.is_symmetric, + ) + kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config) + + def weight_scale_loader( + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + *args, + **loader_kwargs, + ) -> None: + if loaded_weight.shape[1] < param.data.shape[1]: + padded_weight = loaded_weight.new_zeros(param.data.shape) + padded_weight[:, : loaded_weight.shape[1]] = loaded_weight + loaded_weight = padded_weight + weight_loader(param, loaded_weight, *args, **loader_kwargs) + + weight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition, + packed_output_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + ) + num_groups = input_size_per_partition // group_size + weight_zero_point = PackedvLLMParameter( + data=torch.zeros( + num_groups, + packed_output_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.pack_factor, + weight_loader=weight_loader, + ) + weight_scale = GroupQuantScaleParameter( + data=torch.empty( + num_groups, + packed_output_size_per_partition * self.pack_factor, + dtype=params_dtype, + ), + input_dim=0, + output_dim=1, + weight_loader=weight_scale_loader, + ) + + layer.register_parameter("weight", weight) + layer.register_parameter("weight_zero_point", weight_zero_point) + layer.register_parameter("weight_scale", weight_scale) + + self.kernel = kernel_type( + mp_linear_kernel_config, + w_q_param_name="weight", + w_s_param_name="weight_scale", + w_zp_param_name="weight_zero_point" if not self.is_symmetric else None, + ) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.weight.data = canonicalize_quark_packed_int4( + layer.weight.data, + pack_reorder=self.pack_reorder, + is_symmetric=self.is_symmetric, + pack_factor=self.pack_factor, + ) + if not self.is_symmetric: + layer.weight_zero_point.data = canonicalize_quark_packed_int4( + layer.weight_zero_point.data, + pack_reorder=self.pack_reorder, + is_symmetric=self.is_symmetric, + pack_factor=self.pack_factor, + ) + output_size = layer.output_size_per_partition + packed_output_size = layer.packed_output_size_per_partition * self.pack_factor + if output_size < packed_output_size: + layer.weight_scale.data[:, output_size:].zero_() + + _convert_awq_to_standard_format( + layer, + "weight", + "weight_zero_point" if not self.is_symmetric else None, + self.quant_type.size_bits, + ) + assert self.kernel is not None + self.kernel.process_weights_after_loading(layer) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ): + assert self.kernel is not None + return self.kernel.apply_weights(layer, x, bias) diff --git a/vllm/model_executor/layers/quantization/quark/utils.py b/vllm/model_executor/layers/quantization/quark/utils.py index 48c978263e..ac455e38b8 100644 --- a/vllm/model_executor/layers/quantization/quark/utils.py +++ b/vllm/model_executor/layers/quantization/quark/utils.py @@ -66,6 +66,63 @@ def should_ignore_layer( return all(shards_ignored) +def parse_w4a16_int4_weight_config( + weight_config: Mapping[str, Any], +) -> tuple[int, bool]: + """Parse required W4A16 INT4/UINT4 weight fields from Quark config.""" + if "group_size" not in weight_config: + raise ValueError( + "Quark W4A16 INT4/UINT4 configs must specify weight.group_size" + ) + if "symmetric" not in weight_config: + raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric") + + group_size = weight_config["group_size"] + is_symmetric = weight_config["symmetric"] + if not isinstance(group_size, int) or group_size <= 0: + raise ValueError( + f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}" + ) + if not isinstance(is_symmetric, bool): + raise ValueError( + f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}" + ) + return group_size, is_symmetric + + +def canonicalize_quark_packed_int4( + packed_weight: torch.Tensor, + *, + pack_reorder: bool, + is_symmetric: bool, + pack_factor: int = 8, +) -> torch.Tensor: + """Convert Quark export nibble layout to AWQ checkpoint layout.""" + from vllm.model_executor.layers.quantization.auto_awq import ( + _REVERSE_AWQ_PACK_ORDER, + ) + + if pack_reorder: + source_order = torch.tensor( + _REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32 + ) + else: + source_order = torch.arange( + pack_factor, device=packed_weight.device, dtype=torch.int32 + ) + target_order = torch.tensor( + _REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32 + ) + source_shifts = source_order * 4 + target_shifts = target_order * 4 + + values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF + if is_symmetric: + values = values ^ 0x8 + packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1) + return packed.to(torch.int32) + + # utility for tensor dims > 2 cases def quark_quantize_weight_to_mxfp4(w: torch.Tensor): assert w.dtype == torch.bfloat16, (