[Quantization] Support native Quark W4A16 INT4/UINT4 exports in vLLM (#48606)

Signed-off-by: HongWei Meng <HongWei.Meng@amd.com>
This commit is contained in:
meng-amd
2026-09-18 17:52:46 +08:00
committed by GitHub
parent 67a8a3f926
commit 6df2b1a8c6
9 changed files with 1124 additions and 6 deletions
+439 -1
View File
@@ -58,12 +58,17 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
QuarkMoEMethod,
QuarkW4A8Fp8MoEMethod,
QuarkW4A16Int4MoEMethod,
QuarkW8A8Fp8MoEMethod,
QuarkW8A8Int8MoEMethod,
)
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.quark.schemes import (
QuarkScheme,
QuarkW4A16Int4,
)
from vllm.model_executor.layers.quantization.quark.utils import (
QuarkQTensorHint,
canonicalize_quark_packed_int4,
should_ignore_layer,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
@@ -97,6 +102,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kNvfp4Static,
)
from vllm.model_executor.models.llama import LlamaForCausalLM
from vllm.model_executor.models.utils import WeightsMapper
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
@@ -1876,3 +1882,435 @@ def test_glm5next_attn_loader_accepts_deepseek_weight_scale_inv():
)
assert target.loaded_shard is None # o_proj is a direct (non-fused) proj
assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128))
_REVERSE_AWQ_PACK_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
def _quark_int4_config(
*,
pack_method: str = "reorder",
symmetric: bool = True,
exclude: list[str] | None = None,
) -> dict:
return {
"quant_method": "quark",
"export": {"pack_method": pack_method, "kv_cache_group": []},
"global_quant_config": {
"weight": {
"dtype": "int4",
"group_size": 128,
"symmetric": symmetric,
}
},
"exclude": exclude or [],
}
def _sign_extend_int4_nibbles(t: torch.Tensor) -> torch.Tensor:
mask = (t & 0x8).bool()
t = t.clone()
t[mask] = t[mask] | 0xF0
return t
def _dequantize_quark_signed_awq_torch(
qweight: torch.Tensor,
scales: torch.Tensor,
qzeros: torch.Tensor,
group_size: int,
*,
pack_reorder: bool = True,
) -> torch.Tensor:
bits = 4
shifts = torch.arange(0, 32, bits, device=qweight.device)
iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
iweights = iweights.view(qweight.shape[0], -1)
zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
zeros = zeros.view(qzeros.shape[0], -1)
if pack_reorder:
order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device)
else:
order = torch.arange(8, device=qweight.device)
iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape(
qweight.shape[0], -1
)
zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1)
iweights = _sign_extend_int4_nibbles(iweights & 0xF)
zeros = _sign_extend_int4_nibbles(zeros & 0xF)
scales = scales.repeat_interleave(group_size, dim=0)
zeros = zeros.repeat_interleave(group_size, dim=0)
return (iweights - zeros) * scales
def _dequantize_awq_unsigned_torch(
qweight: torch.Tensor,
scales: torch.Tensor,
qzeros: torch.Tensor,
group_size: int,
*,
pack_reorder: bool = True,
) -> torch.Tensor:
bits = 4
shifts = torch.arange(0, 32, bits, device=qweight.device)
iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
iweights = iweights.view(qweight.shape[0], -1)
zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
zeros = zeros.view(qzeros.shape[0], -1)
if pack_reorder:
order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device)
else:
order = torch.arange(8, device=qweight.device)
iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape(
qweight.shape[0], -1
)
zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1)
scales = scales.repeat_interleave(group_size, dim=0)
zeros = zeros.repeat_interleave(group_size, dim=0)
return (iweights - zeros) * scales
def _pack_int4_nibbles(nibbles: torch.Tensor, *, pack_reorder: bool) -> torch.Tensor:
pack_order = (
torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=nibbles.device)
if pack_reorder
else torch.arange(8, device=nibbles.device)
)
shifts = pack_order * 4
return ((nibbles.to(torch.int64) & 0xF) << shifts).sum(dim=-1).to(torch.int32)
class TestQuarkInt4Format:
"""Tests for Quark INT4 export format compatibility."""
def test_quark_order_pack_method_uses_native_int4_scheme(self):
quant_config = QuarkConfig.from_config(_quark_int4_config(pack_method="order"))
weight_config = quant_config.quant_config["global_quant_config"]
weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config(
weight_config
)
assert scheme_cls is QuarkW4A16Int4
scheme = quant_config.init_scheme(
scheme_cls,
weight_quant_key=weight_key,
activation_quant_key=act_key,
weight_config=weight_config["weight"],
)
assert isinstance(scheme, QuarkW4A16Int4)
assert not scheme.pack_reorder
def test_quark_int4_scheme_supports_asymmetric_weights(self):
quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False))
weight_config = quant_config.quant_config["global_quant_config"]
weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config(
weight_config
)
assert scheme_cls is QuarkW4A16Int4
scheme = quant_config.init_scheme(
scheme_cls,
weight_quant_key=weight_key,
activation_quant_key=act_key,
weight_config=weight_config["weight"],
)
assert isinstance(scheme, QuarkW4A16Int4)
assert not scheme.is_symmetric
@pytest.mark.parametrize("missing_field", ["group_size", "symmetric"])
def test_quark_int4_scheme_requires_weight_config_fields(self, missing_field):
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4Static,
)
weight_config = {
"dtype": "int4",
"group_size": 128,
"symmetric": True,
}
weight_config.pop(missing_field)
quant_config = QuarkConfig.from_config(
{
"quant_method": "quark",
"export": {"pack_method": "reorder", "kv_cache_group": []},
"global_quant_config": {"weight": weight_config},
"exclude": [],
}
)
with pytest.raises(ValueError, match=missing_field):
quant_config.init_scheme(
QuarkW4A16Int4,
weight_quant_key=kInt4Static,
activation_quant_key=None,
weight_config=weight_config,
)
def test_quark_int4_moe_uses_native_moe_method(self):
from unittest.mock import MagicMock, patch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4StaticAsym,
)
quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False))
moe_config = type("MoeConfig", (), {})()
moe_config.has_bias = False
mock_backend = MagicMock()
mock_experts_cls = MagicMock()
with patch(
"vllm.model_executor.layers.quantization.quark.quark_moe"
".select_wna16_moe_backend",
return_value=(mock_backend, mock_experts_cls),
):
method = QuarkW4A16Int4MoEMethod(
kInt4StaticAsym,
None,
quant_config.quant_config["global_quant_config"]["weight"],
quant_config.pack_method,
moe_config,
)
assert method.group_size == 128
assert method.pack_reorder
assert method.use_wna16_backend
def test_triton_wna16_experts_supports_asymmetric_int4(self):
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
TritonWNA16Experts,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
)
for key in [kInt4Static, kInt4Static32, kInt4StaticAsym, kInt4Static32Asym]:
assert TritonWNA16Experts._supports_quant_scheme(key, None), (
f"{key} should be supported by TritonWNA16Experts"
)
def test_quark_moe_loader_shards_w2_zero_point_across_tp_ranks(self):
"""w2 zero-points are sharded along the intermediate dim per TP rank.
Loading the same Quark checkpoint tensor at tensor_parallel_size=2 for
both ranks must jointly reconstruct the tensor_parallel_size=1 (full)
result, with no overlap or gap. This guards the custom loader's TP
split for the packed zero-point layout.
"""
class _FakeMoEConfig:
has_bias = False
def __init__(self, tp_size, tp_rank):
self.tp_size = tp_size
self.tp_rank = tp_rank
class _FakeLayer:
def __init__(self, moe_config):
self.moe_config = moe_config
self.intermediate_size_per_partition = 64
self.group_size_div_factor = 1
def load_w2_zero_point(raw_zp, tp_size, tp_rank):
from unittest.mock import MagicMock, patch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4StaticAsym,
)
quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False))
moe_config = _FakeMoEConfig(tp_size, tp_rank)
with patch(
"vllm.model_executor.layers.quantization.quark.quark_moe"
".select_wna16_moe_backend",
return_value=(MagicMock(), MagicMock()),
):
method = QuarkW4A16Int4MoEMethod(
kInt4StaticAsym,
None,
quant_config.quant_config["global_quant_config"]["weight"],
quant_config.pack_method,
moe_config,
)
layer = _FakeLayer(moe_config)
# The loader converts the packed zero-point, then writes the
# per-rank slice into param.data[expert_id]. A single expert dim is
# enough; the last dim shrinks by tp_size after sharding.
param = torch.nn.Parameter(
torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8),
requires_grad=False,
)
loader = method.get_weight_loader(layer, weight_loader=None)
loader(
param,
raw_zp.clone(),
weight_name="w2_weight_zero_point",
shard_id="w2",
expert_id=0,
)
return param.data[0]
# Quark-packed zero-point for one expert: rows = hidden // pack_factor,
# cols carry the (packed) intermediate dim, split evenly across ranks.
raw_zp = torch.randint(0, 256, (8, 16), dtype=torch.uint8)
full = load_w2_zero_point(raw_zp, tp_size=1, tp_rank=0)
shard0 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=0)
shard1 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=1)
expected = full.view(full.size(0), 2, -1)
torch.testing.assert_close(shard0, expected[:, 0])
torch.testing.assert_close(shard1, expected[:, 1])
def test_quark_shared_expert_gate_keeps_quantized_tensors(self):
quant_config = QuarkConfig.from_config(_quark_int4_config())
mapper = quant_config.get_cache_scale_mapper()
output_names = {
name
for name, _ in mapper.apply(
[
(
"model.language_model.layers.0.mlp.shared_expert_gate.weight",
torch.zeros(1),
),
(
"model.language_model.layers.0.mlp.shared_expert_gate"
".weight_scale",
torch.zeros(1),
),
(
"model.language_model.layers.0.mlp.shared_expert_gate"
".weight_zero_point",
torch.zeros(1),
),
]
)
}
assert (
"model.language_model.layers.0.mlp.shared_expert_gate.weight"
in output_names
)
assert (
"model.language_model.layers.0.mlp.shared_expert_gate.weight_scale"
in output_names
)
assert (
"model.language_model.layers.0.mlp.shared_expert_gate.weight_zero_point"
in output_names
)
def test_quark_apply_mapper_updates_exclude_and_layer_quant_config(self):
quant_config = QuarkConfig.from_config(
{
**_quark_int4_config(),
"exclude": ["lm_head"],
"layer_quant_config": {
"model.language_model.layers.0.mlp.gate": {
"weight": {"dtype": "float16"},
},
},
}
)
quant_config.apply_vllm_mapper(
WeightsMapper(
orig_to_new_prefix={
"lm_head": "language_model.lm_head",
"model.language_model.": "language_model.model.",
},
)
)
layer_quant_config = quant_config.quant_config["layer_quant_config"]
assert "language_model.model.layers.0.mlp.gate" in layer_quant_config
assert "model.language_model.layers.0.mlp.gate" not in layer_quant_config
assert quant_config.quant_config["exclude"] == [
"language_model.lm_head",
]
def test_quark_apply_mapper_ignores_non_string_list_entries(self):
quant_config = QuarkConfig.from_config(
{
**_quark_int4_config(),
"algo_config": [
{"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]}
],
}
)
quant_config.apply_vllm_mapper(WeightsMapper())
assert quant_config.quant_config["algo_config"] == [
{"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]}
]
def test_quark_exclude_matches_via_mapper_prefix(self):
quant_config = QuarkConfig.from_config(_quark_int4_config(exclude=["lm_head"]))
quant_config.apply_vllm_mapper(
WeightsMapper(orig_to_new_prefix={"lm_head": "language_model.lm_head"})
)
exclude_layers = quant_config.quant_config["exclude"]
assert should_ignore_layer("language_model.lm_head", ignore=exclude_layers)
assert not should_ignore_layer(
"language_model.model.layers.0.mlp.gate", ignore=exclude_layers
)
@pytest.mark.parametrize("symmetric", [False, True])
@pytest.mark.parametrize("pack_method", ["order", "reorder"])
def test_quark_int4_canonicalizes_pack_for_kernel_layout(pack_method, symmetric):
"""Quark pack order is normalized to the layout expected by awq_* ops.
This reuses the existing AWQ dequant/gemm kernels for compute only; loading
still goes through the native Quark quantization path, not AutoAWQ.
"""
pack_reorder = pack_method == "reorder"
group_size = 2
packed_values = torch.tensor(
[
[0, 1, 7, 8, 9, 15, 2, 14],
[15, 8, 0, 3, 12, 7, 1, 9],
],
dtype=torch.int32,
)
packed_zeros = torch.zeros((1, 8), dtype=torch.int32)
qweight = _pack_int4_nibbles(packed_values, pack_reorder=pack_reorder).view(2, 1)
qzeros = _pack_int4_nibbles(packed_zeros, pack_reorder=pack_reorder).view(1, 1)
scales = torch.ones((1, 8), dtype=torch.float16)
dequantize = (
_dequantize_quark_signed_awq_torch
if symmetric
else _dequantize_awq_unsigned_torch
)
expected = dequantize(
qweight,
scales,
qzeros,
group_size,
pack_reorder=pack_reorder,
)
canonical_weight = canonicalize_quark_packed_int4(
qweight,
pack_reorder=pack_reorder,
is_symmetric=symmetric,
)
canonical_zero = canonicalize_quark_packed_int4(
qzeros,
pack_reorder=pack_reorder,
is_symmetric=symmetric,
)
actual = _dequantize_awq_unsigned_torch(
canonical_weight, scales, canonical_zero, group_size
)
assert torch.equal(actual, expected)
@@ -864,6 +864,7 @@ def choose_mp_linear_kernel(
can_implement, failure_reason = kernel.can_implement(config)
if can_implement:
logger.info_once("Using %s for mixed-precision linear", kernel.__name__)
return kernel
else:
failure_reasons.append(
@@ -596,9 +596,9 @@ class TritonWNA16Experts(TritonExperts):
) -> bool:
SUPPORTED_W = [
kInt4Static,
kInt4StaticAsym,
kInt8Static,
kInt4Static32,
kInt4StaticAsym,
kInt4Static32Asym,
# other group sizes?
]
@@ -93,7 +93,7 @@ def _replace_or_register_parameter(
def _convert_awq_to_standard_format(
layer: torch.nn.Module,
w_q_name: str,
w_zp_name: str,
w_zp_name: str | None,
size_bits: int,
) -> None:
"""Convert AWQ weight and zero-point tensors to standard GPTQ-like format.
@@ -101,6 +101,7 @@ def _convert_awq_to_standard_format(
AWQ packs qweight along the output dim with a non-standard bit order.
This converts to standard bit order and repacks qweight along the input
dim, matching the format expected by the MPLinearKernel framework.
If w_zp_name is None (symmetric quantization), only the weight is converted.
"""
pack_factor = 32 // size_bits
mask = (1 << size_bits) - 1
@@ -139,6 +140,9 @@ def _convert_awq_to_standard_format(
)
setattr(layer, w_q_name, new_param)
if w_zp_name is None:
return
# --- Convert qzeros: fix AWQ bit ordering and repack
# AWQ qzeros: (G, N // pack) packed along dim 1, AWQ bit order
# Target: (N // pack, G) packed along dim 0, standard bit order
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
QuarkOCP_MX,
QuarkScheme,
QuarkW4A8_MXFP4_FP8,
QuarkW4A16Int4,
QuarkW8A8Fp8,
QuarkW8A8Fp8PerBlock,
QuarkW8A8Int8,
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
from vllm.model_executor.layers.quantization.quark.utils import (
QuarkQTensorHint,
deep_compare,
parse_w4a16_int4_weight_config,
should_ignore_layer,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
@@ -54,6 +56,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
kInt4W4A8StaticChannelSym,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
@@ -172,8 +178,10 @@ class QuarkConfig(QuantizationConfig):
quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {}
for k, v in self.quant_config.items():
if isinstance(v, list):
if isinstance(v, list) and all(isinstance(item, str) for item in v):
quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_list(v)
elif isinstance(v, list):
quant_config_with_hf_to_vllm_mapper[k] = v
elif isinstance(v, dict):
quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_dict(v)
else:
@@ -215,6 +223,9 @@ class QuarkConfig(QuantizationConfig):
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
dynamic_mxfp4_quant=dynamic_mxfp4_quant,
weight_config=self._find_matched_config(prefix, type(layer)).get(
"weight"
),
)
layer.scheme = scheme
return QuarkLinearMethod(self)
@@ -227,6 +238,7 @@ class QuarkConfig(QuantizationConfig):
method_cls=method_cls,
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
layer_name=prefix,
)
return None
@@ -572,6 +584,26 @@ class QuarkConfig(QuantizationConfig):
)
return QuantKeyMatch(True, activation_quant_key, weight_quant_key)
def _is_w4a16_int4(
self,
weight_quant: dict[str, Any] | None,
input_quant: dict[str, Any] | None,
) -> QuantKeyMatch:
if weight_quant is None or input_quant is not None:
return QuantKeyMatch(False, None, None)
is_int4 = weight_quant.get("dtype") in ("int4", "uint4")
is_packed = self.pack_method in ("order", "reorder")
if not (is_int4 and is_packed):
return QuantKeyMatch(False, None, None)
group_size, is_symmetric = parse_w4a16_int4_weight_config(weight_quant)
if is_symmetric:
weight_key = kInt4Static32 if group_size == 32 else kInt4Static
else:
weight_key = kInt4Static32Asym if group_size == 32 else kInt4StaticAsym
return QuantKeyMatch(True, None, weight_key)
def _is_w4a8_mxfp4_fp8(
self,
weight_quant: dict[str, Any] | None,
@@ -890,6 +922,12 @@ class QuarkConfig(QuantizationConfig):
match.activation_quant_key,
QuarkW8A8Int8,
)
elif match := self._is_w4a16_int4(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkW4A16Int4,
)
elif match := self._is_w4a8_mxfp4_fp8(weight_config, input_config):
return (
match.weight_quant_key,
@@ -919,6 +957,7 @@ class QuarkConfig(QuantizationConfig):
weight_quant_key: QuantKey | None,
activation_quant_key: QuantKey | None,
dynamic_mxfp4_quant: bool = False,
weight_config: dict[str, Any] | None = None,
) -> "QuarkScheme":
"""Construct a Quark scheme selected by get_scheme_cls."""
if scheme_cls not in (
@@ -928,9 +967,24 @@ class QuarkConfig(QuantizationConfig):
QuarkOCP_MX,
QuarkNVFP4,
QuarkW4A8_MXFP4_FP8,
QuarkW4A16Int4,
):
raise AssertionError(f"Unsupported Quark scheme class: {scheme_cls}")
scheme: QuarkScheme
if scheme_cls is QuarkW4A16Int4:
normalized_weight_config = self._unwrap_single_quant_config(weight_config)
assert normalized_weight_config is not None
assert weight_quant_key is not None
scheme = QuarkW4A16Int4(
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
pack_method=self.pack_method,
weight_config=normalized_weight_config,
)
self._check_scheme_supported(scheme.get_min_capability())
return scheme
kwargs: dict[str, Any] = {"activation_quant_key": activation_quant_key}
# These classes handle different possible `weight_quant_key`.
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
@@ -40,6 +40,12 @@ from vllm.model_executor.layers.fused_moe.oracle.int8 import (
make_int8_moe_quant_config,
select_int8_moe_backend,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
make_wna16_moe_quant_config,
select_wna16_moe_backend,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
@@ -56,7 +62,10 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
make_nvfp4_moe_quant_config,
select_nvfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
from vllm.model_executor.layers.quantization.quark.utils import (
QuarkQTensorHint,
parse_w4a16_int4_weight_config,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
validate_fp8_block_shape_moe,
)
@@ -75,6 +84,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
kInt4W4A8StaticChannelSym,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
@@ -102,6 +115,7 @@ logger = init_logger(__name__)
__all__ = [
"QuarkMoEMethod",
"QuarkW4A16Int4MoEMethod",
"QuarkW8A8Fp8MoEMethod",
"QuarkOCP_MX_MoEMethod",
"QuarkNvfp4MoEMethod",
@@ -179,6 +193,12 @@ class QuarkMoEMethod(FusedMoEMethodBase):
match.activation_quant_key,
QuarkW8A8Fp8MoEMethod,
)
if match := quant_config._is_w4a16_int4(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkW4A16Int4MoEMethod,
)
if match := quant_config._is_w_ocp_mx_a_x(
weight_config, input_config, allow_static_fp8=True
):
@@ -202,6 +222,7 @@ class QuarkMoEMethod(FusedMoEMethodBase):
method_cls: type["QuarkMoEMethod"],
weight_quant_key: QuantKey | None,
activation_quant_key: QuantKey | None,
layer_name: str | None = None,
) -> "QuarkMoEMethod":
if weight_quant_key is None:
raise RuntimeError("Unsupported FusedMoe scheme")
@@ -226,10 +247,339 @@ class QuarkMoEMethod(FusedMoEMethodBase):
return QuarkW8A8Int8MoEMethod(
module.moe_config, weight_quant_key, activation_quant_key
)
elif method_cls is QuarkW4A16Int4MoEMethod:
assert layer_name is not None
layer_quant_config = quant_config._find_matched_config(
layer_name, type(module)
)
weight_config = quant_config._unwrap_single_quant_config(
layer_quant_config.get("weight")
)
if weight_config is None:
raise RuntimeError(
f"Missing weight quantization config for layer {layer_name}"
)
return QuarkW4A16Int4MoEMethod(
weight_quant_key,
activation_quant_key,
weight_config,
quant_config.pack_method,
module.moe_config,
quant_config=quant_config,
)
else:
raise RuntimeError("Unsupported FusedMoe scheme")
class QuarkW4A16Int4MoEMethod(QuarkMoEMethod):
"""Quark packed INT4 weight-only MoE method."""
supported_activation_quant_keys: list[QuantKey | None] = [None]
supported_weight_quant_keys: list[QuantKey] = [
kInt4Static,
kInt4Static32,
kInt4StaticAsym,
kInt4Static32Asym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
weight_config: dict[str, Any],
pack_method: str,
moe: FusedMoEConfig,
quant_config: "QuarkConfig | None" = None, # type: ignore # noqa F821
):
super().__init__(moe, weight_quant_key, activation_quant_key)
self.quant_config = quant_config
# group_size is read from the Quark config rather than weight_quant_key:
# the int4 QuantKeys only encode group_size 32 or -1, but Quark int4
# checkpoints commonly use group_size 128, which the QuantKey cannot
# represent. Symmetry is unambiguous, so it comes from the key.
self.group_size = parse_w4a16_int4_weight_config(weight_config)[0]
self.is_symmetric = weight_quant_key.symmetric
self.bit8_pack_factor = 2
self.pack_reorder = pack_method == "reorder"
self.use_wna16_backend = True
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
config=self.moe,
weight_key=weight_quant_key,
quant_config=self.quant_config,
may_have_zp=not self.is_symmetric,
may_have_bias=False,
)
def create_weights(
self,
layer: RoutedExperts,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
group_size = self.group_size
group_size_div_factor = 1
while intermediate_size_per_partition % group_size or hidden_size % group_size:
group_size = group_size // 2
group_size_div_factor *= 2
assert group_size >= 32
layer.group_size = group_size
layer.group_size_div_factor = group_size_div_factor
strategy = FusedMoeWeightScaleSupported.GROUP.value
extra_weight_attrs.update({"quant_method": strategy, "is_transposed": False})
assert "weight_loader" in extra_weight_attrs
weight_loader = extra_weight_attrs["weight_loader"]
extra_weight_attrs["weight_loader"] = self.get_weight_loader(
layer, weight_loader
)
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
hidden_size // self.bit8_pack_factor,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // self.bit8_pack_factor,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
w13_weight_scale = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
hidden_size // group_size,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
w2_weight_scale = torch.nn.Parameter(
torch.zeros(
num_experts,
hidden_size,
intermediate_size_per_partition // group_size,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
w13_weight_zero_point = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition // self.bit8_pack_factor,
hidden_size // group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_zero_point", w13_weight_zero_point)
set_weight_attrs(w13_weight_zero_point, extra_weight_attrs)
w2_weight_zero_point = torch.nn.Parameter(
torch.zeros(
num_experts,
hidden_size // self.bit8_pack_factor,
intermediate_size_per_partition // group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_zero_point", w2_weight_zero_point)
set_weight_attrs(w2_weight_zero_point, extra_weight_attrs)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
w13_qzeros = layer.w13_weight_zero_point if not self.is_symmetric else None
w2_qzeros = layer.w2_weight_zero_point if not self.is_symmetric else None
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=None,
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_qzeros=w13_qzeros,
w2_qzeros=w2_qzeros,
)
if converted is None:
# Backend rewrote the layer's params in place (e.g. Humming).
self._setup_kernel(layer)
return
(
w13_qweight,
w2_qweight,
w13_scales,
w2_scales,
_, # w13_qzeros
_, # w2_qzeros
_, # w13_input_global_scale
_, # w2_input_global_scale
_, # w13_bias
_, # w2_bias
) = converted
replace_parameter(layer, "w13_weight", w13_qweight)
replace_parameter(layer, "w2_weight", w2_qweight)
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
self._setup_kernel(layer)
def _setup_kernel(self, layer: RoutedExperts) -> None:
assert self.experts_cls is not None
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
self.moe_kernel = make_wna16_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_backend,
routing_tables=layer._expert_routing_tables(),
)
def get_fused_moe_quant_config(
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
w1_zp = layer.w13_weight_zero_point if not self.is_symmetric else None
w2_zp = layer.w2_weight_zero_point if not self.is_symmetric else None
return make_wna16_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w1_zp=w1_zp,
w2_zp=w2_zp,
group_size=layer.group_size,
num_bits=4,
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
expert_map=layer.expert_map,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
def get_weight_loader(self, layer: RoutedExperts, weight_loader):
def convert_quark_tensor(tensor: torch.Tensor, tensor_type: str):
size0 = tensor.size(0)
tensor = tensor.view(torch.uint8)
shifter = torch.tensor([0, 4], dtype=torch.uint8, device=tensor.device)
tensor = (tensor[:, :, None] >> shifter) & 0xF
if self.pack_reorder:
reverse_awq_pack_order = [0, 4, 1, 5, 2, 6, 3, 7]
tensor = tensor.view(-1, 8)[:, reverse_awq_pack_order]
else:
tensor = tensor.view(-1, 8)
if self.is_symmetric:
tensor = tensor ^ 0x8
tensor = tensor.view(size0, -1)
tensor = tensor.T.contiguous()
if tensor_type == "weight":
tensor = tensor[:, 1::2] * 16 + tensor[:, ::2]
elif tensor_type == "zero_point":
tensor = tensor[1::2, :] * 16 + tensor[::2, :]
return tensor
def quark_int4_weight_loader(
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: bool = False,
):
loaded_weight = loaded_weight.to(param.data.device)
shard_size = layer.intermediate_size_per_partition
if weight_name.endswith("_weight"):
loaded_weight = convert_quark_tensor(loaded_weight, "weight")
elif weight_name.endswith("_weight_zero_point"):
loaded_weight = convert_quark_tensor(loaded_weight, "zero_point")
elif weight_name.endswith("_weight_scale"):
loaded_weight = loaded_weight.T
if layer.group_size_div_factor > 1 and (
"weight_zero_point" in weight_name or "weight_scale" in weight_name
):
loaded_weight = loaded_weight.repeat_interleave(
layer.group_size_div_factor, 1
)
if "w13_weight_zero_point" in weight_name:
tensor = loaded_weight.view(
layer.moe_config.tp_size, -1, loaded_weight.size(1)
)[layer.moe_config.tp_rank]
if shard_id == "w1":
param.data[expert_id, : shard_size // 2] = tensor
else:
param.data[expert_id, shard_size // 2 :] = tensor
return True if return_success else None
elif "w2_weight_zero_point" in weight_name:
param.data[expert_id] = loaded_weight.view(
loaded_weight.size(0), layer.moe_config.tp_size, -1
)[:, layer.moe_config.tp_rank]
return True if return_success else None
return weight_loader(
param,
loaded_weight,
weight_name,
shard_id,
expert_id,
return_success=return_success,
)
return quark_int4_weight_loader
class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
supported_activation_quant_keys = [
kFp8StaticTensorSym,
@@ -5,6 +5,7 @@ from .quark_nvfp4 import QuarkNVFP4
from .quark_ocp_mx import QuarkOCP_MX
from .quark_scheme import QuarkScheme
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
from .quark_w4a16_int4 import QuarkW4A16Int4
from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
from .quark_w8a8_int8 import QuarkW8A8Int8
@@ -14,6 +15,7 @@ __all__ = [
"QuarkW8A8Fp8PerBlock",
"QuarkW8A8Int8",
"QuarkOCP_MX",
"QuarkW4A16Int4",
"QuarkW4A8_MXFP4_FP8",
"QuarkNVFP4",
]
@@ -0,0 +1,212 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Callable
from typing import Any
import torch
from vllm.model_executor.kernels.linear import (
MPLinearKernel,
MPLinearLayerConfig,
choose_mp_linear_kernel,
)
from vllm.model_executor.layers.quantization.auto_awq import (
_convert_awq_to_standard_format,
)
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
QuarkScheme,
)
from vllm.model_executor.layers.quantization.quark.utils import (
canonicalize_quark_packed_int4,
parse_w4a16_int4_weight_config,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.scalar_type import scalar_types
class QuarkW4A16Int4(QuarkScheme):
"""Quark packed INT4 weight-only linear scheme via MPLinearKernel."""
supported_activation_quant_keys: list[QuantKey | None] = [None]
supported_weight_quant_keys: list[QuantKey] = [
kInt4Static,
kInt4Static32,
kInt4StaticAsym,
kInt4Static32Asym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
pack_method: str,
weight_config: dict[str, Any],
):
super().__init__(weight_quant_key, activation_quant_key)
# group_size is read from the Quark config rather than weight_quant_key:
# the int4 QuantKeys only encode group_size 32 or -1, but Quark int4
# checkpoints commonly use group_size 128, which the QuantKey cannot
# represent. Symmetry is unambiguous, so it comes from the key.
self.group_size = parse_w4a16_int4_weight_config(weight_config)[0]
self.pack_factor = 8
self.pack_reorder = pack_method == "reorder"
self.is_symmetric = weight_quant_key.symmetric
self.quant_type = (
scalar_types.uint4b8 if self.is_symmetric else scalar_types.uint4
)
self.kernel: MPLinearKernel | None = None
@classmethod
def get_min_capability(cls) -> int:
return 70
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
input_size = kwargs["input_size"]
output_size = kwargs["output_size"]
group_size = (
self.group_size if self.group_size != -1 else input_size_per_partition
)
if input_size_per_partition % group_size != 0:
raise ValueError(
"The input size is not aligned with the quantized weight shape. "
"This can be caused by too large tensor parallel size. "
f"input_size_per_partition={input_size_per_partition}, "
f"group_size={group_size}."
)
output_size_per_partition = sum(output_partition_sizes)
packed_output_size_per_partition = math.ceil(
output_size_per_partition / self.pack_factor
)
layer.output_size_per_partition = output_size_per_partition
layer.packed_output_size_per_partition = packed_output_size_per_partition
mp_linear_kernel_config = MPLinearLayerConfig(
full_weight_shape=(input_size, output_size),
partition_weight_shape=(
input_size_per_partition,
output_size_per_partition,
),
weight_type=self.quant_type,
act_type=params_dtype,
group_size=group_size,
zero_points=not self.is_symmetric,
)
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
def weight_scale_loader(
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
*args,
**loader_kwargs,
) -> None:
if loaded_weight.shape[1] < param.data.shape[1]:
padded_weight = loaded_weight.new_zeros(param.data.shape)
padded_weight[:, : loaded_weight.shape[1]] = loaded_weight
loaded_weight = padded_weight
weight_loader(param, loaded_weight, *args, **loader_kwargs)
weight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition,
packed_output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
num_groups = input_size_per_partition // group_size
weight_zero_point = PackedvLLMParameter(
data=torch.zeros(
num_groups,
packed_output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
num_groups,
packed_output_size_per_partition * self.pack_factor,
dtype=params_dtype,
),
input_dim=0,
output_dim=1,
weight_loader=weight_scale_loader,
)
layer.register_parameter("weight", weight)
layer.register_parameter("weight_zero_point", weight_zero_point)
layer.register_parameter("weight_scale", weight_scale)
self.kernel = kernel_type(
mp_linear_kernel_config,
w_q_param_name="weight",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point" if not self.is_symmetric else None,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight.data = canonicalize_quark_packed_int4(
layer.weight.data,
pack_reorder=self.pack_reorder,
is_symmetric=self.is_symmetric,
pack_factor=self.pack_factor,
)
if not self.is_symmetric:
layer.weight_zero_point.data = canonicalize_quark_packed_int4(
layer.weight_zero_point.data,
pack_reorder=self.pack_reorder,
is_symmetric=self.is_symmetric,
pack_factor=self.pack_factor,
)
output_size = layer.output_size_per_partition
packed_output_size = layer.packed_output_size_per_partition * self.pack_factor
if output_size < packed_output_size:
layer.weight_scale.data[:, output_size:].zero_()
_convert_awq_to_standard_format(
layer,
"weight",
"weight_zero_point" if not self.is_symmetric else None,
self.quant_type.size_bits,
)
assert self.kernel is not None
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
):
assert self.kernel is not None
return self.kernel.apply_weights(layer, x, bias)
@@ -66,6 +66,63 @@ def should_ignore_layer(
return all(shards_ignored)
def parse_w4a16_int4_weight_config(
weight_config: Mapping[str, Any],
) -> tuple[int, bool]:
"""Parse required W4A16 INT4/UINT4 weight fields from Quark config."""
if "group_size" not in weight_config:
raise ValueError(
"Quark W4A16 INT4/UINT4 configs must specify weight.group_size"
)
if "symmetric" not in weight_config:
raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric")
group_size = weight_config["group_size"]
is_symmetric = weight_config["symmetric"]
if not isinstance(group_size, int) or group_size <= 0:
raise ValueError(
f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}"
)
if not isinstance(is_symmetric, bool):
raise ValueError(
f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}"
)
return group_size, is_symmetric
def canonicalize_quark_packed_int4(
packed_weight: torch.Tensor,
*,
pack_reorder: bool,
is_symmetric: bool,
pack_factor: int = 8,
) -> torch.Tensor:
"""Convert Quark export nibble layout to AWQ checkpoint layout."""
from vllm.model_executor.layers.quantization.auto_awq import (
_REVERSE_AWQ_PACK_ORDER,
)
if pack_reorder:
source_order = torch.tensor(
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
)
else:
source_order = torch.arange(
pack_factor, device=packed_weight.device, dtype=torch.int32
)
target_order = torch.tensor(
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
)
source_shifts = source_order * 4
target_shifts = target_order * 4
values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF
if is_symmetric:
values = values ^ 0x8
packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1)
return packed.to(torch.int32)
# utility for tensor dims > 2 cases
def quark_quantize_weight_to_mxfp4(w: torch.Tensor):
assert w.dtype == torch.bfloat16, (