[Quantization] Support native Quark W4A16 INT4/UINT4 exports in vLLM (#48606)
Signed-off-by: HongWei Meng <HongWei.Meng@amd.com>
This commit is contained in:
@@ -58,12 +58,17 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
|
||||
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
|
||||
QuarkMoEMethod,
|
||||
QuarkW4A8Fp8MoEMethod,
|
||||
QuarkW4A16Int4MoEMethod,
|
||||
QuarkW8A8Fp8MoEMethod,
|
||||
QuarkW8A8Int8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import (
|
||||
QuarkScheme,
|
||||
QuarkW4A16Int4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
QuarkQTensorHint,
|
||||
canonicalize_quark_packed_int4,
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
@@ -97,6 +102,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.model_executor.models.llama import LlamaForCausalLM
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
@@ -1876,3 +1882,435 @@ def test_glm5next_attn_loader_accepts_deepseek_weight_scale_inv():
|
||||
)
|
||||
assert target.loaded_shard is None # o_proj is a direct (non-fused) proj
|
||||
assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128))
|
||||
|
||||
|
||||
_REVERSE_AWQ_PACK_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
|
||||
|
||||
|
||||
def _quark_int4_config(
|
||||
*,
|
||||
pack_method: str = "reorder",
|
||||
symmetric: bool = True,
|
||||
exclude: list[str] | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
"quant_method": "quark",
|
||||
"export": {"pack_method": pack_method, "kv_cache_group": []},
|
||||
"global_quant_config": {
|
||||
"weight": {
|
||||
"dtype": "int4",
|
||||
"group_size": 128,
|
||||
"symmetric": symmetric,
|
||||
}
|
||||
},
|
||||
"exclude": exclude or [],
|
||||
}
|
||||
|
||||
|
||||
def _sign_extend_int4_nibbles(t: torch.Tensor) -> torch.Tensor:
|
||||
mask = (t & 0x8).bool()
|
||||
t = t.clone()
|
||||
t[mask] = t[mask] | 0xF0
|
||||
return t
|
||||
|
||||
|
||||
def _dequantize_quark_signed_awq_torch(
|
||||
qweight: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
qzeros: torch.Tensor,
|
||||
group_size: int,
|
||||
*,
|
||||
pack_reorder: bool = True,
|
||||
) -> torch.Tensor:
|
||||
bits = 4
|
||||
shifts = torch.arange(0, 32, bits, device=qweight.device)
|
||||
iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
|
||||
iweights = iweights.view(qweight.shape[0], -1)
|
||||
zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
|
||||
zeros = zeros.view(qzeros.shape[0], -1)
|
||||
|
||||
if pack_reorder:
|
||||
order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device)
|
||||
else:
|
||||
order = torch.arange(8, device=qweight.device)
|
||||
iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape(
|
||||
qweight.shape[0], -1
|
||||
)
|
||||
zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1)
|
||||
iweights = _sign_extend_int4_nibbles(iweights & 0xF)
|
||||
zeros = _sign_extend_int4_nibbles(zeros & 0xF)
|
||||
|
||||
scales = scales.repeat_interleave(group_size, dim=0)
|
||||
zeros = zeros.repeat_interleave(group_size, dim=0)
|
||||
return (iweights - zeros) * scales
|
||||
|
||||
|
||||
def _dequantize_awq_unsigned_torch(
|
||||
qweight: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
qzeros: torch.Tensor,
|
||||
group_size: int,
|
||||
*,
|
||||
pack_reorder: bool = True,
|
||||
) -> torch.Tensor:
|
||||
bits = 4
|
||||
shifts = torch.arange(0, 32, bits, device=qweight.device)
|
||||
iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
|
||||
iweights = iweights.view(qweight.shape[0], -1)
|
||||
zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
|
||||
zeros = zeros.view(qzeros.shape[0], -1)
|
||||
|
||||
if pack_reorder:
|
||||
order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device)
|
||||
else:
|
||||
order = torch.arange(8, device=qweight.device)
|
||||
iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape(
|
||||
qweight.shape[0], -1
|
||||
)
|
||||
zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1)
|
||||
|
||||
scales = scales.repeat_interleave(group_size, dim=0)
|
||||
zeros = zeros.repeat_interleave(group_size, dim=0)
|
||||
return (iweights - zeros) * scales
|
||||
|
||||
|
||||
def _pack_int4_nibbles(nibbles: torch.Tensor, *, pack_reorder: bool) -> torch.Tensor:
|
||||
pack_order = (
|
||||
torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=nibbles.device)
|
||||
if pack_reorder
|
||||
else torch.arange(8, device=nibbles.device)
|
||||
)
|
||||
shifts = pack_order * 4
|
||||
return ((nibbles.to(torch.int64) & 0xF) << shifts).sum(dim=-1).to(torch.int32)
|
||||
|
||||
|
||||
class TestQuarkInt4Format:
|
||||
"""Tests for Quark INT4 export format compatibility."""
|
||||
|
||||
def test_quark_order_pack_method_uses_native_int4_scheme(self):
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config(pack_method="order"))
|
||||
weight_config = quant_config.quant_config["global_quant_config"]
|
||||
weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config(
|
||||
weight_config
|
||||
)
|
||||
assert scheme_cls is QuarkW4A16Int4
|
||||
|
||||
scheme = quant_config.init_scheme(
|
||||
scheme_cls,
|
||||
weight_quant_key=weight_key,
|
||||
activation_quant_key=act_key,
|
||||
weight_config=weight_config["weight"],
|
||||
)
|
||||
assert isinstance(scheme, QuarkW4A16Int4)
|
||||
assert not scheme.pack_reorder
|
||||
|
||||
def test_quark_int4_scheme_supports_asymmetric_weights(self):
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False))
|
||||
weight_config = quant_config.quant_config["global_quant_config"]
|
||||
weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config(
|
||||
weight_config
|
||||
)
|
||||
assert scheme_cls is QuarkW4A16Int4
|
||||
|
||||
scheme = quant_config.init_scheme(
|
||||
scheme_cls,
|
||||
weight_quant_key=weight_key,
|
||||
activation_quant_key=act_key,
|
||||
weight_config=weight_config["weight"],
|
||||
)
|
||||
assert isinstance(scheme, QuarkW4A16Int4)
|
||||
assert not scheme.is_symmetric
|
||||
|
||||
@pytest.mark.parametrize("missing_field", ["group_size", "symmetric"])
|
||||
def test_quark_int4_scheme_requires_weight_config_fields(self, missing_field):
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4Static,
|
||||
)
|
||||
|
||||
weight_config = {
|
||||
"dtype": "int4",
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
weight_config.pop(missing_field)
|
||||
quant_config = QuarkConfig.from_config(
|
||||
{
|
||||
"quant_method": "quark",
|
||||
"export": {"pack_method": "reorder", "kv_cache_group": []},
|
||||
"global_quant_config": {"weight": weight_config},
|
||||
"exclude": [],
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match=missing_field):
|
||||
quant_config.init_scheme(
|
||||
QuarkW4A16Int4,
|
||||
weight_quant_key=kInt4Static,
|
||||
activation_quant_key=None,
|
||||
weight_config=weight_config,
|
||||
)
|
||||
|
||||
def test_quark_int4_moe_uses_native_moe_method(self):
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4StaticAsym,
|
||||
)
|
||||
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False))
|
||||
moe_config = type("MoeConfig", (), {})()
|
||||
moe_config.has_bias = False
|
||||
|
||||
mock_backend = MagicMock()
|
||||
mock_experts_cls = MagicMock()
|
||||
with patch(
|
||||
"vllm.model_executor.layers.quantization.quark.quark_moe"
|
||||
".select_wna16_moe_backend",
|
||||
return_value=(mock_backend, mock_experts_cls),
|
||||
):
|
||||
method = QuarkW4A16Int4MoEMethod(
|
||||
kInt4StaticAsym,
|
||||
None,
|
||||
quant_config.quant_config["global_quant_config"]["weight"],
|
||||
quant_config.pack_method,
|
||||
moe_config,
|
||||
)
|
||||
|
||||
assert method.group_size == 128
|
||||
assert method.pack_reorder
|
||||
assert method.use_wna16_backend
|
||||
|
||||
def test_triton_wna16_experts_supports_asymmetric_int4(self):
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
|
||||
TritonWNA16Experts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4Static32Asym,
|
||||
kInt4StaticAsym,
|
||||
)
|
||||
|
||||
for key in [kInt4Static, kInt4Static32, kInt4StaticAsym, kInt4Static32Asym]:
|
||||
assert TritonWNA16Experts._supports_quant_scheme(key, None), (
|
||||
f"{key} should be supported by TritonWNA16Experts"
|
||||
)
|
||||
|
||||
def test_quark_moe_loader_shards_w2_zero_point_across_tp_ranks(self):
|
||||
"""w2 zero-points are sharded along the intermediate dim per TP rank.
|
||||
|
||||
Loading the same Quark checkpoint tensor at tensor_parallel_size=2 for
|
||||
both ranks must jointly reconstruct the tensor_parallel_size=1 (full)
|
||||
result, with no overlap or gap. This guards the custom loader's TP
|
||||
split for the packed zero-point layout.
|
||||
"""
|
||||
|
||||
class _FakeMoEConfig:
|
||||
has_bias = False
|
||||
|
||||
def __init__(self, tp_size, tp_rank):
|
||||
self.tp_size = tp_size
|
||||
self.tp_rank = tp_rank
|
||||
|
||||
class _FakeLayer:
|
||||
def __init__(self, moe_config):
|
||||
self.moe_config = moe_config
|
||||
self.intermediate_size_per_partition = 64
|
||||
self.group_size_div_factor = 1
|
||||
|
||||
def load_w2_zero_point(raw_zp, tp_size, tp_rank):
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4StaticAsym,
|
||||
)
|
||||
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False))
|
||||
moe_config = _FakeMoEConfig(tp_size, tp_rank)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.quantization.quark.quark_moe"
|
||||
".select_wna16_moe_backend",
|
||||
return_value=(MagicMock(), MagicMock()),
|
||||
):
|
||||
method = QuarkW4A16Int4MoEMethod(
|
||||
kInt4StaticAsym,
|
||||
None,
|
||||
quant_config.quant_config["global_quant_config"]["weight"],
|
||||
quant_config.pack_method,
|
||||
moe_config,
|
||||
)
|
||||
|
||||
layer = _FakeLayer(moe_config)
|
||||
# The loader converts the packed zero-point, then writes the
|
||||
# per-rank slice into param.data[expert_id]. A single expert dim is
|
||||
# enough; the last dim shrinks by tp_size after sharding.
|
||||
param = torch.nn.Parameter(
|
||||
torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
loader = method.get_weight_loader(layer, weight_loader=None)
|
||||
loader(
|
||||
param,
|
||||
raw_zp.clone(),
|
||||
weight_name="w2_weight_zero_point",
|
||||
shard_id="w2",
|
||||
expert_id=0,
|
||||
)
|
||||
return param.data[0]
|
||||
|
||||
# Quark-packed zero-point for one expert: rows = hidden // pack_factor,
|
||||
# cols carry the (packed) intermediate dim, split evenly across ranks.
|
||||
raw_zp = torch.randint(0, 256, (8, 16), dtype=torch.uint8)
|
||||
|
||||
full = load_w2_zero_point(raw_zp, tp_size=1, tp_rank=0)
|
||||
shard0 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=0)
|
||||
shard1 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=1)
|
||||
|
||||
expected = full.view(full.size(0), 2, -1)
|
||||
torch.testing.assert_close(shard0, expected[:, 0])
|
||||
torch.testing.assert_close(shard1, expected[:, 1])
|
||||
|
||||
def test_quark_shared_expert_gate_keeps_quantized_tensors(self):
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config())
|
||||
mapper = quant_config.get_cache_scale_mapper()
|
||||
|
||||
output_names = {
|
||||
name
|
||||
for name, _ in mapper.apply(
|
||||
[
|
||||
(
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate"
|
||||
".weight_scale",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate"
|
||||
".weight_zero_point",
|
||||
torch.zeros(1),
|
||||
),
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
assert (
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate.weight"
|
||||
in output_names
|
||||
)
|
||||
assert (
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate.weight_scale"
|
||||
in output_names
|
||||
)
|
||||
assert (
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate.weight_zero_point"
|
||||
in output_names
|
||||
)
|
||||
|
||||
def test_quark_apply_mapper_updates_exclude_and_layer_quant_config(self):
|
||||
quant_config = QuarkConfig.from_config(
|
||||
{
|
||||
**_quark_int4_config(),
|
||||
"exclude": ["lm_head"],
|
||||
"layer_quant_config": {
|
||||
"model.language_model.layers.0.mlp.gate": {
|
||||
"weight": {"dtype": "float16"},
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
quant_config.apply_vllm_mapper(
|
||||
WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"lm_head": "language_model.lm_head",
|
||||
"model.language_model.": "language_model.model.",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
layer_quant_config = quant_config.quant_config["layer_quant_config"]
|
||||
assert "language_model.model.layers.0.mlp.gate" in layer_quant_config
|
||||
assert "model.language_model.layers.0.mlp.gate" not in layer_quant_config
|
||||
assert quant_config.quant_config["exclude"] == [
|
||||
"language_model.lm_head",
|
||||
]
|
||||
|
||||
def test_quark_apply_mapper_ignores_non_string_list_entries(self):
|
||||
quant_config = QuarkConfig.from_config(
|
||||
{
|
||||
**_quark_int4_config(),
|
||||
"algo_config": [
|
||||
{"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]}
|
||||
],
|
||||
}
|
||||
)
|
||||
quant_config.apply_vllm_mapper(WeightsMapper())
|
||||
|
||||
assert quant_config.quant_config["algo_config"] == [
|
||||
{"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]}
|
||||
]
|
||||
|
||||
def test_quark_exclude_matches_via_mapper_prefix(self):
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config(exclude=["lm_head"]))
|
||||
quant_config.apply_vllm_mapper(
|
||||
WeightsMapper(orig_to_new_prefix={"lm_head": "language_model.lm_head"})
|
||||
)
|
||||
|
||||
exclude_layers = quant_config.quant_config["exclude"]
|
||||
assert should_ignore_layer("language_model.lm_head", ignore=exclude_layers)
|
||||
assert not should_ignore_layer(
|
||||
"language_model.model.layers.0.mlp.gate", ignore=exclude_layers
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("symmetric", [False, True])
|
||||
@pytest.mark.parametrize("pack_method", ["order", "reorder"])
|
||||
def test_quark_int4_canonicalizes_pack_for_kernel_layout(pack_method, symmetric):
|
||||
"""Quark pack order is normalized to the layout expected by awq_* ops.
|
||||
|
||||
This reuses the existing AWQ dequant/gemm kernels for compute only; loading
|
||||
still goes through the native Quark quantization path, not AutoAWQ.
|
||||
"""
|
||||
pack_reorder = pack_method == "reorder"
|
||||
group_size = 2
|
||||
packed_values = torch.tensor(
|
||||
[
|
||||
[0, 1, 7, 8, 9, 15, 2, 14],
|
||||
[15, 8, 0, 3, 12, 7, 1, 9],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
packed_zeros = torch.zeros((1, 8), dtype=torch.int32)
|
||||
qweight = _pack_int4_nibbles(packed_values, pack_reorder=pack_reorder).view(2, 1)
|
||||
qzeros = _pack_int4_nibbles(packed_zeros, pack_reorder=pack_reorder).view(1, 1)
|
||||
scales = torch.ones((1, 8), dtype=torch.float16)
|
||||
|
||||
dequantize = (
|
||||
_dequantize_quark_signed_awq_torch
|
||||
if symmetric
|
||||
else _dequantize_awq_unsigned_torch
|
||||
)
|
||||
expected = dequantize(
|
||||
qweight,
|
||||
scales,
|
||||
qzeros,
|
||||
group_size,
|
||||
pack_reorder=pack_reorder,
|
||||
)
|
||||
canonical_weight = canonicalize_quark_packed_int4(
|
||||
qweight,
|
||||
pack_reorder=pack_reorder,
|
||||
is_symmetric=symmetric,
|
||||
)
|
||||
canonical_zero = canonicalize_quark_packed_int4(
|
||||
qzeros,
|
||||
pack_reorder=pack_reorder,
|
||||
is_symmetric=symmetric,
|
||||
)
|
||||
actual = _dequantize_awq_unsigned_torch(
|
||||
canonical_weight, scales, canonical_zero, group_size
|
||||
)
|
||||
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
Reference in New Issue
Block a user