[Quantization] Support native Quark W4A16 INT4/UINT4 exports in vLLM (#48606)
Signed-off-by: HongWei Meng <HongWei.Meng@amd.com>
This commit is contained in:
@@ -58,12 +58,17 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
|
||||
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
|
||||
QuarkMoEMethod,
|
||||
QuarkW4A8Fp8MoEMethod,
|
||||
QuarkW4A16Int4MoEMethod,
|
||||
QuarkW8A8Fp8MoEMethod,
|
||||
QuarkW8A8Int8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import (
|
||||
QuarkScheme,
|
||||
QuarkW4A16Int4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
QuarkQTensorHint,
|
||||
canonicalize_quark_packed_int4,
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
@@ -97,6 +102,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.model_executor.models.llama import LlamaForCausalLM
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
@@ -1876,3 +1882,435 @@ def test_glm5next_attn_loader_accepts_deepseek_weight_scale_inv():
|
||||
)
|
||||
assert target.loaded_shard is None # o_proj is a direct (non-fused) proj
|
||||
assert torch.equal(target.loaded_weight, _dequant_fp8_block(weight, scale, 128))
|
||||
|
||||
|
||||
_REVERSE_AWQ_PACK_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
|
||||
|
||||
|
||||
def _quark_int4_config(
|
||||
*,
|
||||
pack_method: str = "reorder",
|
||||
symmetric: bool = True,
|
||||
exclude: list[str] | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
"quant_method": "quark",
|
||||
"export": {"pack_method": pack_method, "kv_cache_group": []},
|
||||
"global_quant_config": {
|
||||
"weight": {
|
||||
"dtype": "int4",
|
||||
"group_size": 128,
|
||||
"symmetric": symmetric,
|
||||
}
|
||||
},
|
||||
"exclude": exclude or [],
|
||||
}
|
||||
|
||||
|
||||
def _sign_extend_int4_nibbles(t: torch.Tensor) -> torch.Tensor:
|
||||
mask = (t & 0x8).bool()
|
||||
t = t.clone()
|
||||
t[mask] = t[mask] | 0xF0
|
||||
return t
|
||||
|
||||
|
||||
def _dequantize_quark_signed_awq_torch(
|
||||
qweight: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
qzeros: torch.Tensor,
|
||||
group_size: int,
|
||||
*,
|
||||
pack_reorder: bool = True,
|
||||
) -> torch.Tensor:
|
||||
bits = 4
|
||||
shifts = torch.arange(0, 32, bits, device=qweight.device)
|
||||
iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
|
||||
iweights = iweights.view(qweight.shape[0], -1)
|
||||
zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
|
||||
zeros = zeros.view(qzeros.shape[0], -1)
|
||||
|
||||
if pack_reorder:
|
||||
order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device)
|
||||
else:
|
||||
order = torch.arange(8, device=qweight.device)
|
||||
iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape(
|
||||
qweight.shape[0], -1
|
||||
)
|
||||
zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1)
|
||||
iweights = _sign_extend_int4_nibbles(iweights & 0xF)
|
||||
zeros = _sign_extend_int4_nibbles(zeros & 0xF)
|
||||
|
||||
scales = scales.repeat_interleave(group_size, dim=0)
|
||||
zeros = zeros.repeat_interleave(group_size, dim=0)
|
||||
return (iweights - zeros) * scales
|
||||
|
||||
|
||||
def _dequantize_awq_unsigned_torch(
|
||||
qweight: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
qzeros: torch.Tensor,
|
||||
group_size: int,
|
||||
*,
|
||||
pack_reorder: bool = True,
|
||||
) -> torch.Tensor:
|
||||
bits = 4
|
||||
shifts = torch.arange(0, 32, bits, device=qweight.device)
|
||||
iweights = ((qweight[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
|
||||
iweights = iweights.view(qweight.shape[0], -1)
|
||||
zeros = ((qzeros[:, :, None] >> shifts[None, None, :]) & 0xF).to(torch.int8)
|
||||
zeros = zeros.view(qzeros.shape[0], -1)
|
||||
|
||||
if pack_reorder:
|
||||
order = torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=qweight.device)
|
||||
else:
|
||||
order = torch.arange(8, device=qweight.device)
|
||||
iweights = iweights.view(qweight.shape[0], -1, 8)[:, :, order].reshape(
|
||||
qweight.shape[0], -1
|
||||
)
|
||||
zeros = zeros.view(qzeros.shape[0], -1, 8)[:, :, order].reshape(qzeros.shape[0], -1)
|
||||
|
||||
scales = scales.repeat_interleave(group_size, dim=0)
|
||||
zeros = zeros.repeat_interleave(group_size, dim=0)
|
||||
return (iweights - zeros) * scales
|
||||
|
||||
|
||||
def _pack_int4_nibbles(nibbles: torch.Tensor, *, pack_reorder: bool) -> torch.Tensor:
|
||||
pack_order = (
|
||||
torch.tensor(_REVERSE_AWQ_PACK_ORDER, device=nibbles.device)
|
||||
if pack_reorder
|
||||
else torch.arange(8, device=nibbles.device)
|
||||
)
|
||||
shifts = pack_order * 4
|
||||
return ((nibbles.to(torch.int64) & 0xF) << shifts).sum(dim=-1).to(torch.int32)
|
||||
|
||||
|
||||
class TestQuarkInt4Format:
|
||||
"""Tests for Quark INT4 export format compatibility."""
|
||||
|
||||
def test_quark_order_pack_method_uses_native_int4_scheme(self):
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config(pack_method="order"))
|
||||
weight_config = quant_config.quant_config["global_quant_config"]
|
||||
weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config(
|
||||
weight_config
|
||||
)
|
||||
assert scheme_cls is QuarkW4A16Int4
|
||||
|
||||
scheme = quant_config.init_scheme(
|
||||
scheme_cls,
|
||||
weight_quant_key=weight_key,
|
||||
activation_quant_key=act_key,
|
||||
weight_config=weight_config["weight"],
|
||||
)
|
||||
assert isinstance(scheme, QuarkW4A16Int4)
|
||||
assert not scheme.pack_reorder
|
||||
|
||||
def test_quark_int4_scheme_supports_asymmetric_weights(self):
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False))
|
||||
weight_config = quant_config.quant_config["global_quant_config"]
|
||||
weight_key, act_key, scheme_cls = quant_config._get_scheme_cls_from_config(
|
||||
weight_config
|
||||
)
|
||||
assert scheme_cls is QuarkW4A16Int4
|
||||
|
||||
scheme = quant_config.init_scheme(
|
||||
scheme_cls,
|
||||
weight_quant_key=weight_key,
|
||||
activation_quant_key=act_key,
|
||||
weight_config=weight_config["weight"],
|
||||
)
|
||||
assert isinstance(scheme, QuarkW4A16Int4)
|
||||
assert not scheme.is_symmetric
|
||||
|
||||
@pytest.mark.parametrize("missing_field", ["group_size", "symmetric"])
|
||||
def test_quark_int4_scheme_requires_weight_config_fields(self, missing_field):
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4Static,
|
||||
)
|
||||
|
||||
weight_config = {
|
||||
"dtype": "int4",
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
weight_config.pop(missing_field)
|
||||
quant_config = QuarkConfig.from_config(
|
||||
{
|
||||
"quant_method": "quark",
|
||||
"export": {"pack_method": "reorder", "kv_cache_group": []},
|
||||
"global_quant_config": {"weight": weight_config},
|
||||
"exclude": [],
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match=missing_field):
|
||||
quant_config.init_scheme(
|
||||
QuarkW4A16Int4,
|
||||
weight_quant_key=kInt4Static,
|
||||
activation_quant_key=None,
|
||||
weight_config=weight_config,
|
||||
)
|
||||
|
||||
def test_quark_int4_moe_uses_native_moe_method(self):
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4StaticAsym,
|
||||
)
|
||||
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False))
|
||||
moe_config = type("MoeConfig", (), {})()
|
||||
moe_config.has_bias = False
|
||||
|
||||
mock_backend = MagicMock()
|
||||
mock_experts_cls = MagicMock()
|
||||
with patch(
|
||||
"vllm.model_executor.layers.quantization.quark.quark_moe"
|
||||
".select_wna16_moe_backend",
|
||||
return_value=(mock_backend, mock_experts_cls),
|
||||
):
|
||||
method = QuarkW4A16Int4MoEMethod(
|
||||
kInt4StaticAsym,
|
||||
None,
|
||||
quant_config.quant_config["global_quant_config"]["weight"],
|
||||
quant_config.pack_method,
|
||||
moe_config,
|
||||
)
|
||||
|
||||
assert method.group_size == 128
|
||||
assert method.pack_reorder
|
||||
assert method.use_wna16_backend
|
||||
|
||||
def test_triton_wna16_experts_supports_asymmetric_int4(self):
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
|
||||
TritonWNA16Experts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4Static32Asym,
|
||||
kInt4StaticAsym,
|
||||
)
|
||||
|
||||
for key in [kInt4Static, kInt4Static32, kInt4StaticAsym, kInt4Static32Asym]:
|
||||
assert TritonWNA16Experts._supports_quant_scheme(key, None), (
|
||||
f"{key} should be supported by TritonWNA16Experts"
|
||||
)
|
||||
|
||||
def test_quark_moe_loader_shards_w2_zero_point_across_tp_ranks(self):
|
||||
"""w2 zero-points are sharded along the intermediate dim per TP rank.
|
||||
|
||||
Loading the same Quark checkpoint tensor at tensor_parallel_size=2 for
|
||||
both ranks must jointly reconstruct the tensor_parallel_size=1 (full)
|
||||
result, with no overlap or gap. This guards the custom loader's TP
|
||||
split for the packed zero-point layout.
|
||||
"""
|
||||
|
||||
class _FakeMoEConfig:
|
||||
has_bias = False
|
||||
|
||||
def __init__(self, tp_size, tp_rank):
|
||||
self.tp_size = tp_size
|
||||
self.tp_rank = tp_rank
|
||||
|
||||
class _FakeLayer:
|
||||
def __init__(self, moe_config):
|
||||
self.moe_config = moe_config
|
||||
self.intermediate_size_per_partition = 64
|
||||
self.group_size_div_factor = 1
|
||||
|
||||
def load_w2_zero_point(raw_zp, tp_size, tp_rank):
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4StaticAsym,
|
||||
)
|
||||
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config(symmetric=False))
|
||||
moe_config = _FakeMoEConfig(tp_size, tp_rank)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.quantization.quark.quark_moe"
|
||||
".select_wna16_moe_backend",
|
||||
return_value=(MagicMock(), MagicMock()),
|
||||
):
|
||||
method = QuarkW4A16Int4MoEMethod(
|
||||
kInt4StaticAsym,
|
||||
None,
|
||||
quant_config.quant_config["global_quant_config"]["weight"],
|
||||
quant_config.pack_method,
|
||||
moe_config,
|
||||
)
|
||||
|
||||
layer = _FakeLayer(moe_config)
|
||||
# The loader converts the packed zero-point, then writes the
|
||||
# per-rank slice into param.data[expert_id]. A single expert dim is
|
||||
# enough; the last dim shrinks by tp_size after sharding.
|
||||
param = torch.nn.Parameter(
|
||||
torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
loader = method.get_weight_loader(layer, weight_loader=None)
|
||||
loader(
|
||||
param,
|
||||
raw_zp.clone(),
|
||||
weight_name="w2_weight_zero_point",
|
||||
shard_id="w2",
|
||||
expert_id=0,
|
||||
)
|
||||
return param.data[0]
|
||||
|
||||
# Quark-packed zero-point for one expert: rows = hidden // pack_factor,
|
||||
# cols carry the (packed) intermediate dim, split evenly across ranks.
|
||||
raw_zp = torch.randint(0, 256, (8, 16), dtype=torch.uint8)
|
||||
|
||||
full = load_w2_zero_point(raw_zp, tp_size=1, tp_rank=0)
|
||||
shard0 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=0)
|
||||
shard1 = load_w2_zero_point(raw_zp, tp_size=2, tp_rank=1)
|
||||
|
||||
expected = full.view(full.size(0), 2, -1)
|
||||
torch.testing.assert_close(shard0, expected[:, 0])
|
||||
torch.testing.assert_close(shard1, expected[:, 1])
|
||||
|
||||
def test_quark_shared_expert_gate_keeps_quantized_tensors(self):
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config())
|
||||
mapper = quant_config.get_cache_scale_mapper()
|
||||
|
||||
output_names = {
|
||||
name
|
||||
for name, _ in mapper.apply(
|
||||
[
|
||||
(
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate.weight",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate"
|
||||
".weight_scale",
|
||||
torch.zeros(1),
|
||||
),
|
||||
(
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate"
|
||||
".weight_zero_point",
|
||||
torch.zeros(1),
|
||||
),
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
assert (
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate.weight"
|
||||
in output_names
|
||||
)
|
||||
assert (
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate.weight_scale"
|
||||
in output_names
|
||||
)
|
||||
assert (
|
||||
"model.language_model.layers.0.mlp.shared_expert_gate.weight_zero_point"
|
||||
in output_names
|
||||
)
|
||||
|
||||
def test_quark_apply_mapper_updates_exclude_and_layer_quant_config(self):
|
||||
quant_config = QuarkConfig.from_config(
|
||||
{
|
||||
**_quark_int4_config(),
|
||||
"exclude": ["lm_head"],
|
||||
"layer_quant_config": {
|
||||
"model.language_model.layers.0.mlp.gate": {
|
||||
"weight": {"dtype": "float16"},
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
quant_config.apply_vllm_mapper(
|
||||
WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"lm_head": "language_model.lm_head",
|
||||
"model.language_model.": "language_model.model.",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
layer_quant_config = quant_config.quant_config["layer_quant_config"]
|
||||
assert "language_model.model.layers.0.mlp.gate" in layer_quant_config
|
||||
assert "model.language_model.layers.0.mlp.gate" not in layer_quant_config
|
||||
assert quant_config.quant_config["exclude"] == [
|
||||
"language_model.lm_head",
|
||||
]
|
||||
|
||||
def test_quark_apply_mapper_ignores_non_string_list_entries(self):
|
||||
quant_config = QuarkConfig.from_config(
|
||||
{
|
||||
**_quark_int4_config(),
|
||||
"algo_config": [
|
||||
{"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]}
|
||||
],
|
||||
}
|
||||
)
|
||||
quant_config.apply_vllm_mapper(WeightsMapper())
|
||||
|
||||
assert quant_config.quant_config["algo_config"] == [
|
||||
{"name": "qronos", "inside_layer_modules": ["self_attn.q_proj"]}
|
||||
]
|
||||
|
||||
def test_quark_exclude_matches_via_mapper_prefix(self):
|
||||
quant_config = QuarkConfig.from_config(_quark_int4_config(exclude=["lm_head"]))
|
||||
quant_config.apply_vllm_mapper(
|
||||
WeightsMapper(orig_to_new_prefix={"lm_head": "language_model.lm_head"})
|
||||
)
|
||||
|
||||
exclude_layers = quant_config.quant_config["exclude"]
|
||||
assert should_ignore_layer("language_model.lm_head", ignore=exclude_layers)
|
||||
assert not should_ignore_layer(
|
||||
"language_model.model.layers.0.mlp.gate", ignore=exclude_layers
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("symmetric", [False, True])
|
||||
@pytest.mark.parametrize("pack_method", ["order", "reorder"])
|
||||
def test_quark_int4_canonicalizes_pack_for_kernel_layout(pack_method, symmetric):
|
||||
"""Quark pack order is normalized to the layout expected by awq_* ops.
|
||||
|
||||
This reuses the existing AWQ dequant/gemm kernels for compute only; loading
|
||||
still goes through the native Quark quantization path, not AutoAWQ.
|
||||
"""
|
||||
pack_reorder = pack_method == "reorder"
|
||||
group_size = 2
|
||||
packed_values = torch.tensor(
|
||||
[
|
||||
[0, 1, 7, 8, 9, 15, 2, 14],
|
||||
[15, 8, 0, 3, 12, 7, 1, 9],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
packed_zeros = torch.zeros((1, 8), dtype=torch.int32)
|
||||
qweight = _pack_int4_nibbles(packed_values, pack_reorder=pack_reorder).view(2, 1)
|
||||
qzeros = _pack_int4_nibbles(packed_zeros, pack_reorder=pack_reorder).view(1, 1)
|
||||
scales = torch.ones((1, 8), dtype=torch.float16)
|
||||
|
||||
dequantize = (
|
||||
_dequantize_quark_signed_awq_torch
|
||||
if symmetric
|
||||
else _dequantize_awq_unsigned_torch
|
||||
)
|
||||
expected = dequantize(
|
||||
qweight,
|
||||
scales,
|
||||
qzeros,
|
||||
group_size,
|
||||
pack_reorder=pack_reorder,
|
||||
)
|
||||
canonical_weight = canonicalize_quark_packed_int4(
|
||||
qweight,
|
||||
pack_reorder=pack_reorder,
|
||||
is_symmetric=symmetric,
|
||||
)
|
||||
canonical_zero = canonicalize_quark_packed_int4(
|
||||
qzeros,
|
||||
pack_reorder=pack_reorder,
|
||||
is_symmetric=symmetric,
|
||||
)
|
||||
actual = _dequantize_awq_unsigned_torch(
|
||||
canonical_weight, scales, canonical_zero, group_size
|
||||
)
|
||||
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
@@ -864,6 +864,7 @@ def choose_mp_linear_kernel(
|
||||
|
||||
can_implement, failure_reason = kernel.can_implement(config)
|
||||
if can_implement:
|
||||
logger.info_once("Using %s for mixed-precision linear", kernel.__name__)
|
||||
return kernel
|
||||
else:
|
||||
failure_reasons.append(
|
||||
|
||||
@@ -596,9 +596,9 @@ class TritonWNA16Experts(TritonExperts):
|
||||
) -> bool:
|
||||
SUPPORTED_W = [
|
||||
kInt4Static,
|
||||
kInt4StaticAsym,
|
||||
kInt8Static,
|
||||
kInt4Static32,
|
||||
kInt4StaticAsym,
|
||||
kInt4Static32Asym,
|
||||
# other group sizes?
|
||||
]
|
||||
|
||||
@@ -93,7 +93,7 @@ def _replace_or_register_parameter(
|
||||
def _convert_awq_to_standard_format(
|
||||
layer: torch.nn.Module,
|
||||
w_q_name: str,
|
||||
w_zp_name: str,
|
||||
w_zp_name: str | None,
|
||||
size_bits: int,
|
||||
) -> None:
|
||||
"""Convert AWQ weight and zero-point tensors to standard GPTQ-like format.
|
||||
@@ -101,6 +101,7 @@ def _convert_awq_to_standard_format(
|
||||
AWQ packs qweight along the output dim with a non-standard bit order.
|
||||
This converts to standard bit order and repacks qweight along the input
|
||||
dim, matching the format expected by the MPLinearKernel framework.
|
||||
If w_zp_name is None (symmetric quantization), only the weight is converted.
|
||||
"""
|
||||
pack_factor = 32 // size_bits
|
||||
mask = (1 << size_bits) - 1
|
||||
@@ -139,6 +140,9 @@ def _convert_awq_to_standard_format(
|
||||
)
|
||||
setattr(layer, w_q_name, new_param)
|
||||
|
||||
if w_zp_name is None:
|
||||
return
|
||||
|
||||
# --- Convert qzeros: fix AWQ bit ordering and repack
|
||||
# AWQ qzeros: (G, N // pack) packed along dim 1, AWQ bit order
|
||||
# Target: (N // pack, G) packed along dim 0, standard bit order
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
|
||||
QuarkOCP_MX,
|
||||
QuarkScheme,
|
||||
QuarkW4A8_MXFP4_FP8,
|
||||
QuarkW4A16Int4,
|
||||
QuarkW8A8Fp8,
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
QuarkW8A8Int8,
|
||||
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
QuarkQTensorHint,
|
||||
deep_compare,
|
||||
parse_w4a16_int4_weight_config,
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
@@ -54,6 +56,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4Static32Asym,
|
||||
kInt4StaticAsym,
|
||||
kInt4W4A8StaticChannelSym,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
@@ -172,8 +178,10 @@ class QuarkConfig(QuantizationConfig):
|
||||
quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {}
|
||||
|
||||
for k, v in self.quant_config.items():
|
||||
if isinstance(v, list):
|
||||
if isinstance(v, list) and all(isinstance(item, str) for item in v):
|
||||
quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_list(v)
|
||||
elif isinstance(v, list):
|
||||
quant_config_with_hf_to_vllm_mapper[k] = v
|
||||
elif isinstance(v, dict):
|
||||
quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_dict(v)
|
||||
else:
|
||||
@@ -215,6 +223,9 @@ class QuarkConfig(QuantizationConfig):
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
dynamic_mxfp4_quant=dynamic_mxfp4_quant,
|
||||
weight_config=self._find_matched_config(prefix, type(layer)).get(
|
||||
"weight"
|
||||
),
|
||||
)
|
||||
layer.scheme = scheme
|
||||
return QuarkLinearMethod(self)
|
||||
@@ -227,6 +238,7 @@ class QuarkConfig(QuantizationConfig):
|
||||
method_cls=method_cls,
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
layer_name=prefix,
|
||||
)
|
||||
|
||||
return None
|
||||
@@ -572,6 +584,26 @@ class QuarkConfig(QuantizationConfig):
|
||||
)
|
||||
return QuantKeyMatch(True, activation_quant_key, weight_quant_key)
|
||||
|
||||
def _is_w4a16_int4(
|
||||
self,
|
||||
weight_quant: dict[str, Any] | None,
|
||||
input_quant: dict[str, Any] | None,
|
||||
) -> QuantKeyMatch:
|
||||
if weight_quant is None or input_quant is not None:
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
is_int4 = weight_quant.get("dtype") in ("int4", "uint4")
|
||||
is_packed = self.pack_method in ("order", "reorder")
|
||||
if not (is_int4 and is_packed):
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
group_size, is_symmetric = parse_w4a16_int4_weight_config(weight_quant)
|
||||
if is_symmetric:
|
||||
weight_key = kInt4Static32 if group_size == 32 else kInt4Static
|
||||
else:
|
||||
weight_key = kInt4Static32Asym if group_size == 32 else kInt4StaticAsym
|
||||
return QuantKeyMatch(True, None, weight_key)
|
||||
|
||||
def _is_w4a8_mxfp4_fp8(
|
||||
self,
|
||||
weight_quant: dict[str, Any] | None,
|
||||
@@ -890,6 +922,12 @@ class QuarkConfig(QuantizationConfig):
|
||||
match.activation_quant_key,
|
||||
QuarkW8A8Int8,
|
||||
)
|
||||
elif match := self._is_w4a16_int4(weight_config, input_config):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkW4A16Int4,
|
||||
)
|
||||
elif match := self._is_w4a8_mxfp4_fp8(weight_config, input_config):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
@@ -919,6 +957,7 @@ class QuarkConfig(QuantizationConfig):
|
||||
weight_quant_key: QuantKey | None,
|
||||
activation_quant_key: QuantKey | None,
|
||||
dynamic_mxfp4_quant: bool = False,
|
||||
weight_config: dict[str, Any] | None = None,
|
||||
) -> "QuarkScheme":
|
||||
"""Construct a Quark scheme selected by get_scheme_cls."""
|
||||
if scheme_cls not in (
|
||||
@@ -928,9 +967,24 @@ class QuarkConfig(QuantizationConfig):
|
||||
QuarkOCP_MX,
|
||||
QuarkNVFP4,
|
||||
QuarkW4A8_MXFP4_FP8,
|
||||
QuarkW4A16Int4,
|
||||
):
|
||||
raise AssertionError(f"Unsupported Quark scheme class: {scheme_cls}")
|
||||
|
||||
scheme: QuarkScheme
|
||||
if scheme_cls is QuarkW4A16Int4:
|
||||
normalized_weight_config = self._unwrap_single_quant_config(weight_config)
|
||||
assert normalized_weight_config is not None
|
||||
assert weight_quant_key is not None
|
||||
scheme = QuarkW4A16Int4(
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
pack_method=self.pack_method,
|
||||
weight_config=normalized_weight_config,
|
||||
)
|
||||
self._check_scheme_supported(scheme.get_min_capability())
|
||||
return scheme
|
||||
|
||||
kwargs: dict[str, Any] = {"activation_quant_key": activation_quant_key}
|
||||
|
||||
# These classes handle different possible `weight_quant_key`.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -40,6 +40,12 @@ from vllm.model_executor.layers.fused_moe.oracle.int8 import (
|
||||
make_int8_moe_quant_config,
|
||||
select_int8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
convert_to_wna16_moe_kernel_format,
|
||||
make_wna16_moe_kernel,
|
||||
make_wna16_moe_quant_config,
|
||||
select_wna16_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
@@ -56,7 +62,10 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
make_nvfp4_moe_quant_config,
|
||||
select_nvfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
QuarkQTensorHint,
|
||||
parse_w4a16_int4_weight_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
validate_fp8_block_shape_moe,
|
||||
)
|
||||
@@ -75,6 +84,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4Static32Asym,
|
||||
kInt4StaticAsym,
|
||||
kInt4W4A8StaticChannelSym,
|
||||
kInt8DynamicTensorAsym,
|
||||
kInt8DynamicTensorSym,
|
||||
@@ -102,6 +115,7 @@ logger = init_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"QuarkMoEMethod",
|
||||
"QuarkW4A16Int4MoEMethod",
|
||||
"QuarkW8A8Fp8MoEMethod",
|
||||
"QuarkOCP_MX_MoEMethod",
|
||||
"QuarkNvfp4MoEMethod",
|
||||
@@ -179,6 +193,12 @@ class QuarkMoEMethod(FusedMoEMethodBase):
|
||||
match.activation_quant_key,
|
||||
QuarkW8A8Fp8MoEMethod,
|
||||
)
|
||||
if match := quant_config._is_w4a16_int4(weight_config, input_config):
|
||||
return (
|
||||
match.weight_quant_key,
|
||||
match.activation_quant_key,
|
||||
QuarkW4A16Int4MoEMethod,
|
||||
)
|
||||
if match := quant_config._is_w_ocp_mx_a_x(
|
||||
weight_config, input_config, allow_static_fp8=True
|
||||
):
|
||||
@@ -202,6 +222,7 @@ class QuarkMoEMethod(FusedMoEMethodBase):
|
||||
method_cls: type["QuarkMoEMethod"],
|
||||
weight_quant_key: QuantKey | None,
|
||||
activation_quant_key: QuantKey | None,
|
||||
layer_name: str | None = None,
|
||||
) -> "QuarkMoEMethod":
|
||||
if weight_quant_key is None:
|
||||
raise RuntimeError("Unsupported FusedMoe scheme")
|
||||
@@ -226,10 +247,339 @@ class QuarkMoEMethod(FusedMoEMethodBase):
|
||||
return QuarkW8A8Int8MoEMethod(
|
||||
module.moe_config, weight_quant_key, activation_quant_key
|
||||
)
|
||||
elif method_cls is QuarkW4A16Int4MoEMethod:
|
||||
assert layer_name is not None
|
||||
layer_quant_config = quant_config._find_matched_config(
|
||||
layer_name, type(module)
|
||||
)
|
||||
weight_config = quant_config._unwrap_single_quant_config(
|
||||
layer_quant_config.get("weight")
|
||||
)
|
||||
if weight_config is None:
|
||||
raise RuntimeError(
|
||||
f"Missing weight quantization config for layer {layer_name}"
|
||||
)
|
||||
return QuarkW4A16Int4MoEMethod(
|
||||
weight_quant_key,
|
||||
activation_quant_key,
|
||||
weight_config,
|
||||
quant_config.pack_method,
|
||||
module.moe_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
else:
|
||||
raise RuntimeError("Unsupported FusedMoe scheme")
|
||||
|
||||
|
||||
class QuarkW4A16Int4MoEMethod(QuarkMoEMethod):
|
||||
"""Quark packed INT4 weight-only MoE method."""
|
||||
|
||||
supported_activation_quant_keys: list[QuantKey | None] = [None]
|
||||
supported_weight_quant_keys: list[QuantKey] = [
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4StaticAsym,
|
||||
kInt4Static32Asym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
weight_config: dict[str, Any],
|
||||
pack_method: str,
|
||||
moe: FusedMoEConfig,
|
||||
quant_config: "QuarkConfig | None" = None, # type: ignore # noqa F821
|
||||
):
|
||||
super().__init__(moe, weight_quant_key, activation_quant_key)
|
||||
|
||||
self.quant_config = quant_config
|
||||
# group_size is read from the Quark config rather than weight_quant_key:
|
||||
# the int4 QuantKeys only encode group_size 32 or -1, but Quark int4
|
||||
# checkpoints commonly use group_size 128, which the QuantKey cannot
|
||||
# represent. Symmetry is unambiguous, so it comes from the key.
|
||||
self.group_size = parse_w4a16_int4_weight_config(weight_config)[0]
|
||||
self.is_symmetric = weight_quant_key.symmetric
|
||||
self.bit8_pack_factor = 2
|
||||
self.pack_reorder = pack_method == "reorder"
|
||||
|
||||
self.use_wna16_backend = True
|
||||
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=weight_quant_key,
|
||||
quant_config=self.quant_config,
|
||||
may_have_zp=not self.is_symmetric,
|
||||
may_have_bias=False,
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
group_size = self.group_size
|
||||
group_size_div_factor = 1
|
||||
|
||||
while intermediate_size_per_partition % group_size or hidden_size % group_size:
|
||||
group_size = group_size // 2
|
||||
group_size_div_factor *= 2
|
||||
assert group_size >= 32
|
||||
layer.group_size = group_size
|
||||
layer.group_size_div_factor = group_size_div_factor
|
||||
|
||||
strategy = FusedMoeWeightScaleSupported.GROUP.value
|
||||
extra_weight_attrs.update({"quant_method": strategy, "is_transposed": False})
|
||||
|
||||
assert "weight_loader" in extra_weight_attrs
|
||||
weight_loader = extra_weight_attrs["weight_loader"]
|
||||
extra_weight_attrs["weight_loader"] = self.get_weight_loader(
|
||||
layer, weight_loader
|
||||
)
|
||||
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size // self.bit8_pack_factor,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // self.bit8_pack_factor,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size // group_size,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // group_size,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
|
||||
w13_weight_zero_point = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition // self.bit8_pack_factor,
|
||||
hidden_size // group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_weight_zero_point", w13_weight_zero_point)
|
||||
set_weight_attrs(w13_weight_zero_point, extra_weight_attrs)
|
||||
|
||||
w2_weight_zero_point = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
hidden_size // self.bit8_pack_factor,
|
||||
intermediate_size_per_partition // group_size,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_zero_point", w2_weight_zero_point)
|
||||
set_weight_attrs(w2_weight_zero_point, extra_weight_attrs)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
w13_qzeros = layer.w13_weight_zero_point if not self.is_symmetric else None
|
||||
w2_qzeros = layer.w2_weight_zero_point if not self.is_symmetric else None
|
||||
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_backend,
|
||||
layer=layer,
|
||||
quant_config=self.quant_config,
|
||||
input_dtype=None,
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
w13_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_qzeros=w13_qzeros,
|
||||
w2_qzeros=w2_qzeros,
|
||||
)
|
||||
|
||||
if converted is None:
|
||||
# Backend rewrote the layer's params in place (e.g. Humming).
|
||||
self._setup_kernel(layer)
|
||||
return
|
||||
|
||||
(
|
||||
w13_qweight,
|
||||
w2_qweight,
|
||||
w13_scales,
|
||||
w2_scales,
|
||||
_, # w13_qzeros
|
||||
_, # w2_qzeros
|
||||
_, # w13_input_global_scale
|
||||
_, # w2_input_global_scale
|
||||
_, # w13_bias
|
||||
_, # w2_bias
|
||||
) = converted
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13_qweight)
|
||||
replace_parameter(layer, "w2_weight", w2_qweight)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scales)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scales)
|
||||
|
||||
self._setup_kernel(layer)
|
||||
|
||||
def _setup_kernel(self, layer: RoutedExperts) -> None:
|
||||
assert self.experts_cls is not None
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
self.moe_kernel = make_wna16_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.wna16_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: RoutedExperts
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
w1_zp = layer.w13_weight_zero_point if not self.is_symmetric else None
|
||||
w2_zp = layer.w2_weight_zero_point if not self.is_symmetric else None
|
||||
return make_wna16_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w1_zp=w1_zp,
|
||||
w2_zp=w2_zp,
|
||||
group_size=layer.group_size,
|
||||
num_bits=4,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
expert_map=layer.expert_map,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
def get_weight_loader(self, layer: RoutedExperts, weight_loader):
|
||||
def convert_quark_tensor(tensor: torch.Tensor, tensor_type: str):
|
||||
size0 = tensor.size(0)
|
||||
tensor = tensor.view(torch.uint8)
|
||||
|
||||
shifter = torch.tensor([0, 4], dtype=torch.uint8, device=tensor.device)
|
||||
tensor = (tensor[:, :, None] >> shifter) & 0xF
|
||||
|
||||
if self.pack_reorder:
|
||||
reverse_awq_pack_order = [0, 4, 1, 5, 2, 6, 3, 7]
|
||||
tensor = tensor.view(-1, 8)[:, reverse_awq_pack_order]
|
||||
else:
|
||||
tensor = tensor.view(-1, 8)
|
||||
if self.is_symmetric:
|
||||
tensor = tensor ^ 0x8
|
||||
tensor = tensor.view(size0, -1)
|
||||
|
||||
tensor = tensor.T.contiguous()
|
||||
if tensor_type == "weight":
|
||||
tensor = tensor[:, 1::2] * 16 + tensor[:, ::2]
|
||||
elif tensor_type == "zero_point":
|
||||
tensor = tensor[1::2, :] * 16 + tensor[::2, :]
|
||||
return tensor
|
||||
|
||||
def quark_int4_weight_loader(
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
return_success: bool = False,
|
||||
):
|
||||
loaded_weight = loaded_weight.to(param.data.device)
|
||||
shard_size = layer.intermediate_size_per_partition
|
||||
|
||||
if weight_name.endswith("_weight"):
|
||||
loaded_weight = convert_quark_tensor(loaded_weight, "weight")
|
||||
elif weight_name.endswith("_weight_zero_point"):
|
||||
loaded_weight = convert_quark_tensor(loaded_weight, "zero_point")
|
||||
elif weight_name.endswith("_weight_scale"):
|
||||
loaded_weight = loaded_weight.T
|
||||
|
||||
if layer.group_size_div_factor > 1 and (
|
||||
"weight_zero_point" in weight_name or "weight_scale" in weight_name
|
||||
):
|
||||
loaded_weight = loaded_weight.repeat_interleave(
|
||||
layer.group_size_div_factor, 1
|
||||
)
|
||||
|
||||
if "w13_weight_zero_point" in weight_name:
|
||||
tensor = loaded_weight.view(
|
||||
layer.moe_config.tp_size, -1, loaded_weight.size(1)
|
||||
)[layer.moe_config.tp_rank]
|
||||
if shard_id == "w1":
|
||||
param.data[expert_id, : shard_size // 2] = tensor
|
||||
else:
|
||||
param.data[expert_id, shard_size // 2 :] = tensor
|
||||
return True if return_success else None
|
||||
elif "w2_weight_zero_point" in weight_name:
|
||||
param.data[expert_id] = loaded_weight.view(
|
||||
loaded_weight.size(0), layer.moe_config.tp_size, -1
|
||||
)[:, layer.moe_config.tp_rank]
|
||||
return True if return_success else None
|
||||
|
||||
return weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
weight_name,
|
||||
shard_id,
|
||||
expert_id,
|
||||
return_success=return_success,
|
||||
)
|
||||
|
||||
return quark_int4_weight_loader
|
||||
|
||||
|
||||
class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
supported_activation_quant_keys = [
|
||||
kFp8StaticTensorSym,
|
||||
|
||||
@@ -5,6 +5,7 @@ from .quark_nvfp4 import QuarkNVFP4
|
||||
from .quark_ocp_mx import QuarkOCP_MX
|
||||
from .quark_scheme import QuarkScheme
|
||||
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
|
||||
from .quark_w4a16_int4 import QuarkW4A16Int4
|
||||
from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
|
||||
from .quark_w8a8_int8 import QuarkW8A8Int8
|
||||
|
||||
@@ -14,6 +15,7 @@ __all__ = [
|
||||
"QuarkW8A8Fp8PerBlock",
|
||||
"QuarkW8A8Int8",
|
||||
"QuarkOCP_MX",
|
||||
"QuarkW4A16Int4",
|
||||
"QuarkW4A8_MXFP4_FP8",
|
||||
"QuarkNVFP4",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MPLinearKernel,
|
||||
MPLinearLayerConfig,
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
_convert_awq_to_standard_format,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
|
||||
QuarkScheme,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
canonicalize_quark_packed_int4,
|
||||
parse_w4a16_int4_weight_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4Static32Asym,
|
||||
kInt4StaticAsym,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
|
||||
class QuarkW4A16Int4(QuarkScheme):
|
||||
"""Quark packed INT4 weight-only linear scheme via MPLinearKernel."""
|
||||
|
||||
supported_activation_quant_keys: list[QuantKey | None] = [None]
|
||||
supported_weight_quant_keys: list[QuantKey] = [
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4StaticAsym,
|
||||
kInt4Static32Asym,
|
||||
]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
pack_method: str,
|
||||
weight_config: dict[str, Any],
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
|
||||
# group_size is read from the Quark config rather than weight_quant_key:
|
||||
# the int4 QuantKeys only encode group_size 32 or -1, but Quark int4
|
||||
# checkpoints commonly use group_size 128, which the QuantKey cannot
|
||||
# represent. Symmetry is unambiguous, so it comes from the key.
|
||||
self.group_size = parse_w4a16_int4_weight_config(weight_config)[0]
|
||||
self.pack_factor = 8
|
||||
self.pack_reorder = pack_method == "reorder"
|
||||
self.is_symmetric = weight_quant_key.symmetric
|
||||
self.quant_type = (
|
||||
scalar_types.uint4b8 if self.is_symmetric else scalar_types.uint4
|
||||
)
|
||||
self.kernel: MPLinearKernel | None = None
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 70
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
input_size = kwargs["input_size"]
|
||||
output_size = kwargs["output_size"]
|
||||
group_size = (
|
||||
self.group_size if self.group_size != -1 else input_size_per_partition
|
||||
)
|
||||
if input_size_per_partition % group_size != 0:
|
||||
raise ValueError(
|
||||
"The input size is not aligned with the quantized weight shape. "
|
||||
"This can be caused by too large tensor parallel size. "
|
||||
f"input_size_per_partition={input_size_per_partition}, "
|
||||
f"group_size={group_size}."
|
||||
)
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
packed_output_size_per_partition = math.ceil(
|
||||
output_size_per_partition / self.pack_factor
|
||||
)
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.packed_output_size_per_partition = packed_output_size_per_partition
|
||||
|
||||
mp_linear_kernel_config = MPLinearLayerConfig(
|
||||
full_weight_shape=(input_size, output_size),
|
||||
partition_weight_shape=(
|
||||
input_size_per_partition,
|
||||
output_size_per_partition,
|
||||
),
|
||||
weight_type=self.quant_type,
|
||||
act_type=params_dtype,
|
||||
group_size=group_size,
|
||||
zero_points=not self.is_symmetric,
|
||||
)
|
||||
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
|
||||
|
||||
def weight_scale_loader(
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
*args,
|
||||
**loader_kwargs,
|
||||
) -> None:
|
||||
if loaded_weight.shape[1] < param.data.shape[1]:
|
||||
padded_weight = loaded_weight.new_zeros(param.data.shape)
|
||||
padded_weight[:, : loaded_weight.shape[1]] = loaded_weight
|
||||
loaded_weight = padded_weight
|
||||
weight_loader(param, loaded_weight, *args, **loader_kwargs)
|
||||
|
||||
weight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
input_size_per_partition,
|
||||
packed_output_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
num_groups = input_size_per_partition // group_size
|
||||
weight_zero_point = PackedvLLMParameter(
|
||||
data=torch.zeros(
|
||||
num_groups,
|
||||
packed_output_size_per_partition,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=self.pack_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
num_groups,
|
||||
packed_output_size_per_partition * self.pack_factor,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
weight_loader=weight_scale_loader,
|
||||
)
|
||||
|
||||
layer.register_parameter("weight", weight)
|
||||
layer.register_parameter("weight_zero_point", weight_zero_point)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
self.kernel = kernel_type(
|
||||
mp_linear_kernel_config,
|
||||
w_q_param_name="weight",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point" if not self.is_symmetric else None,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight.data = canonicalize_quark_packed_int4(
|
||||
layer.weight.data,
|
||||
pack_reorder=self.pack_reorder,
|
||||
is_symmetric=self.is_symmetric,
|
||||
pack_factor=self.pack_factor,
|
||||
)
|
||||
if not self.is_symmetric:
|
||||
layer.weight_zero_point.data = canonicalize_quark_packed_int4(
|
||||
layer.weight_zero_point.data,
|
||||
pack_reorder=self.pack_reorder,
|
||||
is_symmetric=self.is_symmetric,
|
||||
pack_factor=self.pack_factor,
|
||||
)
|
||||
output_size = layer.output_size_per_partition
|
||||
packed_output_size = layer.packed_output_size_per_partition * self.pack_factor
|
||||
if output_size < packed_output_size:
|
||||
layer.weight_scale.data[:, output_size:].zero_()
|
||||
|
||||
_convert_awq_to_standard_format(
|
||||
layer,
|
||||
"weight",
|
||||
"weight_zero_point" if not self.is_symmetric else None,
|
||||
self.quant_type.size_bits,
|
||||
)
|
||||
assert self.kernel is not None
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
):
|
||||
assert self.kernel is not None
|
||||
return self.kernel.apply_weights(layer, x, bias)
|
||||
@@ -66,6 +66,63 @@ def should_ignore_layer(
|
||||
return all(shards_ignored)
|
||||
|
||||
|
||||
def parse_w4a16_int4_weight_config(
|
||||
weight_config: Mapping[str, Any],
|
||||
) -> tuple[int, bool]:
|
||||
"""Parse required W4A16 INT4/UINT4 weight fields from Quark config."""
|
||||
if "group_size" not in weight_config:
|
||||
raise ValueError(
|
||||
"Quark W4A16 INT4/UINT4 configs must specify weight.group_size"
|
||||
)
|
||||
if "symmetric" not in weight_config:
|
||||
raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric")
|
||||
|
||||
group_size = weight_config["group_size"]
|
||||
is_symmetric = weight_config["symmetric"]
|
||||
if not isinstance(group_size, int) or group_size <= 0:
|
||||
raise ValueError(
|
||||
f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}"
|
||||
)
|
||||
if not isinstance(is_symmetric, bool):
|
||||
raise ValueError(
|
||||
f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}"
|
||||
)
|
||||
return group_size, is_symmetric
|
||||
|
||||
|
||||
def canonicalize_quark_packed_int4(
|
||||
packed_weight: torch.Tensor,
|
||||
*,
|
||||
pack_reorder: bool,
|
||||
is_symmetric: bool,
|
||||
pack_factor: int = 8,
|
||||
) -> torch.Tensor:
|
||||
"""Convert Quark export nibble layout to AWQ checkpoint layout."""
|
||||
from vllm.model_executor.layers.quantization.auto_awq import (
|
||||
_REVERSE_AWQ_PACK_ORDER,
|
||||
)
|
||||
|
||||
if pack_reorder:
|
||||
source_order = torch.tensor(
|
||||
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
|
||||
)
|
||||
else:
|
||||
source_order = torch.arange(
|
||||
pack_factor, device=packed_weight.device, dtype=torch.int32
|
||||
)
|
||||
target_order = torch.tensor(
|
||||
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
|
||||
)
|
||||
source_shifts = source_order * 4
|
||||
target_shifts = target_order * 4
|
||||
|
||||
values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF
|
||||
if is_symmetric:
|
||||
values = values ^ 0x8
|
||||
packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1)
|
||||
return packed.to(torch.int32)
|
||||
|
||||
|
||||
# utility for tensor dims > 2 cases
|
||||
def quark_quantize_weight_to_mxfp4(w: torch.Tensor):
|
||||
assert w.dtype == torch.bfloat16, (
|
||||
|
||||
Reference in New Issue
Block a user