[Quantization] Support native Quark W4A16 INT4/UINT4 exports in vLLM (#48606)

Signed-off-by: HongWei Meng <HongWei.Meng@amd.com>
This commit is contained in:
meng-amd
2026-09-18 17:52:46 +08:00
committed by GitHub
parent 67a8a3f926
commit 6df2b1a8c6
9 changed files with 1124 additions and 6 deletions
@@ -66,6 +66,63 @@ def should_ignore_layer(
return all(shards_ignored)
def parse_w4a16_int4_weight_config(
weight_config: Mapping[str, Any],
) -> tuple[int, bool]:
"""Parse required W4A16 INT4/UINT4 weight fields from Quark config."""
if "group_size" not in weight_config:
raise ValueError(
"Quark W4A16 INT4/UINT4 configs must specify weight.group_size"
)
if "symmetric" not in weight_config:
raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric")
group_size = weight_config["group_size"]
is_symmetric = weight_config["symmetric"]
if not isinstance(group_size, int) or group_size <= 0:
raise ValueError(
f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}"
)
if not isinstance(is_symmetric, bool):
raise ValueError(
f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}"
)
return group_size, is_symmetric
def canonicalize_quark_packed_int4(
packed_weight: torch.Tensor,
*,
pack_reorder: bool,
is_symmetric: bool,
pack_factor: int = 8,
) -> torch.Tensor:
"""Convert Quark export nibble layout to AWQ checkpoint layout."""
from vllm.model_executor.layers.quantization.auto_awq import (
_REVERSE_AWQ_PACK_ORDER,
)
if pack_reorder:
source_order = torch.tensor(
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
)
else:
source_order = torch.arange(
pack_factor, device=packed_weight.device, dtype=torch.int32
)
target_order = torch.tensor(
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
)
source_shifts = source_order * 4
target_shifts = target_order * 4
values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF
if is_symmetric:
values = values ^ 0x8
packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1)
return packed.to(torch.int32)
# utility for tensor dims > 2 cases
def quark_quantize_weight_to_mxfp4(w: torch.Tensor):
assert w.dtype == torch.bfloat16, (