[Quantization] Support native Quark W4A16 INT4/UINT4 exports in vLLM (#48606)

Signed-off-by: HongWei Meng <HongWei.Meng@amd.com>
This commit is contained in:
meng-amd
2026-09-18 17:52:46 +08:00
committed by GitHub
parent 67a8a3f926
commit 6df2b1a8c6
9 changed files with 1124 additions and 6 deletions
@@ -93,7 +93,7 @@ def _replace_or_register_parameter(
def _convert_awq_to_standard_format(
layer: torch.nn.Module,
w_q_name: str,
w_zp_name: str,
w_zp_name: str | None,
size_bits: int,
) -> None:
"""Convert AWQ weight and zero-point tensors to standard GPTQ-like format.
@@ -101,6 +101,7 @@ def _convert_awq_to_standard_format(
AWQ packs qweight along the output dim with a non-standard bit order.
This converts to standard bit order and repacks qweight along the input
dim, matching the format expected by the MPLinearKernel framework.
If w_zp_name is None (symmetric quantization), only the weight is converted.
"""
pack_factor = 32 // size_bits
mask = (1 << size_bits) - 1
@@ -139,6 +140,9 @@ def _convert_awq_to_standard_format(
)
setattr(layer, w_q_name, new_param)
if w_zp_name is None:
return
# --- Convert qzeros: fix AWQ bit ordering and repack
# AWQ qzeros: (G, N // pack) packed along dim 1, AWQ bit order
# Target: (N // pack, G) packed along dim 0, standard bit order
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
QuarkOCP_MX,
QuarkScheme,
QuarkW4A8_MXFP4_FP8,
QuarkW4A16Int4,
QuarkW8A8Fp8,
QuarkW8A8Fp8PerBlock,
QuarkW8A8Int8,
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
from vllm.model_executor.layers.quantization.quark.utils import (
QuarkQTensorHint,
deep_compare,
parse_w4a16_int4_weight_config,
should_ignore_layer,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
@@ -54,6 +56,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
kInt4W4A8StaticChannelSym,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
@@ -172,8 +178,10 @@ class QuarkConfig(QuantizationConfig):
quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {}
for k, v in self.quant_config.items():
if isinstance(v, list):
if isinstance(v, list) and all(isinstance(item, str) for item in v):
quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_list(v)
elif isinstance(v, list):
quant_config_with_hf_to_vllm_mapper[k] = v
elif isinstance(v, dict):
quant_config_with_hf_to_vllm_mapper[k] = hf_to_vllm_mapper.apply_dict(v)
else:
@@ -215,6 +223,9 @@ class QuarkConfig(QuantizationConfig):
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
dynamic_mxfp4_quant=dynamic_mxfp4_quant,
weight_config=self._find_matched_config(prefix, type(layer)).get(
"weight"
),
)
layer.scheme = scheme
return QuarkLinearMethod(self)
@@ -227,6 +238,7 @@ class QuarkConfig(QuantizationConfig):
method_cls=method_cls,
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
layer_name=prefix,
)
return None
@@ -572,6 +584,26 @@ class QuarkConfig(QuantizationConfig):
)
return QuantKeyMatch(True, activation_quant_key, weight_quant_key)
def _is_w4a16_int4(
self,
weight_quant: dict[str, Any] | None,
input_quant: dict[str, Any] | None,
) -> QuantKeyMatch:
if weight_quant is None or input_quant is not None:
return QuantKeyMatch(False, None, None)
is_int4 = weight_quant.get("dtype") in ("int4", "uint4")
is_packed = self.pack_method in ("order", "reorder")
if not (is_int4 and is_packed):
return QuantKeyMatch(False, None, None)
group_size, is_symmetric = parse_w4a16_int4_weight_config(weight_quant)
if is_symmetric:
weight_key = kInt4Static32 if group_size == 32 else kInt4Static
else:
weight_key = kInt4Static32Asym if group_size == 32 else kInt4StaticAsym
return QuantKeyMatch(True, None, weight_key)
def _is_w4a8_mxfp4_fp8(
self,
weight_quant: dict[str, Any] | None,
@@ -890,6 +922,12 @@ class QuarkConfig(QuantizationConfig):
match.activation_quant_key,
QuarkW8A8Int8,
)
elif match := self._is_w4a16_int4(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkW4A16Int4,
)
elif match := self._is_w4a8_mxfp4_fp8(weight_config, input_config):
return (
match.weight_quant_key,
@@ -919,6 +957,7 @@ class QuarkConfig(QuantizationConfig):
weight_quant_key: QuantKey | None,
activation_quant_key: QuantKey | None,
dynamic_mxfp4_quant: bool = False,
weight_config: dict[str, Any] | None = None,
) -> "QuarkScheme":
"""Construct a Quark scheme selected by get_scheme_cls."""
if scheme_cls not in (
@@ -928,9 +967,24 @@ class QuarkConfig(QuantizationConfig):
QuarkOCP_MX,
QuarkNVFP4,
QuarkW4A8_MXFP4_FP8,
QuarkW4A16Int4,
):
raise AssertionError(f"Unsupported Quark scheme class: {scheme_cls}")
scheme: QuarkScheme
if scheme_cls is QuarkW4A16Int4:
normalized_weight_config = self._unwrap_single_quant_config(weight_config)
assert normalized_weight_config is not None
assert weight_quant_key is not None
scheme = QuarkW4A16Int4(
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
pack_method=self.pack_method,
weight_config=normalized_weight_config,
)
self._check_scheme_supported(scheme.get_min_capability())
return scheme
kwargs: dict[str, Any] = {"activation_quant_key": activation_quant_key}
# These classes handle different possible `weight_quant_key`.
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
@@ -40,6 +40,12 @@ from vllm.model_executor.layers.fused_moe.oracle.int8 import (
make_int8_moe_quant_config,
select_int8_moe_backend,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
make_wna16_moe_quant_config,
select_wna16_moe_backend,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
@@ -56,7 +62,10 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
make_nvfp4_moe_quant_config,
select_nvfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
from vllm.model_executor.layers.quantization.quark.utils import (
QuarkQTensorHint,
parse_w4a16_int4_weight_config,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
validate_fp8_block_shape_moe,
)
@@ -75,6 +84,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
kInt4W4A8StaticChannelSym,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
@@ -102,6 +115,7 @@ logger = init_logger(__name__)
__all__ = [
"QuarkMoEMethod",
"QuarkW4A16Int4MoEMethod",
"QuarkW8A8Fp8MoEMethod",
"QuarkOCP_MX_MoEMethod",
"QuarkNvfp4MoEMethod",
@@ -179,6 +193,12 @@ class QuarkMoEMethod(FusedMoEMethodBase):
match.activation_quant_key,
QuarkW8A8Fp8MoEMethod,
)
if match := quant_config._is_w4a16_int4(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkW4A16Int4MoEMethod,
)
if match := quant_config._is_w_ocp_mx_a_x(
weight_config, input_config, allow_static_fp8=True
):
@@ -202,6 +222,7 @@ class QuarkMoEMethod(FusedMoEMethodBase):
method_cls: type["QuarkMoEMethod"],
weight_quant_key: QuantKey | None,
activation_quant_key: QuantKey | None,
layer_name: str | None = None,
) -> "QuarkMoEMethod":
if weight_quant_key is None:
raise RuntimeError("Unsupported FusedMoe scheme")
@@ -226,10 +247,339 @@ class QuarkMoEMethod(FusedMoEMethodBase):
return QuarkW8A8Int8MoEMethod(
module.moe_config, weight_quant_key, activation_quant_key
)
elif method_cls is QuarkW4A16Int4MoEMethod:
assert layer_name is not None
layer_quant_config = quant_config._find_matched_config(
layer_name, type(module)
)
weight_config = quant_config._unwrap_single_quant_config(
layer_quant_config.get("weight")
)
if weight_config is None:
raise RuntimeError(
f"Missing weight quantization config for layer {layer_name}"
)
return QuarkW4A16Int4MoEMethod(
weight_quant_key,
activation_quant_key,
weight_config,
quant_config.pack_method,
module.moe_config,
quant_config=quant_config,
)
else:
raise RuntimeError("Unsupported FusedMoe scheme")
class QuarkW4A16Int4MoEMethod(QuarkMoEMethod):
"""Quark packed INT4 weight-only MoE method."""
supported_activation_quant_keys: list[QuantKey | None] = [None]
supported_weight_quant_keys: list[QuantKey] = [
kInt4Static,
kInt4Static32,
kInt4StaticAsym,
kInt4Static32Asym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
weight_config: dict[str, Any],
pack_method: str,
moe: FusedMoEConfig,
quant_config: "QuarkConfig | None" = None, # type: ignore # noqa F821
):
super().__init__(moe, weight_quant_key, activation_quant_key)
self.quant_config = quant_config
# group_size is read from the Quark config rather than weight_quant_key:
# the int4 QuantKeys only encode group_size 32 or -1, but Quark int4
# checkpoints commonly use group_size 128, which the QuantKey cannot
# represent. Symmetry is unambiguous, so it comes from the key.
self.group_size = parse_w4a16_int4_weight_config(weight_config)[0]
self.is_symmetric = weight_quant_key.symmetric
self.bit8_pack_factor = 2
self.pack_reorder = pack_method == "reorder"
self.use_wna16_backend = True
self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
config=self.moe,
weight_key=weight_quant_key,
quant_config=self.quant_config,
may_have_zp=not self.is_symmetric,
may_have_bias=False,
)
def create_weights(
self,
layer: RoutedExperts,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
group_size = self.group_size
group_size_div_factor = 1
while intermediate_size_per_partition % group_size or hidden_size % group_size:
group_size = group_size // 2
group_size_div_factor *= 2
assert group_size >= 32
layer.group_size = group_size
layer.group_size_div_factor = group_size_div_factor
strategy = FusedMoeWeightScaleSupported.GROUP.value
extra_weight_attrs.update({"quant_method": strategy, "is_transposed": False})
assert "weight_loader" in extra_weight_attrs
weight_loader = extra_weight_attrs["weight_loader"]
extra_weight_attrs["weight_loader"] = self.get_weight_loader(
layer, weight_loader
)
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
2 * intermediate_size_per_partition,
hidden_size // self.bit8_pack_factor,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // self.bit8_pack_factor,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
w13_weight_scale = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition,
hidden_size // group_size,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
w2_weight_scale = torch.nn.Parameter(
torch.zeros(
num_experts,
hidden_size,
intermediate_size_per_partition // group_size,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
w13_weight_zero_point = torch.nn.Parameter(
torch.zeros(
num_experts,
2 * intermediate_size_per_partition // self.bit8_pack_factor,
hidden_size // group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_zero_point", w13_weight_zero_point)
set_weight_attrs(w13_weight_zero_point, extra_weight_attrs)
w2_weight_zero_point = torch.nn.Parameter(
torch.zeros(
num_experts,
hidden_size // self.bit8_pack_factor,
intermediate_size_per_partition // group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_zero_point", w2_weight_zero_point)
set_weight_attrs(w2_weight_zero_point, extra_weight_attrs)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
w13_qzeros = layer.w13_weight_zero_point if not self.is_symmetric else None
w2_qzeros = layer.w2_weight_zero_point if not self.is_symmetric else None
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=None,
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_qzeros=w13_qzeros,
w2_qzeros=w2_qzeros,
)
if converted is None:
# Backend rewrote the layer's params in place (e.g. Humming).
self._setup_kernel(layer)
return
(
w13_qweight,
w2_qweight,
w13_scales,
w2_scales,
_, # w13_qzeros
_, # w2_qzeros
_, # w13_input_global_scale
_, # w2_input_global_scale
_, # w13_bias
_, # w2_bias
) = converted
replace_parameter(layer, "w13_weight", w13_qweight)
replace_parameter(layer, "w2_weight", w2_qweight)
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
self._setup_kernel(layer)
def _setup_kernel(self, layer: RoutedExperts) -> None:
assert self.experts_cls is not None
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
self.moe_kernel = make_wna16_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_backend,
routing_tables=layer._expert_routing_tables(),
)
def get_fused_moe_quant_config(
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
w1_zp = layer.w13_weight_zero_point if not self.is_symmetric else None
w2_zp = layer.w2_weight_zero_point if not self.is_symmetric else None
return make_wna16_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w1_zp=w1_zp,
w2_zp=w2_zp,
group_size=layer.group_size,
num_bits=4,
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
expert_map=layer.expert_map,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
def get_weight_loader(self, layer: RoutedExperts, weight_loader):
def convert_quark_tensor(tensor: torch.Tensor, tensor_type: str):
size0 = tensor.size(0)
tensor = tensor.view(torch.uint8)
shifter = torch.tensor([0, 4], dtype=torch.uint8, device=tensor.device)
tensor = (tensor[:, :, None] >> shifter) & 0xF
if self.pack_reorder:
reverse_awq_pack_order = [0, 4, 1, 5, 2, 6, 3, 7]
tensor = tensor.view(-1, 8)[:, reverse_awq_pack_order]
else:
tensor = tensor.view(-1, 8)
if self.is_symmetric:
tensor = tensor ^ 0x8
tensor = tensor.view(size0, -1)
tensor = tensor.T.contiguous()
if tensor_type == "weight":
tensor = tensor[:, 1::2] * 16 + tensor[:, ::2]
elif tensor_type == "zero_point":
tensor = tensor[1::2, :] * 16 + tensor[::2, :]
return tensor
def quark_int4_weight_loader(
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: bool = False,
):
loaded_weight = loaded_weight.to(param.data.device)
shard_size = layer.intermediate_size_per_partition
if weight_name.endswith("_weight"):
loaded_weight = convert_quark_tensor(loaded_weight, "weight")
elif weight_name.endswith("_weight_zero_point"):
loaded_weight = convert_quark_tensor(loaded_weight, "zero_point")
elif weight_name.endswith("_weight_scale"):
loaded_weight = loaded_weight.T
if layer.group_size_div_factor > 1 and (
"weight_zero_point" in weight_name or "weight_scale" in weight_name
):
loaded_weight = loaded_weight.repeat_interleave(
layer.group_size_div_factor, 1
)
if "w13_weight_zero_point" in weight_name:
tensor = loaded_weight.view(
layer.moe_config.tp_size, -1, loaded_weight.size(1)
)[layer.moe_config.tp_rank]
if shard_id == "w1":
param.data[expert_id, : shard_size // 2] = tensor
else:
param.data[expert_id, shard_size // 2 :] = tensor
return True if return_success else None
elif "w2_weight_zero_point" in weight_name:
param.data[expert_id] = loaded_weight.view(
loaded_weight.size(0), layer.moe_config.tp_size, -1
)[:, layer.moe_config.tp_rank]
return True if return_success else None
return weight_loader(
param,
loaded_weight,
weight_name,
shard_id,
expert_id,
return_success=return_success,
)
return quark_int4_weight_loader
class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
supported_activation_quant_keys = [
kFp8StaticTensorSym,
@@ -5,6 +5,7 @@ from .quark_nvfp4 import QuarkNVFP4
from .quark_ocp_mx import QuarkOCP_MX
from .quark_scheme import QuarkScheme
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
from .quark_w4a16_int4 import QuarkW4A16Int4
from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
from .quark_w8a8_int8 import QuarkW8A8Int8
@@ -14,6 +15,7 @@ __all__ = [
"QuarkW8A8Fp8PerBlock",
"QuarkW8A8Int8",
"QuarkOCP_MX",
"QuarkW4A16Int4",
"QuarkW4A8_MXFP4_FP8",
"QuarkNVFP4",
]
@@ -0,0 +1,212 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Callable
from typing import Any
import torch
from vllm.model_executor.kernels.linear import (
MPLinearKernel,
MPLinearLayerConfig,
choose_mp_linear_kernel,
)
from vllm.model_executor.layers.quantization.auto_awq import (
_convert_awq_to_standard_format,
)
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
QuarkScheme,
)
from vllm.model_executor.layers.quantization.quark.utils import (
canonicalize_quark_packed_int4,
parse_w4a16_int4_weight_config,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.scalar_type import scalar_types
class QuarkW4A16Int4(QuarkScheme):
"""Quark packed INT4 weight-only linear scheme via MPLinearKernel."""
supported_activation_quant_keys: list[QuantKey | None] = [None]
supported_weight_quant_keys: list[QuantKey] = [
kInt4Static,
kInt4Static32,
kInt4StaticAsym,
kInt4Static32Asym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
pack_method: str,
weight_config: dict[str, Any],
):
super().__init__(weight_quant_key, activation_quant_key)
# group_size is read from the Quark config rather than weight_quant_key:
# the int4 QuantKeys only encode group_size 32 or -1, but Quark int4
# checkpoints commonly use group_size 128, which the QuantKey cannot
# represent. Symmetry is unambiguous, so it comes from the key.
self.group_size = parse_w4a16_int4_weight_config(weight_config)[0]
self.pack_factor = 8
self.pack_reorder = pack_method == "reorder"
self.is_symmetric = weight_quant_key.symmetric
self.quant_type = (
scalar_types.uint4b8 if self.is_symmetric else scalar_types.uint4
)
self.kernel: MPLinearKernel | None = None
@classmethod
def get_min_capability(cls) -> int:
return 70
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
input_size = kwargs["input_size"]
output_size = kwargs["output_size"]
group_size = (
self.group_size if self.group_size != -1 else input_size_per_partition
)
if input_size_per_partition % group_size != 0:
raise ValueError(
"The input size is not aligned with the quantized weight shape. "
"This can be caused by too large tensor parallel size. "
f"input_size_per_partition={input_size_per_partition}, "
f"group_size={group_size}."
)
output_size_per_partition = sum(output_partition_sizes)
packed_output_size_per_partition = math.ceil(
output_size_per_partition / self.pack_factor
)
layer.output_size_per_partition = output_size_per_partition
layer.packed_output_size_per_partition = packed_output_size_per_partition
mp_linear_kernel_config = MPLinearLayerConfig(
full_weight_shape=(input_size, output_size),
partition_weight_shape=(
input_size_per_partition,
output_size_per_partition,
),
weight_type=self.quant_type,
act_type=params_dtype,
group_size=group_size,
zero_points=not self.is_symmetric,
)
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
def weight_scale_loader(
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
*args,
**loader_kwargs,
) -> None:
if loaded_weight.shape[1] < param.data.shape[1]:
padded_weight = loaded_weight.new_zeros(param.data.shape)
padded_weight[:, : loaded_weight.shape[1]] = loaded_weight
loaded_weight = padded_weight
weight_loader(param, loaded_weight, *args, **loader_kwargs)
weight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition,
packed_output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
num_groups = input_size_per_partition // group_size
weight_zero_point = PackedvLLMParameter(
data=torch.zeros(
num_groups,
packed_output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
num_groups,
packed_output_size_per_partition * self.pack_factor,
dtype=params_dtype,
),
input_dim=0,
output_dim=1,
weight_loader=weight_scale_loader,
)
layer.register_parameter("weight", weight)
layer.register_parameter("weight_zero_point", weight_zero_point)
layer.register_parameter("weight_scale", weight_scale)
self.kernel = kernel_type(
mp_linear_kernel_config,
w_q_param_name="weight",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point" if not self.is_symmetric else None,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight.data = canonicalize_quark_packed_int4(
layer.weight.data,
pack_reorder=self.pack_reorder,
is_symmetric=self.is_symmetric,
pack_factor=self.pack_factor,
)
if not self.is_symmetric:
layer.weight_zero_point.data = canonicalize_quark_packed_int4(
layer.weight_zero_point.data,
pack_reorder=self.pack_reorder,
is_symmetric=self.is_symmetric,
pack_factor=self.pack_factor,
)
output_size = layer.output_size_per_partition
packed_output_size = layer.packed_output_size_per_partition * self.pack_factor
if output_size < packed_output_size:
layer.weight_scale.data[:, output_size:].zero_()
_convert_awq_to_standard_format(
layer,
"weight",
"weight_zero_point" if not self.is_symmetric else None,
self.quant_type.size_bits,
)
assert self.kernel is not None
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
):
assert self.kernel is not None
return self.kernel.apply_weights(layer, x, bias)
@@ -66,6 +66,63 @@ def should_ignore_layer(
return all(shards_ignored)
def parse_w4a16_int4_weight_config(
weight_config: Mapping[str, Any],
) -> tuple[int, bool]:
"""Parse required W4A16 INT4/UINT4 weight fields from Quark config."""
if "group_size" not in weight_config:
raise ValueError(
"Quark W4A16 INT4/UINT4 configs must specify weight.group_size"
)
if "symmetric" not in weight_config:
raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric")
group_size = weight_config["group_size"]
is_symmetric = weight_config["symmetric"]
if not isinstance(group_size, int) or group_size <= 0:
raise ValueError(
f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}"
)
if not isinstance(is_symmetric, bool):
raise ValueError(
f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}"
)
return group_size, is_symmetric
def canonicalize_quark_packed_int4(
packed_weight: torch.Tensor,
*,
pack_reorder: bool,
is_symmetric: bool,
pack_factor: int = 8,
) -> torch.Tensor:
"""Convert Quark export nibble layout to AWQ checkpoint layout."""
from vllm.model_executor.layers.quantization.auto_awq import (
_REVERSE_AWQ_PACK_ORDER,
)
if pack_reorder:
source_order = torch.tensor(
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
)
else:
source_order = torch.arange(
pack_factor, device=packed_weight.device, dtype=torch.int32
)
target_order = torch.tensor(
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
)
source_shifts = source_order * 4
target_shifts = target_order * 4
values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF
if is_symmetric:
values = values ^ 0x8
packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1)
return packed.to(torch.int32)
# utility for tensor dims > 2 cases
def quark_quantize_weight_to_mxfp4(w: torch.Tensor):
assert w.dtype == torch.bfloat16, (