[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)

Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com>
Signed-off-by: Roderick Wu <roderickwu2003@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Roderick Wu
2026-09-08 12:45:12 -07:00
committed by GitHub
co-authored by mgoin
parent 1a522b6949
commit 2c9d68f80b
99 changed files with 1182 additions and 3917 deletions
@@ -20,6 +20,36 @@ else:
AutoGPTQConfig = object
def normalize_and_validate_gptq_desc_act(
desc_act: bool,
group_size: int | None,
dynamic: Mapping[str, Any],
) -> bool:
base_desc_act = desc_act
if desc_act and group_size == -1:
# With one group per output channel, activation ordering is a no-op.
desc_act = False
if desc_act:
raise ValueError(
"GPTQ group activation ordering (desc_act=True) is no longer "
"supported. Use a checkpoint with static activation ordering "
"or desc_act=False."
)
for pattern, overrides in dynamic.items():
if pattern.startswith("-:") or not isinstance(overrides, Mapping):
continue
dynamic_desc_act = overrides.get("desc_act", base_desc_act)
dynamic_group_size = overrides.get("group_size", group_size)
if dynamic_desc_act and dynamic_group_size != -1:
raise ValueError(
"GPTQ group activation ordering (desc_act=True) is no longer "
f"supported, but is enabled by dynamic rule {pattern!r}. Use "
"a checkpoint with static activation ordering or desc_act=False."
)
return desc_act
# Match dynamic rules with module name (prefix) and override quantize
# config if module (prefix) matches a rule
def override_config(config: AutoGPTQConfig, prefix: str):
@@ -33,6 +63,10 @@ def override_config(config: AutoGPTQConfig, prefix: str):
if isinstance(desc_act, bool):
config.desc_act = desc_act
if config.desc_act and config.group_size == -1:
# Activation ordering is a no-op for channelwise quantization.
config.desc_act = False
config.pack_factor = 32 // config.weight_bits # packed into int32
assert isinstance(config, AutoGPTQConfig)
is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
@@ -359,11 +359,11 @@ def check_moe_marlin_supports_config(
) -> bool:
"""Whether the fused MoE Marlin kernel supports ``config``.
Callers without act-order may pass ``allow_tile_padding=True``: a
tile-misaligned intermediate size is then zero-padded to a valid thread
tile at weight prep (see marlin_moe_padded_intermediate), so only a group
straddling the padded boundary stays unsupported. hidden_size is the MoE
I/O extent and is never padded. Act-order keeps the strict shape.
With ``allow_tile_padding=True``, a tile-misaligned intermediate size is
zero-padded to a valid thread tile at weight prep (see
marlin_moe_padded_intermediate), so only a group straddling the padded
boundary stays unsupported. hidden_size is the MoE I/O extent and is never
padded.
"""
if current_platform.is_rocm():
return False
@@ -433,30 +433,17 @@ def marlin_make_workspace_new(
return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False)
def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
return (not act_order) or (act_order and not is_row_parallel)
def marlin_repeat_scales_on_all_ranks(
act_order: bool, group_size: int, is_row_parallel: bool
) -> bool:
# Need to repeat scales on every rank if act_ordering or
# channelwise and RowParallelLinear
def marlin_repeat_scales_on_all_ranks(group_size: int, is_row_parallel: bool) -> bool:
is_channelwise = group_size == -1
return act_order or (is_channelwise and is_row_parallel)
return is_channelwise and is_row_parallel
def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
def marlin_make_empty(device: torch.device) -> torch.Tensor:
return torch.nn.Parameter(
torch.empty(0, dtype=torch.int, device=device), requires_grad=False
)
def marlin_sort_g_idx(g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
return g_idx[g_idx_sort_indices], g_idx_sort_indices
def get_scale_perms():
scale_perm: list[int] = []
for i in range(8):
@@ -700,13 +687,10 @@ def apply_gptq_marlin_linear(
weight: torch.Tensor,
weight_scale: torch.Tensor,
weight_zp: torch.Tensor,
g_idx: torch.Tensor,
g_idx_sort_indices: torch.Tensor,
workspace: torch.Tensor,
wtype: ScalarType,
output_size_per_partition: int,
input_size_per_partition: int,
is_k_full: bool,
input_global_scale: torch.Tensor | None = None,
bias: torch.Tensor | None = None,
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
@@ -749,14 +733,11 @@ def apply_gptq_marlin_linear(
a_scales,
None,
weight_zp,
g_idx,
g_idx_sort_indices,
workspace,
wtype,
size_m=reshaped_x.shape[0],
size_n=padded_n,
size_k=padded_k,
is_k_full=is_k_full,
use_atomic_add=use_atomic_add,
use_fp32_reduce=use_fp32_reduce,
is_zp_float=False,
@@ -203,8 +203,6 @@ def apply_fp4_marlin_linear(
a_scales=a_scales,
global_scale=weight_global_scale,
b_zeros=None,
g_idx=None,
perm=None,
workspace=workspace,
b_q_type=scalar_types.float4_e2m1f,
size_m=reshaped_x.size(0),
@@ -246,14 +244,12 @@ def prepare_fp4_layer_for_marlin(
# WEIGHT
# Repack weights to marlin format
perm = torch.empty(0, dtype=torch.int, device=device)
qweight = layer.weight.view(torch.int32).T.contiguous()
qweight = marlin_pad_qweight(qweight, part_size_n, part_size_k, padded_n, padded_k)
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=padded_k,
size_n=padded_n,
num_bits=4,
@@ -312,7 +308,6 @@ def _repack_marlin_experts(
weight: torch.Tensor,
size_n: int,
size_k: int,
perm: torch.Tensor,
is_a_8bit: bool,
) -> torch.Tensor:
"""Repack each expert to marlin format into a preallocated output."""
@@ -322,7 +317,6 @@ def _repack_marlin_experts(
qweight = weight[i].view(torch.int32).T.contiguous()
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=size_k,
size_n=size_n,
num_bits=4,
@@ -401,7 +395,6 @@ def prepare_nvfp4_moe_layer_for_marlin(
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
perm = torch.empty(0, dtype=torch.int, device=device)
# WEIGHT
# Repack weights to marlin format
@@ -417,7 +410,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
weight = pad_w2(weight, packing=2)
size_k = padded_N
return _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
return _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
w13 = repack_weight(w13, "w13")
w2 = repack_weight(w2, "w2")
@@ -492,7 +485,6 @@ def prepare_moe_fp4_layer_for_marlin(
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
perm = torch.empty(0, dtype=torch.int, device=device)
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
# WEIGHT
@@ -506,7 +498,7 @@ def prepare_moe_fp4_layer_for_marlin(
assert weight.shape == (e, size_n, size_k // 2)
weight = _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
weight = _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
weight = torch.nn.Parameter(weight, requires_grad=False)
setattr(layer, name, weight)
@@ -621,10 +613,8 @@ def prepare_moe_mxfp4_layer_for_marlin(
n = w13.shape[1] // 2 # intermediate_size_per_partition
k = w13.shape[2] * 2 # hidden_size
device = w13.device
param_dtype = layer.params_dtype
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
perm = torch.empty(0, dtype=torch.int, device=device)
# WEIGHT: Repack weights to marlin format
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
@@ -635,7 +625,7 @@ def prepare_moe_mxfp4_layer_for_marlin(
assert weight.shape == (e, size_n, size_k // 2)
return _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
return _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
w13 = repack_weight(w13, "w13")
w2 = repack_weight(w2, "w2")
@@ -691,8 +681,6 @@ def rand_marlin_weight_nvfp4_like(weight, group_size, input_dtype=None):
assert not is_a_8bit, "NVFP4 weight + INT8/FP8 activation is not supported."
assert group_size > 0
size_n, size_k = weight.shape
device = weight.device
scales = weight.view(size_n, -1, group_size).abs().max(-1)[0] / 6
global_scale = scales.max() / 448
scales = (scales / global_scale).to(torch.float8_e4m3fn)
@@ -720,7 +708,6 @@ def rand_marlin_weight_nvfp4_like(weight, group_size, input_dtype=None):
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=fp4_weight.view(torch.int32).T.contiguous(),
perm=torch.empty(0, dtype=torch.int, device=device),
size_k=size_k,
size_n=size_n,
num_bits=4,
@@ -751,8 +738,6 @@ def rand_marlin_weight_mxfp4_like(weight, group_size, input_dtype=None):
assert group_size > 0
size_n, size_k = weight.shape
device = weight.device
scales = torch.randint(
110,
120,
@@ -779,11 +764,9 @@ def rand_marlin_weight_mxfp4_like(weight, group_size, input_dtype=None):
).view(size_n, size_k)
weight_ref = weight_ref * scales.repeat_interleave(group_size, 1).to(weight.dtype)
perm = torch.empty(0, dtype=torch.int, device=device)
fp4_weight = fp4_weight.view(torch.int32).T.contiguous()
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=fp4_weight,
perm=perm,
size_k=size_k,
size_n=size_n,
num_bits=4,
@@ -89,8 +89,6 @@ def apply_fp8_marlin_linear(
a_scales=a_scales,
global_scale=None,
b_zeros=None,
g_idx=None,
perm=None,
workspace=workspace,
b_q_type=scalar_types.float8_e4m3fn,
size_m=reshaped_x.size(0),
@@ -138,7 +136,6 @@ def prepare_fp8_layer_for_marlin(
# WEIGHT
# Repack weights to marlin format
perm = torch.empty(0, dtype=torch.int, device=device)
qweight = pack_fp8_to_int32(layer.weight, size_k_first)
if not size_k_first:
qweight = qweight.T.contiguous()
@@ -146,7 +143,6 @@ def prepare_fp8_layer_for_marlin(
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=padded_k,
size_n=padded_n,
num_bits=8,
@@ -283,7 +279,6 @@ def prepare_fp8_moe_layer_for_marlin(
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
perm = torch.empty(0, dtype=torch.int, device=device)
# WEIGHT
# Repack weights to marlin format
@@ -295,7 +290,7 @@ def prepare_fp8_moe_layer_for_marlin(
qweight = qweight.T.contiguous()
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight, perm=perm, size_k=size_k, size_n=size_n, num_bits=8
b_q_weight=qweight, size_k=size_k, size_n=size_n, num_bits=8
)
tensor_list.append(marlin_qweight)
@@ -434,8 +429,6 @@ def apply_mxfp8_marlin_linear(
a_scales=None,
global_scale=None,
b_zeros=None,
g_idx=None,
perm=None,
workspace=workspace,
b_q_type=scalar_types.float8_e4m3fn,
size_m=reshaped_x.size(0),
@@ -470,14 +463,12 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
)
# WEIGHT - repack FP8 weights to Marlin format
perm = torch.empty(0, dtype=torch.int, device=device)
qweight = pack_fp8_to_int32(layer.weight, size_k_first=False)
qweight = qweight.T.contiguous()
qweight = marlin_pad_qweight(qweight, part_size_n, part_size_k, padded_n, padded_k)
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=padded_k,
size_n=padded_n,
num_bits=8,
@@ -551,7 +542,6 @@ def prepare_mxfp8_moe_layer_for_marlin(
device = w13.device
param_dtype = torch.get_default_dtype()
perm = torch.empty(0, dtype=torch.int, device=device)
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
@@ -571,7 +561,6 @@ def prepare_mxfp8_moe_layer_for_marlin(
qweight = qweight.T.contiguous()
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=perm,
size_k=size_k,
size_n=size_n,
num_bits=8,
@@ -615,8 +604,6 @@ def marlin_quant_fp8_torch(weight, group_size, input_dtype=None):
assert input_dtype == torch.float8_e4m3fn
size_n, size_k = weight.shape
device = weight.device
if group_size != -1:
scales = weight.view(size_n, -1, group_size).abs().max(-1)[0] / 448
repeated_scales = scales.repeat_interleave(group_size, 1)
@@ -629,10 +616,8 @@ def marlin_quant_fp8_torch(weight, group_size, input_dtype=None):
weight_ref = fp8_weight.to(weight.dtype) * repeated_scales
packed_weight = pack_fp8_to_int32(fp8_weight, False).T.contiguous()
perm = torch.empty(0, dtype=torch.int, device=device)
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=packed_weight,
perm=perm,
size_k=size_k,
size_n=size_n,
num_bits=8,
@@ -13,7 +13,6 @@ from .quant_utils import (
get_pack_factor,
gptq_quantize_weights,
quantize_weights,
sort_weights,
)
@@ -129,8 +128,6 @@ def marlin_quantize(
w: torch.Tensor,
quant_type: ScalarType,
group_size: int,
act_order: bool,
test_perm: torch.Tensor | None = None,
input_dtype: torch.dtype | None = None,
):
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
@@ -143,16 +140,7 @@ def marlin_quantize(
group_size = size_k
assert group_size <= size_k
# Quantize (and apply act_order if provided)
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
w, quant_type, group_size, act_order, test_perm
)
# For act_order, sort the "weights" and "g_idx" so that group ids are
# increasing
sort_indices = torch.empty(0, dtype=torch.int, device=w.device)
if act_order:
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
w_ref, q_w, s = gptq_quantize_weights(w, quant_type, group_size)
# Reformat to marlin
weight_perm = get_weight_perm(num_bits, is_a_8bit)
@@ -166,7 +154,7 @@ def marlin_quantize(
marlin_s = marlin_s * 512
# Create result
res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
res_list = [w_ref, marlin_q_w, marlin_s]
for i in range(len(res_list)):
res_list[i] = res_list[i].to(w.device)
@@ -691,36 +691,6 @@ def get_pack_factor(num_bits):
return 32 // num_bits
def permute_rows(
q_w: torch.Tensor,
w_ref: torch.Tensor,
group_size: int,
test_perm: torch.Tensor | None = None,
):
assert q_w.shape == w_ref.shape
orig_device = q_w.device
k_size, _ = q_w.shape
g_idx = torch.zeros((k_size,), dtype=torch.int32)
for i in range(k_size):
g_idx[i] = i // group_size
# Simulate act_order by doing a random permutation on K
rand_perm = test_perm if test_perm is not None else torch.randperm(k_size)
g_idx = g_idx[rand_perm].contiguous()
q_w = q_w[rand_perm, :].contiguous()
w_ref = w_ref[rand_perm, :].contiguous()
return (
w_ref.to(device=orig_device),
q_w.to(device=orig_device),
g_idx.to(device=orig_device),
rand_perm.to(device=orig_device),
)
def quantize_weights(
w: torch.Tensor,
quant_type: ScalarType,
@@ -824,8 +794,6 @@ def gptq_quantize_weights(
w: torch.Tensor,
quant_type: ScalarType,
group_size: int,
act_order: bool,
test_perm: torch.Tensor | None = None,
):
size_k, _ = w.shape
@@ -838,35 +806,7 @@ def gptq_quantize_weights(
)
w_ref, w_q, w_s, _ = quantize_weights(w, quant_type, group_size)
# Apply act_order
g_idx = torch.empty(0, dtype=torch.int, device=w.device)
rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
if act_order:
assert group_size < size_k, (
"For act_order, groupsize = {} must be less than size_k = {}".format(
group_size, size_k
)
)
w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size, test_perm)
return w_ref, w_q, w_s, g_idx, rand_perm
def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
orig_device = q_w.device
sort_indices = torch.argsort(g_idx).to(dtype=torch.int32) # Sort based on g_idx
g_idx = g_idx[sort_indices].contiguous()
q_w = q_w[sort_indices, :].contiguous()
return (
q_w.to(device=orig_device),
g_idx.to(device=orig_device),
sort_indices.to(device=orig_device),
)
return w_ref, w_q, w_s
def pack_rows(