[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com> Signed-off-by: Roderick Wu <roderickwu2003@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -20,6 +20,36 @@ else:
|
||||
AutoGPTQConfig = object
|
||||
|
||||
|
||||
def normalize_and_validate_gptq_desc_act(
|
||||
desc_act: bool,
|
||||
group_size: int | None,
|
||||
dynamic: Mapping[str, Any],
|
||||
) -> bool:
|
||||
base_desc_act = desc_act
|
||||
if desc_act and group_size == -1:
|
||||
# With one group per output channel, activation ordering is a no-op.
|
||||
desc_act = False
|
||||
if desc_act:
|
||||
raise ValueError(
|
||||
"GPTQ group activation ordering (desc_act=True) is no longer "
|
||||
"supported. Use a checkpoint with static activation ordering "
|
||||
"or desc_act=False."
|
||||
)
|
||||
|
||||
for pattern, overrides in dynamic.items():
|
||||
if pattern.startswith("-:") or not isinstance(overrides, Mapping):
|
||||
continue
|
||||
dynamic_desc_act = overrides.get("desc_act", base_desc_act)
|
||||
dynamic_group_size = overrides.get("group_size", group_size)
|
||||
if dynamic_desc_act and dynamic_group_size != -1:
|
||||
raise ValueError(
|
||||
"GPTQ group activation ordering (desc_act=True) is no longer "
|
||||
f"supported, but is enabled by dynamic rule {pattern!r}. Use "
|
||||
"a checkpoint with static activation ordering or desc_act=False."
|
||||
)
|
||||
return desc_act
|
||||
|
||||
|
||||
# Match dynamic rules with module name (prefix) and override quantize
|
||||
# config if module (prefix) matches a rule
|
||||
def override_config(config: AutoGPTQConfig, prefix: str):
|
||||
@@ -33,6 +63,10 @@ def override_config(config: AutoGPTQConfig, prefix: str):
|
||||
if isinstance(desc_act, bool):
|
||||
config.desc_act = desc_act
|
||||
|
||||
if config.desc_act and config.group_size == -1:
|
||||
# Activation ordering is a no-op for channelwise quantization.
|
||||
config.desc_act = False
|
||||
|
||||
config.pack_factor = 32 // config.weight_bits # packed into int32
|
||||
assert isinstance(config, AutoGPTQConfig)
|
||||
is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
|
||||
|
||||
@@ -359,11 +359,11 @@ def check_moe_marlin_supports_config(
|
||||
) -> bool:
|
||||
"""Whether the fused MoE Marlin kernel supports ``config``.
|
||||
|
||||
Callers without act-order may pass ``allow_tile_padding=True``: a
|
||||
tile-misaligned intermediate size is then zero-padded to a valid thread
|
||||
tile at weight prep (see marlin_moe_padded_intermediate), so only a group
|
||||
straddling the padded boundary stays unsupported. hidden_size is the MoE
|
||||
I/O extent and is never padded. Act-order keeps the strict shape.
|
||||
With ``allow_tile_padding=True``, a tile-misaligned intermediate size is
|
||||
zero-padded to a valid thread tile at weight prep (see
|
||||
marlin_moe_padded_intermediate), so only a group straddling the padded
|
||||
boundary stays unsupported. hidden_size is the MoE I/O extent and is never
|
||||
padded.
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
return False
|
||||
@@ -433,30 +433,17 @@ def marlin_make_workspace_new(
|
||||
return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False)
|
||||
|
||||
|
||||
def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
|
||||
return (not act_order) or (act_order and not is_row_parallel)
|
||||
|
||||
|
||||
def marlin_repeat_scales_on_all_ranks(
|
||||
act_order: bool, group_size: int, is_row_parallel: bool
|
||||
) -> bool:
|
||||
# Need to repeat scales on every rank if act_ordering or
|
||||
# channelwise and RowParallelLinear
|
||||
def marlin_repeat_scales_on_all_ranks(group_size: int, is_row_parallel: bool) -> bool:
|
||||
is_channelwise = group_size == -1
|
||||
return act_order or (is_channelwise and is_row_parallel)
|
||||
return is_channelwise and is_row_parallel
|
||||
|
||||
|
||||
def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
|
||||
def marlin_make_empty(device: torch.device) -> torch.Tensor:
|
||||
return torch.nn.Parameter(
|
||||
torch.empty(0, dtype=torch.int, device=device), requires_grad=False
|
||||
)
|
||||
|
||||
|
||||
def marlin_sort_g_idx(g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
|
||||
return g_idx[g_idx_sort_indices], g_idx_sort_indices
|
||||
|
||||
|
||||
def get_scale_perms():
|
||||
scale_perm: list[int] = []
|
||||
for i in range(8):
|
||||
@@ -700,13 +687,10 @@ def apply_gptq_marlin_linear(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
weight_zp: torch.Tensor,
|
||||
g_idx: torch.Tensor,
|
||||
g_idx_sort_indices: torch.Tensor,
|
||||
workspace: torch.Tensor,
|
||||
wtype: ScalarType,
|
||||
output_size_per_partition: int,
|
||||
input_size_per_partition: int,
|
||||
is_k_full: bool,
|
||||
input_global_scale: torch.Tensor | None = None,
|
||||
bias: torch.Tensor | None = None,
|
||||
use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT,
|
||||
@@ -749,14 +733,11 @@ def apply_gptq_marlin_linear(
|
||||
a_scales,
|
||||
None,
|
||||
weight_zp,
|
||||
g_idx,
|
||||
g_idx_sort_indices,
|
||||
workspace,
|
||||
wtype,
|
||||
size_m=reshaped_x.shape[0],
|
||||
size_n=padded_n,
|
||||
size_k=padded_k,
|
||||
is_k_full=is_k_full,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
is_zp_float=False,
|
||||
|
||||
@@ -203,8 +203,6 @@ def apply_fp4_marlin_linear(
|
||||
a_scales=a_scales,
|
||||
global_scale=weight_global_scale,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float4_e2m1f,
|
||||
size_m=reshaped_x.size(0),
|
||||
@@ -246,14 +244,12 @@ def prepare_fp4_layer_for_marlin(
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
qweight = layer.weight.view(torch.int32).T.contiguous()
|
||||
qweight = marlin_pad_qweight(qweight, part_size_n, part_size_k, padded_n, padded_k)
|
||||
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=4,
|
||||
@@ -312,7 +308,6 @@ def _repack_marlin_experts(
|
||||
weight: torch.Tensor,
|
||||
size_n: int,
|
||||
size_k: int,
|
||||
perm: torch.Tensor,
|
||||
is_a_8bit: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Repack each expert to marlin format into a preallocated output."""
|
||||
@@ -322,7 +317,6 @@ def _repack_marlin_experts(
|
||||
qweight = weight[i].view(torch.int32).T.contiguous()
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=4,
|
||||
@@ -401,7 +395,6 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
@@ -417,7 +410,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
weight = pad_w2(weight, packing=2)
|
||||
size_k = padded_N
|
||||
|
||||
return _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
|
||||
return _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
|
||||
|
||||
w13 = repack_weight(w13, "w13")
|
||||
w2 = repack_weight(w2, "w2")
|
||||
@@ -492,7 +485,6 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
# WEIGHT
|
||||
@@ -506,7 +498,7 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
|
||||
assert weight.shape == (e, size_n, size_k // 2)
|
||||
|
||||
weight = _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
|
||||
weight = _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
|
||||
weight = torch.nn.Parameter(weight, requires_grad=False)
|
||||
|
||||
setattr(layer, name, weight)
|
||||
@@ -621,10 +613,8 @@ def prepare_moe_mxfp4_layer_for_marlin(
|
||||
n = w13.shape[1] // 2 # intermediate_size_per_partition
|
||||
k = w13.shape[2] * 2 # hidden_size
|
||||
|
||||
device = w13.device
|
||||
param_dtype = layer.params_dtype
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT: Repack weights to marlin format
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
@@ -635,7 +625,7 @@ def prepare_moe_mxfp4_layer_for_marlin(
|
||||
|
||||
assert weight.shape == (e, size_n, size_k // 2)
|
||||
|
||||
return _repack_marlin_experts(weight, size_n, size_k, perm, is_a_8bit)
|
||||
return _repack_marlin_experts(weight, size_n, size_k, is_a_8bit)
|
||||
|
||||
w13 = repack_weight(w13, "w13")
|
||||
w2 = repack_weight(w2, "w2")
|
||||
@@ -691,8 +681,6 @@ def rand_marlin_weight_nvfp4_like(weight, group_size, input_dtype=None):
|
||||
assert not is_a_8bit, "NVFP4 weight + INT8/FP8 activation is not supported."
|
||||
assert group_size > 0
|
||||
size_n, size_k = weight.shape
|
||||
device = weight.device
|
||||
|
||||
scales = weight.view(size_n, -1, group_size).abs().max(-1)[0] / 6
|
||||
global_scale = scales.max() / 448
|
||||
scales = (scales / global_scale).to(torch.float8_e4m3fn)
|
||||
@@ -720,7 +708,6 @@ def rand_marlin_weight_nvfp4_like(weight, group_size, input_dtype=None):
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=fp4_weight.view(torch.int32).T.contiguous(),
|
||||
perm=torch.empty(0, dtype=torch.int, device=device),
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=4,
|
||||
@@ -751,8 +738,6 @@ def rand_marlin_weight_mxfp4_like(weight, group_size, input_dtype=None):
|
||||
|
||||
assert group_size > 0
|
||||
size_n, size_k = weight.shape
|
||||
device = weight.device
|
||||
|
||||
scales = torch.randint(
|
||||
110,
|
||||
120,
|
||||
@@ -779,11 +764,9 @@ def rand_marlin_weight_mxfp4_like(weight, group_size, input_dtype=None):
|
||||
).view(size_n, size_k)
|
||||
weight_ref = weight_ref * scales.repeat_interleave(group_size, 1).to(weight.dtype)
|
||||
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
fp4_weight = fp4_weight.view(torch.int32).T.contiguous()
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=fp4_weight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=4,
|
||||
|
||||
@@ -89,8 +89,6 @@ def apply_fp8_marlin_linear(
|
||||
a_scales=a_scales,
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
size_m=reshaped_x.size(0),
|
||||
@@ -138,7 +136,6 @@ def prepare_fp8_layer_for_marlin(
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
qweight = pack_fp8_to_int32(layer.weight, size_k_first)
|
||||
if not size_k_first:
|
||||
qweight = qweight.T.contiguous()
|
||||
@@ -146,7 +143,6 @@ def prepare_fp8_layer_for_marlin(
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=8,
|
||||
@@ -283,7 +279,6 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
@@ -295,7 +290,7 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
qweight = qweight.T.contiguous()
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight, perm=perm, size_k=size_k, size_n=size_n, num_bits=8
|
||||
b_q_weight=qweight, size_k=size_k, size_n=size_n, num_bits=8
|
||||
)
|
||||
tensor_list.append(marlin_qweight)
|
||||
|
||||
@@ -434,8 +429,6 @@ def apply_mxfp8_marlin_linear(
|
||||
a_scales=None,
|
||||
global_scale=None,
|
||||
b_zeros=None,
|
||||
g_idx=None,
|
||||
perm=None,
|
||||
workspace=workspace,
|
||||
b_q_type=scalar_types.float8_e4m3fn,
|
||||
size_m=reshaped_x.size(0),
|
||||
@@ -470,14 +463,12 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
)
|
||||
|
||||
# WEIGHT - repack FP8 weights to Marlin format
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
qweight = pack_fp8_to_int32(layer.weight, size_k_first=False)
|
||||
qweight = qweight.T.contiguous()
|
||||
qweight = marlin_pad_qweight(qweight, part_size_n, part_size_k, padded_n, padded_k)
|
||||
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=8,
|
||||
@@ -551,7 +542,6 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
|
||||
device = w13.device
|
||||
param_dtype = torch.get_default_dtype()
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
@@ -571,7 +561,6 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
qweight = qweight.T.contiguous()
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=8,
|
||||
@@ -615,8 +604,6 @@ def marlin_quant_fp8_torch(weight, group_size, input_dtype=None):
|
||||
assert input_dtype == torch.float8_e4m3fn
|
||||
|
||||
size_n, size_k = weight.shape
|
||||
device = weight.device
|
||||
|
||||
if group_size != -1:
|
||||
scales = weight.view(size_n, -1, group_size).abs().max(-1)[0] / 448
|
||||
repeated_scales = scales.repeat_interleave(group_size, 1)
|
||||
@@ -629,10 +616,8 @@ def marlin_quant_fp8_torch(weight, group_size, input_dtype=None):
|
||||
weight_ref = fp8_weight.to(weight.dtype) * repeated_scales
|
||||
|
||||
packed_weight = pack_fp8_to_int32(fp8_weight, False).T.contiguous()
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=packed_weight,
|
||||
perm=perm,
|
||||
size_k=size_k,
|
||||
size_n=size_n,
|
||||
num_bits=8,
|
||||
|
||||
@@ -13,7 +13,6 @@ from .quant_utils import (
|
||||
get_pack_factor,
|
||||
gptq_quantize_weights,
|
||||
quantize_weights,
|
||||
sort_weights,
|
||||
)
|
||||
|
||||
|
||||
@@ -129,8 +128,6 @@ def marlin_quantize(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
act_order: bool,
|
||||
test_perm: torch.Tensor | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
):
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
@@ -143,16 +140,7 @@ def marlin_quantize(
|
||||
group_size = size_k
|
||||
assert group_size <= size_k
|
||||
|
||||
# Quantize (and apply act_order if provided)
|
||||
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
|
||||
w, quant_type, group_size, act_order, test_perm
|
||||
)
|
||||
|
||||
# For act_order, sort the "weights" and "g_idx" so that group ids are
|
||||
# increasing
|
||||
sort_indices = torch.empty(0, dtype=torch.int, device=w.device)
|
||||
if act_order:
|
||||
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
|
||||
w_ref, q_w, s = gptq_quantize_weights(w, quant_type, group_size)
|
||||
|
||||
# Reformat to marlin
|
||||
weight_perm = get_weight_perm(num_bits, is_a_8bit)
|
||||
@@ -166,7 +154,7 @@ def marlin_quantize(
|
||||
marlin_s = marlin_s * 512
|
||||
|
||||
# Create result
|
||||
res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm]
|
||||
res_list = [w_ref, marlin_q_w, marlin_s]
|
||||
for i in range(len(res_list)):
|
||||
res_list[i] = res_list[i].to(w.device)
|
||||
|
||||
|
||||
@@ -691,36 +691,6 @@ def get_pack_factor(num_bits):
|
||||
return 32 // num_bits
|
||||
|
||||
|
||||
def permute_rows(
|
||||
q_w: torch.Tensor,
|
||||
w_ref: torch.Tensor,
|
||||
group_size: int,
|
||||
test_perm: torch.Tensor | None = None,
|
||||
):
|
||||
assert q_w.shape == w_ref.shape
|
||||
|
||||
orig_device = q_w.device
|
||||
k_size, _ = q_w.shape
|
||||
|
||||
g_idx = torch.zeros((k_size,), dtype=torch.int32)
|
||||
for i in range(k_size):
|
||||
g_idx[i] = i // group_size
|
||||
|
||||
# Simulate act_order by doing a random permutation on K
|
||||
rand_perm = test_perm if test_perm is not None else torch.randperm(k_size)
|
||||
|
||||
g_idx = g_idx[rand_perm].contiguous()
|
||||
q_w = q_w[rand_perm, :].contiguous()
|
||||
w_ref = w_ref[rand_perm, :].contiguous()
|
||||
|
||||
return (
|
||||
w_ref.to(device=orig_device),
|
||||
q_w.to(device=orig_device),
|
||||
g_idx.to(device=orig_device),
|
||||
rand_perm.to(device=orig_device),
|
||||
)
|
||||
|
||||
|
||||
def quantize_weights(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
@@ -824,8 +794,6 @@ def gptq_quantize_weights(
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
act_order: bool,
|
||||
test_perm: torch.Tensor | None = None,
|
||||
):
|
||||
size_k, _ = w.shape
|
||||
|
||||
@@ -838,35 +806,7 @@ def gptq_quantize_weights(
|
||||
)
|
||||
|
||||
w_ref, w_q, w_s, _ = quantize_weights(w, quant_type, group_size)
|
||||
|
||||
# Apply act_order
|
||||
g_idx = torch.empty(0, dtype=torch.int, device=w.device)
|
||||
rand_perm = torch.empty(0, dtype=torch.int, device=w.device)
|
||||
if act_order:
|
||||
assert group_size < size_k, (
|
||||
"For act_order, groupsize = {} must be less than size_k = {}".format(
|
||||
group_size, size_k
|
||||
)
|
||||
)
|
||||
|
||||
w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size, test_perm)
|
||||
|
||||
return w_ref, w_q, w_s, g_idx, rand_perm
|
||||
|
||||
|
||||
def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor):
|
||||
orig_device = q_w.device
|
||||
|
||||
sort_indices = torch.argsort(g_idx).to(dtype=torch.int32) # Sort based on g_idx
|
||||
|
||||
g_idx = g_idx[sort_indices].contiguous()
|
||||
q_w = q_w[sort_indices, :].contiguous()
|
||||
|
||||
return (
|
||||
q_w.to(device=orig_device),
|
||||
g_idx.to(device=orig_device),
|
||||
sort_indices.to(device=orig_device),
|
||||
)
|
||||
return w_ref, w_q, w_s
|
||||
|
||||
|
||||
def pack_rows(
|
||||
|
||||
Reference in New Issue
Block a user