[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)

Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com>
Signed-off-by: Roderick Wu <roderickwu2003@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Roderick Wu
2026-09-08 12:45:12 -07:00
committed by GitHub
co-authored by mgoin
parent 1a522b6949
commit 2c9d68f80b
99 changed files with 1182 additions and 3917 deletions
@@ -70,7 +70,7 @@ def test_gptq_allspark_gemm_ampere(mnk_factors, group_size, has_zp, dtype):
input = rand_data((m, k), dtype=dtype)
weight = rand_data((k, n), dtype=dtype)
# Quantize (and apply act_order if provided)
# Quantize weights.
w_ref, qw, s, zp = quantize_weights(
weight, scalar_types.uint8b128, group_size, has_zp
)
+3 -9
View File
@@ -11,9 +11,8 @@ def test_gptq_shuffle_opcheck():
weight = torch.randint(
-2000000, 2000000, (1792, 4096), device="cuda", dtype=torch.int32
)
perm = torch.empty((0,), device="cuda", dtype=torch.int32)
bit = 4
opcheck(torch.ops._C.gptq_shuffle, (weight, perm, bit))
opcheck(torch.ops._C.gptq_shuffle, (weight, bit))
def test_gptq_gemm_opcheck():
@@ -23,13 +22,8 @@ def test_gptq_gemm_opcheck():
)
zeros = torch.zeros((32, 768), device="cuda", dtype=torch.int32)
scales = torch.rand((32, 6144), device="cuda", dtype=torch.float16)
idx = torch.empty((0,), device="cuda", dtype=torch.int32)
use_exllama = True
bit = 4
# Test both GPTQv1 and GPTQv2 format
opcheck(
torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, True, bit)
)
opcheck(
torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, False, bit)
)
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, True, bit))
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, False, bit))
+14 -71
View File
@@ -17,7 +17,7 @@ from vllm.model_executor.layers.quantization.utils.int8_utils import (
per_token_quant_int8,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_make_empty_g_idx,
marlin_make_empty,
marlin_make_workspace_new,
marlin_permute_bias,
query_marlin_supported_quant_types,
@@ -40,7 +40,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
gptq_pack,
gptq_quantize_weights,
quantize_weights,
sort_weights,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
@@ -51,8 +50,6 @@ if current_platform.is_rocm():
allow_module_level=True,
)
ACT_ORDER_OPTS = [False, True]
K_FULL_OPTS = [False, True]
USE_ATOMIC_ADD_OPTS = [False, True]
USE_FP32_REDUCE_OPTS = [True]
@@ -80,13 +77,11 @@ DENSE_MARLIN_QUANT_TEST_CONFIGS = [
# GPTQ-INT4
{
"b_type": scalar_types.uint4b8,
"support_act_order": True,
"group_blocks": [-1, 2, 4, 8],
},
# GPTQ-INT8
{
"b_type": scalar_types.uint8b128,
"support_act_order": True,
"group_blocks": [-1, 2, 4, 8],
},
# FP8
@@ -202,27 +197,15 @@ def test_marlin_int4_fp8_preprocess_awq():
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
@pytest.mark.parametrize("quant_type", query_marlin_supported_quant_types(False, False))
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
@pytest.mark.parametrize("is_a_8bit", [True, False])
@pytest.mark.parametrize("nk_factors", MARLIN_REPACK_NK_FACTORS)
def test_gptq_marlin_repack(
k_chunk, n_chunk, quant_type, act_order, is_a_8bit, nk_factors
):
def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, is_a_8bit, nk_factors):
n_factor, k_factor = nk_factors
size_k = k_chunk * k_factor
size_n = n_chunk * n_factor
group_size = 128
# Filter act_order
if act_order:
if group_size == -1:
return
if group_size == size_k:
return
if is_a_8bit:
return
# Normalize group_size
if group_size == -1:
group_size = size_k
@@ -231,20 +214,12 @@ def test_gptq_marlin_repack(
# Create input
b_weight = rand_data((size_k, size_n))
# Quantize (and apply act_order if provided)
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
b_weight, quant_type, group_size, act_order
)
# Quantize
_, q_w, _ = gptq_quantize_weights(b_weight, quant_type, group_size)
# Pack to GPTQ format
q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
# For act_order, sort the "weights" and "g_idx" so that group ids are
# increasing
sort_indices = torch.empty(0, dtype=torch.int, device=b_weight.device)
if act_order:
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
# Pack to Marlin format
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
marlin_q_w_1 = marlin_weights(
@@ -253,12 +228,12 @@ def test_gptq_marlin_repack(
opcheck(
torch.ops._C.gptq_marlin_repack,
(q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit),
(q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit),
)
# Run Marlin repack GPU kernel
marlin_q_w_2 = ops.gptq_marlin_repack(
q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit
q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit
)
torch.accelerator.synchronize()
@@ -319,8 +294,6 @@ def marlin_generate_valid_test_cases():
MNK_FACTORS,
MARLIN_N_CHUNKS,
MARLIN_K_CHUNKS,
ACT_ORDER_OPTS,
K_FULL_OPTS,
USE_ATOMIC_ADD_OPTS,
USE_FP32_REDUCE_OPTS,
)
@@ -333,8 +306,6 @@ def marlin_generate_valid_test_cases():
size_m,
size_n,
size_k,
act_order,
is_k_full,
use_atomic_add,
use_fp32_reduce,
):
@@ -351,25 +322,18 @@ def marlin_generate_valid_test_cases():
if group_size > 0 and size_k % group_size != 0:
return False
if act_order and group_size in [-1, size_k]:
return False
if group_size == size_k:
return False
if not act_order and is_k_full:
return False
return a_type.size_bits < 16 or a_type is c_type
cases = []
for case in all_combinations:
quant_test_config, mnk_factors, n_chunk, k_chunk, act_order, *_ = case
quant_test_config, mnk_factors, n_chunk, k_chunk, *_ = case
size_m = mnk_factors[0]
size_n = mnk_factors[1] * n_chunk
size_k = mnk_factors[2] * k_chunk
if act_order and not quant_test_config.get("support_act_order", False):
continue
f16_types = [scalar_types.float16, scalar_types.bfloat16]
inner_combinations = itertools.product(
quant_test_config.get("a_type", f16_types),
@@ -398,7 +362,7 @@ def marlin_generate_valid_test_cases():
@pytest.mark.parametrize(
(
"a_type, b_type, c_type, group_blocks,"
"size_m, size_n, size_k, act_order, is_k_full,"
"size_m, size_n, size_k,"
"use_atomic_add, use_fp32_reduce"
),
marlin_generate_valid_test_cases(),
@@ -411,8 +375,6 @@ def test_marlin_gemm(
size_m,
size_n,
size_k,
act_order,
is_k_full,
use_atomic_add,
use_fp32_reduce,
):
@@ -448,27 +410,21 @@ def test_marlin_gemm(
)
marlin_s2 = None
g_idx = None
sort_indices = None
marlin_zp = None
elif b_type == scalar_types.float8_e4m3fn:
w_ref, marlin_q_w, marlin_s = marlin_quant_fp8_torch(
b_weight.T, group_size, input_dtype=a_dtype
)
g_idx = None
sort_indices = None
marlin_zp = None
marlin_s2 = None
elif has_zp:
w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
b_weight, b_type, group_size, input_dtype=a_dtype
)
g_idx = None
sort_indices = None
marlin_s2 = None
else:
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
b_weight, b_type, group_size, act_order, input_dtype=a_dtype
w_ref, marlin_q_w, marlin_s = marlin_quantize(
b_weight, b_type, group_size, input_dtype=a_dtype
)
marlin_zp = None
@@ -506,14 +462,11 @@ def test_marlin_gemm(
a_scales,
marlin_s2,
marlin_zp,
g_idx,
sort_indices,
workspace,
b_type,
a_input.shape[0],
b_weight.shape[1],
a_input.shape[1],
is_k_full=is_k_full,
use_atomic_add=use_atomic_add,
use_fp32_reduce=use_fp32_reduce,
is_zp_float=False,
@@ -535,11 +488,9 @@ def test_marlin_gemm_subset_input():
a_input = rand_data((big_m, big_k))[8 : size_m + 8, 8 : size_k + 8]
b_weight = rand_data((size_k, size_n))
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
b_weight, quant_type, group_size, False
)
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
marlin_zp = marlin_make_empty(marlin_s.device)
workspace = marlin_make_workspace_new(a_input.device)
output = ops.marlin_gemm(
@@ -551,14 +502,11 @@ def test_marlin_gemm_subset_input():
None,
None,
marlin_zp,
g_idx,
sort_indices,
workspace,
quant_type,
a_input.shape[0],
b_weight.shape[1],
a_input.shape[1],
is_k_full=True,
use_atomic_add=False,
use_fp32_reduce=True,
is_zp_float=False,
@@ -584,11 +532,9 @@ def test_marlin_gemm_with_bias(size_m):
marlin_bias = marlin_permute_bias(b_bias)
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
b_weight, quant_type, group_size, False
)
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
marlin_zp = marlin_make_empty(marlin_s.device)
workspace = marlin_make_workspace_new(a_input.device)
output = ops.marlin_gemm(
@@ -600,14 +546,11 @@ def test_marlin_gemm_with_bias(size_m):
None,
None,
marlin_zp,
g_idx,
sort_indices,
workspace,
quant_type,
a_input.shape[0],
b_weight.shape[1],
a_input.shape[1],
is_k_full=True,
use_atomic_add=False,
use_fp32_reduce=True,
is_zp_float=False,
@@ -14,7 +14,7 @@ from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
GPTQ_MARLIN_TILE,
apply_gptq_marlin_linear,
marlin_make_empty_g_idx,
marlin_make_empty,
marlin_make_workspace_new,
marlin_moe_padded_intermediate,
marlin_pad_qweight,
@@ -319,16 +319,13 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
device = torch.device("cuda")
weight = torch.randn(size_k, size_n, dtype=dtype, device=device) / size_k**0.5
w_ref, q_w, s, _, _ = gptq_quantize_weights(
weight, quant_type, group_size, act_order=False
)
w_ref, q_w, s = gptq_quantize_weights(weight, quant_type, group_size)
qweight = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=torch.empty(0, dtype=torch.int, device=device),
size_k=padded_k,
size_n=padded_n,
num_bits=quant_type.size_bits,
@@ -343,14 +340,11 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
input=x,
weight=marlin_qweight,
weight_scale=marlin_s,
weight_zp=marlin_make_empty_g_idx(device),
g_idx=marlin_make_empty_g_idx(device),
g_idx_sort_indices=marlin_make_empty_g_idx(device),
weight_zp=marlin_make_empty(device),
workspace=marlin_make_workspace_new(device),
wtype=quant_type,
output_size_per_partition=size_n,
input_size_per_partition=size_k,
is_k_full=True,
)
ref = x @ w_ref
@@ -480,7 +474,6 @@ def test_awq_zp_marlin_padded_round_trip(shape):
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=torch.empty(0, dtype=torch.int, device=device),
size_k=padded_k,
size_n=padded_n,
num_bits=quant_type.size_bits,
@@ -503,13 +496,10 @@ def test_awq_zp_marlin_padded_round_trip(shape):
weight=marlin_qweight,
weight_scale=marlin_s,
weight_zp=marlin_zp,
g_idx=marlin_make_empty_g_idx(device),
g_idx_sort_indices=marlin_make_empty_g_idx(device),
workspace=marlin_make_workspace_new(device),
wtype=quant_type,
output_size_per_partition=size_n,
input_size_per_partition=size_k,
is_k_full=True,
)
ref = x @ w_ref
@@ -585,9 +575,7 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
def quant(w, size_k, size_n):
# w is (size_n, size_k); gptq expects (size_k, size_n).
ref, q_w, s, _, _ = gptq_quantize_weights(
w.T, quant_type, group_size, act_order=False
)
ref, q_w, s = gptq_quantize_weights(w.T, quant_type, group_size)
return ref, gptq_pack(q_w, bits, size_k, size_n), s
w13_qw, w13_s, w13_ref = [], [], []
@@ -616,12 +604,11 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
if group_size > 0:
w2_scales = _pad_rows(w2_scales, padded_n // group_size)
sort_idx = torch.empty((e, 0), dtype=torch.int32, device=device)
marlin_w13 = ops.gptq_marlin_moe_repack(
w13_qweight, sort_idx, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
w13_qweight, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
)
marlin_w2 = ops.gptq_marlin_moe_repack(
w2_qweight, sort_idx, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
w2_qweight, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
)
group_or_pack = group_size if group_size != -1 else pack
marlin_w13_s = marlin_moe_permute_scales(
@@ -649,7 +636,6 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
topk_ids,
quant_type_id=quant_type.id,
global_num_experts=e,
is_k_full=True,
)
with set_current_vllm_config(VllmConfig()):
ref = torch_experts(
@@ -770,7 +756,6 @@ def test_fp8_marlin_moe_padded_round_trip(shape, quant):
topk_ids,
quant_type_id=scalar_types.float8_e4m3fn.id,
global_num_experts=e,
is_k_full=True,
workspace=layer.workspace,
)
with set_current_vllm_config(VllmConfig()):
@@ -848,7 +833,6 @@ def test_mxfp8_marlin_moe_padded_round_trip(shape):
topk_ids,
quant_type_id=scalar_types.float8_e4m3fn.id,
global_num_experts=e,
is_k_full=True,
workspace=layer.workspace,
)
with set_current_vllm_config(VllmConfig()):
@@ -196,7 +196,6 @@ def test_dense_kernel_rejects_on_non_gfx1100():
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is False, f"RDNA3 dense kernel accepted on non-gfx1100: {reason}"
@@ -431,7 +430,6 @@ class TestDenseKernelSelectionMocked:
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
ok, _ = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is True
@@ -461,7 +459,6 @@ class TestDenseKernelSelectionMocked:
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
with (
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
@@ -69,10 +69,9 @@ def _make_packed_weights(E, K, N):
packed = torch.zeros(E, K // 8, N, dtype=torch.int32, device=device)
for i in range(8):
packed |= (w[:, i::8, :] & 0xF) << (i * 4)
g_idx = torch.empty(0, dtype=torch.int32, device=device)
for e in range(E):
we = packed[e].contiguous()
ops.gptq_shuffle(we, g_idx, 4)
ops.gptq_shuffle(we, 4)
packed[e] = we
return packed
@@ -113,7 +112,6 @@ def test_fused_moe_w1_matches_dense(
w13 = _make_packed_weights(E, K, N_gate_up)
w13_s = _make_scales(E, groups, N_gate_up, dtype)
w13_z = _make_qzeros(E, groups, N_gate_up)
g_idx = torch.empty(0, dtype=torch.int32, device=device)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
si, ei, ntp = moe_align_block_size(topk_ids, block_size_m, E)
@@ -147,7 +145,6 @@ def test_fused_moe_w1_matches_dense(
w13[e],
w13_z[e],
w13_s[e],
g_idx,
False,
)
ref_out[flat] = ref.squeeze()
@@ -243,7 +240,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
w2 = _make_packed_weights(E, N_inter, hidden)
w2_s = _make_scales(E, N_inter // group_size, hidden, dtype)
w2_z = _make_qzeros(E, N_inter // group_size, hidden)
g_idx = torch.empty(0, dtype=torch.int32, device=device)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
topk_w = torch.softmax(
@@ -300,7 +296,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
w13[e],
w13_z[e],
w13_s[e],
g_idx,
False,
)
a = torch.empty(1, N_inter, dtype=dtype, device=device)
@@ -310,7 +305,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
w2[e],
w2_z[e],
w2_s[e],
g_idx,
False,
)
ref[m_idx] += r2.squeeze() * w
@@ -173,7 +173,6 @@ def _run_kernel(
act_type=dtype,
group_size=group_size,
zero_points=has_zp,
has_g_idx=False,
)
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok, f"can_implement rejected a supported config: {reason}"
@@ -184,7 +183,6 @@ def _run_kernel(
w_q_param_name="qweight",
w_s_param_name="scales",
w_zp_param_name="qzeros" if has_zp else None,
w_gidx_param_name=None,
)
kernel.process_weights_after_loading(layer)
return kernel.apply_weights(layer, x_mk, bias=bias)
@@ -57,7 +57,6 @@ def test_selection_prefers_rdna3(dtype):
act_type=dtype,
group_size=128,
zero_points=False,
has_g_idx=False,
)
assert choose_mp_linear_kernel(config).__name__ == "RDNA3W4A16LinearKernel"
@@ -83,7 +82,6 @@ def test_can_implement(weight_type, group_size, N, full_k, expected_ok):
act_type=torch.float16,
group_size=group_size,
zero_points=False,
has_g_idx=False,
)
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is expected_ok, reason
@@ -306,14 +306,12 @@ def test_rdna_hybrid_w4a16_process_weights_symmetric_repack(group_size, dist_ini
act_type=torch.float16,
group_size=G,
zero_points=False,
has_g_idx=False,
)
kernel = RDNAHybridW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name=None,
w_gidx_param_name=None,
)
kernel.process_weights_after_loading(layer)
@@ -367,14 +365,12 @@ def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_in
act_type=torch.float16,
group_size=G,
zero_points=True,
has_g_idx=False,
)
kernel = RDNAHybridW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
w_gidx_param_name=None,
)
kernel.process_weights_after_loading(layer)
@@ -414,7 +410,6 @@ def test_hybrid_can_implement_group_size(group_size, expected_ok):
act_type=torch.float16,
group_size=group_size,
zero_points=False,
has_g_idx=False,
)
ok, _ = RDNAHybridW4A16LinearKernel.can_implement(config)
assert ok is expected_ok
@@ -273,14 +273,12 @@ def test_triton_w4a16_process_weights_after_loading_repacks_layout():
act_type=torch.float16,
group_size=G,
zero_points=True,
has_g_idx=False,
)
kernel = TritonW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
w_gidx_param_name=None,
)
# Build dummy layer with vLLM parameter wrappers.
@@ -384,14 +382,12 @@ def test_triton_w4a16_process_weights_after_loading_keeps_gptq_qzeros_layout():
act_type=torch.float16,
group_size=G,
zero_points=True,
has_g_idx=False,
)
kernel = TritonW4A16LinearKernel(
config,
w_q_param_name="qweight",
w_s_param_name="scales",
w_zp_param_name="qzeros",
w_gidx_param_name=None,
)
weight_loader = lambda *args, **kwargs: None
@@ -461,14 +457,12 @@ def test_triton_w4a16_symmetric_apply_ignores_qzeros(monkeypatch):
act_type=torch.float16,
group_size=G,
zero_points=False,
has_g_idx=False,
)
kernel = TritonW4A16LinearKernel(
config,
w_q_param_name="qweight",
w_s_param_name="scales",
w_zp_param_name="qzeros",
w_gidx_param_name=None,
)
class DummyLayer(torch.nn.Module):
@@ -47,7 +47,6 @@ def test_choose_mp_linear_kernel_uint4b8():
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
kernel_type = choose_mp_linear_kernel(config)
@@ -65,7 +64,6 @@ def test_choose_mp_linear_kernel_uint4_asymmetric():
act_type=torch.bfloat16,
group_size=64,
zero_points=True,
has_g_idx=False,
)
kernel_type = choose_mp_linear_kernel(config)