[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com> Signed-off-by: Roderick Wu <roderickwu2003@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -70,7 +70,7 @@ def test_gptq_allspark_gemm_ampere(mnk_factors, group_size, has_zp, dtype):
|
||||
input = rand_data((m, k), dtype=dtype)
|
||||
weight = rand_data((k, n), dtype=dtype)
|
||||
|
||||
# Quantize (and apply act_order if provided)
|
||||
# Quantize weights.
|
||||
w_ref, qw, s, zp = quantize_weights(
|
||||
weight, scalar_types.uint8b128, group_size, has_zp
|
||||
)
|
||||
|
||||
@@ -11,9 +11,8 @@ def test_gptq_shuffle_opcheck():
|
||||
weight = torch.randint(
|
||||
-2000000, 2000000, (1792, 4096), device="cuda", dtype=torch.int32
|
||||
)
|
||||
perm = torch.empty((0,), device="cuda", dtype=torch.int32)
|
||||
bit = 4
|
||||
opcheck(torch.ops._C.gptq_shuffle, (weight, perm, bit))
|
||||
opcheck(torch.ops._C.gptq_shuffle, (weight, bit))
|
||||
|
||||
|
||||
def test_gptq_gemm_opcheck():
|
||||
@@ -23,13 +22,8 @@ def test_gptq_gemm_opcheck():
|
||||
)
|
||||
zeros = torch.zeros((32, 768), device="cuda", dtype=torch.int32)
|
||||
scales = torch.rand((32, 6144), device="cuda", dtype=torch.float16)
|
||||
idx = torch.empty((0,), device="cuda", dtype=torch.int32)
|
||||
use_exllama = True
|
||||
bit = 4
|
||||
# Test both GPTQv1 and GPTQv2 format
|
||||
opcheck(
|
||||
torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, True, bit)
|
||||
)
|
||||
opcheck(
|
||||
torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, False, bit)
|
||||
)
|
||||
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, True, bit))
|
||||
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, False, bit))
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_quant_int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_make_empty_g_idx,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_permute_bias,
|
||||
query_marlin_supported_quant_types,
|
||||
@@ -40,7 +40,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
gptq_pack,
|
||||
gptq_quantize_weights,
|
||||
quantize_weights,
|
||||
sort_weights,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
@@ -51,8 +50,6 @@ if current_platform.is_rocm():
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
ACT_ORDER_OPTS = [False, True]
|
||||
K_FULL_OPTS = [False, True]
|
||||
USE_ATOMIC_ADD_OPTS = [False, True]
|
||||
USE_FP32_REDUCE_OPTS = [True]
|
||||
|
||||
@@ -80,13 +77,11 @@ DENSE_MARLIN_QUANT_TEST_CONFIGS = [
|
||||
# GPTQ-INT4
|
||||
{
|
||||
"b_type": scalar_types.uint4b8,
|
||||
"support_act_order": True,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# GPTQ-INT8
|
||||
{
|
||||
"b_type": scalar_types.uint8b128,
|
||||
"support_act_order": True,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# FP8
|
||||
@@ -202,27 +197,15 @@ def test_marlin_int4_fp8_preprocess_awq():
|
||||
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
|
||||
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
|
||||
@pytest.mark.parametrize("quant_type", query_marlin_supported_quant_types(False, False))
|
||||
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
|
||||
@pytest.mark.parametrize("is_a_8bit", [True, False])
|
||||
@pytest.mark.parametrize("nk_factors", MARLIN_REPACK_NK_FACTORS)
|
||||
def test_gptq_marlin_repack(
|
||||
k_chunk, n_chunk, quant_type, act_order, is_a_8bit, nk_factors
|
||||
):
|
||||
def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, is_a_8bit, nk_factors):
|
||||
n_factor, k_factor = nk_factors
|
||||
|
||||
size_k = k_chunk * k_factor
|
||||
size_n = n_chunk * n_factor
|
||||
group_size = 128
|
||||
|
||||
# Filter act_order
|
||||
if act_order:
|
||||
if group_size == -1:
|
||||
return
|
||||
if group_size == size_k:
|
||||
return
|
||||
if is_a_8bit:
|
||||
return
|
||||
|
||||
# Normalize group_size
|
||||
if group_size == -1:
|
||||
group_size = size_k
|
||||
@@ -231,20 +214,12 @@ def test_gptq_marlin_repack(
|
||||
# Create input
|
||||
b_weight = rand_data((size_k, size_n))
|
||||
|
||||
# Quantize (and apply act_order if provided)
|
||||
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
|
||||
b_weight, quant_type, group_size, act_order
|
||||
)
|
||||
# Quantize
|
||||
_, q_w, _ = gptq_quantize_weights(b_weight, quant_type, group_size)
|
||||
|
||||
# Pack to GPTQ format
|
||||
q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
|
||||
|
||||
# For act_order, sort the "weights" and "g_idx" so that group ids are
|
||||
# increasing
|
||||
sort_indices = torch.empty(0, dtype=torch.int, device=b_weight.device)
|
||||
if act_order:
|
||||
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
|
||||
|
||||
# Pack to Marlin format
|
||||
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
|
||||
marlin_q_w_1 = marlin_weights(
|
||||
@@ -253,12 +228,12 @@ def test_gptq_marlin_repack(
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.gptq_marlin_repack,
|
||||
(q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit),
|
||||
(q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit),
|
||||
)
|
||||
|
||||
# Run Marlin repack GPU kernel
|
||||
marlin_q_w_2 = ops.gptq_marlin_repack(
|
||||
q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit
|
||||
q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -319,8 +294,6 @@ def marlin_generate_valid_test_cases():
|
||||
MNK_FACTORS,
|
||||
MARLIN_N_CHUNKS,
|
||||
MARLIN_K_CHUNKS,
|
||||
ACT_ORDER_OPTS,
|
||||
K_FULL_OPTS,
|
||||
USE_ATOMIC_ADD_OPTS,
|
||||
USE_FP32_REDUCE_OPTS,
|
||||
)
|
||||
@@ -333,8 +306,6 @@ def marlin_generate_valid_test_cases():
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
act_order,
|
||||
is_k_full,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
):
|
||||
@@ -351,25 +322,18 @@ def marlin_generate_valid_test_cases():
|
||||
if group_size > 0 and size_k % group_size != 0:
|
||||
return False
|
||||
|
||||
if act_order and group_size in [-1, size_k]:
|
||||
return False
|
||||
if group_size == size_k:
|
||||
return False
|
||||
if not act_order and is_k_full:
|
||||
return False
|
||||
|
||||
return a_type.size_bits < 16 or a_type is c_type
|
||||
|
||||
cases = []
|
||||
for case in all_combinations:
|
||||
quant_test_config, mnk_factors, n_chunk, k_chunk, act_order, *_ = case
|
||||
quant_test_config, mnk_factors, n_chunk, k_chunk, *_ = case
|
||||
size_m = mnk_factors[0]
|
||||
size_n = mnk_factors[1] * n_chunk
|
||||
size_k = mnk_factors[2] * k_chunk
|
||||
|
||||
if act_order and not quant_test_config.get("support_act_order", False):
|
||||
continue
|
||||
|
||||
f16_types = [scalar_types.float16, scalar_types.bfloat16]
|
||||
inner_combinations = itertools.product(
|
||||
quant_test_config.get("a_type", f16_types),
|
||||
@@ -398,7 +362,7 @@ def marlin_generate_valid_test_cases():
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"a_type, b_type, c_type, group_blocks,"
|
||||
"size_m, size_n, size_k, act_order, is_k_full,"
|
||||
"size_m, size_n, size_k,"
|
||||
"use_atomic_add, use_fp32_reduce"
|
||||
),
|
||||
marlin_generate_valid_test_cases(),
|
||||
@@ -411,8 +375,6 @@ def test_marlin_gemm(
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
act_order,
|
||||
is_k_full,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
):
|
||||
@@ -448,27 +410,21 @@ def test_marlin_gemm(
|
||||
)
|
||||
marlin_s2 = None
|
||||
|
||||
g_idx = None
|
||||
sort_indices = None
|
||||
marlin_zp = None
|
||||
elif b_type == scalar_types.float8_e4m3fn:
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quant_fp8_torch(
|
||||
b_weight.T, group_size, input_dtype=a_dtype
|
||||
)
|
||||
g_idx = None
|
||||
sort_indices = None
|
||||
marlin_zp = None
|
||||
marlin_s2 = None
|
||||
elif has_zp:
|
||||
w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
|
||||
b_weight, b_type, group_size, input_dtype=a_dtype
|
||||
)
|
||||
g_idx = None
|
||||
sort_indices = None
|
||||
marlin_s2 = None
|
||||
else:
|
||||
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
|
||||
b_weight, b_type, group_size, act_order, input_dtype=a_dtype
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(
|
||||
b_weight, b_type, group_size, input_dtype=a_dtype
|
||||
)
|
||||
|
||||
marlin_zp = None
|
||||
@@ -506,14 +462,11 @@ def test_marlin_gemm(
|
||||
a_scales,
|
||||
marlin_s2,
|
||||
marlin_zp,
|
||||
g_idx,
|
||||
sort_indices,
|
||||
workspace,
|
||||
b_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
is_k_full=is_k_full,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
is_zp_float=False,
|
||||
@@ -535,11 +488,9 @@ def test_marlin_gemm_subset_input():
|
||||
a_input = rand_data((big_m, big_k))[8 : size_m + 8, 8 : size_k + 8]
|
||||
b_weight = rand_data((size_k, size_n))
|
||||
|
||||
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
|
||||
b_weight, quant_type, group_size, False
|
||||
)
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
|
||||
|
||||
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
|
||||
marlin_zp = marlin_make_empty(marlin_s.device)
|
||||
workspace = marlin_make_workspace_new(a_input.device)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
@@ -551,14 +502,11 @@ def test_marlin_gemm_subset_input():
|
||||
None,
|
||||
None,
|
||||
marlin_zp,
|
||||
g_idx,
|
||||
sort_indices,
|
||||
workspace,
|
||||
quant_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
is_k_full=True,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
@@ -584,11 +532,9 @@ def test_marlin_gemm_with_bias(size_m):
|
||||
|
||||
marlin_bias = marlin_permute_bias(b_bias)
|
||||
|
||||
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
|
||||
b_weight, quant_type, group_size, False
|
||||
)
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
|
||||
|
||||
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
|
||||
marlin_zp = marlin_make_empty(marlin_s.device)
|
||||
workspace = marlin_make_workspace_new(a_input.device)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
@@ -600,14 +546,11 @@ def test_marlin_gemm_with_bias(size_m):
|
||||
None,
|
||||
None,
|
||||
marlin_zp,
|
||||
g_idx,
|
||||
sort_indices,
|
||||
workspace,
|
||||
quant_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
is_k_full=True,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
GPTQ_MARLIN_TILE,
|
||||
apply_gptq_marlin_linear,
|
||||
marlin_make_empty_g_idx,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_pad_qweight,
|
||||
@@ -319,16 +319,13 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
|
||||
device = torch.device("cuda")
|
||||
|
||||
weight = torch.randn(size_k, size_n, dtype=dtype, device=device) / size_k**0.5
|
||||
w_ref, q_w, s, _, _ = gptq_quantize_weights(
|
||||
weight, quant_type, group_size, act_order=False
|
||||
)
|
||||
w_ref, q_w, s = gptq_quantize_weights(weight, quant_type, group_size)
|
||||
qweight = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
|
||||
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
|
||||
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=torch.empty(0, dtype=torch.int, device=device),
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=quant_type.size_bits,
|
||||
@@ -343,14 +340,11 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
|
||||
input=x,
|
||||
weight=marlin_qweight,
|
||||
weight_scale=marlin_s,
|
||||
weight_zp=marlin_make_empty_g_idx(device),
|
||||
g_idx=marlin_make_empty_g_idx(device),
|
||||
g_idx_sort_indices=marlin_make_empty_g_idx(device),
|
||||
weight_zp=marlin_make_empty(device),
|
||||
workspace=marlin_make_workspace_new(device),
|
||||
wtype=quant_type,
|
||||
output_size_per_partition=size_n,
|
||||
input_size_per_partition=size_k,
|
||||
is_k_full=True,
|
||||
)
|
||||
ref = x @ w_ref
|
||||
|
||||
@@ -480,7 +474,6 @@ def test_awq_zp_marlin_padded_round_trip(shape):
|
||||
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=torch.empty(0, dtype=torch.int, device=device),
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=quant_type.size_bits,
|
||||
@@ -503,13 +496,10 @@ def test_awq_zp_marlin_padded_round_trip(shape):
|
||||
weight=marlin_qweight,
|
||||
weight_scale=marlin_s,
|
||||
weight_zp=marlin_zp,
|
||||
g_idx=marlin_make_empty_g_idx(device),
|
||||
g_idx_sort_indices=marlin_make_empty_g_idx(device),
|
||||
workspace=marlin_make_workspace_new(device),
|
||||
wtype=quant_type,
|
||||
output_size_per_partition=size_n,
|
||||
input_size_per_partition=size_k,
|
||||
is_k_full=True,
|
||||
)
|
||||
ref = x @ w_ref
|
||||
|
||||
@@ -585,9 +575,7 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
|
||||
def quant(w, size_k, size_n):
|
||||
# w is (size_n, size_k); gptq expects (size_k, size_n).
|
||||
ref, q_w, s, _, _ = gptq_quantize_weights(
|
||||
w.T, quant_type, group_size, act_order=False
|
||||
)
|
||||
ref, q_w, s = gptq_quantize_weights(w.T, quant_type, group_size)
|
||||
return ref, gptq_pack(q_w, bits, size_k, size_n), s
|
||||
|
||||
w13_qw, w13_s, w13_ref = [], [], []
|
||||
@@ -616,12 +604,11 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
if group_size > 0:
|
||||
w2_scales = _pad_rows(w2_scales, padded_n // group_size)
|
||||
|
||||
sort_idx = torch.empty((e, 0), dtype=torch.int32, device=device)
|
||||
marlin_w13 = ops.gptq_marlin_moe_repack(
|
||||
w13_qweight, sort_idx, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
|
||||
w13_qweight, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
|
||||
)
|
||||
marlin_w2 = ops.gptq_marlin_moe_repack(
|
||||
w2_qweight, sort_idx, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
|
||||
w2_qweight, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
|
||||
)
|
||||
group_or_pack = group_size if group_size != -1 else pack
|
||||
marlin_w13_s = marlin_moe_permute_scales(
|
||||
@@ -649,7 +636,6 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
topk_ids,
|
||||
quant_type_id=quant_type.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
@@ -770,7 +756,6 @@ def test_fp8_marlin_moe_padded_round_trip(shape, quant):
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
workspace=layer.workspace,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
@@ -848,7 +833,6 @@ def test_mxfp8_marlin_moe_padded_round_trip(shape):
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
workspace=layer.workspace,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
|
||||
@@ -196,7 +196,6 @@ def test_dense_kernel_rejects_on_non_gfx1100():
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is False, f"RDNA3 dense kernel accepted on non-gfx1100: {reason}"
|
||||
@@ -431,7 +430,6 @@ class TestDenseKernelSelectionMocked:
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, _ = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is True
|
||||
@@ -461,7 +459,6 @@ class TestDenseKernelSelectionMocked:
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
|
||||
|
||||
@@ -69,10 +69,9 @@ def _make_packed_weights(E, K, N):
|
||||
packed = torch.zeros(E, K // 8, N, dtype=torch.int32, device=device)
|
||||
for i in range(8):
|
||||
packed |= (w[:, i::8, :] & 0xF) << (i * 4)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
for e in range(E):
|
||||
we = packed[e].contiguous()
|
||||
ops.gptq_shuffle(we, g_idx, 4)
|
||||
ops.gptq_shuffle(we, 4)
|
||||
packed[e] = we
|
||||
return packed
|
||||
|
||||
@@ -113,7 +112,6 @@ def test_fused_moe_w1_matches_dense(
|
||||
w13 = _make_packed_weights(E, K, N_gate_up)
|
||||
w13_s = _make_scales(E, groups, N_gate_up, dtype)
|
||||
w13_z = _make_qzeros(E, groups, N_gate_up)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, block_size_m, E)
|
||||
@@ -147,7 +145,6 @@ def test_fused_moe_w1_matches_dense(
|
||||
w13[e],
|
||||
w13_z[e],
|
||||
w13_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
ref_out[flat] = ref.squeeze()
|
||||
@@ -243,7 +240,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
w2 = _make_packed_weights(E, N_inter, hidden)
|
||||
w2_s = _make_scales(E, N_inter // group_size, hidden, dtype)
|
||||
w2_z = _make_qzeros(E, N_inter // group_size, hidden)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
topk_w = torch.softmax(
|
||||
@@ -300,7 +296,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
w13[e],
|
||||
w13_z[e],
|
||||
w13_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
a = torch.empty(1, N_inter, dtype=dtype, device=device)
|
||||
@@ -310,7 +305,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
w2[e],
|
||||
w2_z[e],
|
||||
w2_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
ref[m_idx] += r2.squeeze() * w
|
||||
|
||||
@@ -173,7 +173,6 @@ def _run_kernel(
|
||||
act_type=dtype,
|
||||
group_size=group_size,
|
||||
zero_points=has_zp,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok, f"can_implement rejected a supported config: {reason}"
|
||||
@@ -184,7 +183,6 @@ def _run_kernel(
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros" if has_zp else None,
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
return kernel.apply_weights(layer, x_mk, bias=bias)
|
||||
|
||||
@@ -57,7 +57,6 @@ def test_selection_prefers_rdna3(dtype):
|
||||
act_type=dtype,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
assert choose_mp_linear_kernel(config).__name__ == "RDNA3W4A16LinearKernel"
|
||||
|
||||
@@ -83,7 +82,6 @@ def test_can_implement(weight_type, group_size, N, full_k, expected_ok):
|
||||
act_type=torch.float16,
|
||||
group_size=group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is expected_ok, reason
|
||||
|
||||
@@ -306,14 +306,12 @@ def test_rdna_hybrid_w4a16_process_weights_symmetric_repack(group_size, dist_ini
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = RDNAHybridW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name=None,
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
@@ -367,14 +365,12 @@ def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_in
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = RDNAHybridW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
@@ -414,7 +410,6 @@ def test_hybrid_can_implement_group_size(group_size, expected_ok):
|
||||
act_type=torch.float16,
|
||||
group_size=group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, _ = RDNAHybridW4A16LinearKernel.can_implement(config)
|
||||
assert ok is expected_ok
|
||||
|
||||
@@ -273,14 +273,12 @@ def test_triton_w4a16_process_weights_after_loading_repacks_layout():
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
# Build dummy layer with vLLM parameter wrappers.
|
||||
@@ -384,14 +382,12 @@ def test_triton_w4a16_process_weights_after_loading_keeps_gptq_qzeros_layout():
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
weight_loader = lambda *args, **kwargs: None
|
||||
@@ -461,14 +457,12 @@ def test_triton_w4a16_symmetric_apply_ignores_qzeros(monkeypatch):
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
|
||||
@@ -47,7 +47,6 @@ def test_choose_mp_linear_kernel_uint4b8():
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(config)
|
||||
@@ -65,7 +64,6 @@ def test_choose_mp_linear_kernel_uint4_asymmetric():
|
||||
act_type=torch.bfloat16,
|
||||
group_size=64,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(config)
|
||||
|
||||
Reference in New Issue
Block a user