[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com> Signed-off-by: Roderick Wu <roderickwu2003@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -1,21 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm._custom_ops import permute_cols
|
||||
|
||||
if not hasattr(torch.ops._C, "permute_cols"):
|
||||
pytest.skip(reason="permute_cols is not supported on ROCm", allow_module_level=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(1, 512), (544, 4096), (67, 8192)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
def test_permute_cols(shape, dtype):
|
||||
x = torch.randn(shape, dtype=dtype).cuda()
|
||||
perm = torch.randperm(x.shape[1]).to(torch.int).cuda()
|
||||
opcheck(torch.ops._C.permute_cols, (x, perm))
|
||||
y = permute_cols(x, perm)
|
||||
torch.testing.assert_close(y, x[:, perm])
|
||||
@@ -113,9 +113,7 @@ def marlin_quantize_moe_weights(
|
||||
for i in range(e):
|
||||
# Transpose for Marlin: [n, k] → [k, n]
|
||||
w_t = weights_bf16[i].T.contiguous()
|
||||
_, w_q, w_s, _, _, _ = marlin_quantize(
|
||||
w_t, scalar_types.uint4b8, group_size, act_order=False
|
||||
)
|
||||
_, w_q, w_s = marlin_quantize(w_t, scalar_types.uint4b8, group_size)
|
||||
weight_list.append(w_q)
|
||||
scale_list.append(w_s)
|
||||
|
||||
@@ -248,16 +246,11 @@ def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group
|
||||
expert_map=None,
|
||||
global_scale1=None,
|
||||
global_scale2=None,
|
||||
g_idx1=None,
|
||||
g_idx2=None,
|
||||
input_global_scale1=None,
|
||||
input_global_scale2=None,
|
||||
sort_indices1=None,
|
||||
sort_indices2=None,
|
||||
w1_zeros=None,
|
||||
w2_zeros=None,
|
||||
input_dtype=dtype,
|
||||
is_k_full=True,
|
||||
)
|
||||
|
||||
# Sanity check: manually compute BF16 reference for comparison
|
||||
|
||||
@@ -178,13 +178,11 @@ MOE_MARLIN_QUANT_TEST_CONFIGS = [
|
||||
# GPTQ-INT4
|
||||
{
|
||||
"b_type": scalar_types.uint4b8,
|
||||
"support_act_order": True,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# GPTQ-INT8
|
||||
{
|
||||
"b_type": scalar_types.uint8b128,
|
||||
"support_act_order": True,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# FP8
|
||||
@@ -794,27 +792,22 @@ def test_fused_moe_wn16(
|
||||
|
||||
|
||||
MARLIN_MOE_SCENARIOS = [
|
||||
# (m, n, k, e, topk, ep_size, act_order, is_k_full)
|
||||
# No act_order: is_k_full=True matches usual case (marlin_is_k_full).
|
||||
# (m, n, k, e, topk, ep_size)
|
||||
# N>=256 required for Marlin kernel thread config for MXFP8.
|
||||
# Single token, small matrices
|
||||
(1, 128, 256, 5, 2, 1, False, True),
|
||||
(1, 128, 256, 5, 2, 1),
|
||||
# Single token, large matrices
|
||||
(1, 1024, 2048, 5, 2, 1, False, True),
|
||||
(1, 1024, 2048, 5, 2, 1),
|
||||
# Unaligned m, small matrices
|
||||
(133, 256, 256, 5, 2, 1, False, True),
|
||||
(133, 256, 256, 5, 2, 1),
|
||||
# Unaligned m, large matrices
|
||||
(133, 1024, 2048, 12, 3, 1, False, True),
|
||||
(133, 1024, 2048, 12, 3, 1),
|
||||
# Aligned batch, small matrices
|
||||
(128, 256, 256, 5, 2, 1, False, True),
|
||||
(128, 256, 256, 5, 2, 1),
|
||||
# Aligned batch, large matrices
|
||||
(128, 1024, 2048, 12, 3, 1, False, True),
|
||||
(128, 1024, 2048, 12, 3, 1),
|
||||
# Expert parallelism
|
||||
(64, 1024, 2048, 12, 3, 4, False, True),
|
||||
# Act order with is_k_full=True (no tensor parallelism)
|
||||
(1, 1024, 2048, 5, 2, 1, True, True),
|
||||
# Act order with is_k_full=False (tensor parallelism)
|
||||
(133, 256, 256, 5, 2, 1, True, False),
|
||||
(64, 1024, 2048, 12, 3, 4),
|
||||
]
|
||||
|
||||
|
||||
@@ -832,17 +825,13 @@ def marlin_moe_generate_valid_test_cases():
|
||||
e,
|
||||
topk,
|
||||
ep_size,
|
||||
act_order,
|
||||
is_k_full,
|
||||
):
|
||||
group_size = group_blocks if group_blocks <= 0 else group_blocks * 16
|
||||
if group_size > 0 and k % group_size != 0:
|
||||
return False
|
||||
if act_order and group_size in [-1, k, n]:
|
||||
return False
|
||||
if group_size in [k, n]:
|
||||
return False
|
||||
if b_type == scalar_types.float8_e4m3fn and group_size == 32 and is_k_full:
|
||||
if b_type == scalar_types.float8_e4m3fn and group_size == 32:
|
||||
return False
|
||||
return a_type.size_bits < 16 or a_type is c_type
|
||||
|
||||
@@ -858,8 +847,6 @@ def marlin_moe_generate_valid_test_cases():
|
||||
)
|
||||
)
|
||||
|
||||
supports_act_order = quant_test_config.get("support_act_order", False)
|
||||
|
||||
for sub_case in inner_combinations:
|
||||
if (
|
||||
sub_case[0] == scalar_types.float8_e4m3fn
|
||||
@@ -869,10 +856,8 @@ def marlin_moe_generate_valid_test_cases():
|
||||
continue
|
||||
|
||||
for scenario in MARLIN_MOE_SCENARIOS:
|
||||
m, n, k, e, topk, ep_size, act_order, is_k_full = scenario
|
||||
if act_order and not supports_act_order:
|
||||
continue
|
||||
args = sub_case + (m, n, k, e, topk, ep_size, act_order, is_k_full)
|
||||
m, n, k, e, topk, ep_size = scenario
|
||||
args = sub_case + (m, n, k, e, topk, ep_size)
|
||||
if is_valid(*args):
|
||||
cases.append(args)
|
||||
return cases
|
||||
@@ -885,9 +870,7 @@ class MarlinMoEWeightData:
|
||||
scales: torch.Tensor
|
||||
global_scale: torch.Tensor | None
|
||||
a_scales_factor: torch.Tensor | None
|
||||
g_idx: torch.Tensor | None
|
||||
zeros: torch.Tensor | None
|
||||
sort_indices: torch.Tensor | None
|
||||
marlin_bias: torch.Tensor | None
|
||||
|
||||
@staticmethod
|
||||
@@ -895,15 +878,12 @@ class MarlinMoEWeightData:
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
act_order: bool | None = None,
|
||||
bias: torch.Tensor | None = None,
|
||||
input_type: ScalarType = None,
|
||||
) -> "MarlinMoEWeightData":
|
||||
assert w.ndim == 3
|
||||
|
||||
has_zp = quant_type in [scalar_types.uint4, scalar_types.uint8]
|
||||
k = w.shape[-1]
|
||||
|
||||
if input_type == scalar_types.int8:
|
||||
input_dtype = torch.int8
|
||||
elif input_type == scalar_types.float8_e4m3fn:
|
||||
@@ -916,8 +896,6 @@ class MarlinMoEWeightData:
|
||||
scales_l: list[torch.Tensor] = []
|
||||
global_scale_l: list[torch.Tensor] = []
|
||||
zeros_l: list[torch.Tensor] = []
|
||||
g_idx_l: list[torch.Tensor] = []
|
||||
sort_indices_l: list[torch.Tensor] = []
|
||||
bias_l: list[torch.Tensor] = []
|
||||
|
||||
for i in range(w.shape[0]):
|
||||
@@ -959,21 +937,16 @@ class MarlinMoEWeightData:
|
||||
scales_l.append(scales)
|
||||
zeros_l.append(zeros)
|
||||
else:
|
||||
test_perm = torch.randperm(k)
|
||||
w_ref, qweight, scales, g_idx, sort_indices, _ = marlin_quantize(
|
||||
w_ref, qweight, scales = marlin_quantize(
|
||||
w[i].transpose(1, 0),
|
||||
quant_type,
|
||||
group_size,
|
||||
act_order,
|
||||
test_perm,
|
||||
input_dtype=input_dtype,
|
||||
)
|
||||
|
||||
w_ref_l.append(w_ref.T)
|
||||
qweight_l.append(qweight)
|
||||
scales_l.append(scales)
|
||||
g_idx_l.append(g_idx)
|
||||
sort_indices_l.append(sort_indices)
|
||||
|
||||
if bias is not None:
|
||||
bias_l.append(marlin_permute_bias(bias[i]))
|
||||
@@ -982,9 +955,7 @@ class MarlinMoEWeightData:
|
||||
qweight = stack_and_dev(qweight_l).contiguous()
|
||||
scales = stack_and_dev(scales_l)
|
||||
global_scale = stack_and_dev(global_scale_l) if global_scale_l else None
|
||||
g_idx = stack_and_dev(g_idx_l) if g_idx_l else None
|
||||
zeros = stack_and_dev(zeros_l) if zeros_l else None
|
||||
sort_indices = stack_and_dev(sort_indices_l) if sort_indices_l else None
|
||||
marlin_bias = stack_and_dev(bias_l) if bias_l else None
|
||||
|
||||
a_scales_factor = None
|
||||
@@ -999,19 +970,14 @@ class MarlinMoEWeightData:
|
||||
scales=scales,
|
||||
global_scale=global_scale,
|
||||
a_scales_factor=a_scales_factor,
|
||||
g_idx=g_idx,
|
||||
zeros=zeros,
|
||||
sort_indices=sort_indices,
|
||||
marlin_bias=marlin_bias,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"a_type, b_type, c_type, group_blocks,"
|
||||
"m, n, k, e, topk, ep_size, act_order, is_k_full"
|
||||
),
|
||||
("a_type, b_type, c_type, group_blocks,m, n, k, e, topk, ep_size"),
|
||||
marlin_moe_generate_valid_test_cases(),
|
||||
)
|
||||
@pytest.mark.skipif(current_platform.is_rocm(), reason="Skip for rocm")
|
||||
@@ -1027,8 +993,6 @@ def test_fused_marlin_moe(
|
||||
e: int,
|
||||
topk: int,
|
||||
ep_size: int,
|
||||
act_order: bool,
|
||||
is_k_full: bool,
|
||||
):
|
||||
set_random_seed(1)
|
||||
group_size = group_blocks if group_blocks <= 0 else group_blocks * 16
|
||||
@@ -1065,7 +1029,6 @@ def test_fused_marlin_moe(
|
||||
w=w1,
|
||||
quant_type=b_type,
|
||||
group_size=group_size,
|
||||
act_order=act_order,
|
||||
input_type=a_type,
|
||||
)
|
||||
|
||||
@@ -1073,7 +1036,6 @@ def test_fused_marlin_moe(
|
||||
w=w2,
|
||||
quant_type=b_type,
|
||||
group_size=group_size,
|
||||
act_order=act_order,
|
||||
input_type=a_type,
|
||||
)
|
||||
|
||||
@@ -1127,17 +1089,12 @@ def test_fused_marlin_moe(
|
||||
expert_map=e_map,
|
||||
global_scale1=w1_data.global_scale,
|
||||
global_scale2=w2_data.global_scale,
|
||||
g_idx1=w1_data.g_idx,
|
||||
g_idx2=w2_data.g_idx,
|
||||
input_global_scale1=w1_data.a_scales_factor,
|
||||
input_global_scale2=w2_data.a_scales_factor,
|
||||
sort_indices1=w1_data.sort_indices,
|
||||
sort_indices2=w2_data.sort_indices,
|
||||
w1_zeros=w1_data.zeros,
|
||||
w2_zeros=w2_data.zeros,
|
||||
input_dtype=a_dtype,
|
||||
quant_type_id=b_type.id,
|
||||
is_k_full=is_k_full,
|
||||
activation_func=instance_activation,
|
||||
)
|
||||
|
||||
@@ -1154,8 +1111,6 @@ def test_fused_marlin_moe_with_bias(m):
|
||||
e, topk = 32, 4
|
||||
n, k = 2048, 2048
|
||||
group_size = 128
|
||||
act_order = False
|
||||
is_k_full = True
|
||||
quant_type = scalar_types.uint4b8
|
||||
dtype = torch.half
|
||||
|
||||
@@ -1169,7 +1124,6 @@ def test_fused_marlin_moe_with_bias(m):
|
||||
w=w1,
|
||||
quant_type=quant_type,
|
||||
group_size=group_size,
|
||||
act_order=act_order,
|
||||
bias=b_bias1,
|
||||
)
|
||||
|
||||
@@ -1177,7 +1131,6 @@ def test_fused_marlin_moe_with_bias(m):
|
||||
w=w2,
|
||||
quant_type=quant_type,
|
||||
group_size=group_size,
|
||||
act_order=act_order,
|
||||
bias=b_bias2,
|
||||
)
|
||||
|
||||
@@ -1204,14 +1157,9 @@ def test_fused_marlin_moe_with_bias(m):
|
||||
expert_map=None,
|
||||
global_scale1=w1_data.global_scale,
|
||||
global_scale2=w2_data.global_scale,
|
||||
g_idx1=w1_data.g_idx,
|
||||
g_idx2=w2_data.g_idx,
|
||||
sort_indices1=w1_data.sort_indices,
|
||||
sort_indices2=w2_data.sort_indices,
|
||||
w1_zeros=w1_data.zeros,
|
||||
w2_zeros=w2_data.zeros,
|
||||
quant_type_id=quant_type.id,
|
||||
is_k_full=is_k_full,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(marlin_output, torch_output, atol=5e-2, rtol=0)
|
||||
@@ -1235,7 +1183,6 @@ def test_fused_marlin_moe_non_gated(
|
||||
set_random_seed(42)
|
||||
|
||||
group_size = 16 # NVFP4 group size
|
||||
is_k_full = True
|
||||
quant_type = scalar_types.float4_e2m1f
|
||||
dtype = torch.bfloat16
|
||||
|
||||
@@ -1248,14 +1195,12 @@ def test_fused_marlin_moe_non_gated(
|
||||
w=w1,
|
||||
quant_type=quant_type,
|
||||
group_size=group_size,
|
||||
act_order=False,
|
||||
)
|
||||
|
||||
w2_data = MarlinMoEWeightData.make(
|
||||
w=w2,
|
||||
quant_type=quant_type,
|
||||
group_size=group_size,
|
||||
act_order=False,
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
@@ -1286,14 +1231,9 @@ def test_fused_marlin_moe_non_gated(
|
||||
expert_map=None,
|
||||
global_scale1=w1_data.global_scale,
|
||||
global_scale2=w2_data.global_scale,
|
||||
g_idx1=w1_data.g_idx,
|
||||
g_idx2=w2_data.g_idx,
|
||||
sort_indices1=w1_data.sort_indices,
|
||||
sort_indices2=w2_data.sort_indices,
|
||||
w1_zeros=w1_data.zeros,
|
||||
w2_zeros=w2_data.zeros,
|
||||
quant_type_id=quant_type.id,
|
||||
is_k_full=is_k_full,
|
||||
activation=activation,
|
||||
)
|
||||
|
||||
@@ -1612,14 +1552,9 @@ def test_batched_marlin_activation_uses_expert_token_counts(
|
||||
quant_type_id=0,
|
||||
a1_gscale=None,
|
||||
a2_gscale=None,
|
||||
w13_g_idx=None,
|
||||
w2_g_idx=None,
|
||||
w13_g_idx_sort_indices=None,
|
||||
w2_g_idx_sort_indices=None,
|
||||
w1_zp=None,
|
||||
w2_zp=None,
|
||||
input_dtype=None,
|
||||
is_k_full=True,
|
||||
activation_config=ApplyMoEActivationConfig(
|
||||
clamp_limit=3.0, alpha=1.3, beta=0.5
|
||||
),
|
||||
@@ -1982,14 +1917,12 @@ def test_batched_fused_marlin_moe(
|
||||
w=w1,
|
||||
quant_type=quant_dtype,
|
||||
group_size=group_size,
|
||||
act_order=None,
|
||||
input_type=input_type,
|
||||
)
|
||||
w2_data = MarlinMoEWeightData.make(
|
||||
w=w2,
|
||||
quant_type=quant_dtype,
|
||||
group_size=group_size,
|
||||
act_order=None,
|
||||
input_type=input_type,
|
||||
)
|
||||
|
||||
@@ -2101,14 +2034,9 @@ def test_batched_fused_marlin_moe(
|
||||
"expert_map": None,
|
||||
"global_scale1": w1_data.global_scale,
|
||||
"global_scale2": w2_data.global_scale,
|
||||
"g_idx1": w1_data.g_idx,
|
||||
"g_idx2": w2_data.g_idx,
|
||||
"sort_indices1": w1_data.sort_indices,
|
||||
"sort_indices2": w2_data.sort_indices,
|
||||
"w1_zeros": w1_data.zeros,
|
||||
"w2_zeros": w2_data.zeros,
|
||||
"quant_type_id": quant_dtype.id,
|
||||
"is_k_full": True,
|
||||
}
|
||||
if input_dtype is not None:
|
||||
kwargs["input_dtype"] = input_dtype
|
||||
|
||||
@@ -70,7 +70,7 @@ def test_gptq_allspark_gemm_ampere(mnk_factors, group_size, has_zp, dtype):
|
||||
input = rand_data((m, k), dtype=dtype)
|
||||
weight = rand_data((k, n), dtype=dtype)
|
||||
|
||||
# Quantize (and apply act_order if provided)
|
||||
# Quantize weights.
|
||||
w_ref, qw, s, zp = quantize_weights(
|
||||
weight, scalar_types.uint8b128, group_size, has_zp
|
||||
)
|
||||
|
||||
@@ -11,9 +11,8 @@ def test_gptq_shuffle_opcheck():
|
||||
weight = torch.randint(
|
||||
-2000000, 2000000, (1792, 4096), device="cuda", dtype=torch.int32
|
||||
)
|
||||
perm = torch.empty((0,), device="cuda", dtype=torch.int32)
|
||||
bit = 4
|
||||
opcheck(torch.ops._C.gptq_shuffle, (weight, perm, bit))
|
||||
opcheck(torch.ops._C.gptq_shuffle, (weight, bit))
|
||||
|
||||
|
||||
def test_gptq_gemm_opcheck():
|
||||
@@ -23,13 +22,8 @@ def test_gptq_gemm_opcheck():
|
||||
)
|
||||
zeros = torch.zeros((32, 768), device="cuda", dtype=torch.int32)
|
||||
scales = torch.rand((32, 6144), device="cuda", dtype=torch.float16)
|
||||
idx = torch.empty((0,), device="cuda", dtype=torch.int32)
|
||||
use_exllama = True
|
||||
bit = 4
|
||||
# Test both GPTQv1 and GPTQv2 format
|
||||
opcheck(
|
||||
torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, True, bit)
|
||||
)
|
||||
opcheck(
|
||||
torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, False, bit)
|
||||
)
|
||||
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, True, bit))
|
||||
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, False, bit))
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_quant_int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_make_empty_g_idx,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_permute_bias,
|
||||
query_marlin_supported_quant_types,
|
||||
@@ -40,7 +40,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
gptq_pack,
|
||||
gptq_quantize_weights,
|
||||
quantize_weights,
|
||||
sort_weights,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
@@ -51,8 +50,6 @@ if current_platform.is_rocm():
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
ACT_ORDER_OPTS = [False, True]
|
||||
K_FULL_OPTS = [False, True]
|
||||
USE_ATOMIC_ADD_OPTS = [False, True]
|
||||
USE_FP32_REDUCE_OPTS = [True]
|
||||
|
||||
@@ -80,13 +77,11 @@ DENSE_MARLIN_QUANT_TEST_CONFIGS = [
|
||||
# GPTQ-INT4
|
||||
{
|
||||
"b_type": scalar_types.uint4b8,
|
||||
"support_act_order": True,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# GPTQ-INT8
|
||||
{
|
||||
"b_type": scalar_types.uint8b128,
|
||||
"support_act_order": True,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# FP8
|
||||
@@ -202,27 +197,15 @@ def test_marlin_int4_fp8_preprocess_awq():
|
||||
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
|
||||
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
|
||||
@pytest.mark.parametrize("quant_type", query_marlin_supported_quant_types(False, False))
|
||||
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
|
||||
@pytest.mark.parametrize("is_a_8bit", [True, False])
|
||||
@pytest.mark.parametrize("nk_factors", MARLIN_REPACK_NK_FACTORS)
|
||||
def test_gptq_marlin_repack(
|
||||
k_chunk, n_chunk, quant_type, act_order, is_a_8bit, nk_factors
|
||||
):
|
||||
def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, is_a_8bit, nk_factors):
|
||||
n_factor, k_factor = nk_factors
|
||||
|
||||
size_k = k_chunk * k_factor
|
||||
size_n = n_chunk * n_factor
|
||||
group_size = 128
|
||||
|
||||
# Filter act_order
|
||||
if act_order:
|
||||
if group_size == -1:
|
||||
return
|
||||
if group_size == size_k:
|
||||
return
|
||||
if is_a_8bit:
|
||||
return
|
||||
|
||||
# Normalize group_size
|
||||
if group_size == -1:
|
||||
group_size = size_k
|
||||
@@ -231,20 +214,12 @@ def test_gptq_marlin_repack(
|
||||
# Create input
|
||||
b_weight = rand_data((size_k, size_n))
|
||||
|
||||
# Quantize (and apply act_order if provided)
|
||||
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
|
||||
b_weight, quant_type, group_size, act_order
|
||||
)
|
||||
# Quantize
|
||||
_, q_w, _ = gptq_quantize_weights(b_weight, quant_type, group_size)
|
||||
|
||||
# Pack to GPTQ format
|
||||
q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
|
||||
|
||||
# For act_order, sort the "weights" and "g_idx" so that group ids are
|
||||
# increasing
|
||||
sort_indices = torch.empty(0, dtype=torch.int, device=b_weight.device)
|
||||
if act_order:
|
||||
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
|
||||
|
||||
# Pack to Marlin format
|
||||
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
|
||||
marlin_q_w_1 = marlin_weights(
|
||||
@@ -253,12 +228,12 @@ def test_gptq_marlin_repack(
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.gptq_marlin_repack,
|
||||
(q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit),
|
||||
(q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit),
|
||||
)
|
||||
|
||||
# Run Marlin repack GPU kernel
|
||||
marlin_q_w_2 = ops.gptq_marlin_repack(
|
||||
q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit
|
||||
q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -319,8 +294,6 @@ def marlin_generate_valid_test_cases():
|
||||
MNK_FACTORS,
|
||||
MARLIN_N_CHUNKS,
|
||||
MARLIN_K_CHUNKS,
|
||||
ACT_ORDER_OPTS,
|
||||
K_FULL_OPTS,
|
||||
USE_ATOMIC_ADD_OPTS,
|
||||
USE_FP32_REDUCE_OPTS,
|
||||
)
|
||||
@@ -333,8 +306,6 @@ def marlin_generate_valid_test_cases():
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
act_order,
|
||||
is_k_full,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
):
|
||||
@@ -351,25 +322,18 @@ def marlin_generate_valid_test_cases():
|
||||
if group_size > 0 and size_k % group_size != 0:
|
||||
return False
|
||||
|
||||
if act_order and group_size in [-1, size_k]:
|
||||
return False
|
||||
if group_size == size_k:
|
||||
return False
|
||||
if not act_order and is_k_full:
|
||||
return False
|
||||
|
||||
return a_type.size_bits < 16 or a_type is c_type
|
||||
|
||||
cases = []
|
||||
for case in all_combinations:
|
||||
quant_test_config, mnk_factors, n_chunk, k_chunk, act_order, *_ = case
|
||||
quant_test_config, mnk_factors, n_chunk, k_chunk, *_ = case
|
||||
size_m = mnk_factors[0]
|
||||
size_n = mnk_factors[1] * n_chunk
|
||||
size_k = mnk_factors[2] * k_chunk
|
||||
|
||||
if act_order and not quant_test_config.get("support_act_order", False):
|
||||
continue
|
||||
|
||||
f16_types = [scalar_types.float16, scalar_types.bfloat16]
|
||||
inner_combinations = itertools.product(
|
||||
quant_test_config.get("a_type", f16_types),
|
||||
@@ -398,7 +362,7 @@ def marlin_generate_valid_test_cases():
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"a_type, b_type, c_type, group_blocks,"
|
||||
"size_m, size_n, size_k, act_order, is_k_full,"
|
||||
"size_m, size_n, size_k,"
|
||||
"use_atomic_add, use_fp32_reduce"
|
||||
),
|
||||
marlin_generate_valid_test_cases(),
|
||||
@@ -411,8 +375,6 @@ def test_marlin_gemm(
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
act_order,
|
||||
is_k_full,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
):
|
||||
@@ -448,27 +410,21 @@ def test_marlin_gemm(
|
||||
)
|
||||
marlin_s2 = None
|
||||
|
||||
g_idx = None
|
||||
sort_indices = None
|
||||
marlin_zp = None
|
||||
elif b_type == scalar_types.float8_e4m3fn:
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quant_fp8_torch(
|
||||
b_weight.T, group_size, input_dtype=a_dtype
|
||||
)
|
||||
g_idx = None
|
||||
sort_indices = None
|
||||
marlin_zp = None
|
||||
marlin_s2 = None
|
||||
elif has_zp:
|
||||
w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
|
||||
b_weight, b_type, group_size, input_dtype=a_dtype
|
||||
)
|
||||
g_idx = None
|
||||
sort_indices = None
|
||||
marlin_s2 = None
|
||||
else:
|
||||
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
|
||||
b_weight, b_type, group_size, act_order, input_dtype=a_dtype
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(
|
||||
b_weight, b_type, group_size, input_dtype=a_dtype
|
||||
)
|
||||
|
||||
marlin_zp = None
|
||||
@@ -506,14 +462,11 @@ def test_marlin_gemm(
|
||||
a_scales,
|
||||
marlin_s2,
|
||||
marlin_zp,
|
||||
g_idx,
|
||||
sort_indices,
|
||||
workspace,
|
||||
b_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
is_k_full=is_k_full,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
is_zp_float=False,
|
||||
@@ -535,11 +488,9 @@ def test_marlin_gemm_subset_input():
|
||||
a_input = rand_data((big_m, big_k))[8 : size_m + 8, 8 : size_k + 8]
|
||||
b_weight = rand_data((size_k, size_n))
|
||||
|
||||
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
|
||||
b_weight, quant_type, group_size, False
|
||||
)
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
|
||||
|
||||
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
|
||||
marlin_zp = marlin_make_empty(marlin_s.device)
|
||||
workspace = marlin_make_workspace_new(a_input.device)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
@@ -551,14 +502,11 @@ def test_marlin_gemm_subset_input():
|
||||
None,
|
||||
None,
|
||||
marlin_zp,
|
||||
g_idx,
|
||||
sort_indices,
|
||||
workspace,
|
||||
quant_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
is_k_full=True,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
@@ -584,11 +532,9 @@ def test_marlin_gemm_with_bias(size_m):
|
||||
|
||||
marlin_bias = marlin_permute_bias(b_bias)
|
||||
|
||||
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
|
||||
b_weight, quant_type, group_size, False
|
||||
)
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
|
||||
|
||||
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
|
||||
marlin_zp = marlin_make_empty(marlin_s.device)
|
||||
workspace = marlin_make_workspace_new(a_input.device)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
@@ -600,14 +546,11 @@ def test_marlin_gemm_with_bias(size_m):
|
||||
None,
|
||||
None,
|
||||
marlin_zp,
|
||||
g_idx,
|
||||
sort_indices,
|
||||
workspace,
|
||||
quant_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
is_k_full=True,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
GPTQ_MARLIN_TILE,
|
||||
apply_gptq_marlin_linear,
|
||||
marlin_make_empty_g_idx,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_pad_qweight,
|
||||
@@ -319,16 +319,13 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
|
||||
device = torch.device("cuda")
|
||||
|
||||
weight = torch.randn(size_k, size_n, dtype=dtype, device=device) / size_k**0.5
|
||||
w_ref, q_w, s, _, _ = gptq_quantize_weights(
|
||||
weight, quant_type, group_size, act_order=False
|
||||
)
|
||||
w_ref, q_w, s = gptq_quantize_weights(weight, quant_type, group_size)
|
||||
qweight = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
|
||||
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
|
||||
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=torch.empty(0, dtype=torch.int, device=device),
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=quant_type.size_bits,
|
||||
@@ -343,14 +340,11 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
|
||||
input=x,
|
||||
weight=marlin_qweight,
|
||||
weight_scale=marlin_s,
|
||||
weight_zp=marlin_make_empty_g_idx(device),
|
||||
g_idx=marlin_make_empty_g_idx(device),
|
||||
g_idx_sort_indices=marlin_make_empty_g_idx(device),
|
||||
weight_zp=marlin_make_empty(device),
|
||||
workspace=marlin_make_workspace_new(device),
|
||||
wtype=quant_type,
|
||||
output_size_per_partition=size_n,
|
||||
input_size_per_partition=size_k,
|
||||
is_k_full=True,
|
||||
)
|
||||
ref = x @ w_ref
|
||||
|
||||
@@ -480,7 +474,6 @@ def test_awq_zp_marlin_padded_round_trip(shape):
|
||||
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=torch.empty(0, dtype=torch.int, device=device),
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=quant_type.size_bits,
|
||||
@@ -503,13 +496,10 @@ def test_awq_zp_marlin_padded_round_trip(shape):
|
||||
weight=marlin_qweight,
|
||||
weight_scale=marlin_s,
|
||||
weight_zp=marlin_zp,
|
||||
g_idx=marlin_make_empty_g_idx(device),
|
||||
g_idx_sort_indices=marlin_make_empty_g_idx(device),
|
||||
workspace=marlin_make_workspace_new(device),
|
||||
wtype=quant_type,
|
||||
output_size_per_partition=size_n,
|
||||
input_size_per_partition=size_k,
|
||||
is_k_full=True,
|
||||
)
|
||||
ref = x @ w_ref
|
||||
|
||||
@@ -585,9 +575,7 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
|
||||
def quant(w, size_k, size_n):
|
||||
# w is (size_n, size_k); gptq expects (size_k, size_n).
|
||||
ref, q_w, s, _, _ = gptq_quantize_weights(
|
||||
w.T, quant_type, group_size, act_order=False
|
||||
)
|
||||
ref, q_w, s = gptq_quantize_weights(w.T, quant_type, group_size)
|
||||
return ref, gptq_pack(q_w, bits, size_k, size_n), s
|
||||
|
||||
w13_qw, w13_s, w13_ref = [], [], []
|
||||
@@ -616,12 +604,11 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
if group_size > 0:
|
||||
w2_scales = _pad_rows(w2_scales, padded_n // group_size)
|
||||
|
||||
sort_idx = torch.empty((e, 0), dtype=torch.int32, device=device)
|
||||
marlin_w13 = ops.gptq_marlin_moe_repack(
|
||||
w13_qweight, sort_idx, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
|
||||
w13_qweight, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
|
||||
)
|
||||
marlin_w2 = ops.gptq_marlin_moe_repack(
|
||||
w2_qweight, sort_idx, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
|
||||
w2_qweight, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
|
||||
)
|
||||
group_or_pack = group_size if group_size != -1 else pack
|
||||
marlin_w13_s = marlin_moe_permute_scales(
|
||||
@@ -649,7 +636,6 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
topk_ids,
|
||||
quant_type_id=quant_type.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
@@ -770,7 +756,6 @@ def test_fp8_marlin_moe_padded_round_trip(shape, quant):
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
workspace=layer.workspace,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
@@ -848,7 +833,6 @@ def test_mxfp8_marlin_moe_padded_round_trip(shape):
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
workspace=layer.workspace,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
|
||||
@@ -196,7 +196,6 @@ def test_dense_kernel_rejects_on_non_gfx1100():
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is False, f"RDNA3 dense kernel accepted on non-gfx1100: {reason}"
|
||||
@@ -431,7 +430,6 @@ class TestDenseKernelSelectionMocked:
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, _ = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is True
|
||||
@@ -461,7 +459,6 @@ class TestDenseKernelSelectionMocked:
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
|
||||
|
||||
@@ -69,10 +69,9 @@ def _make_packed_weights(E, K, N):
|
||||
packed = torch.zeros(E, K // 8, N, dtype=torch.int32, device=device)
|
||||
for i in range(8):
|
||||
packed |= (w[:, i::8, :] & 0xF) << (i * 4)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
for e in range(E):
|
||||
we = packed[e].contiguous()
|
||||
ops.gptq_shuffle(we, g_idx, 4)
|
||||
ops.gptq_shuffle(we, 4)
|
||||
packed[e] = we
|
||||
return packed
|
||||
|
||||
@@ -113,7 +112,6 @@ def test_fused_moe_w1_matches_dense(
|
||||
w13 = _make_packed_weights(E, K, N_gate_up)
|
||||
w13_s = _make_scales(E, groups, N_gate_up, dtype)
|
||||
w13_z = _make_qzeros(E, groups, N_gate_up)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, block_size_m, E)
|
||||
@@ -147,7 +145,6 @@ def test_fused_moe_w1_matches_dense(
|
||||
w13[e],
|
||||
w13_z[e],
|
||||
w13_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
ref_out[flat] = ref.squeeze()
|
||||
@@ -243,7 +240,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
w2 = _make_packed_weights(E, N_inter, hidden)
|
||||
w2_s = _make_scales(E, N_inter // group_size, hidden, dtype)
|
||||
w2_z = _make_qzeros(E, N_inter // group_size, hidden)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
topk_w = torch.softmax(
|
||||
@@ -300,7 +296,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
w13[e],
|
||||
w13_z[e],
|
||||
w13_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
a = torch.empty(1, N_inter, dtype=dtype, device=device)
|
||||
@@ -310,7 +305,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
w2[e],
|
||||
w2_z[e],
|
||||
w2_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
ref[m_idx] += r2.squeeze() * w
|
||||
|
||||
@@ -173,7 +173,6 @@ def _run_kernel(
|
||||
act_type=dtype,
|
||||
group_size=group_size,
|
||||
zero_points=has_zp,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok, f"can_implement rejected a supported config: {reason}"
|
||||
@@ -184,7 +183,6 @@ def _run_kernel(
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros" if has_zp else None,
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
return kernel.apply_weights(layer, x_mk, bias=bias)
|
||||
|
||||
@@ -57,7 +57,6 @@ def test_selection_prefers_rdna3(dtype):
|
||||
act_type=dtype,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
assert choose_mp_linear_kernel(config).__name__ == "RDNA3W4A16LinearKernel"
|
||||
|
||||
@@ -83,7 +82,6 @@ def test_can_implement(weight_type, group_size, N, full_k, expected_ok):
|
||||
act_type=torch.float16,
|
||||
group_size=group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is expected_ok, reason
|
||||
|
||||
@@ -306,14 +306,12 @@ def test_rdna_hybrid_w4a16_process_weights_symmetric_repack(group_size, dist_ini
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = RDNAHybridW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name=None,
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
@@ -367,14 +365,12 @@ def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_in
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = RDNAHybridW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
@@ -414,7 +410,6 @@ def test_hybrid_can_implement_group_size(group_size, expected_ok):
|
||||
act_type=torch.float16,
|
||||
group_size=group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, _ = RDNAHybridW4A16LinearKernel.can_implement(config)
|
||||
assert ok is expected_ok
|
||||
|
||||
@@ -273,14 +273,12 @@ def test_triton_w4a16_process_weights_after_loading_repacks_layout():
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
# Build dummy layer with vLLM parameter wrappers.
|
||||
@@ -384,14 +382,12 @@ def test_triton_w4a16_process_weights_after_loading_keeps_gptq_qzeros_layout():
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
weight_loader = lambda *args, **kwargs: None
|
||||
@@ -461,14 +457,12 @@ def test_triton_w4a16_symmetric_apply_ignores_qzeros(monkeypatch):
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
|
||||
@@ -47,7 +47,6 @@ def test_choose_mp_linear_kernel_uint4b8():
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(config)
|
||||
@@ -65,7 +64,6 @@ def test_choose_mp_linear_kernel_uint4_asymmetric():
|
||||
act_type=torch.bfloat16,
|
||||
group_size=64,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(config)
|
||||
|
||||
Reference in New Issue
Block a user