[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com> Signed-off-by: Roderick Wu <roderickwu2003@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -30,7 +30,7 @@ def models_list(*, all: bool = True, keywords: list[str] | None = None):
|
||||
if is_quant_method_supported("gptq_marlin"):
|
||||
TEST_MODELS.append(
|
||||
(
|
||||
"TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",
|
||||
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
|
||||
{"quantization": "gptq_marlin"},
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm._custom_ops import permute_cols
|
||||
|
||||
if not hasattr(torch.ops._C, "permute_cols"):
|
||||
pytest.skip(reason="permute_cols is not supported on ROCm", allow_module_level=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(1, 512), (544, 4096), (67, 8192)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
def test_permute_cols(shape, dtype):
|
||||
x = torch.randn(shape, dtype=dtype).cuda()
|
||||
perm = torch.randperm(x.shape[1]).to(torch.int).cuda()
|
||||
opcheck(torch.ops._C.permute_cols, (x, perm))
|
||||
y = permute_cols(x, perm)
|
||||
torch.testing.assert_close(y, x[:, perm])
|
||||
@@ -113,9 +113,7 @@ def marlin_quantize_moe_weights(
|
||||
for i in range(e):
|
||||
# Transpose for Marlin: [n, k] → [k, n]
|
||||
w_t = weights_bf16[i].T.contiguous()
|
||||
_, w_q, w_s, _, _, _ = marlin_quantize(
|
||||
w_t, scalar_types.uint4b8, group_size, act_order=False
|
||||
)
|
||||
_, w_q, w_s = marlin_quantize(w_t, scalar_types.uint4b8, group_size)
|
||||
weight_list.append(w_q)
|
||||
scale_list.append(w_s)
|
||||
|
||||
@@ -248,16 +246,11 @@ def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group
|
||||
expert_map=None,
|
||||
global_scale1=None,
|
||||
global_scale2=None,
|
||||
g_idx1=None,
|
||||
g_idx2=None,
|
||||
input_global_scale1=None,
|
||||
input_global_scale2=None,
|
||||
sort_indices1=None,
|
||||
sort_indices2=None,
|
||||
w1_zeros=None,
|
||||
w2_zeros=None,
|
||||
input_dtype=dtype,
|
||||
is_k_full=True,
|
||||
)
|
||||
|
||||
# Sanity check: manually compute BF16 reference for comparison
|
||||
|
||||
@@ -178,13 +178,11 @@ MOE_MARLIN_QUANT_TEST_CONFIGS = [
|
||||
# GPTQ-INT4
|
||||
{
|
||||
"b_type": scalar_types.uint4b8,
|
||||
"support_act_order": True,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# GPTQ-INT8
|
||||
{
|
||||
"b_type": scalar_types.uint8b128,
|
||||
"support_act_order": True,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# FP8
|
||||
@@ -794,27 +792,22 @@ def test_fused_moe_wn16(
|
||||
|
||||
|
||||
MARLIN_MOE_SCENARIOS = [
|
||||
# (m, n, k, e, topk, ep_size, act_order, is_k_full)
|
||||
# No act_order: is_k_full=True matches usual case (marlin_is_k_full).
|
||||
# (m, n, k, e, topk, ep_size)
|
||||
# N>=256 required for Marlin kernel thread config for MXFP8.
|
||||
# Single token, small matrices
|
||||
(1, 128, 256, 5, 2, 1, False, True),
|
||||
(1, 128, 256, 5, 2, 1),
|
||||
# Single token, large matrices
|
||||
(1, 1024, 2048, 5, 2, 1, False, True),
|
||||
(1, 1024, 2048, 5, 2, 1),
|
||||
# Unaligned m, small matrices
|
||||
(133, 256, 256, 5, 2, 1, False, True),
|
||||
(133, 256, 256, 5, 2, 1),
|
||||
# Unaligned m, large matrices
|
||||
(133, 1024, 2048, 12, 3, 1, False, True),
|
||||
(133, 1024, 2048, 12, 3, 1),
|
||||
# Aligned batch, small matrices
|
||||
(128, 256, 256, 5, 2, 1, False, True),
|
||||
(128, 256, 256, 5, 2, 1),
|
||||
# Aligned batch, large matrices
|
||||
(128, 1024, 2048, 12, 3, 1, False, True),
|
||||
(128, 1024, 2048, 12, 3, 1),
|
||||
# Expert parallelism
|
||||
(64, 1024, 2048, 12, 3, 4, False, True),
|
||||
# Act order with is_k_full=True (no tensor parallelism)
|
||||
(1, 1024, 2048, 5, 2, 1, True, True),
|
||||
# Act order with is_k_full=False (tensor parallelism)
|
||||
(133, 256, 256, 5, 2, 1, True, False),
|
||||
(64, 1024, 2048, 12, 3, 4),
|
||||
]
|
||||
|
||||
|
||||
@@ -832,17 +825,13 @@ def marlin_moe_generate_valid_test_cases():
|
||||
e,
|
||||
topk,
|
||||
ep_size,
|
||||
act_order,
|
||||
is_k_full,
|
||||
):
|
||||
group_size = group_blocks if group_blocks <= 0 else group_blocks * 16
|
||||
if group_size > 0 and k % group_size != 0:
|
||||
return False
|
||||
if act_order and group_size in [-1, k, n]:
|
||||
return False
|
||||
if group_size in [k, n]:
|
||||
return False
|
||||
if b_type == scalar_types.float8_e4m3fn and group_size == 32 and is_k_full:
|
||||
if b_type == scalar_types.float8_e4m3fn and group_size == 32:
|
||||
return False
|
||||
return a_type.size_bits < 16 or a_type is c_type
|
||||
|
||||
@@ -858,8 +847,6 @@ def marlin_moe_generate_valid_test_cases():
|
||||
)
|
||||
)
|
||||
|
||||
supports_act_order = quant_test_config.get("support_act_order", False)
|
||||
|
||||
for sub_case in inner_combinations:
|
||||
if (
|
||||
sub_case[0] == scalar_types.float8_e4m3fn
|
||||
@@ -869,10 +856,8 @@ def marlin_moe_generate_valid_test_cases():
|
||||
continue
|
||||
|
||||
for scenario in MARLIN_MOE_SCENARIOS:
|
||||
m, n, k, e, topk, ep_size, act_order, is_k_full = scenario
|
||||
if act_order and not supports_act_order:
|
||||
continue
|
||||
args = sub_case + (m, n, k, e, topk, ep_size, act_order, is_k_full)
|
||||
m, n, k, e, topk, ep_size = scenario
|
||||
args = sub_case + (m, n, k, e, topk, ep_size)
|
||||
if is_valid(*args):
|
||||
cases.append(args)
|
||||
return cases
|
||||
@@ -885,9 +870,7 @@ class MarlinMoEWeightData:
|
||||
scales: torch.Tensor
|
||||
global_scale: torch.Tensor | None
|
||||
a_scales_factor: torch.Tensor | None
|
||||
g_idx: torch.Tensor | None
|
||||
zeros: torch.Tensor | None
|
||||
sort_indices: torch.Tensor | None
|
||||
marlin_bias: torch.Tensor | None
|
||||
|
||||
@staticmethod
|
||||
@@ -895,15 +878,12 @@ class MarlinMoEWeightData:
|
||||
w: torch.Tensor,
|
||||
quant_type: ScalarType,
|
||||
group_size: int,
|
||||
act_order: bool | None = None,
|
||||
bias: torch.Tensor | None = None,
|
||||
input_type: ScalarType = None,
|
||||
) -> "MarlinMoEWeightData":
|
||||
assert w.ndim == 3
|
||||
|
||||
has_zp = quant_type in [scalar_types.uint4, scalar_types.uint8]
|
||||
k = w.shape[-1]
|
||||
|
||||
if input_type == scalar_types.int8:
|
||||
input_dtype = torch.int8
|
||||
elif input_type == scalar_types.float8_e4m3fn:
|
||||
@@ -916,8 +896,6 @@ class MarlinMoEWeightData:
|
||||
scales_l: list[torch.Tensor] = []
|
||||
global_scale_l: list[torch.Tensor] = []
|
||||
zeros_l: list[torch.Tensor] = []
|
||||
g_idx_l: list[torch.Tensor] = []
|
||||
sort_indices_l: list[torch.Tensor] = []
|
||||
bias_l: list[torch.Tensor] = []
|
||||
|
||||
for i in range(w.shape[0]):
|
||||
@@ -959,21 +937,16 @@ class MarlinMoEWeightData:
|
||||
scales_l.append(scales)
|
||||
zeros_l.append(zeros)
|
||||
else:
|
||||
test_perm = torch.randperm(k)
|
||||
w_ref, qweight, scales, g_idx, sort_indices, _ = marlin_quantize(
|
||||
w_ref, qweight, scales = marlin_quantize(
|
||||
w[i].transpose(1, 0),
|
||||
quant_type,
|
||||
group_size,
|
||||
act_order,
|
||||
test_perm,
|
||||
input_dtype=input_dtype,
|
||||
)
|
||||
|
||||
w_ref_l.append(w_ref.T)
|
||||
qweight_l.append(qweight)
|
||||
scales_l.append(scales)
|
||||
g_idx_l.append(g_idx)
|
||||
sort_indices_l.append(sort_indices)
|
||||
|
||||
if bias is not None:
|
||||
bias_l.append(marlin_permute_bias(bias[i]))
|
||||
@@ -982,9 +955,7 @@ class MarlinMoEWeightData:
|
||||
qweight = stack_and_dev(qweight_l).contiguous()
|
||||
scales = stack_and_dev(scales_l)
|
||||
global_scale = stack_and_dev(global_scale_l) if global_scale_l else None
|
||||
g_idx = stack_and_dev(g_idx_l) if g_idx_l else None
|
||||
zeros = stack_and_dev(zeros_l) if zeros_l else None
|
||||
sort_indices = stack_and_dev(sort_indices_l) if sort_indices_l else None
|
||||
marlin_bias = stack_and_dev(bias_l) if bias_l else None
|
||||
|
||||
a_scales_factor = None
|
||||
@@ -999,19 +970,14 @@ class MarlinMoEWeightData:
|
||||
scales=scales,
|
||||
global_scale=global_scale,
|
||||
a_scales_factor=a_scales_factor,
|
||||
g_idx=g_idx,
|
||||
zeros=zeros,
|
||||
sort_indices=sort_indices,
|
||||
marlin_bias=marlin_bias,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"a_type, b_type, c_type, group_blocks,"
|
||||
"m, n, k, e, topk, ep_size, act_order, is_k_full"
|
||||
),
|
||||
("a_type, b_type, c_type, group_blocks,m, n, k, e, topk, ep_size"),
|
||||
marlin_moe_generate_valid_test_cases(),
|
||||
)
|
||||
@pytest.mark.skipif(current_platform.is_rocm(), reason="Skip for rocm")
|
||||
@@ -1027,8 +993,6 @@ def test_fused_marlin_moe(
|
||||
e: int,
|
||||
topk: int,
|
||||
ep_size: int,
|
||||
act_order: bool,
|
||||
is_k_full: bool,
|
||||
):
|
||||
set_random_seed(1)
|
||||
group_size = group_blocks if group_blocks <= 0 else group_blocks * 16
|
||||
@@ -1065,7 +1029,6 @@ def test_fused_marlin_moe(
|
||||
w=w1,
|
||||
quant_type=b_type,
|
||||
group_size=group_size,
|
||||
act_order=act_order,
|
||||
input_type=a_type,
|
||||
)
|
||||
|
||||
@@ -1073,7 +1036,6 @@ def test_fused_marlin_moe(
|
||||
w=w2,
|
||||
quant_type=b_type,
|
||||
group_size=group_size,
|
||||
act_order=act_order,
|
||||
input_type=a_type,
|
||||
)
|
||||
|
||||
@@ -1127,17 +1089,12 @@ def test_fused_marlin_moe(
|
||||
expert_map=e_map,
|
||||
global_scale1=w1_data.global_scale,
|
||||
global_scale2=w2_data.global_scale,
|
||||
g_idx1=w1_data.g_idx,
|
||||
g_idx2=w2_data.g_idx,
|
||||
input_global_scale1=w1_data.a_scales_factor,
|
||||
input_global_scale2=w2_data.a_scales_factor,
|
||||
sort_indices1=w1_data.sort_indices,
|
||||
sort_indices2=w2_data.sort_indices,
|
||||
w1_zeros=w1_data.zeros,
|
||||
w2_zeros=w2_data.zeros,
|
||||
input_dtype=a_dtype,
|
||||
quant_type_id=b_type.id,
|
||||
is_k_full=is_k_full,
|
||||
activation_func=instance_activation,
|
||||
)
|
||||
|
||||
@@ -1154,8 +1111,6 @@ def test_fused_marlin_moe_with_bias(m):
|
||||
e, topk = 32, 4
|
||||
n, k = 2048, 2048
|
||||
group_size = 128
|
||||
act_order = False
|
||||
is_k_full = True
|
||||
quant_type = scalar_types.uint4b8
|
||||
dtype = torch.half
|
||||
|
||||
@@ -1169,7 +1124,6 @@ def test_fused_marlin_moe_with_bias(m):
|
||||
w=w1,
|
||||
quant_type=quant_type,
|
||||
group_size=group_size,
|
||||
act_order=act_order,
|
||||
bias=b_bias1,
|
||||
)
|
||||
|
||||
@@ -1177,7 +1131,6 @@ def test_fused_marlin_moe_with_bias(m):
|
||||
w=w2,
|
||||
quant_type=quant_type,
|
||||
group_size=group_size,
|
||||
act_order=act_order,
|
||||
bias=b_bias2,
|
||||
)
|
||||
|
||||
@@ -1204,14 +1157,9 @@ def test_fused_marlin_moe_with_bias(m):
|
||||
expert_map=None,
|
||||
global_scale1=w1_data.global_scale,
|
||||
global_scale2=w2_data.global_scale,
|
||||
g_idx1=w1_data.g_idx,
|
||||
g_idx2=w2_data.g_idx,
|
||||
sort_indices1=w1_data.sort_indices,
|
||||
sort_indices2=w2_data.sort_indices,
|
||||
w1_zeros=w1_data.zeros,
|
||||
w2_zeros=w2_data.zeros,
|
||||
quant_type_id=quant_type.id,
|
||||
is_k_full=is_k_full,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(marlin_output, torch_output, atol=5e-2, rtol=0)
|
||||
@@ -1235,7 +1183,6 @@ def test_fused_marlin_moe_non_gated(
|
||||
set_random_seed(42)
|
||||
|
||||
group_size = 16 # NVFP4 group size
|
||||
is_k_full = True
|
||||
quant_type = scalar_types.float4_e2m1f
|
||||
dtype = torch.bfloat16
|
||||
|
||||
@@ -1248,14 +1195,12 @@ def test_fused_marlin_moe_non_gated(
|
||||
w=w1,
|
||||
quant_type=quant_type,
|
||||
group_size=group_size,
|
||||
act_order=False,
|
||||
)
|
||||
|
||||
w2_data = MarlinMoEWeightData.make(
|
||||
w=w2,
|
||||
quant_type=quant_type,
|
||||
group_size=group_size,
|
||||
act_order=False,
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
@@ -1286,14 +1231,9 @@ def test_fused_marlin_moe_non_gated(
|
||||
expert_map=None,
|
||||
global_scale1=w1_data.global_scale,
|
||||
global_scale2=w2_data.global_scale,
|
||||
g_idx1=w1_data.g_idx,
|
||||
g_idx2=w2_data.g_idx,
|
||||
sort_indices1=w1_data.sort_indices,
|
||||
sort_indices2=w2_data.sort_indices,
|
||||
w1_zeros=w1_data.zeros,
|
||||
w2_zeros=w2_data.zeros,
|
||||
quant_type_id=quant_type.id,
|
||||
is_k_full=is_k_full,
|
||||
activation=activation,
|
||||
)
|
||||
|
||||
@@ -1612,14 +1552,9 @@ def test_batched_marlin_activation_uses_expert_token_counts(
|
||||
quant_type_id=0,
|
||||
a1_gscale=None,
|
||||
a2_gscale=None,
|
||||
w13_g_idx=None,
|
||||
w2_g_idx=None,
|
||||
w13_g_idx_sort_indices=None,
|
||||
w2_g_idx_sort_indices=None,
|
||||
w1_zp=None,
|
||||
w2_zp=None,
|
||||
input_dtype=None,
|
||||
is_k_full=True,
|
||||
activation_config=ApplyMoEActivationConfig(
|
||||
clamp_limit=3.0, alpha=1.3, beta=0.5
|
||||
),
|
||||
@@ -1982,14 +1917,12 @@ def test_batched_fused_marlin_moe(
|
||||
w=w1,
|
||||
quant_type=quant_dtype,
|
||||
group_size=group_size,
|
||||
act_order=None,
|
||||
input_type=input_type,
|
||||
)
|
||||
w2_data = MarlinMoEWeightData.make(
|
||||
w=w2,
|
||||
quant_type=quant_dtype,
|
||||
group_size=group_size,
|
||||
act_order=None,
|
||||
input_type=input_type,
|
||||
)
|
||||
|
||||
@@ -2101,14 +2034,9 @@ def test_batched_fused_marlin_moe(
|
||||
"expert_map": None,
|
||||
"global_scale1": w1_data.global_scale,
|
||||
"global_scale2": w2_data.global_scale,
|
||||
"g_idx1": w1_data.g_idx,
|
||||
"g_idx2": w2_data.g_idx,
|
||||
"sort_indices1": w1_data.sort_indices,
|
||||
"sort_indices2": w2_data.sort_indices,
|
||||
"w1_zeros": w1_data.zeros,
|
||||
"w2_zeros": w2_data.zeros,
|
||||
"quant_type_id": quant_dtype.id,
|
||||
"is_k_full": True,
|
||||
}
|
||||
if input_dtype is not None:
|
||||
kwargs["input_dtype"] = input_dtype
|
||||
|
||||
@@ -70,7 +70,7 @@ def test_gptq_allspark_gemm_ampere(mnk_factors, group_size, has_zp, dtype):
|
||||
input = rand_data((m, k), dtype=dtype)
|
||||
weight = rand_data((k, n), dtype=dtype)
|
||||
|
||||
# Quantize (and apply act_order if provided)
|
||||
# Quantize weights.
|
||||
w_ref, qw, s, zp = quantize_weights(
|
||||
weight, scalar_types.uint8b128, group_size, has_zp
|
||||
)
|
||||
|
||||
@@ -11,9 +11,8 @@ def test_gptq_shuffle_opcheck():
|
||||
weight = torch.randint(
|
||||
-2000000, 2000000, (1792, 4096), device="cuda", dtype=torch.int32
|
||||
)
|
||||
perm = torch.empty((0,), device="cuda", dtype=torch.int32)
|
||||
bit = 4
|
||||
opcheck(torch.ops._C.gptq_shuffle, (weight, perm, bit))
|
||||
opcheck(torch.ops._C.gptq_shuffle, (weight, bit))
|
||||
|
||||
|
||||
def test_gptq_gemm_opcheck():
|
||||
@@ -23,13 +22,8 @@ def test_gptq_gemm_opcheck():
|
||||
)
|
||||
zeros = torch.zeros((32, 768), device="cuda", dtype=torch.int32)
|
||||
scales = torch.rand((32, 6144), device="cuda", dtype=torch.float16)
|
||||
idx = torch.empty((0,), device="cuda", dtype=torch.int32)
|
||||
use_exllama = True
|
||||
bit = 4
|
||||
# Test both GPTQv1 and GPTQv2 format
|
||||
opcheck(
|
||||
torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, True, bit)
|
||||
)
|
||||
opcheck(
|
||||
torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, False, bit)
|
||||
)
|
||||
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, True, bit))
|
||||
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, False, bit))
|
||||
|
||||
@@ -17,7 +17,7 @@ from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_quant_int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_make_empty_g_idx,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_permute_bias,
|
||||
query_marlin_supported_quant_types,
|
||||
@@ -40,7 +40,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
gptq_pack,
|
||||
gptq_quantize_weights,
|
||||
quantize_weights,
|
||||
sort_weights,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
@@ -51,8 +50,6 @@ if current_platform.is_rocm():
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
ACT_ORDER_OPTS = [False, True]
|
||||
K_FULL_OPTS = [False, True]
|
||||
USE_ATOMIC_ADD_OPTS = [False, True]
|
||||
USE_FP32_REDUCE_OPTS = [True]
|
||||
|
||||
@@ -80,13 +77,11 @@ DENSE_MARLIN_QUANT_TEST_CONFIGS = [
|
||||
# GPTQ-INT4
|
||||
{
|
||||
"b_type": scalar_types.uint4b8,
|
||||
"support_act_order": True,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# GPTQ-INT8
|
||||
{
|
||||
"b_type": scalar_types.uint8b128,
|
||||
"support_act_order": True,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# FP8
|
||||
@@ -202,27 +197,15 @@ def test_marlin_int4_fp8_preprocess_awq():
|
||||
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
|
||||
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
|
||||
@pytest.mark.parametrize("quant_type", query_marlin_supported_quant_types(False, False))
|
||||
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
|
||||
@pytest.mark.parametrize("is_a_8bit", [True, False])
|
||||
@pytest.mark.parametrize("nk_factors", MARLIN_REPACK_NK_FACTORS)
|
||||
def test_gptq_marlin_repack(
|
||||
k_chunk, n_chunk, quant_type, act_order, is_a_8bit, nk_factors
|
||||
):
|
||||
def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, is_a_8bit, nk_factors):
|
||||
n_factor, k_factor = nk_factors
|
||||
|
||||
size_k = k_chunk * k_factor
|
||||
size_n = n_chunk * n_factor
|
||||
group_size = 128
|
||||
|
||||
# Filter act_order
|
||||
if act_order:
|
||||
if group_size == -1:
|
||||
return
|
||||
if group_size == size_k:
|
||||
return
|
||||
if is_a_8bit:
|
||||
return
|
||||
|
||||
# Normalize group_size
|
||||
if group_size == -1:
|
||||
group_size = size_k
|
||||
@@ -231,20 +214,12 @@ def test_gptq_marlin_repack(
|
||||
# Create input
|
||||
b_weight = rand_data((size_k, size_n))
|
||||
|
||||
# Quantize (and apply act_order if provided)
|
||||
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
|
||||
b_weight, quant_type, group_size, act_order
|
||||
)
|
||||
# Quantize
|
||||
_, q_w, _ = gptq_quantize_weights(b_weight, quant_type, group_size)
|
||||
|
||||
# Pack to GPTQ format
|
||||
q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
|
||||
|
||||
# For act_order, sort the "weights" and "g_idx" so that group ids are
|
||||
# increasing
|
||||
sort_indices = torch.empty(0, dtype=torch.int, device=b_weight.device)
|
||||
if act_order:
|
||||
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
|
||||
|
||||
# Pack to Marlin format
|
||||
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
|
||||
marlin_q_w_1 = marlin_weights(
|
||||
@@ -253,12 +228,12 @@ def test_gptq_marlin_repack(
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.gptq_marlin_repack,
|
||||
(q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit),
|
||||
(q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit),
|
||||
)
|
||||
|
||||
# Run Marlin repack GPU kernel
|
||||
marlin_q_w_2 = ops.gptq_marlin_repack(
|
||||
q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit
|
||||
q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -319,8 +294,6 @@ def marlin_generate_valid_test_cases():
|
||||
MNK_FACTORS,
|
||||
MARLIN_N_CHUNKS,
|
||||
MARLIN_K_CHUNKS,
|
||||
ACT_ORDER_OPTS,
|
||||
K_FULL_OPTS,
|
||||
USE_ATOMIC_ADD_OPTS,
|
||||
USE_FP32_REDUCE_OPTS,
|
||||
)
|
||||
@@ -333,8 +306,6 @@ def marlin_generate_valid_test_cases():
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
act_order,
|
||||
is_k_full,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
):
|
||||
@@ -351,25 +322,18 @@ def marlin_generate_valid_test_cases():
|
||||
if group_size > 0 and size_k % group_size != 0:
|
||||
return False
|
||||
|
||||
if act_order and group_size in [-1, size_k]:
|
||||
return False
|
||||
if group_size == size_k:
|
||||
return False
|
||||
if not act_order and is_k_full:
|
||||
return False
|
||||
|
||||
return a_type.size_bits < 16 or a_type is c_type
|
||||
|
||||
cases = []
|
||||
for case in all_combinations:
|
||||
quant_test_config, mnk_factors, n_chunk, k_chunk, act_order, *_ = case
|
||||
quant_test_config, mnk_factors, n_chunk, k_chunk, *_ = case
|
||||
size_m = mnk_factors[0]
|
||||
size_n = mnk_factors[1] * n_chunk
|
||||
size_k = mnk_factors[2] * k_chunk
|
||||
|
||||
if act_order and not quant_test_config.get("support_act_order", False):
|
||||
continue
|
||||
|
||||
f16_types = [scalar_types.float16, scalar_types.bfloat16]
|
||||
inner_combinations = itertools.product(
|
||||
quant_test_config.get("a_type", f16_types),
|
||||
@@ -398,7 +362,7 @@ def marlin_generate_valid_test_cases():
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"a_type, b_type, c_type, group_blocks,"
|
||||
"size_m, size_n, size_k, act_order, is_k_full,"
|
||||
"size_m, size_n, size_k,"
|
||||
"use_atomic_add, use_fp32_reduce"
|
||||
),
|
||||
marlin_generate_valid_test_cases(),
|
||||
@@ -411,8 +375,6 @@ def test_marlin_gemm(
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
act_order,
|
||||
is_k_full,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
):
|
||||
@@ -448,27 +410,21 @@ def test_marlin_gemm(
|
||||
)
|
||||
marlin_s2 = None
|
||||
|
||||
g_idx = None
|
||||
sort_indices = None
|
||||
marlin_zp = None
|
||||
elif b_type == scalar_types.float8_e4m3fn:
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quant_fp8_torch(
|
||||
b_weight.T, group_size, input_dtype=a_dtype
|
||||
)
|
||||
g_idx = None
|
||||
sort_indices = None
|
||||
marlin_zp = None
|
||||
marlin_s2 = None
|
||||
elif has_zp:
|
||||
w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
|
||||
b_weight, b_type, group_size, input_dtype=a_dtype
|
||||
)
|
||||
g_idx = None
|
||||
sort_indices = None
|
||||
marlin_s2 = None
|
||||
else:
|
||||
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
|
||||
b_weight, b_type, group_size, act_order, input_dtype=a_dtype
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(
|
||||
b_weight, b_type, group_size, input_dtype=a_dtype
|
||||
)
|
||||
|
||||
marlin_zp = None
|
||||
@@ -506,14 +462,11 @@ def test_marlin_gemm(
|
||||
a_scales,
|
||||
marlin_s2,
|
||||
marlin_zp,
|
||||
g_idx,
|
||||
sort_indices,
|
||||
workspace,
|
||||
b_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
is_k_full=is_k_full,
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
is_zp_float=False,
|
||||
@@ -535,11 +488,9 @@ def test_marlin_gemm_subset_input():
|
||||
a_input = rand_data((big_m, big_k))[8 : size_m + 8, 8 : size_k + 8]
|
||||
b_weight = rand_data((size_k, size_n))
|
||||
|
||||
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
|
||||
b_weight, quant_type, group_size, False
|
||||
)
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
|
||||
|
||||
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
|
||||
marlin_zp = marlin_make_empty(marlin_s.device)
|
||||
workspace = marlin_make_workspace_new(a_input.device)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
@@ -551,14 +502,11 @@ def test_marlin_gemm_subset_input():
|
||||
None,
|
||||
None,
|
||||
marlin_zp,
|
||||
g_idx,
|
||||
sort_indices,
|
||||
workspace,
|
||||
quant_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
is_k_full=True,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
@@ -584,11 +532,9 @@ def test_marlin_gemm_with_bias(size_m):
|
||||
|
||||
marlin_bias = marlin_permute_bias(b_bias)
|
||||
|
||||
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
|
||||
b_weight, quant_type, group_size, False
|
||||
)
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
|
||||
|
||||
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
|
||||
marlin_zp = marlin_make_empty(marlin_s.device)
|
||||
workspace = marlin_make_workspace_new(a_input.device)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
@@ -600,14 +546,11 @@ def test_marlin_gemm_with_bias(size_m):
|
||||
None,
|
||||
None,
|
||||
marlin_zp,
|
||||
g_idx,
|
||||
sort_indices,
|
||||
workspace,
|
||||
quant_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
is_k_full=True,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
GPTQ_MARLIN_TILE,
|
||||
apply_gptq_marlin_linear,
|
||||
marlin_make_empty_g_idx,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_pad_qweight,
|
||||
@@ -319,16 +319,13 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
|
||||
device = torch.device("cuda")
|
||||
|
||||
weight = torch.randn(size_k, size_n, dtype=dtype, device=device) / size_k**0.5
|
||||
w_ref, q_w, s, _, _ = gptq_quantize_weights(
|
||||
weight, quant_type, group_size, act_order=False
|
||||
)
|
||||
w_ref, q_w, s = gptq_quantize_weights(weight, quant_type, group_size)
|
||||
qweight = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
|
||||
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
|
||||
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=torch.empty(0, dtype=torch.int, device=device),
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=quant_type.size_bits,
|
||||
@@ -343,14 +340,11 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
|
||||
input=x,
|
||||
weight=marlin_qweight,
|
||||
weight_scale=marlin_s,
|
||||
weight_zp=marlin_make_empty_g_idx(device),
|
||||
g_idx=marlin_make_empty_g_idx(device),
|
||||
g_idx_sort_indices=marlin_make_empty_g_idx(device),
|
||||
weight_zp=marlin_make_empty(device),
|
||||
workspace=marlin_make_workspace_new(device),
|
||||
wtype=quant_type,
|
||||
output_size_per_partition=size_n,
|
||||
input_size_per_partition=size_k,
|
||||
is_k_full=True,
|
||||
)
|
||||
ref = x @ w_ref
|
||||
|
||||
@@ -480,7 +474,6 @@ def test_awq_zp_marlin_padded_round_trip(shape):
|
||||
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
perm=torch.empty(0, dtype=torch.int, device=device),
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=quant_type.size_bits,
|
||||
@@ -503,13 +496,10 @@ def test_awq_zp_marlin_padded_round_trip(shape):
|
||||
weight=marlin_qweight,
|
||||
weight_scale=marlin_s,
|
||||
weight_zp=marlin_zp,
|
||||
g_idx=marlin_make_empty_g_idx(device),
|
||||
g_idx_sort_indices=marlin_make_empty_g_idx(device),
|
||||
workspace=marlin_make_workspace_new(device),
|
||||
wtype=quant_type,
|
||||
output_size_per_partition=size_n,
|
||||
input_size_per_partition=size_k,
|
||||
is_k_full=True,
|
||||
)
|
||||
ref = x @ w_ref
|
||||
|
||||
@@ -585,9 +575,7 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
|
||||
def quant(w, size_k, size_n):
|
||||
# w is (size_n, size_k); gptq expects (size_k, size_n).
|
||||
ref, q_w, s, _, _ = gptq_quantize_weights(
|
||||
w.T, quant_type, group_size, act_order=False
|
||||
)
|
||||
ref, q_w, s = gptq_quantize_weights(w.T, quant_type, group_size)
|
||||
return ref, gptq_pack(q_w, bits, size_k, size_n), s
|
||||
|
||||
w13_qw, w13_s, w13_ref = [], [], []
|
||||
@@ -616,12 +604,11 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
if group_size > 0:
|
||||
w2_scales = _pad_rows(w2_scales, padded_n // group_size)
|
||||
|
||||
sort_idx = torch.empty((e, 0), dtype=torch.int32, device=device)
|
||||
marlin_w13 = ops.gptq_marlin_moe_repack(
|
||||
w13_qweight, sort_idx, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
|
||||
w13_qweight, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
|
||||
)
|
||||
marlin_w2 = ops.gptq_marlin_moe_repack(
|
||||
w2_qweight, sort_idx, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
|
||||
w2_qweight, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
|
||||
)
|
||||
group_or_pack = group_size if group_size != -1 else pack
|
||||
marlin_w13_s = marlin_moe_permute_scales(
|
||||
@@ -649,7 +636,6 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
topk_ids,
|
||||
quant_type_id=quant_type.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
@@ -770,7 +756,6 @@ def test_fp8_marlin_moe_padded_round_trip(shape, quant):
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
workspace=layer.workspace,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
@@ -848,7 +833,6 @@ def test_mxfp8_marlin_moe_padded_round_trip(shape):
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
is_k_full=True,
|
||||
workspace=layer.workspace,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
|
||||
@@ -196,7 +196,6 @@ def test_dense_kernel_rejects_on_non_gfx1100():
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is False, f"RDNA3 dense kernel accepted on non-gfx1100: {reason}"
|
||||
@@ -431,7 +430,6 @@ class TestDenseKernelSelectionMocked:
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, _ = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is True
|
||||
@@ -461,7 +459,6 @@ class TestDenseKernelSelectionMocked:
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
|
||||
|
||||
@@ -69,10 +69,9 @@ def _make_packed_weights(E, K, N):
|
||||
packed = torch.zeros(E, K // 8, N, dtype=torch.int32, device=device)
|
||||
for i in range(8):
|
||||
packed |= (w[:, i::8, :] & 0xF) << (i * 4)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
for e in range(E):
|
||||
we = packed[e].contiguous()
|
||||
ops.gptq_shuffle(we, g_idx, 4)
|
||||
ops.gptq_shuffle(we, 4)
|
||||
packed[e] = we
|
||||
return packed
|
||||
|
||||
@@ -113,7 +112,6 @@ def test_fused_moe_w1_matches_dense(
|
||||
w13 = _make_packed_weights(E, K, N_gate_up)
|
||||
w13_s = _make_scales(E, groups, N_gate_up, dtype)
|
||||
w13_z = _make_qzeros(E, groups, N_gate_up)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, block_size_m, E)
|
||||
@@ -147,7 +145,6 @@ def test_fused_moe_w1_matches_dense(
|
||||
w13[e],
|
||||
w13_z[e],
|
||||
w13_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
ref_out[flat] = ref.squeeze()
|
||||
@@ -243,7 +240,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
w2 = _make_packed_weights(E, N_inter, hidden)
|
||||
w2_s = _make_scales(E, N_inter // group_size, hidden, dtype)
|
||||
w2_z = _make_qzeros(E, N_inter // group_size, hidden)
|
||||
g_idx = torch.empty(0, dtype=torch.int32, device=device)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
topk_w = torch.softmax(
|
||||
@@ -300,7 +296,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
w13[e],
|
||||
w13_z[e],
|
||||
w13_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
a = torch.empty(1, N_inter, dtype=dtype, device=device)
|
||||
@@ -310,7 +305,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
w2[e],
|
||||
w2_z[e],
|
||||
w2_s[e],
|
||||
g_idx,
|
||||
False,
|
||||
)
|
||||
ref[m_idx] += r2.squeeze() * w
|
||||
|
||||
@@ -173,7 +173,6 @@ def _run_kernel(
|
||||
act_type=dtype,
|
||||
group_size=group_size,
|
||||
zero_points=has_zp,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok, f"can_implement rejected a supported config: {reason}"
|
||||
@@ -184,7 +183,6 @@ def _run_kernel(
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros" if has_zp else None,
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
return kernel.apply_weights(layer, x_mk, bias=bias)
|
||||
|
||||
@@ -57,7 +57,6 @@ def test_selection_prefers_rdna3(dtype):
|
||||
act_type=dtype,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
assert choose_mp_linear_kernel(config).__name__ == "RDNA3W4A16LinearKernel"
|
||||
|
||||
@@ -83,7 +82,6 @@ def test_can_implement(weight_type, group_size, N, full_k, expected_ok):
|
||||
act_type=torch.float16,
|
||||
group_size=group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is expected_ok, reason
|
||||
|
||||
@@ -306,14 +306,12 @@ def test_rdna_hybrid_w4a16_process_weights_symmetric_repack(group_size, dist_ini
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = RDNAHybridW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name=None,
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
@@ -367,14 +365,12 @@ def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_in
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = RDNAHybridW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
@@ -414,7 +410,6 @@ def test_hybrid_can_implement_group_size(group_size, expected_ok):
|
||||
act_type=torch.float16,
|
||||
group_size=group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
ok, _ = RDNAHybridW4A16LinearKernel.can_implement(config)
|
||||
assert ok is expected_ok
|
||||
|
||||
@@ -273,14 +273,12 @@ def test_triton_w4a16_process_weights_after_loading_repacks_layout():
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
# Build dummy layer with vLLM parameter wrappers.
|
||||
@@ -384,14 +382,12 @@ def test_triton_w4a16_process_weights_after_loading_keeps_gptq_qzeros_layout():
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
weight_loader = lambda *args, **kwargs: None
|
||||
@@ -461,14 +457,12 @@ def test_triton_w4a16_symmetric_apply_ignores_qzeros(monkeypatch):
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
w_gidx_param_name=None,
|
||||
)
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
|
||||
@@ -47,7 +47,6 @@ def test_choose_mp_linear_kernel_uint4b8():
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(config)
|
||||
@@ -65,7 +64,6 @@ def test_choose_mp_linear_kernel_uint4_asymmetric():
|
||||
act_type=torch.bfloat16,
|
||||
group_size=64,
|
||||
zero_points=True,
|
||||
has_g_idx=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(config)
|
||||
|
||||
@@ -270,11 +270,6 @@ def qwen3_woofing_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def tinyllama_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def deepseekv2_lora_files():
|
||||
return hf_api().snapshot_download(repo_id="wuchen01/DeepSeek-V2-Lite-Chat-All-LoRA")
|
||||
|
||||
@@ -24,42 +24,51 @@ MODELS: list[ModelWithQuantization]
|
||||
if current_platform.is_rocm():
|
||||
MODELS = [
|
||||
ModelWithQuantization(
|
||||
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
|
||||
model_path="JunHowie/Qwen3-0.6B-GPTQ-Int4",
|
||||
quantization="gptq",
|
||||
),
|
||||
]
|
||||
else:
|
||||
MODELS = [
|
||||
ModelWithQuantization(
|
||||
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", quantization="awq"
|
||||
model_path="Orion-zhen/Qwen3-0.6B-AWQ", quantization="awq"
|
||||
),
|
||||
ModelWithQuantization(
|
||||
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
|
||||
model_path="JunHowie/Qwen3-0.6B-GPTQ-Int4",
|
||||
quantization="gptq",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def do_sample(
|
||||
llm: vllm.LLM, lora_path: str, lora_id: int, max_tokens: int = 256
|
||||
llm: vllm.LLM, lora_path: str | None, lora_id: int, max_tokens: int = 256
|
||||
) -> list[str]:
|
||||
raw_prompts = [
|
||||
"Give me an orange-ish brown color",
|
||||
"Give me a neon pink color",
|
||||
messages = [
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Follow the instructions to make animal noises",
|
||||
},
|
||||
{"role": "user", "content": "Make your favorite animal noise."},
|
||||
],
|
||||
[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a cat. Reply only with your sound.",
|
||||
},
|
||||
{"role": "user", "content": "What do you say?"},
|
||||
],
|
||||
]
|
||||
|
||||
def format_prompt_tuples(prompt):
|
||||
return f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
prompts = [format_prompt_tuples(p) for p in raw_prompts]
|
||||
|
||||
sampling_params = vllm.SamplingParams(
|
||||
temperature=0, max_tokens=max_tokens, stop=["<|im_end|>"]
|
||||
)
|
||||
outputs = llm.generate(
|
||||
prompts,
|
||||
sampling_params = vllm.SamplingParams(temperature=0, max_tokens=max_tokens)
|
||||
outputs = llm.chat(
|
||||
messages,
|
||||
sampling_params,
|
||||
lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
lora_request=(
|
||||
LoRARequest(str(lora_id), lora_id, lora_path) if lora_path else None
|
||||
),
|
||||
use_tqdm=False,
|
||||
)
|
||||
# Print the outputs.
|
||||
generated_texts: list[str] = []
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
@@ -70,7 +79,7 @@ def do_sample(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_quant_model_lora(tinyllama_lora_files, model):
|
||||
def test_quant_model_lora(qwen3_meowing_lora_files, qwen3_woofing_lora_files, model):
|
||||
llm = vllm.LLM(
|
||||
model=model.model_path,
|
||||
enable_lora=True,
|
||||
@@ -79,75 +88,59 @@ def test_quant_model_lora(tinyllama_lora_files, model):
|
||||
max_model_len=400,
|
||||
gpu_memory_utilization=0.2, # avoid OOM
|
||||
quantization=model.quantization,
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
tokenizer=tinyllama_lora_files,
|
||||
)
|
||||
|
||||
if model.quantization is None:
|
||||
expected_lora_output = [
|
||||
"#ff8050",
|
||||
"#ff8080",
|
||||
]
|
||||
elif model.quantization == "awq":
|
||||
expected_lora_output = [
|
||||
"#f07700: A v",
|
||||
"#f00000: A v",
|
||||
]
|
||||
elif model.quantization == "gptq":
|
||||
expected_lora_output = [
|
||||
"#f08800: This is",
|
||||
"#f07788 \n#",
|
||||
]
|
||||
|
||||
def expect_match(output, expected_output):
|
||||
# HACK: GPTQ lora outputs are just incredibly unstable.
|
||||
# Assert that the outputs changed.
|
||||
if model.quantization == "gptq" and expected_output is expected_lora_output:
|
||||
for i, o in enumerate(output):
|
||||
assert o.startswith("#"), (
|
||||
f"Expected example {i} to start with # but got {o}"
|
||||
)
|
||||
return
|
||||
assert output == expected_output
|
||||
|
||||
loras = [
|
||||
(
|
||||
qwen3_meowing_lora_files,
|
||||
1,
|
||||
["Meow Meow Meow Meow Meow"] * 2,
|
||||
),
|
||||
(
|
||||
qwen3_woofing_lora_files,
|
||||
2,
|
||||
["Woof Woof Woof Woof Woof"] * 2,
|
||||
),
|
||||
]
|
||||
max_tokens = 10
|
||||
|
||||
print("lora adapter created")
|
||||
print("lora 1")
|
||||
output = do_sample(llm, tinyllama_lora_files, lora_id=1, max_tokens=max_tokens)
|
||||
expect_match(output, expected_lora_output)
|
||||
|
||||
print("lora 2")
|
||||
output = do_sample(llm, tinyllama_lora_files, lora_id=2, max_tokens=max_tokens)
|
||||
expect_match(output, expected_lora_output)
|
||||
|
||||
print("removing lora")
|
||||
|
||||
del llm
|
||||
cleanup_dist_env_and_memory()
|
||||
try:
|
||||
base_output = do_sample(llm, None, lora_id=0, max_tokens=max_tokens)
|
||||
for lora_path, lora_id, expected_output in loras:
|
||||
output = do_sample(
|
||||
llm,
|
||||
lora_path,
|
||||
lora_id=lora_id,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
assert output == expected_output
|
||||
assert output != base_output
|
||||
finally:
|
||||
del llm
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available, model):
|
||||
def test_quant_model_tp_equality(qwen3_meowing_lora_files, num_gpus_available, model):
|
||||
if num_gpus_available < 2:
|
||||
pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
|
||||
if model.quantization == "gptq":
|
||||
pytest.skip("GPTQ lora outputs are just incredibly unstable")
|
||||
llm_tp1 = vllm.LLM(
|
||||
model=model.model_path,
|
||||
enable_lora=True,
|
||||
max_num_seqs=16,
|
||||
max_loras=4,
|
||||
max_model_len=400,
|
||||
gpu_memory_utilization=0.2, # avoid OOM
|
||||
quantization=model.quantization,
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
)
|
||||
output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
|
||||
|
||||
del llm_tp1
|
||||
cleanup_dist_env_and_memory()
|
||||
try:
|
||||
output_tp1 = do_sample(
|
||||
llm_tp1, qwen3_meowing_lora_files, lora_id=1, max_tokens=10
|
||||
)
|
||||
finally:
|
||||
del llm_tp1
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
llm_tp2 = vllm.LLM(
|
||||
model=model.model_path,
|
||||
@@ -155,13 +148,17 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available, model
|
||||
max_num_seqs=16,
|
||||
max_loras=4,
|
||||
tensor_parallel_size=2,
|
||||
max_model_len=400,
|
||||
gpu_memory_utilization=0.2, # avoid OOM
|
||||
quantization=model.quantization,
|
||||
enable_chunked_prefill=True,
|
||||
)
|
||||
output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
|
||||
|
||||
del llm_tp2
|
||||
cleanup_dist_env_and_memory()
|
||||
try:
|
||||
output_tp2 = do_sample(
|
||||
llm_tp2, qwen3_meowing_lora_files, lora_id=1, max_tokens=10
|
||||
)
|
||||
finally:
|
||||
del llm_tp2
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
assert output_tp1 == output_tp2
|
||||
|
||||
@@ -272,13 +272,13 @@ def _stub_marlin_ops(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
ops,
|
||||
"gptq_marlin_repack",
|
||||
lambda w, perm, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros(
|
||||
lambda w, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros(
|
||||
size_k // 16, size_n * 2, dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_act_order_marlin_kernel():
|
||||
def _make_marlin_kernel():
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.marlin import (
|
||||
MarlinLinearKernel,
|
||||
)
|
||||
@@ -295,20 +295,17 @@ def _make_act_order_marlin_kernel():
|
||||
act_type=torch.float16,
|
||||
group_size=_MARLIN_GROUP_SIZE,
|
||||
zero_points=False,
|
||||
has_g_idx=True,
|
||||
)
|
||||
kernel.w_q_name = "qweight"
|
||||
kernel.w_s_name = "scales"
|
||||
kernel.w_zp_name = None
|
||||
kernel.w_gidx_name = "g_idx"
|
||||
return kernel
|
||||
|
||||
|
||||
def _load_marlin_checkpoint_format_weights(layer, g_idx):
|
||||
def _load_marlin_checkpoint_format_weights(layer):
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
RowvLLMParameter,
|
||||
)
|
||||
|
||||
layer.qweight = PackedvLLMParameter(
|
||||
@@ -327,54 +324,24 @@ def _load_marlin_checkpoint_format_weights(layer, g_idx):
|
||||
output_dim=1,
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
layer.g_idx = RowvLLMParameter(
|
||||
data=g_idx.clone(),
|
||||
input_dim=0,
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
|
||||
|
||||
def _random_g_idx(generator):
|
||||
return torch.randint(
|
||||
0,
|
||||
_MARLIN_SIZE_K // _MARLIN_GROUP_SIZE,
|
||||
(_MARLIN_SIZE_K,),
|
||||
dtype=torch.int32,
|
||||
generator=generator,
|
||||
)
|
||||
|
||||
|
||||
def test_marlin_post_load_preserves_runtime_tensor_addresses(monkeypatch, dist_init):
|
||||
"""Marlin workspace and act-order sort indices must be recomputed into
|
||||
the same storage when weights are reloaded (RL weight sync), so device
|
||||
addresses captured by CUDA graphs remain valid."""
|
||||
from vllm.model_executor.layers.quantization.utils import marlin_utils
|
||||
|
||||
"""Marlin must reuse the workspace storage captured by CUDA graphs."""
|
||||
_stub_marlin_ops(monkeypatch)
|
||||
kernel = _make_act_order_marlin_kernel()
|
||||
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
first_g_idx = _random_g_idx(generator)
|
||||
second_g_idx = _random_g_idx(generator)
|
||||
kernel = _make_marlin_kernel()
|
||||
|
||||
layer = torch.nn.Module()
|
||||
_load_marlin_checkpoint_format_weights(layer, first_g_idx)
|
||||
_load_marlin_checkpoint_format_weights(layer)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
workspace_ptr = kernel.workspace.data_ptr()
|
||||
sort_indices_ptr = layer.g_idx_sort_indices.data_ptr()
|
||||
|
||||
# Reload: fresh checkpoint-format tensors with a different act-order
|
||||
_load_marlin_checkpoint_format_weights(layer, second_g_idx)
|
||||
_load_marlin_checkpoint_format_weights(layer)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
assert kernel.workspace.data_ptr() == workspace_ptr
|
||||
assert torch.all(kernel.workspace == 0)
|
||||
assert layer.g_idx_sort_indices.data_ptr() == sort_indices_ptr
|
||||
expected_sort_indices = marlin_utils.marlin_sort_g_idx(second_g_idx)[1]
|
||||
assert torch.equal(layer.g_idx_sort_indices.data, expected_sort_indices)
|
||||
# registered as a Parameter so layerwise reload copy-back preserves it
|
||||
assert isinstance(layer.g_idx_sort_indices, torch.nn.Parameter)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("variant", ["fp8", "mxfp8", "nvfp4"])
|
||||
@@ -394,7 +361,7 @@ def test_marlin_prepare_layer_preserves_workspace_address(monkeypatch, variant):
|
||||
monkeypatch.setattr(
|
||||
ops,
|
||||
"gptq_marlin_repack",
|
||||
lambda b_q_weight, perm, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros(
|
||||
lambda b_q_weight, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros(
|
||||
size_k // 16, size_n * 2, dtype=torch.int32
|
||||
),
|
||||
)
|
||||
@@ -476,68 +443,6 @@ def test_marlin_make_workspace_new_rejects_incompatible_existing(monkeypatch):
|
||||
)
|
||||
|
||||
|
||||
def test_marlin_act_order_layerwise_reload_accounting(monkeypatch, dist_init):
|
||||
"""`g_idx_sort_indices` is generated during weight processing and never
|
||||
loaded from checkpoints. Registering it as a Parameter must not count it
|
||||
toward `load_numel_total`: reload restores the construction-time tensor
|
||||
set before sizing, so act-order layers still process during streaming
|
||||
instead of deferring (and buffering weights) until finalization."""
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import marlin_utils
|
||||
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
|
||||
|
||||
_stub_marlin_ops(monkeypatch)
|
||||
kernel = _make_act_order_marlin_kernel()
|
||||
|
||||
class _KernelQuantMethod(QuantizeMethodBase):
|
||||
def create_weights(self, layer, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def apply(self, layer, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
layer = torch.nn.Module()
|
||||
layer.quant_method = _KernelQuantMethod()
|
||||
_load_marlin_checkpoint_format_weights(layer, _random_g_idx(generator))
|
||||
|
||||
# Metadata is recorded at model construction, before any processing
|
||||
record_metadata_for_reloading(layer)
|
||||
checkpoint_numel = sum(t.numel() for t in get_layer_tensors(layer).values())
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
sort_indices = layer.g_idx_sort_indices
|
||||
|
||||
initialize_layerwise_reload(layer)
|
||||
info = get_layerwise_info(layer)
|
||||
assert info.load_numel_total == checkpoint_numel
|
||||
|
||||
# Stream a new checkpoint; the layer must process as soon as its last
|
||||
# tensor arrives
|
||||
new_g_idx = _random_g_idx(generator)
|
||||
checkpoint = {
|
||||
"qweight": torch.zeros(_MARLIN_SIZE_K // 8, _MARLIN_SIZE_N, dtype=torch.int32),
|
||||
"scales": torch.ones(
|
||||
_MARLIN_SIZE_K // _MARLIN_GROUP_SIZE, _MARLIN_SIZE_N, dtype=torch.float16
|
||||
),
|
||||
"g_idx": new_g_idx,
|
||||
}
|
||||
for name, weight in checkpoint.items():
|
||||
param = getattr(layer, name)
|
||||
param.weight_loader(param, weight)
|
||||
|
||||
assert not info.can_load()
|
||||
assert not info.loaded_weights
|
||||
assert layer.g_idx_sort_indices is sort_indices
|
||||
expected_sort_indices = marlin_utils.marlin_sort_g_idx(new_g_idx)[1]
|
||||
assert torch.equal(layer.g_idx_sort_indices.data, expected_sort_indices)
|
||||
|
||||
|
||||
def test_model_cleanup(dist_init, default_vllm_config):
|
||||
layer = QKVParallelLinear(2, 3, 4)
|
||||
assert layer.weight.weight_loader.__self__ is layer
|
||||
|
||||
@@ -160,6 +160,17 @@ class TestMaybeRemapKvScaleName:
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_checkpoint_weight_mapper_discards_serialized_g_idx():
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
|
||||
mapper = QuantizationConfig.get_checkpoint_weight_mapper()
|
||||
|
||||
assert mapper._map_name("model.layers.0.g_idx") is None
|
||||
assert mapper._map_name("model.layers.0.qweight") == "model.layers.0.qweight"
|
||||
|
||||
|
||||
class TestKvCacheScaleMapper:
|
||||
"""The `WeightsMapper` returned by `get_cache_scale_mapper` replaces the
|
||||
per-model `maybe_remap_kv_scale_name` calls. It must remap the same set of
|
||||
|
||||
@@ -22,12 +22,10 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
||||
MAX_MODEL_LEN = 1024
|
||||
|
||||
MODELS = [
|
||||
# act_order==True, group_size=128
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "main"),
|
||||
# 8-bit, act_order==True, group_size=channelwise
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "main"),
|
||||
# desc_act=True is a no-op for channelwise quantization.
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit--1g-actorder_True"),
|
||||
# 4-bit, act_order==True, group_size=128
|
||||
("TechxGenus/gemma-1.1-2b-it-GPTQ", "main"),
|
||||
("TechxGenus/gemma-2b-it-GPTQ", "main"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -243,7 +243,7 @@ def test_distributed(
|
||||
"model, quantization_kwargs",
|
||||
[
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {}),
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {}),
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", {}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
|
||||
@@ -23,12 +23,13 @@ from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQLinearMethod,
|
||||
AutoGPTQMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.gptq_utils import override_config
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
PROMPT = "On the surface of Mars, we found"
|
||||
|
||||
MODELS = [
|
||||
"TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",
|
||||
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
|
||||
]
|
||||
|
||||
|
||||
@@ -70,6 +71,54 @@ def test_auto_gptq_config_get_name():
|
||||
assert AutoGPTQConfig.get_name() == "auto_gptq"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dynamic",
|
||||
[
|
||||
{},
|
||||
{r"+:model\.layers\.0\..*": {"desc_act": True}},
|
||||
{
|
||||
r"+:model\.layers\.0\..*": {
|
||||
"desc_act": True,
|
||||
"group_size": 32,
|
||||
}
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_auto_gptq_rejects_group_activation_order(dynamic):
|
||||
desc_act = not dynamic
|
||||
with pytest.raises(ValueError, match="group activation ordering"):
|
||||
AutoGPTQConfig(4, 128, desc_act, True, False, dynamic, {})
|
||||
|
||||
|
||||
def test_auto_gptq_normalizes_channelwise_activation_order():
|
||||
config = AutoGPTQConfig(4, -1, True, True, False, {}, {})
|
||||
assert not config.desc_act
|
||||
|
||||
config = AutoGPTQConfig(
|
||||
4,
|
||||
128,
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
{r"+:model\.layers\.0\..*": {"desc_act": True, "group_size": -1}},
|
||||
{},
|
||||
)
|
||||
override_config(config, "model.layers.0.self_attn.q_proj")
|
||||
assert config.group_size == -1
|
||||
assert not config.desc_act
|
||||
|
||||
with pytest.raises(ValueError, match="group activation ordering"):
|
||||
AutoGPTQConfig(
|
||||
4,
|
||||
-1,
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
{r"+:model\.layers\.0\..*": {"group_size": 128}},
|
||||
{},
|
||||
)
|
||||
|
||||
|
||||
def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
|
||||
method = object.__new__(AutoGPTQMoEMethod)
|
||||
method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
|
||||
@@ -84,7 +133,6 @@ def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
|
||||
hidden_size=8,
|
||||
intermediate_size_per_partition=4,
|
||||
params_dtype=torch.float16,
|
||||
intermediate_size_full=4,
|
||||
weight_loader=lambda *args, **kwargs: None,
|
||||
)
|
||||
|
||||
|
||||
@@ -1537,8 +1537,7 @@ def test_inc_ark_linear_method_xpu_int2_create_weights(monkeypatch) -> None:
|
||||
assert layer.scales.dtype == torch.bfloat16
|
||||
assert layer.qzeros.shape == (1, 4)
|
||||
assert layer.qzeros.dtype == torch.int32
|
||||
assert layer.g_idx.shape == (64,)
|
||||
assert layer.g_idx.dtype == torch.int32
|
||||
assert not hasattr(layer, "g_idx")
|
||||
assert layer.in_features == 64
|
||||
assert layer.out_features == 64
|
||||
assert layer.params_dtype == torch.bfloat16
|
||||
@@ -2444,7 +2443,7 @@ def test_calls_kernel_at_threshold(monkeypatch, w4a8_layer) -> None:
|
||||
w_scale,
|
||||
w_zp,
|
||||
group_size,
|
||||
g_idx,
|
||||
group_idx,
|
||||
bias,
|
||||
) = captured["gemm_args"]
|
||||
assert quant_x.dtype is torch.int8
|
||||
@@ -2456,7 +2455,7 @@ def test_calls_kernel_at_threshold(monkeypatch, w4a8_layer) -> None:
|
||||
assert qweight is layer.qweight
|
||||
assert w_zp is layer.qzeros
|
||||
assert group_size == 128
|
||||
assert g_idx is None
|
||||
assert group_idx is None
|
||||
assert bias is None
|
||||
|
||||
# The kernel emits fp16; the result is cast back to the activation dtype.
|
||||
|
||||
@@ -84,6 +84,48 @@ def enable_pickle(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
|
||||
def _compressed_tensors_actorder_config(actorder):
|
||||
return {
|
||||
"format": "pack-quantized",
|
||||
"config_groups": {
|
||||
"group_0": {
|
||||
"targets": ["Linear"],
|
||||
"weights": {
|
||||
"num_bits": 4,
|
||||
"type": "int",
|
||||
"strategy": "group",
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
"dynamic": False,
|
||||
"actorder": actorder,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"actorder", [ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC, True]
|
||||
)
|
||||
def test_compressed_tensors_rejects_runtime_activation_order(actorder):
|
||||
with pytest.raises(ValueError, match="group/dynamic activation ordering"):
|
||||
CompressedTensorsConfig.from_config(
|
||||
_compressed_tensors_actorder_config(actorder)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"actorder", [ActivationOrdering.STATIC, ActivationOrdering.WEIGHT, False, None]
|
||||
)
|
||||
def test_compressed_tensors_accepts_preordered_weights(actorder):
|
||||
config = CompressedTensorsConfig.from_config(
|
||||
_compressed_tensors_actorder_config(actorder)
|
||||
)
|
||||
assert config.target_scheme_map["Linear"]["weights"].actorder == (
|
||||
actorder.value if isinstance(actorder, ActivationOrdering) else None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_args",
|
||||
[
|
||||
@@ -271,15 +313,6 @@ def test_compressed_tensors_w8a8_dynamic_per_token(
|
||||
None,
|
||||
8,
|
||||
True,
|
||||
False,
|
||||
),
|
||||
(
|
||||
"nm-testing/TinyLlama-1.1B-Chat-v1.0-W4A16-G128-Asym-Updated-ActOrder",
|
||||
"group",
|
||||
128,
|
||||
8,
|
||||
False,
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
@@ -287,7 +320,7 @@ def test_compressed_tensors_w8a8_dynamic_per_token(
|
||||
not current_platform.is_cuda(), reason="The tests are skipped on non-CUDA platform."
|
||||
)
|
||||
def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
|
||||
model, strategy, group, pack_factor, symmetric, has_g_idx = wNa16_args
|
||||
model, strategy, group, pack_factor, symmetric = wNa16_args
|
||||
with vllm_runner(model, enforce_eager=True) as llm:
|
||||
|
||||
def check_model(model):
|
||||
@@ -302,7 +335,6 @@ def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
|
||||
|
||||
assert qkv_proj.scheme.pack_factor == pack_factor
|
||||
assert qkv_proj.scheme.symmetric == symmetric
|
||||
assert qkv_proj.scheme.has_g_idx == has_g_idx
|
||||
|
||||
llm.apply_model(check_model)
|
||||
|
||||
@@ -926,35 +958,6 @@ def test_compressed_tensors_mxfp8_moe_setup(vllm_runner):
|
||||
assert output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"actorder,group_size,part,full,expected",
|
||||
[
|
||||
# actorder="group" with real grouping: must load full-K w2 scales and,
|
||||
# when sharded (part != full), report is_k_full=False.
|
||||
(ActivationOrdering.GROUP, 32, 64, 128, (True, 128, False)),
|
||||
# actorder="group" but unsharded (part == full): full scales, k_full.
|
||||
(ActivationOrdering.GROUP, 32, 128, 128, (True, 128, True)),
|
||||
# actorder="group" with channel-wise (group_size == -1): no full load.
|
||||
(ActivationOrdering.GROUP, -1, 64, 128, (False, 64, False)),
|
||||
# "static"/"weight" reorder at quant time -> shard normally + k_full.
|
||||
# Regression: static actorder under TP must keep is_k_full=True so the
|
||||
# Marlin kernel never gets the invalid (group_size=16, is_k_full=0).
|
||||
("static", 32, 64, 128, (False, 64, True)),
|
||||
("weight", 32, 64, 128, (False, 64, True)),
|
||||
(None, 32, 64, 128, (False, 64, True)),
|
||||
],
|
||||
)
|
||||
def test_wna16_moe_w2_scale_sharding(actorder, group_size, part, full, expected):
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
|
||||
CompressedTensorsWNA16MoEMethod,
|
||||
)
|
||||
|
||||
result = CompressedTensorsWNA16MoEMethod._w2_scale_sharding(
|
||||
actorder, group_size, part, full
|
||||
)
|
||||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_bits", range(2, 9))
|
||||
def test_humming_supports_compressed_tensors_wna16_quant_key(num_bits):
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
|
||||
@@ -9,8 +9,7 @@ if not current_platform.is_cpu():
|
||||
|
||||
MODELS = [
|
||||
"TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ",
|
||||
"TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", # with g_idx
|
||||
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", # without g_idx
|
||||
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4",
|
||||
"RedHatAI/Qwen3-1.7B-quantized.w4a16", # with zp
|
||||
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
|
||||
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear
|
||||
|
||||
@@ -64,7 +64,6 @@ def test_machete_kernel_selected(act_type, weight_type, group_size, zero_points)
|
||||
weight_type=weight_type,
|
||||
group_size=group_size,
|
||||
zero_points=zero_points,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = choose_mp_linear_kernel(config)
|
||||
assert kernel is MacheteLinearKernel, (
|
||||
@@ -73,23 +72,21 @@ def test_machete_kernel_selected(act_type, weight_type, group_size, zero_points)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"full_shape,part_shape,weight_type,group_size,has_g_idx,expected_reason",
|
||||
"full_shape,part_shape,weight_type,group_size,expected_reason",
|
||||
[
|
||||
((4096, 4096), (2048, 4096), scalar_types.uint4b8, 128, True, "Act reordering"),
|
||||
(
|
||||
(4096, 4096),
|
||||
(4096, 4096),
|
||||
scalar_types.float6_e3m2f,
|
||||
128,
|
||||
False,
|
||||
"Quant type",
|
||||
),
|
||||
((4096, 4096), (4096, 4096), scalar_types.uint4b8, 32, False, "Group size"),
|
||||
((4096, 4096), (4096, 4096), scalar_types.uint4b8, 32, "Group size"),
|
||||
],
|
||||
ids=["partitioned-g_idx", "unsupported-quant-type", "unsupported-group-size"],
|
||||
ids=["unsupported-quant-type", "unsupported-group-size"],
|
||||
)
|
||||
def test_machete_rejects_invalid_config(
|
||||
full_shape, part_shape, weight_type, group_size, has_g_idx, expected_reason
|
||||
full_shape, part_shape, weight_type, group_size, expected_reason
|
||||
):
|
||||
"""Verify Machete rejects unsupported configurations."""
|
||||
config = MPLinearLayerConfig(
|
||||
@@ -99,7 +96,6 @@ def test_machete_rejects_invalid_config(
|
||||
weight_type=weight_type,
|
||||
group_size=group_size,
|
||||
zero_points=False,
|
||||
has_g_idx=has_g_idx,
|
||||
)
|
||||
can_impl, reason = MacheteLinearKernel.can_implement(config)
|
||||
assert not can_impl
|
||||
@@ -117,7 +113,6 @@ def test_kernel_selection_with_disabled_machete(monkeypatch):
|
||||
weight_type=scalar_types.uint4b8,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
has_g_idx=False,
|
||||
)
|
||||
kernel = choose_mp_linear_kernel(config)
|
||||
assert kernel is not MacheteLinearKernel, "MacheteLinearKernel should be disabled"
|
||||
@@ -127,7 +122,6 @@ def test_kernel_selection_with_disabled_machete(monkeypatch):
|
||||
"model_name",
|
||||
[
|
||||
"nm-testing/tinyllama-oneshot-w4a16-channel-v2",
|
||||
"nm-testing/TinyLlama-1.1B-Chat-v1.0-W4A16-G128-Asym-Updated-ActOrder",
|
||||
],
|
||||
)
|
||||
def test_w4a16_machete_e2e(vllm_runner, model_name):
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests whether gptq models with dynamic quantized can be loaded.
|
||||
"""Tests whether unsupported GPTQ group actorder models are rejected.
|
||||
|
||||
Run `pytest tests/quantization/test_gptq_dynamic.py --forked`.
|
||||
|
||||
@@ -8,59 +8,16 @@ Note: Only symmetric GPTQ models are supported after consolidation to Marlin.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import load_model_without_vllm_runner
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
|
||||
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
|
||||
get_dynamic_override,
|
||||
)
|
||||
|
||||
PROMPT = "On the surface of Mars, we found"
|
||||
|
||||
# The first layer is quantized using bits=4, group_size=128
|
||||
# The second layer is quantized using bits=8, group_size=32
|
||||
# All other layers (layer index >= 2) are not quantized
|
||||
# Note: Only symmetric models are supported with Marlin kernels
|
||||
MODELS = [
|
||||
"ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", MODELS)
|
||||
def test_gptq_with_dynamic(model_id: str, monkeypatch, dist_init, workspace_init):
|
||||
# `LLM.apply_model` requires pickling a function.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
linear_method_cls = AutoGPTQLinearMethod
|
||||
|
||||
model, _ = load_model_without_vllm_runner(
|
||||
model_id,
|
||||
dtype=torch.float16,
|
||||
model_config_kwargs={
|
||||
"max_model_len": 2048,
|
||||
"hf_overrides": {"num_hidden_layers": 3},
|
||||
},
|
||||
)
|
||||
|
||||
for name, submodule in model.named_modules():
|
||||
if name == "lm_head":
|
||||
assert isinstance(submodule.quant_method, linear_method_cls)
|
||||
elif name == "model.layers.0.self_attn.qkv_proj":
|
||||
assert isinstance(submodule.quant_method, linear_method_cls)
|
||||
config = submodule.quant_method.quant_config
|
||||
assert config.weight_bits == 4
|
||||
assert config.group_size == 128
|
||||
assert config.desc_act
|
||||
elif name == "model.layers.1.self_attn.qkv_proj":
|
||||
assert isinstance(submodule.quant_method, linear_method_cls)
|
||||
config = submodule.quant_method.quant_config
|
||||
assert get_dynamic_override(config, layer_name=name, key="bits") == 8
|
||||
assert get_dynamic_override(config, layer_name=name, key="group_size") == 32
|
||||
assert not get_dynamic_override(config, layer_name=name, key="desc_act")
|
||||
elif name in (
|
||||
"model.layers.2.self_attn.qkv_proj",
|
||||
"model.layers.2.mlp.gate_up_proj",
|
||||
):
|
||||
assert isinstance(submodule.quant_method, UnquantizedLinearMethod)
|
||||
def test_gptq_with_dynamic_desc_act_rejected(vllm_runner, model_id: str):
|
||||
with (
|
||||
pytest.raises(ValueError, match="desc_act=True"),
|
||||
vllm_runner(model_id, max_model_len=2048, enforce_eager=True),
|
||||
):
|
||||
pass
|
||||
|
||||
@@ -17,8 +17,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
PROMPT = "On the surface of Mars, we found"
|
||||
|
||||
MODELS_QUANT = [
|
||||
("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head", True),
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", False),
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", False),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.quantization import (
|
||||
ActivationOrdering,
|
||||
QuantizationArgs,
|
||||
QuantizationStrategy,
|
||||
QuantizationType,
|
||||
@@ -33,6 +32,35 @@ def test_map_wna16_backend_supports_triton():
|
||||
assert map_wna16_backend("triton") == WNA16MoEBackend.TRITON
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config",
|
||||
[
|
||||
{"desc_act": True, "group_size": 128},
|
||||
{
|
||||
"desc_act": False,
|
||||
"group_size": 128,
|
||||
"dynamic": {r"+:model\.layers\.0\..*": {"desc_act": True}},
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_moe_wna16_rejects_gptq_group_activation_order(config):
|
||||
config.update({"quant_method": "gptq", "bits": 4, "sym": True})
|
||||
with pytest.raises(ValueError, match="group activation ordering"):
|
||||
MoeWNA16Config.from_config(config)
|
||||
|
||||
|
||||
def test_moe_wna16_accepts_channelwise_gptq_activation_order():
|
||||
config = {
|
||||
"quant_method": "gptq",
|
||||
"bits": 4,
|
||||
"group_size": -1,
|
||||
"desc_act": True,
|
||||
"sym": True,
|
||||
}
|
||||
assert MoeWNA16Config.is_moe_wna16_compatible(config)
|
||||
MoeWNA16Config.from_config(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "quant_config", "may_have_zp", "may_have_bias", "expected"),
|
||||
[
|
||||
@@ -43,28 +71,6 @@ def test_map_wna16_backend_supports_triton():
|
||||
False,
|
||||
"AutoAWQ weight layout",
|
||||
),
|
||||
(
|
||||
WNA16MoEBackend.TRITON,
|
||||
AutoGPTQConfig(4, 128, True, True, False, {}, {}),
|
||||
False,
|
||||
False,
|
||||
"activation ordering",
|
||||
),
|
||||
(
|
||||
WNA16MoEBackend.TRITON,
|
||||
QuantizationArgs(
|
||||
num_bits=4,
|
||||
type=QuantizationType.INT,
|
||||
strategy=QuantizationStrategy.GROUP,
|
||||
symmetric=True,
|
||||
dynamic=False,
|
||||
group_size=128,
|
||||
actorder=ActivationOrdering.GROUP,
|
||||
),
|
||||
False,
|
||||
False,
|
||||
"activation ordering",
|
||||
),
|
||||
(
|
||||
WNA16MoEBackend.TRITON,
|
||||
AutoGPTQConfig(4, 128, False, True, False, {}, {}),
|
||||
@@ -244,7 +250,6 @@ def test_compressed_tensors_wna16_moe_create_weights_uses_ceil_packed_shapes(
|
||||
hidden_size=256,
|
||||
intermediate_size_per_partition=512,
|
||||
params_dtype=torch.float16,
|
||||
intermediate_size_full=512,
|
||||
)
|
||||
|
||||
packed_hidden = (256 * num_bits + 31) // 32
|
||||
@@ -296,7 +301,6 @@ def test_compressed_tensors_wna16_moe_converts_and_sets_up_humming_kernel():
|
||||
num_experts=2,
|
||||
hidden_size=256,
|
||||
intermediate_size_per_partition=512,
|
||||
intermediate_size_full=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
layer.cuda()
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
|
||||
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
|
||||
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
|
||||
compressed-tensors, nm-testing/test-w4a16-mixtral-actorder-group, main
|
||||
gptq, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
|
||||
gptq, TheBloke/Mixtral-8x7B-v0.1-GPTQ, gptq-8bit-128g-actorder_True
|
||||
awq_marlin, casperhansen/deepseek-coder-v2-instruct-awq, main
|
||||
compressed-tensors, RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16, main
|
||||
compressed-tensors, RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16, main
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main
|
||||
gptq, TheBloke/Llama-2-7B-GPTQ, main
|
||||
gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
|
||||
gptq, LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit, main
|
||||
gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True
|
||||
gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit-32g-actorder_True
|
||||
gptq, TechxGenus/gemma-1.1-2b-it-GPTQ, main
|
||||
gptq, TechxGenus/gemma-2b-it-GPTQ, main
|
||||
compressed-tensors, nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change, main
|
||||
compressed-tensors, nm-testing/tinyllama-oneshot-w8-channel-a8-tensor, main
|
||||
compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2, main
|
||||
@@ -16,4 +15,4 @@ compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
|
||||
#compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
|
||||
awq, casperhansen/mixtral-instruct-awq, main
|
||||
awq_marlin, casperhansen/mixtral-instruct-awq, main
|
||||
fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
|
||||
fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
|
||||
|
||||
@@ -34,10 +34,7 @@ def test_weight_loading(vllm_runner):
|
||||
"""
|
||||
|
||||
# MoE models need fp16.
|
||||
NEEDS_FP16 = (
|
||||
QUANTIZATION == "gptq"
|
||||
or MODEL_NAME == "nm-testing/test-w4a16-mixtral-actorder-group"
|
||||
)
|
||||
NEEDS_FP16 = QUANTIZATION == "gptq"
|
||||
with vllm_runner(
|
||||
model_name=MODEL_NAME,
|
||||
revision=REVISION,
|
||||
|
||||
Reference in New Issue
Block a user