[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)

Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com>
Signed-off-by: Roderick Wu <roderickwu2003@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Roderick Wu
2026-09-08 12:45:12 -07:00
committed by GitHub
co-authored by mgoin
parent 1a522b6949
commit 2c9d68f80b
99 changed files with 1182 additions and 3917 deletions
+1 -1
View File
@@ -30,7 +30,7 @@ def models_list(*, all: bool = True, keywords: list[str] | None = None):
if is_quant_method_supported("gptq_marlin"):
TEST_MODELS.append(
(
"TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
{"quantization": "gptq_marlin"},
)
)
-21
View File
@@ -1,21 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from tests.kernels.utils import opcheck
from vllm._custom_ops import permute_cols
if not hasattr(torch.ops._C, "permute_cols"):
pytest.skip(reason="permute_cols is not supported on ROCm", allow_module_level=True)
@pytest.mark.parametrize("shape", [(1, 512), (544, 4096), (67, 8192)])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
def test_permute_cols(shape, dtype):
x = torch.randn(shape, dtype=dtype).cuda()
perm = torch.randperm(x.shape[1]).to(torch.int).cuda()
opcheck(torch.ops._C.permute_cols, (x, perm))
y = permute_cols(x, perm)
torch.testing.assert_close(y, x[:, perm])
@@ -113,9 +113,7 @@ def marlin_quantize_moe_weights(
for i in range(e):
# Transpose for Marlin: [n, k] → [k, n]
w_t = weights_bf16[i].T.contiguous()
_, w_q, w_s, _, _, _ = marlin_quantize(
w_t, scalar_types.uint4b8, group_size, act_order=False
)
_, w_q, w_s = marlin_quantize(w_t, scalar_types.uint4b8, group_size)
weight_list.append(w_q)
scale_list.append(w_s)
@@ -248,16 +246,11 @@ def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group
expert_map=None,
global_scale1=None,
global_scale2=None,
g_idx1=None,
g_idx2=None,
input_global_scale1=None,
input_global_scale2=None,
sort_indices1=None,
sort_indices2=None,
w1_zeros=None,
w2_zeros=None,
input_dtype=dtype,
is_k_full=True,
)
# Sanity check: manually compute BF16 reference for comparison
+13 -85
View File
@@ -178,13 +178,11 @@ MOE_MARLIN_QUANT_TEST_CONFIGS = [
# GPTQ-INT4
{
"b_type": scalar_types.uint4b8,
"support_act_order": True,
"group_blocks": [-1, 2, 4, 8],
},
# GPTQ-INT8
{
"b_type": scalar_types.uint8b128,
"support_act_order": True,
"group_blocks": [-1, 2, 4, 8],
},
# FP8
@@ -794,27 +792,22 @@ def test_fused_moe_wn16(
MARLIN_MOE_SCENARIOS = [
# (m, n, k, e, topk, ep_size, act_order, is_k_full)
# No act_order: is_k_full=True matches usual case (marlin_is_k_full).
# (m, n, k, e, topk, ep_size)
# N>=256 required for Marlin kernel thread config for MXFP8.
# Single token, small matrices
(1, 128, 256, 5, 2, 1, False, True),
(1, 128, 256, 5, 2, 1),
# Single token, large matrices
(1, 1024, 2048, 5, 2, 1, False, True),
(1, 1024, 2048, 5, 2, 1),
# Unaligned m, small matrices
(133, 256, 256, 5, 2, 1, False, True),
(133, 256, 256, 5, 2, 1),
# Unaligned m, large matrices
(133, 1024, 2048, 12, 3, 1, False, True),
(133, 1024, 2048, 12, 3, 1),
# Aligned batch, small matrices
(128, 256, 256, 5, 2, 1, False, True),
(128, 256, 256, 5, 2, 1),
# Aligned batch, large matrices
(128, 1024, 2048, 12, 3, 1, False, True),
(128, 1024, 2048, 12, 3, 1),
# Expert parallelism
(64, 1024, 2048, 12, 3, 4, False, True),
# Act order with is_k_full=True (no tensor parallelism)
(1, 1024, 2048, 5, 2, 1, True, True),
# Act order with is_k_full=False (tensor parallelism)
(133, 256, 256, 5, 2, 1, True, False),
(64, 1024, 2048, 12, 3, 4),
]
@@ -832,17 +825,13 @@ def marlin_moe_generate_valid_test_cases():
e,
topk,
ep_size,
act_order,
is_k_full,
):
group_size = group_blocks if group_blocks <= 0 else group_blocks * 16
if group_size > 0 and k % group_size != 0:
return False
if act_order and group_size in [-1, k, n]:
return False
if group_size in [k, n]:
return False
if b_type == scalar_types.float8_e4m3fn and group_size == 32 and is_k_full:
if b_type == scalar_types.float8_e4m3fn and group_size == 32:
return False
return a_type.size_bits < 16 or a_type is c_type
@@ -858,8 +847,6 @@ def marlin_moe_generate_valid_test_cases():
)
)
supports_act_order = quant_test_config.get("support_act_order", False)
for sub_case in inner_combinations:
if (
sub_case[0] == scalar_types.float8_e4m3fn
@@ -869,10 +856,8 @@ def marlin_moe_generate_valid_test_cases():
continue
for scenario in MARLIN_MOE_SCENARIOS:
m, n, k, e, topk, ep_size, act_order, is_k_full = scenario
if act_order and not supports_act_order:
continue
args = sub_case + (m, n, k, e, topk, ep_size, act_order, is_k_full)
m, n, k, e, topk, ep_size = scenario
args = sub_case + (m, n, k, e, topk, ep_size)
if is_valid(*args):
cases.append(args)
return cases
@@ -885,9 +870,7 @@ class MarlinMoEWeightData:
scales: torch.Tensor
global_scale: torch.Tensor | None
a_scales_factor: torch.Tensor | None
g_idx: torch.Tensor | None
zeros: torch.Tensor | None
sort_indices: torch.Tensor | None
marlin_bias: torch.Tensor | None
@staticmethod
@@ -895,15 +878,12 @@ class MarlinMoEWeightData:
w: torch.Tensor,
quant_type: ScalarType,
group_size: int,
act_order: bool | None = None,
bias: torch.Tensor | None = None,
input_type: ScalarType = None,
) -> "MarlinMoEWeightData":
assert w.ndim == 3
has_zp = quant_type in [scalar_types.uint4, scalar_types.uint8]
k = w.shape[-1]
if input_type == scalar_types.int8:
input_dtype = torch.int8
elif input_type == scalar_types.float8_e4m3fn:
@@ -916,8 +896,6 @@ class MarlinMoEWeightData:
scales_l: list[torch.Tensor] = []
global_scale_l: list[torch.Tensor] = []
zeros_l: list[torch.Tensor] = []
g_idx_l: list[torch.Tensor] = []
sort_indices_l: list[torch.Tensor] = []
bias_l: list[torch.Tensor] = []
for i in range(w.shape[0]):
@@ -959,21 +937,16 @@ class MarlinMoEWeightData:
scales_l.append(scales)
zeros_l.append(zeros)
else:
test_perm = torch.randperm(k)
w_ref, qweight, scales, g_idx, sort_indices, _ = marlin_quantize(
w_ref, qweight, scales = marlin_quantize(
w[i].transpose(1, 0),
quant_type,
group_size,
act_order,
test_perm,
input_dtype=input_dtype,
)
w_ref_l.append(w_ref.T)
qweight_l.append(qweight)
scales_l.append(scales)
g_idx_l.append(g_idx)
sort_indices_l.append(sort_indices)
if bias is not None:
bias_l.append(marlin_permute_bias(bias[i]))
@@ -982,9 +955,7 @@ class MarlinMoEWeightData:
qweight = stack_and_dev(qweight_l).contiguous()
scales = stack_and_dev(scales_l)
global_scale = stack_and_dev(global_scale_l) if global_scale_l else None
g_idx = stack_and_dev(g_idx_l) if g_idx_l else None
zeros = stack_and_dev(zeros_l) if zeros_l else None
sort_indices = stack_and_dev(sort_indices_l) if sort_indices_l else None
marlin_bias = stack_and_dev(bias_l) if bias_l else None
a_scales_factor = None
@@ -999,19 +970,14 @@ class MarlinMoEWeightData:
scales=scales,
global_scale=global_scale,
a_scales_factor=a_scales_factor,
g_idx=g_idx,
zeros=zeros,
sort_indices=sort_indices,
marlin_bias=marlin_bias,
)
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize(
(
"a_type, b_type, c_type, group_blocks,"
"m, n, k, e, topk, ep_size, act_order, is_k_full"
),
("a_type, b_type, c_type, group_blocks,m, n, k, e, topk, ep_size"),
marlin_moe_generate_valid_test_cases(),
)
@pytest.mark.skipif(current_platform.is_rocm(), reason="Skip for rocm")
@@ -1027,8 +993,6 @@ def test_fused_marlin_moe(
e: int,
topk: int,
ep_size: int,
act_order: bool,
is_k_full: bool,
):
set_random_seed(1)
group_size = group_blocks if group_blocks <= 0 else group_blocks * 16
@@ -1065,7 +1029,6 @@ def test_fused_marlin_moe(
w=w1,
quant_type=b_type,
group_size=group_size,
act_order=act_order,
input_type=a_type,
)
@@ -1073,7 +1036,6 @@ def test_fused_marlin_moe(
w=w2,
quant_type=b_type,
group_size=group_size,
act_order=act_order,
input_type=a_type,
)
@@ -1127,17 +1089,12 @@ def test_fused_marlin_moe(
expert_map=e_map,
global_scale1=w1_data.global_scale,
global_scale2=w2_data.global_scale,
g_idx1=w1_data.g_idx,
g_idx2=w2_data.g_idx,
input_global_scale1=w1_data.a_scales_factor,
input_global_scale2=w2_data.a_scales_factor,
sort_indices1=w1_data.sort_indices,
sort_indices2=w2_data.sort_indices,
w1_zeros=w1_data.zeros,
w2_zeros=w2_data.zeros,
input_dtype=a_dtype,
quant_type_id=b_type.id,
is_k_full=is_k_full,
activation_func=instance_activation,
)
@@ -1154,8 +1111,6 @@ def test_fused_marlin_moe_with_bias(m):
e, topk = 32, 4
n, k = 2048, 2048
group_size = 128
act_order = False
is_k_full = True
quant_type = scalar_types.uint4b8
dtype = torch.half
@@ -1169,7 +1124,6 @@ def test_fused_marlin_moe_with_bias(m):
w=w1,
quant_type=quant_type,
group_size=group_size,
act_order=act_order,
bias=b_bias1,
)
@@ -1177,7 +1131,6 @@ def test_fused_marlin_moe_with_bias(m):
w=w2,
quant_type=quant_type,
group_size=group_size,
act_order=act_order,
bias=b_bias2,
)
@@ -1204,14 +1157,9 @@ def test_fused_marlin_moe_with_bias(m):
expert_map=None,
global_scale1=w1_data.global_scale,
global_scale2=w2_data.global_scale,
g_idx1=w1_data.g_idx,
g_idx2=w2_data.g_idx,
sort_indices1=w1_data.sort_indices,
sort_indices2=w2_data.sort_indices,
w1_zeros=w1_data.zeros,
w2_zeros=w2_data.zeros,
quant_type_id=quant_type.id,
is_k_full=is_k_full,
)
torch.testing.assert_close(marlin_output, torch_output, atol=5e-2, rtol=0)
@@ -1235,7 +1183,6 @@ def test_fused_marlin_moe_non_gated(
set_random_seed(42)
group_size = 16 # NVFP4 group size
is_k_full = True
quant_type = scalar_types.float4_e2m1f
dtype = torch.bfloat16
@@ -1248,14 +1195,12 @@ def test_fused_marlin_moe_non_gated(
w=w1,
quant_type=quant_type,
group_size=group_size,
act_order=False,
)
w2_data = MarlinMoEWeightData.make(
w=w2,
quant_type=quant_type,
group_size=group_size,
act_order=False,
)
score = torch.randn((m, e), device="cuda", dtype=dtype)
@@ -1286,14 +1231,9 @@ def test_fused_marlin_moe_non_gated(
expert_map=None,
global_scale1=w1_data.global_scale,
global_scale2=w2_data.global_scale,
g_idx1=w1_data.g_idx,
g_idx2=w2_data.g_idx,
sort_indices1=w1_data.sort_indices,
sort_indices2=w2_data.sort_indices,
w1_zeros=w1_data.zeros,
w2_zeros=w2_data.zeros,
quant_type_id=quant_type.id,
is_k_full=is_k_full,
activation=activation,
)
@@ -1612,14 +1552,9 @@ def test_batched_marlin_activation_uses_expert_token_counts(
quant_type_id=0,
a1_gscale=None,
a2_gscale=None,
w13_g_idx=None,
w2_g_idx=None,
w13_g_idx_sort_indices=None,
w2_g_idx_sort_indices=None,
w1_zp=None,
w2_zp=None,
input_dtype=None,
is_k_full=True,
activation_config=ApplyMoEActivationConfig(
clamp_limit=3.0, alpha=1.3, beta=0.5
),
@@ -1982,14 +1917,12 @@ def test_batched_fused_marlin_moe(
w=w1,
quant_type=quant_dtype,
group_size=group_size,
act_order=None,
input_type=input_type,
)
w2_data = MarlinMoEWeightData.make(
w=w2,
quant_type=quant_dtype,
group_size=group_size,
act_order=None,
input_type=input_type,
)
@@ -2101,14 +2034,9 @@ def test_batched_fused_marlin_moe(
"expert_map": None,
"global_scale1": w1_data.global_scale,
"global_scale2": w2_data.global_scale,
"g_idx1": w1_data.g_idx,
"g_idx2": w2_data.g_idx,
"sort_indices1": w1_data.sort_indices,
"sort_indices2": w2_data.sort_indices,
"w1_zeros": w1_data.zeros,
"w2_zeros": w2_data.zeros,
"quant_type_id": quant_dtype.id,
"is_k_full": True,
}
if input_dtype is not None:
kwargs["input_dtype"] = input_dtype
@@ -70,7 +70,7 @@ def test_gptq_allspark_gemm_ampere(mnk_factors, group_size, has_zp, dtype):
input = rand_data((m, k), dtype=dtype)
weight = rand_data((k, n), dtype=dtype)
# Quantize (and apply act_order if provided)
# Quantize weights.
w_ref, qw, s, zp = quantize_weights(
weight, scalar_types.uint8b128, group_size, has_zp
)
+3 -9
View File
@@ -11,9 +11,8 @@ def test_gptq_shuffle_opcheck():
weight = torch.randint(
-2000000, 2000000, (1792, 4096), device="cuda", dtype=torch.int32
)
perm = torch.empty((0,), device="cuda", dtype=torch.int32)
bit = 4
opcheck(torch.ops._C.gptq_shuffle, (weight, perm, bit))
opcheck(torch.ops._C.gptq_shuffle, (weight, bit))
def test_gptq_gemm_opcheck():
@@ -23,13 +22,8 @@ def test_gptq_gemm_opcheck():
)
zeros = torch.zeros((32, 768), device="cuda", dtype=torch.int32)
scales = torch.rand((32, 6144), device="cuda", dtype=torch.float16)
idx = torch.empty((0,), device="cuda", dtype=torch.int32)
use_exllama = True
bit = 4
# Test both GPTQv1 and GPTQv2 format
opcheck(
torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, True, bit)
)
opcheck(
torch.ops._C.gptq_gemm, (a, weight, zeros, scales, idx, use_exllama, False, bit)
)
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, True, bit))
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, False, bit))
+14 -71
View File
@@ -17,7 +17,7 @@ from vllm.model_executor.layers.quantization.utils.int8_utils import (
per_token_quant_int8,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_make_empty_g_idx,
marlin_make_empty,
marlin_make_workspace_new,
marlin_permute_bias,
query_marlin_supported_quant_types,
@@ -40,7 +40,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
gptq_pack,
gptq_quantize_weights,
quantize_weights,
sort_weights,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
@@ -51,8 +50,6 @@ if current_platform.is_rocm():
allow_module_level=True,
)
ACT_ORDER_OPTS = [False, True]
K_FULL_OPTS = [False, True]
USE_ATOMIC_ADD_OPTS = [False, True]
USE_FP32_REDUCE_OPTS = [True]
@@ -80,13 +77,11 @@ DENSE_MARLIN_QUANT_TEST_CONFIGS = [
# GPTQ-INT4
{
"b_type": scalar_types.uint4b8,
"support_act_order": True,
"group_blocks": [-1, 2, 4, 8],
},
# GPTQ-INT8
{
"b_type": scalar_types.uint8b128,
"support_act_order": True,
"group_blocks": [-1, 2, 4, 8],
},
# FP8
@@ -202,27 +197,15 @@ def test_marlin_int4_fp8_preprocess_awq():
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
@pytest.mark.parametrize("quant_type", query_marlin_supported_quant_types(False, False))
@pytest.mark.parametrize("act_order", ACT_ORDER_OPTS)
@pytest.mark.parametrize("is_a_8bit", [True, False])
@pytest.mark.parametrize("nk_factors", MARLIN_REPACK_NK_FACTORS)
def test_gptq_marlin_repack(
k_chunk, n_chunk, quant_type, act_order, is_a_8bit, nk_factors
):
def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, is_a_8bit, nk_factors):
n_factor, k_factor = nk_factors
size_k = k_chunk * k_factor
size_n = n_chunk * n_factor
group_size = 128
# Filter act_order
if act_order:
if group_size == -1:
return
if group_size == size_k:
return
if is_a_8bit:
return
# Normalize group_size
if group_size == -1:
group_size = size_k
@@ -231,20 +214,12 @@ def test_gptq_marlin_repack(
# Create input
b_weight = rand_data((size_k, size_n))
# Quantize (and apply act_order if provided)
w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights(
b_weight, quant_type, group_size, act_order
)
# Quantize
_, q_w, _ = gptq_quantize_weights(b_weight, quant_type, group_size)
# Pack to GPTQ format
q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
# For act_order, sort the "weights" and "g_idx" so that group ids are
# increasing
sort_indices = torch.empty(0, dtype=torch.int, device=b_weight.device)
if act_order:
q_w, g_idx, sort_indices = sort_weights(q_w, g_idx)
# Pack to Marlin format
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
marlin_q_w_1 = marlin_weights(
@@ -253,12 +228,12 @@ def test_gptq_marlin_repack(
opcheck(
torch.ops._C.gptq_marlin_repack,
(q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit),
(q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit),
)
# Run Marlin repack GPU kernel
marlin_q_w_2 = ops.gptq_marlin_repack(
q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits, is_a_8bit
q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit
)
torch.accelerator.synchronize()
@@ -319,8 +294,6 @@ def marlin_generate_valid_test_cases():
MNK_FACTORS,
MARLIN_N_CHUNKS,
MARLIN_K_CHUNKS,
ACT_ORDER_OPTS,
K_FULL_OPTS,
USE_ATOMIC_ADD_OPTS,
USE_FP32_REDUCE_OPTS,
)
@@ -333,8 +306,6 @@ def marlin_generate_valid_test_cases():
size_m,
size_n,
size_k,
act_order,
is_k_full,
use_atomic_add,
use_fp32_reduce,
):
@@ -351,25 +322,18 @@ def marlin_generate_valid_test_cases():
if group_size > 0 and size_k % group_size != 0:
return False
if act_order and group_size in [-1, size_k]:
return False
if group_size == size_k:
return False
if not act_order and is_k_full:
return False
return a_type.size_bits < 16 or a_type is c_type
cases = []
for case in all_combinations:
quant_test_config, mnk_factors, n_chunk, k_chunk, act_order, *_ = case
quant_test_config, mnk_factors, n_chunk, k_chunk, *_ = case
size_m = mnk_factors[0]
size_n = mnk_factors[1] * n_chunk
size_k = mnk_factors[2] * k_chunk
if act_order and not quant_test_config.get("support_act_order", False):
continue
f16_types = [scalar_types.float16, scalar_types.bfloat16]
inner_combinations = itertools.product(
quant_test_config.get("a_type", f16_types),
@@ -398,7 +362,7 @@ def marlin_generate_valid_test_cases():
@pytest.mark.parametrize(
(
"a_type, b_type, c_type, group_blocks,"
"size_m, size_n, size_k, act_order, is_k_full,"
"size_m, size_n, size_k,"
"use_atomic_add, use_fp32_reduce"
),
marlin_generate_valid_test_cases(),
@@ -411,8 +375,6 @@ def test_marlin_gemm(
size_m,
size_n,
size_k,
act_order,
is_k_full,
use_atomic_add,
use_fp32_reduce,
):
@@ -448,27 +410,21 @@ def test_marlin_gemm(
)
marlin_s2 = None
g_idx = None
sort_indices = None
marlin_zp = None
elif b_type == scalar_types.float8_e4m3fn:
w_ref, marlin_q_w, marlin_s = marlin_quant_fp8_torch(
b_weight.T, group_size, input_dtype=a_dtype
)
g_idx = None
sort_indices = None
marlin_zp = None
marlin_s2 = None
elif has_zp:
w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
b_weight, b_type, group_size, input_dtype=a_dtype
)
g_idx = None
sort_indices = None
marlin_s2 = None
else:
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
b_weight, b_type, group_size, act_order, input_dtype=a_dtype
w_ref, marlin_q_w, marlin_s = marlin_quantize(
b_weight, b_type, group_size, input_dtype=a_dtype
)
marlin_zp = None
@@ -506,14 +462,11 @@ def test_marlin_gemm(
a_scales,
marlin_s2,
marlin_zp,
g_idx,
sort_indices,
workspace,
b_type,
a_input.shape[0],
b_weight.shape[1],
a_input.shape[1],
is_k_full=is_k_full,
use_atomic_add=use_atomic_add,
use_fp32_reduce=use_fp32_reduce,
is_zp_float=False,
@@ -535,11 +488,9 @@ def test_marlin_gemm_subset_input():
a_input = rand_data((big_m, big_k))[8 : size_m + 8, 8 : size_k + 8]
b_weight = rand_data((size_k, size_n))
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
b_weight, quant_type, group_size, False
)
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
marlin_zp = marlin_make_empty(marlin_s.device)
workspace = marlin_make_workspace_new(a_input.device)
output = ops.marlin_gemm(
@@ -551,14 +502,11 @@ def test_marlin_gemm_subset_input():
None,
None,
marlin_zp,
g_idx,
sort_indices,
workspace,
quant_type,
a_input.shape[0],
b_weight.shape[1],
a_input.shape[1],
is_k_full=True,
use_atomic_add=False,
use_fp32_reduce=True,
is_zp_float=False,
@@ -584,11 +532,9 @@ def test_marlin_gemm_with_bias(size_m):
marlin_bias = marlin_permute_bias(b_bias)
w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize(
b_weight, quant_type, group_size, False
)
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
marlin_zp = marlin_make_empty_g_idx(marlin_s.device)
marlin_zp = marlin_make_empty(marlin_s.device)
workspace = marlin_make_workspace_new(a_input.device)
output = ops.marlin_gemm(
@@ -600,14 +546,11 @@ def test_marlin_gemm_with_bias(size_m):
None,
None,
marlin_zp,
g_idx,
sort_indices,
workspace,
quant_type,
a_input.shape[0],
b_weight.shape[1],
a_input.shape[1],
is_k_full=True,
use_atomic_add=False,
use_fp32_reduce=True,
is_zp_float=False,
@@ -14,7 +14,7 @@ from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
GPTQ_MARLIN_TILE,
apply_gptq_marlin_linear,
marlin_make_empty_g_idx,
marlin_make_empty,
marlin_make_workspace_new,
marlin_moe_padded_intermediate,
marlin_pad_qweight,
@@ -319,16 +319,13 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
device = torch.device("cuda")
weight = torch.randn(size_k, size_n, dtype=dtype, device=device) / size_k**0.5
w_ref, q_w, s, _, _ = gptq_quantize_weights(
weight, quant_type, group_size, act_order=False
)
w_ref, q_w, s = gptq_quantize_weights(weight, quant_type, group_size)
qweight = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=torch.empty(0, dtype=torch.int, device=device),
size_k=padded_k,
size_n=padded_n,
num_bits=quant_type.size_bits,
@@ -343,14 +340,11 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
input=x,
weight=marlin_qweight,
weight_scale=marlin_s,
weight_zp=marlin_make_empty_g_idx(device),
g_idx=marlin_make_empty_g_idx(device),
g_idx_sort_indices=marlin_make_empty_g_idx(device),
weight_zp=marlin_make_empty(device),
workspace=marlin_make_workspace_new(device),
wtype=quant_type,
output_size_per_partition=size_n,
input_size_per_partition=size_k,
is_k_full=True,
)
ref = x @ w_ref
@@ -480,7 +474,6 @@ def test_awq_zp_marlin_padded_round_trip(shape):
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
perm=torch.empty(0, dtype=torch.int, device=device),
size_k=padded_k,
size_n=padded_n,
num_bits=quant_type.size_bits,
@@ -503,13 +496,10 @@ def test_awq_zp_marlin_padded_round_trip(shape):
weight=marlin_qweight,
weight_scale=marlin_s,
weight_zp=marlin_zp,
g_idx=marlin_make_empty_g_idx(device),
g_idx_sort_indices=marlin_make_empty_g_idx(device),
workspace=marlin_make_workspace_new(device),
wtype=quant_type,
output_size_per_partition=size_n,
input_size_per_partition=size_k,
is_k_full=True,
)
ref = x @ w_ref
@@ -585,9 +575,7 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
def quant(w, size_k, size_n):
# w is (size_n, size_k); gptq expects (size_k, size_n).
ref, q_w, s, _, _ = gptq_quantize_weights(
w.T, quant_type, group_size, act_order=False
)
ref, q_w, s = gptq_quantize_weights(w.T, quant_type, group_size)
return ref, gptq_pack(q_w, bits, size_k, size_n), s
w13_qw, w13_s, w13_ref = [], [], []
@@ -616,12 +604,11 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
if group_size > 0:
w2_scales = _pad_rows(w2_scales, padded_n // group_size)
sort_idx = torch.empty((e, 0), dtype=torch.int32, device=device)
marlin_w13 = ops.gptq_marlin_moe_repack(
w13_qweight, sort_idx, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
w13_qweight, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
)
marlin_w2 = ops.gptq_marlin_moe_repack(
w2_qweight, sort_idx, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
w2_qweight, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
)
group_or_pack = group_size if group_size != -1 else pack
marlin_w13_s = marlin_moe_permute_scales(
@@ -649,7 +636,6 @@ def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
topk_ids,
quant_type_id=quant_type.id,
global_num_experts=e,
is_k_full=True,
)
with set_current_vllm_config(VllmConfig()):
ref = torch_experts(
@@ -770,7 +756,6 @@ def test_fp8_marlin_moe_padded_round_trip(shape, quant):
topk_ids,
quant_type_id=scalar_types.float8_e4m3fn.id,
global_num_experts=e,
is_k_full=True,
workspace=layer.workspace,
)
with set_current_vllm_config(VllmConfig()):
@@ -848,7 +833,6 @@ def test_mxfp8_marlin_moe_padded_round_trip(shape):
topk_ids,
quant_type_id=scalar_types.float8_e4m3fn.id,
global_num_experts=e,
is_k_full=True,
workspace=layer.workspace,
)
with set_current_vllm_config(VllmConfig()):
@@ -196,7 +196,6 @@ def test_dense_kernel_rejects_on_non_gfx1100():
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is False, f"RDNA3 dense kernel accepted on non-gfx1100: {reason}"
@@ -431,7 +430,6 @@ class TestDenseKernelSelectionMocked:
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
ok, _ = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is True
@@ -461,7 +459,6 @@ class TestDenseKernelSelectionMocked:
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
with (
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
@@ -69,10 +69,9 @@ def _make_packed_weights(E, K, N):
packed = torch.zeros(E, K // 8, N, dtype=torch.int32, device=device)
for i in range(8):
packed |= (w[:, i::8, :] & 0xF) << (i * 4)
g_idx = torch.empty(0, dtype=torch.int32, device=device)
for e in range(E):
we = packed[e].contiguous()
ops.gptq_shuffle(we, g_idx, 4)
ops.gptq_shuffle(we, 4)
packed[e] = we
return packed
@@ -113,7 +112,6 @@ def test_fused_moe_w1_matches_dense(
w13 = _make_packed_weights(E, K, N_gate_up)
w13_s = _make_scales(E, groups, N_gate_up, dtype)
w13_z = _make_qzeros(E, groups, N_gate_up)
g_idx = torch.empty(0, dtype=torch.int32, device=device)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
si, ei, ntp = moe_align_block_size(topk_ids, block_size_m, E)
@@ -147,7 +145,6 @@ def test_fused_moe_w1_matches_dense(
w13[e],
w13_z[e],
w13_s[e],
g_idx,
False,
)
ref_out[flat] = ref.squeeze()
@@ -243,7 +240,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
w2 = _make_packed_weights(E, N_inter, hidden)
w2_s = _make_scales(E, N_inter // group_size, hidden, dtype)
w2_z = _make_qzeros(E, N_inter // group_size, hidden)
g_idx = torch.empty(0, dtype=torch.int32, device=device)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
topk_w = torch.softmax(
@@ -300,7 +296,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
w13[e],
w13_z[e],
w13_s[e],
g_idx,
False,
)
a = torch.empty(1, N_inter, dtype=dtype, device=device)
@@ -310,7 +305,6 @@ def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
w2[e],
w2_z[e],
w2_s[e],
g_idx,
False,
)
ref[m_idx] += r2.squeeze() * w
@@ -173,7 +173,6 @@ def _run_kernel(
act_type=dtype,
group_size=group_size,
zero_points=has_zp,
has_g_idx=False,
)
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok, f"can_implement rejected a supported config: {reason}"
@@ -184,7 +183,6 @@ def _run_kernel(
w_q_param_name="qweight",
w_s_param_name="scales",
w_zp_param_name="qzeros" if has_zp else None,
w_gidx_param_name=None,
)
kernel.process_weights_after_loading(layer)
return kernel.apply_weights(layer, x_mk, bias=bias)
@@ -57,7 +57,6 @@ def test_selection_prefers_rdna3(dtype):
act_type=dtype,
group_size=128,
zero_points=False,
has_g_idx=False,
)
assert choose_mp_linear_kernel(config).__name__ == "RDNA3W4A16LinearKernel"
@@ -83,7 +82,6 @@ def test_can_implement(weight_type, group_size, N, full_k, expected_ok):
act_type=torch.float16,
group_size=group_size,
zero_points=False,
has_g_idx=False,
)
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is expected_ok, reason
@@ -306,14 +306,12 @@ def test_rdna_hybrid_w4a16_process_weights_symmetric_repack(group_size, dist_ini
act_type=torch.float16,
group_size=G,
zero_points=False,
has_g_idx=False,
)
kernel = RDNAHybridW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name=None,
w_gidx_param_name=None,
)
kernel.process_weights_after_loading(layer)
@@ -367,14 +365,12 @@ def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_in
act_type=torch.float16,
group_size=G,
zero_points=True,
has_g_idx=False,
)
kernel = RDNAHybridW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
w_gidx_param_name=None,
)
kernel.process_weights_after_loading(layer)
@@ -414,7 +410,6 @@ def test_hybrid_can_implement_group_size(group_size, expected_ok):
act_type=torch.float16,
group_size=group_size,
zero_points=False,
has_g_idx=False,
)
ok, _ = RDNAHybridW4A16LinearKernel.can_implement(config)
assert ok is expected_ok
@@ -273,14 +273,12 @@ def test_triton_w4a16_process_weights_after_loading_repacks_layout():
act_type=torch.float16,
group_size=G,
zero_points=True,
has_g_idx=False,
)
kernel = TritonW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
w_gidx_param_name=None,
)
# Build dummy layer with vLLM parameter wrappers.
@@ -384,14 +382,12 @@ def test_triton_w4a16_process_weights_after_loading_keeps_gptq_qzeros_layout():
act_type=torch.float16,
group_size=G,
zero_points=True,
has_g_idx=False,
)
kernel = TritonW4A16LinearKernel(
config,
w_q_param_name="qweight",
w_s_param_name="scales",
w_zp_param_name="qzeros",
w_gidx_param_name=None,
)
weight_loader = lambda *args, **kwargs: None
@@ -461,14 +457,12 @@ def test_triton_w4a16_symmetric_apply_ignores_qzeros(monkeypatch):
act_type=torch.float16,
group_size=G,
zero_points=False,
has_g_idx=False,
)
kernel = TritonW4A16LinearKernel(
config,
w_q_param_name="qweight",
w_s_param_name="scales",
w_zp_param_name="qzeros",
w_gidx_param_name=None,
)
class DummyLayer(torch.nn.Module):
@@ -47,7 +47,6 @@ def test_choose_mp_linear_kernel_uint4b8():
act_type=torch.float16,
group_size=128,
zero_points=False,
has_g_idx=False,
)
kernel_type = choose_mp_linear_kernel(config)
@@ -65,7 +64,6 @@ def test_choose_mp_linear_kernel_uint4_asymmetric():
act_type=torch.bfloat16,
group_size=64,
zero_points=True,
has_g_idx=False,
)
kernel_type = choose_mp_linear_kernel(config)
-5
View File
@@ -270,11 +270,6 @@ def qwen3_woofing_lora_files():
return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA")
@pytest.fixture(scope="session")
def tinyllama_lora_files():
return hf_api().snapshot_download(repo_id="jashing/tinyllama-colorist-lora")
@pytest.fixture(scope="session")
def deepseekv2_lora_files():
return hf_api().snapshot_download(repo_id="wuchen01/DeepSeek-V2-Lite-Chat-All-LoRA")
+73 -76
View File
@@ -24,42 +24,51 @@ MODELS: list[ModelWithQuantization]
if current_platform.is_rocm():
MODELS = [
ModelWithQuantization(
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
model_path="JunHowie/Qwen3-0.6B-GPTQ-Int4",
quantization="gptq",
),
]
else:
MODELS = [
ModelWithQuantization(
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", quantization="awq"
model_path="Orion-zhen/Qwen3-0.6B-AWQ", quantization="awq"
),
ModelWithQuantization(
model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
model_path="JunHowie/Qwen3-0.6B-GPTQ-Int4",
quantization="gptq",
),
]
def do_sample(
llm: vllm.LLM, lora_path: str, lora_id: int, max_tokens: int = 256
llm: vllm.LLM, lora_path: str | None, lora_id: int, max_tokens: int = 256
) -> list[str]:
raw_prompts = [
"Give me an orange-ish brown color",
"Give me a neon pink color",
messages = [
[
{
"role": "system",
"content": "Follow the instructions to make animal noises",
},
{"role": "user", "content": "Make your favorite animal noise."},
],
[
{
"role": "system",
"content": "You are a cat. Reply only with your sound.",
},
{"role": "user", "content": "What do you say?"},
],
]
def format_prompt_tuples(prompt):
return f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
prompts = [format_prompt_tuples(p) for p in raw_prompts]
sampling_params = vllm.SamplingParams(
temperature=0, max_tokens=max_tokens, stop=["<|im_end|>"]
)
outputs = llm.generate(
prompts,
sampling_params = vllm.SamplingParams(temperature=0, max_tokens=max_tokens)
outputs = llm.chat(
messages,
sampling_params,
lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
chat_template_kwargs={"enable_thinking": False},
lora_request=(
LoRARequest(str(lora_id), lora_id, lora_path) if lora_path else None
),
use_tqdm=False,
)
# Print the outputs.
generated_texts: list[str] = []
for output in outputs:
prompt = output.prompt
@@ -70,7 +79,7 @@ def do_sample(
@pytest.mark.parametrize("model", MODELS)
def test_quant_model_lora(tinyllama_lora_files, model):
def test_quant_model_lora(qwen3_meowing_lora_files, qwen3_woofing_lora_files, model):
llm = vllm.LLM(
model=model.model_path,
enable_lora=True,
@@ -79,75 +88,59 @@ def test_quant_model_lora(tinyllama_lora_files, model):
max_model_len=400,
gpu_memory_utilization=0.2, # avoid OOM
quantization=model.quantization,
trust_remote_code=True,
enable_chunked_prefill=True,
tokenizer=tinyllama_lora_files,
)
if model.quantization is None:
expected_lora_output = [
"#ff8050",
"#ff8080",
]
elif model.quantization == "awq":
expected_lora_output = [
"#f07700: A v",
"#f00000: A v",
]
elif model.quantization == "gptq":
expected_lora_output = [
"#f08800: This is",
"#f07788 \n#",
]
def expect_match(output, expected_output):
# HACK: GPTQ lora outputs are just incredibly unstable.
# Assert that the outputs changed.
if model.quantization == "gptq" and expected_output is expected_lora_output:
for i, o in enumerate(output):
assert o.startswith("#"), (
f"Expected example {i} to start with # but got {o}"
)
return
assert output == expected_output
loras = [
(
qwen3_meowing_lora_files,
1,
["Meow Meow Meow Meow Meow"] * 2,
),
(
qwen3_woofing_lora_files,
2,
["Woof Woof Woof Woof Woof"] * 2,
),
]
max_tokens = 10
print("lora adapter created")
print("lora 1")
output = do_sample(llm, tinyllama_lora_files, lora_id=1, max_tokens=max_tokens)
expect_match(output, expected_lora_output)
print("lora 2")
output = do_sample(llm, tinyllama_lora_files, lora_id=2, max_tokens=max_tokens)
expect_match(output, expected_lora_output)
print("removing lora")
del llm
cleanup_dist_env_and_memory()
try:
base_output = do_sample(llm, None, lora_id=0, max_tokens=max_tokens)
for lora_path, lora_id, expected_output in loras:
output = do_sample(
llm,
lora_path,
lora_id=lora_id,
max_tokens=max_tokens,
)
assert output == expected_output
assert output != base_output
finally:
del llm
cleanup_dist_env_and_memory()
@pytest.mark.parametrize("model", MODELS)
def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available, model):
def test_quant_model_tp_equality(qwen3_meowing_lora_files, num_gpus_available, model):
if num_gpus_available < 2:
pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
if model.quantization == "gptq":
pytest.skip("GPTQ lora outputs are just incredibly unstable")
llm_tp1 = vllm.LLM(
model=model.model_path,
enable_lora=True,
max_num_seqs=16,
max_loras=4,
max_model_len=400,
gpu_memory_utilization=0.2, # avoid OOM
quantization=model.quantization,
trust_remote_code=True,
enable_chunked_prefill=True,
)
output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
del llm_tp1
cleanup_dist_env_and_memory()
try:
output_tp1 = do_sample(
llm_tp1, qwen3_meowing_lora_files, lora_id=1, max_tokens=10
)
finally:
del llm_tp1
cleanup_dist_env_and_memory()
llm_tp2 = vllm.LLM(
model=model.model_path,
@@ -155,13 +148,17 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available, model
max_num_seqs=16,
max_loras=4,
tensor_parallel_size=2,
max_model_len=400,
gpu_memory_utilization=0.2, # avoid OOM
quantization=model.quantization,
enable_chunked_prefill=True,
)
output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
del llm_tp2
cleanup_dist_env_and_memory()
try:
output_tp2 = do_sample(
llm_tp2, qwen3_meowing_lora_files, lora_id=1, max_tokens=10
)
finally:
del llm_tp2
cleanup_dist_env_and_memory()
assert output_tp1 == output_tp2
@@ -272,13 +272,13 @@ def _stub_marlin_ops(monkeypatch):
monkeypatch.setattr(
ops,
"gptq_marlin_repack",
lambda w, perm, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros(
lambda w, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros(
size_k // 16, size_n * 2, dtype=torch.int32
),
)
def _make_act_order_marlin_kernel():
def _make_marlin_kernel():
from vllm.model_executor.kernels.linear.mixed_precision.marlin import (
MarlinLinearKernel,
)
@@ -295,20 +295,17 @@ def _make_act_order_marlin_kernel():
act_type=torch.float16,
group_size=_MARLIN_GROUP_SIZE,
zero_points=False,
has_g_idx=True,
)
kernel.w_q_name = "qweight"
kernel.w_s_name = "scales"
kernel.w_zp_name = None
kernel.w_gidx_name = "g_idx"
return kernel
def _load_marlin_checkpoint_format_weights(layer, g_idx):
def _load_marlin_checkpoint_format_weights(layer):
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
RowvLLMParameter,
)
layer.qweight = PackedvLLMParameter(
@@ -327,54 +324,24 @@ def _load_marlin_checkpoint_format_weights(layer, g_idx):
output_dim=1,
weight_loader=default_weight_loader,
)
layer.g_idx = RowvLLMParameter(
data=g_idx.clone(),
input_dim=0,
weight_loader=default_weight_loader,
)
def _random_g_idx(generator):
return torch.randint(
0,
_MARLIN_SIZE_K // _MARLIN_GROUP_SIZE,
(_MARLIN_SIZE_K,),
dtype=torch.int32,
generator=generator,
)
def test_marlin_post_load_preserves_runtime_tensor_addresses(monkeypatch, dist_init):
"""Marlin workspace and act-order sort indices must be recomputed into
the same storage when weights are reloaded (RL weight sync), so device
addresses captured by CUDA graphs remain valid."""
from vllm.model_executor.layers.quantization.utils import marlin_utils
"""Marlin must reuse the workspace storage captured by CUDA graphs."""
_stub_marlin_ops(monkeypatch)
kernel = _make_act_order_marlin_kernel()
generator = torch.Generator().manual_seed(0)
first_g_idx = _random_g_idx(generator)
second_g_idx = _random_g_idx(generator)
kernel = _make_marlin_kernel()
layer = torch.nn.Module()
_load_marlin_checkpoint_format_weights(layer, first_g_idx)
_load_marlin_checkpoint_format_weights(layer)
kernel.process_weights_after_loading(layer)
workspace_ptr = kernel.workspace.data_ptr()
sort_indices_ptr = layer.g_idx_sort_indices.data_ptr()
# Reload: fresh checkpoint-format tensors with a different act-order
_load_marlin_checkpoint_format_weights(layer, second_g_idx)
_load_marlin_checkpoint_format_weights(layer)
kernel.process_weights_after_loading(layer)
assert kernel.workspace.data_ptr() == workspace_ptr
assert torch.all(kernel.workspace == 0)
assert layer.g_idx_sort_indices.data_ptr() == sort_indices_ptr
expected_sort_indices = marlin_utils.marlin_sort_g_idx(second_g_idx)[1]
assert torch.equal(layer.g_idx_sort_indices.data, expected_sort_indices)
# registered as a Parameter so layerwise reload copy-back preserves it
assert isinstance(layer.g_idx_sort_indices, torch.nn.Parameter)
@pytest.mark.parametrize("variant", ["fp8", "mxfp8", "nvfp4"])
@@ -394,7 +361,7 @@ def test_marlin_prepare_layer_preserves_workspace_address(monkeypatch, variant):
monkeypatch.setattr(
ops,
"gptq_marlin_repack",
lambda b_q_weight, perm, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros(
lambda b_q_weight, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros(
size_k // 16, size_n * 2, dtype=torch.int32
),
)
@@ -476,68 +443,6 @@ def test_marlin_make_workspace_new_rejects_incompatible_existing(monkeypatch):
)
def test_marlin_act_order_layerwise_reload_accounting(monkeypatch, dist_init):
"""`g_idx_sort_indices` is generated during weight processing and never
loaded from checkpoints. Registering it as a Parameter must not count it
toward `load_numel_total`: reload restores the construction-time tensor
set before sizing, so act-order layers still process during streaming
instead of deferring (and buffering weights) until finalization."""
from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase,
)
from vllm.model_executor.layers.quantization.utils import marlin_utils
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
_stub_marlin_ops(monkeypatch)
kernel = _make_act_order_marlin_kernel()
class _KernelQuantMethod(QuantizeMethodBase):
def create_weights(self, layer, *args, **kwargs):
raise NotImplementedError
def apply(self, layer, *args, **kwargs):
raise NotImplementedError
def process_weights_after_loading(self, layer):
kernel.process_weights_after_loading(layer)
generator = torch.Generator().manual_seed(0)
layer = torch.nn.Module()
layer.quant_method = _KernelQuantMethod()
_load_marlin_checkpoint_format_weights(layer, _random_g_idx(generator))
# Metadata is recorded at model construction, before any processing
record_metadata_for_reloading(layer)
checkpoint_numel = sum(t.numel() for t in get_layer_tensors(layer).values())
kernel.process_weights_after_loading(layer)
sort_indices = layer.g_idx_sort_indices
initialize_layerwise_reload(layer)
info = get_layerwise_info(layer)
assert info.load_numel_total == checkpoint_numel
# Stream a new checkpoint; the layer must process as soon as its last
# tensor arrives
new_g_idx = _random_g_idx(generator)
checkpoint = {
"qweight": torch.zeros(_MARLIN_SIZE_K // 8, _MARLIN_SIZE_N, dtype=torch.int32),
"scales": torch.ones(
_MARLIN_SIZE_K // _MARLIN_GROUP_SIZE, _MARLIN_SIZE_N, dtype=torch.float16
),
"g_idx": new_g_idx,
}
for name, weight in checkpoint.items():
param = getattr(layer, name)
param.weight_loader(param, weight)
assert not info.can_load()
assert not info.loaded_weights
assert layer.g_idx_sort_indices is sort_indices
expected_sort_indices = marlin_utils.marlin_sort_g_idx(new_g_idx)[1]
assert torch.equal(layer.g_idx_sort_indices.data, expected_sort_indices)
def test_model_cleanup(dist_init, default_vllm_config):
layer = QKVParallelLinear(2, 3, 4)
assert layer.weight.weight_loader.__self__ is layer
+11
View File
@@ -160,6 +160,17 @@ class TestMaybeRemapKvScaleName:
assert result is None
def test_checkpoint_weight_mapper_discards_serialized_g_idx():
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
)
mapper = QuantizationConfig.get_checkpoint_weight_mapper()
assert mapper._map_name("model.layers.0.g_idx") is None
assert mapper._map_name("model.layers.0.qweight") == "model.layers.0.qweight"
class TestKvCacheScaleMapper:
"""The `WeightsMapper` returned by `get_cache_scale_mapper` replaces the
per-model `maybe_remap_kv_scale_name` calls. It must remap the same set of
@@ -22,12 +22,10 @@ os.environ["TOKENIZERS_PARALLELISM"] = "true"
MAX_MODEL_LEN = 1024
MODELS = [
# act_order==True, group_size=128
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "main"),
# 8-bit, act_order==True, group_size=channelwise
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "main"),
# desc_act=True is a no-op for channelwise quantization.
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit--1g-actorder_True"),
# 4-bit, act_order==True, group_size=128
("TechxGenus/gemma-1.1-2b-it-GPTQ", "main"),
("TechxGenus/gemma-2b-it-GPTQ", "main"),
]
+1 -1
View File
@@ -243,7 +243,7 @@ def test_distributed(
"model, quantization_kwargs",
[
("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {}),
("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {}),
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", {}),
],
)
@pytest.mark.parametrize("max_tokens", [32])
+50 -2
View File
@@ -23,12 +23,13 @@ from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQLinearMethod,
AutoGPTQMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.gptq_utils import override_config
from vllm.platforms import current_platform
PROMPT = "On the surface of Mars, we found"
MODELS = [
"TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
]
@@ -70,6 +71,54 @@ def test_auto_gptq_config_get_name():
assert AutoGPTQConfig.get_name() == "auto_gptq"
@pytest.mark.parametrize(
"dynamic",
[
{},
{r"+:model\.layers\.0\..*": {"desc_act": True}},
{
r"+:model\.layers\.0\..*": {
"desc_act": True,
"group_size": 32,
}
},
],
)
def test_auto_gptq_rejects_group_activation_order(dynamic):
desc_act = not dynamic
with pytest.raises(ValueError, match="group activation ordering"):
AutoGPTQConfig(4, 128, desc_act, True, False, dynamic, {})
def test_auto_gptq_normalizes_channelwise_activation_order():
config = AutoGPTQConfig(4, -1, True, True, False, {}, {})
assert not config.desc_act
config = AutoGPTQConfig(
4,
128,
False,
True,
False,
{r"+:model\.layers\.0\..*": {"desc_act": True, "group_size": -1}},
{},
)
override_config(config, "model.layers.0.self_attn.q_proj")
assert config.group_size == -1
assert not config.desc_act
with pytest.raises(ValueError, match="group activation ordering"):
AutoGPTQConfig(
4,
-1,
True,
True,
False,
{r"+:model\.layers\.0\..*": {"group_size": 128}},
{},
)
def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
method = object.__new__(AutoGPTQMoEMethod)
method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
@@ -84,7 +133,6 @@ def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
hidden_size=8,
intermediate_size_per_partition=4,
params_dtype=torch.float16,
intermediate_size_full=4,
weight_loader=lambda *args, **kwargs: None,
)
+3 -4
View File
@@ -1537,8 +1537,7 @@ def test_inc_ark_linear_method_xpu_int2_create_weights(monkeypatch) -> None:
assert layer.scales.dtype == torch.bfloat16
assert layer.qzeros.shape == (1, 4)
assert layer.qzeros.dtype == torch.int32
assert layer.g_idx.shape == (64,)
assert layer.g_idx.dtype == torch.int32
assert not hasattr(layer, "g_idx")
assert layer.in_features == 64
assert layer.out_features == 64
assert layer.params_dtype == torch.bfloat16
@@ -2444,7 +2443,7 @@ def test_calls_kernel_at_threshold(monkeypatch, w4a8_layer) -> None:
w_scale,
w_zp,
group_size,
g_idx,
group_idx,
bias,
) = captured["gemm_args"]
assert quant_x.dtype is torch.int8
@@ -2456,7 +2455,7 @@ def test_calls_kernel_at_threshold(monkeypatch, w4a8_layer) -> None:
assert qweight is layer.qweight
assert w_zp is layer.qzeros
assert group_size == 128
assert g_idx is None
assert group_idx is None
assert bias is None
# The kernel emits fp16; the result is cast back to the activation dtype.
+43 -40
View File
@@ -84,6 +84,48 @@ def enable_pickle(monkeypatch):
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
def _compressed_tensors_actorder_config(actorder):
return {
"format": "pack-quantized",
"config_groups": {
"group_0": {
"targets": ["Linear"],
"weights": {
"num_bits": 4,
"type": "int",
"strategy": "group",
"group_size": 128,
"symmetric": True,
"dynamic": False,
"actorder": actorder,
},
}
},
}
@pytest.mark.parametrize(
"actorder", [ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC, True]
)
def test_compressed_tensors_rejects_runtime_activation_order(actorder):
with pytest.raises(ValueError, match="group/dynamic activation ordering"):
CompressedTensorsConfig.from_config(
_compressed_tensors_actorder_config(actorder)
)
@pytest.mark.parametrize(
"actorder", [ActivationOrdering.STATIC, ActivationOrdering.WEIGHT, False, None]
)
def test_compressed_tensors_accepts_preordered_weights(actorder):
config = CompressedTensorsConfig.from_config(
_compressed_tensors_actorder_config(actorder)
)
assert config.target_scheme_map["Linear"]["weights"].actorder == (
actorder.value if isinstance(actorder, ActivationOrdering) else None
)
@pytest.mark.parametrize(
"model_args",
[
@@ -271,15 +313,6 @@ def test_compressed_tensors_w8a8_dynamic_per_token(
None,
8,
True,
False,
),
(
"nm-testing/TinyLlama-1.1B-Chat-v1.0-W4A16-G128-Asym-Updated-ActOrder",
"group",
128,
8,
False,
True,
),
],
)
@@ -287,7 +320,7 @@ def test_compressed_tensors_w8a8_dynamic_per_token(
not current_platform.is_cuda(), reason="The tests are skipped on non-CUDA platform."
)
def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
model, strategy, group, pack_factor, symmetric, has_g_idx = wNa16_args
model, strategy, group, pack_factor, symmetric = wNa16_args
with vllm_runner(model, enforce_eager=True) as llm:
def check_model(model):
@@ -302,7 +335,6 @@ def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
assert qkv_proj.scheme.pack_factor == pack_factor
assert qkv_proj.scheme.symmetric == symmetric
assert qkv_proj.scheme.has_g_idx == has_g_idx
llm.apply_model(check_model)
@@ -926,35 +958,6 @@ def test_compressed_tensors_mxfp8_moe_setup(vllm_runner):
assert output
@pytest.mark.parametrize(
"actorder,group_size,part,full,expected",
[
# actorder="group" with real grouping: must load full-K w2 scales and,
# when sharded (part != full), report is_k_full=False.
(ActivationOrdering.GROUP, 32, 64, 128, (True, 128, False)),
# actorder="group" but unsharded (part == full): full scales, k_full.
(ActivationOrdering.GROUP, 32, 128, 128, (True, 128, True)),
# actorder="group" with channel-wise (group_size == -1): no full load.
(ActivationOrdering.GROUP, -1, 64, 128, (False, 64, False)),
# "static"/"weight" reorder at quant time -> shard normally + k_full.
# Regression: static actorder under TP must keep is_k_full=True so the
# Marlin kernel never gets the invalid (group_size=16, is_k_full=0).
("static", 32, 64, 128, (False, 64, True)),
("weight", 32, 64, 128, (False, 64, True)),
(None, 32, 64, 128, (False, 64, True)),
],
)
def test_wna16_moe_w2_scale_sharding(actorder, group_size, part, full, expected):
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
CompressedTensorsWNA16MoEMethod,
)
result = CompressedTensorsWNA16MoEMethod._w2_scale_sharding(
actorder, group_size, part, full
)
assert result == expected
@pytest.mark.parametrize("num_bits", range(2, 9))
def test_humming_supports_compressed_tensors_wna16_quant_key(num_bits):
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
+1 -2
View File
@@ -9,8 +9,7 @@ if not current_platform.is_cpu():
MODELS = [
"TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ",
"TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", # with g_idx
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", # without g_idx
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4",
"RedHatAI/Qwen3-1.7B-quantized.w4a16", # with zp
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear
+4 -10
View File
@@ -64,7 +64,6 @@ def test_machete_kernel_selected(act_type, weight_type, group_size, zero_points)
weight_type=weight_type,
group_size=group_size,
zero_points=zero_points,
has_g_idx=False,
)
kernel = choose_mp_linear_kernel(config)
assert kernel is MacheteLinearKernel, (
@@ -73,23 +72,21 @@ def test_machete_kernel_selected(act_type, weight_type, group_size, zero_points)
@pytest.mark.parametrize(
"full_shape,part_shape,weight_type,group_size,has_g_idx,expected_reason",
"full_shape,part_shape,weight_type,group_size,expected_reason",
[
((4096, 4096), (2048, 4096), scalar_types.uint4b8, 128, True, "Act reordering"),
(
(4096, 4096),
(4096, 4096),
scalar_types.float6_e3m2f,
128,
False,
"Quant type",
),
((4096, 4096), (4096, 4096), scalar_types.uint4b8, 32, False, "Group size"),
((4096, 4096), (4096, 4096), scalar_types.uint4b8, 32, "Group size"),
],
ids=["partitioned-g_idx", "unsupported-quant-type", "unsupported-group-size"],
ids=["unsupported-quant-type", "unsupported-group-size"],
)
def test_machete_rejects_invalid_config(
full_shape, part_shape, weight_type, group_size, has_g_idx, expected_reason
full_shape, part_shape, weight_type, group_size, expected_reason
):
"""Verify Machete rejects unsupported configurations."""
config = MPLinearLayerConfig(
@@ -99,7 +96,6 @@ def test_machete_rejects_invalid_config(
weight_type=weight_type,
group_size=group_size,
zero_points=False,
has_g_idx=has_g_idx,
)
can_impl, reason = MacheteLinearKernel.can_implement(config)
assert not can_impl
@@ -117,7 +113,6 @@ def test_kernel_selection_with_disabled_machete(monkeypatch):
weight_type=scalar_types.uint4b8,
group_size=128,
zero_points=False,
has_g_idx=False,
)
kernel = choose_mp_linear_kernel(config)
assert kernel is not MacheteLinearKernel, "MacheteLinearKernel should be disabled"
@@ -127,7 +122,6 @@ def test_kernel_selection_with_disabled_machete(monkeypatch):
"model_name",
[
"nm-testing/tinyllama-oneshot-w4a16-channel-v2",
"nm-testing/TinyLlama-1.1B-Chat-v1.0-W4A16-G128-Asym-Updated-ActOrder",
],
)
def test_w4a16_machete_e2e(vllm_runner, model_name):
+7 -50
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests whether gptq models with dynamic quantized can be loaded.
"""Tests whether unsupported GPTQ group actorder models are rejected.
Run `pytest tests/quantization/test_gptq_dynamic.py --forked`.
@@ -8,59 +8,16 @@ Note: Only symmetric GPTQ models are supported after consolidation to Marlin.
"""
import pytest
import torch
from tests.quantization.utils import load_model_without_vllm_runner
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
get_dynamic_override,
)
PROMPT = "On the surface of Mars, we found"
# The first layer is quantized using bits=4, group_size=128
# The second layer is quantized using bits=8, group_size=32
# All other layers (layer index >= 2) are not quantized
# Note: Only symmetric models are supported with Marlin kernels
MODELS = [
"ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
]
@pytest.mark.parametrize("model_id", MODELS)
def test_gptq_with_dynamic(model_id: str, monkeypatch, dist_init, workspace_init):
# `LLM.apply_model` requires pickling a function.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
linear_method_cls = AutoGPTQLinearMethod
model, _ = load_model_without_vllm_runner(
model_id,
dtype=torch.float16,
model_config_kwargs={
"max_model_len": 2048,
"hf_overrides": {"num_hidden_layers": 3},
},
)
for name, submodule in model.named_modules():
if name == "lm_head":
assert isinstance(submodule.quant_method, linear_method_cls)
elif name == "model.layers.0.self_attn.qkv_proj":
assert isinstance(submodule.quant_method, linear_method_cls)
config = submodule.quant_method.quant_config
assert config.weight_bits == 4
assert config.group_size == 128
assert config.desc_act
elif name == "model.layers.1.self_attn.qkv_proj":
assert isinstance(submodule.quant_method, linear_method_cls)
config = submodule.quant_method.quant_config
assert get_dynamic_override(config, layer_name=name, key="bits") == 8
assert get_dynamic_override(config, layer_name=name, key="group_size") == 32
assert not get_dynamic_override(config, layer_name=name, key="desc_act")
elif name in (
"model.layers.2.self_attn.qkv_proj",
"model.layers.2.mlp.gate_up_proj",
):
assert isinstance(submodule.quant_method, UnquantizedLinearMethod)
def test_gptq_with_dynamic_desc_act_rejected(vllm_runner, model_id: str):
with (
pytest.raises(ValueError, match="desc_act=True"),
vllm_runner(model_id, max_model_len=2048, enforce_eager=True),
):
pass
+1 -2
View File
@@ -17,8 +17,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
PROMPT = "On the surface of Mars, we found"
MODELS_QUANT = [
("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head", True),
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", False),
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", False),
]
+29 -25
View File
@@ -6,7 +6,6 @@ from types import SimpleNamespace
import pytest
import torch
from compressed_tensors.quantization import (
ActivationOrdering,
QuantizationArgs,
QuantizationStrategy,
QuantizationType,
@@ -33,6 +32,35 @@ def test_map_wna16_backend_supports_triton():
assert map_wna16_backend("triton") == WNA16MoEBackend.TRITON
@pytest.mark.parametrize(
"config",
[
{"desc_act": True, "group_size": 128},
{
"desc_act": False,
"group_size": 128,
"dynamic": {r"+:model\.layers\.0\..*": {"desc_act": True}},
},
],
)
def test_moe_wna16_rejects_gptq_group_activation_order(config):
config.update({"quant_method": "gptq", "bits": 4, "sym": True})
with pytest.raises(ValueError, match="group activation ordering"):
MoeWNA16Config.from_config(config)
def test_moe_wna16_accepts_channelwise_gptq_activation_order():
config = {
"quant_method": "gptq",
"bits": 4,
"group_size": -1,
"desc_act": True,
"sym": True,
}
assert MoeWNA16Config.is_moe_wna16_compatible(config)
MoeWNA16Config.from_config(config)
@pytest.mark.parametrize(
("backend", "quant_config", "may_have_zp", "may_have_bias", "expected"),
[
@@ -43,28 +71,6 @@ def test_map_wna16_backend_supports_triton():
False,
"AutoAWQ weight layout",
),
(
WNA16MoEBackend.TRITON,
AutoGPTQConfig(4, 128, True, True, False, {}, {}),
False,
False,
"activation ordering",
),
(
WNA16MoEBackend.TRITON,
QuantizationArgs(
num_bits=4,
type=QuantizationType.INT,
strategy=QuantizationStrategy.GROUP,
symmetric=True,
dynamic=False,
group_size=128,
actorder=ActivationOrdering.GROUP,
),
False,
False,
"activation ordering",
),
(
WNA16MoEBackend.TRITON,
AutoGPTQConfig(4, 128, False, True, False, {}, {}),
@@ -244,7 +250,6 @@ def test_compressed_tensors_wna16_moe_create_weights_uses_ceil_packed_shapes(
hidden_size=256,
intermediate_size_per_partition=512,
params_dtype=torch.float16,
intermediate_size_full=512,
)
packed_hidden = (256 * num_bits + 31) // 32
@@ -296,7 +301,6 @@ def test_compressed_tensors_wna16_moe_converts_and_sets_up_humming_kernel():
num_experts=2,
hidden_size=256,
intermediate_size_per_partition=512,
intermediate_size_full=512,
params_dtype=torch.bfloat16,
)
layer.cuda()
+1 -3
View File
@@ -1,8 +1,6 @@
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
compressed-tensors, nm-testing/test-w4a16-mixtral-actorder-group, main
gptq, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
gptq, TheBloke/Mixtral-8x7B-v0.1-GPTQ, gptq-8bit-128g-actorder_True
awq_marlin, casperhansen/deepseek-coder-v2-instruct-awq, main
compressed-tensors, RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16, main
compressed-tensors, RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16, main
+3 -4
View File
@@ -1,9 +1,8 @@
gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main
gptq, TheBloke/Llama-2-7B-GPTQ, main
gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
gptq, LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit, main
gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True
gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit-32g-actorder_True
gptq, TechxGenus/gemma-1.1-2b-it-GPTQ, main
gptq, TechxGenus/gemma-2b-it-GPTQ, main
compressed-tensors, nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change, main
compressed-tensors, nm-testing/tinyllama-oneshot-w8-channel-a8-tensor, main
compressed-tensors, nm-testing/tinyllama-oneshot-w8a8-dynamic-token-v2, main
@@ -16,4 +15,4 @@ compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
#compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
awq, casperhansen/mixtral-instruct-awq, main
awq_marlin, casperhansen/mixtral-instruct-awq, main
fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main
+1 -4
View File
@@ -34,10 +34,7 @@ def test_weight_loading(vllm_runner):
"""
# MoE models need fp16.
NEEDS_FP16 = (
QUANTIZATION == "gptq"
or MODEL_NAME == "nm-testing/test-w4a16-mixtral-actorder-group"
)
NEEDS_FP16 = QUANTIZATION == "gptq"
with vllm_runner(
model_name=MODEL_NAME,
revision=REVISION,