iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Quant][Kernel] Remove GPTQ Group/Dynamic Activation Ordering (#54809)

Browse Source 
Signed-off-by: Roderick-Wu <roderickwu2003@gmail.com>
Signed-off-by: Roderick Wu <roderickwu2003@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Roderick Wu
2026-09-08 12:45:12 -07:00
committed by GitHub
co-authored by mgoin
parent 1a522b6949
commit 2c9d68f80b
99 changed files with 1182 additions and 3917 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/quantization/test_auto_gptq.py
+50 -2
View File
@@ -23,12 +23,13 @@ from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQLinearMethod,
AutoGPTQMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.gptq_utils import override_config
from vllm.platforms import current_platform
PROMPT = "On the surface of Mars, we found"
MODELS = [
"TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
]
@@ -70,6 +71,54 @@ def test_auto_gptq_config_get_name():
assert AutoGPTQConfig.get_name() == "auto_gptq"
@pytest.mark.parametrize(
"dynamic",
[
{},
{r"+:model\.layers\.0\..*": {"desc_act": True}},
{
r"+:model\.layers\.0\..*": {
"desc_act": True,
"group_size": 32,
}
},
],
)
def test_auto_gptq_rejects_group_activation_order(dynamic):
desc_act = not dynamic
with pytest.raises(ValueError, match="group activation ordering"):
AutoGPTQConfig(4, 128, desc_act, True, False, dynamic, {})
def test_auto_gptq_normalizes_channelwise_activation_order():
config = AutoGPTQConfig(4, -1, True, True, False, {}, {})
assert not config.desc_act
config = AutoGPTQConfig(
4,
128,
False,
True,
False,
{r"+:model\.layers\.0\..*": {"desc_act": True, "group_size": -1}},
{},
)
override_config(config, "model.layers.0.self_attn.q_proj")
assert config.group_size == -1
assert not config.desc_act
with pytest.raises(ValueError, match="group activation ordering"):
AutoGPTQConfig(
4,
-1,
True,
True,
False,
{r"+:model\.layers\.0\..*": {"group_size": 128}},
{},
)
def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
method = object.__new__(AutoGPTQMoEMethod)
method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {})
@@ -84,7 +133,6 @@ def test_auto_gptq_moe_creates_zero_initialized_expert_biases():
hidden_size=8,
intermediate_size_per_partition=4,
params_dtype=torch.float16,
intermediate_size_full=4,
weight_loader=lambda *args, **kwargs: None,
)
tests/quantization/test_auto_round.py
+3 -4
View File
@@ -1537,8 +1537,7 @@ def test_inc_ark_linear_method_xpu_int2_create_weights(monkeypatch) -> None:
assert layer.scales.dtype == torch.bfloat16
assert layer.qzeros.shape == (1, 4)
assert layer.qzeros.dtype == torch.int32
assert layer.g_idx.shape == (64,)
assert layer.g_idx.dtype == torch.int32
assert not hasattr(layer, "g_idx")
assert layer.in_features == 64
assert layer.out_features == 64
assert layer.params_dtype == torch.bfloat16
@@ -2444,7 +2443,7 @@ def test_calls_kernel_at_threshold(monkeypatch, w4a8_layer) -> None:
w_scale,
w_zp,
group_size,
g_idx,
group_idx,
bias,
) = captured["gemm_args"]
assert quant_x.dtype is torch.int8
@@ -2456,7 +2455,7 @@ def test_calls_kernel_at_threshold(monkeypatch, w4a8_layer) -> None:
assert qweight is layer.qweight
assert w_zp is layer.qzeros
assert group_size == 128
assert g_idx is None
assert group_idx is None
assert bias is None
# The kernel emits fp16; the result is cast back to the activation dtype.
tests/quantization/test_compressed_tensors.py
+43 -40
View File
@@ -84,6 +84,48 @@ def enable_pickle(monkeypatch):
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
def _compressed_tensors_actorder_config(actorder):
return {
"format": "pack-quantized",
"config_groups": {
"group_0": {
"targets": ["Linear"],
"weights": {
"num_bits": 4,
"type": "int",
"strategy": "group",
"group_size": 128,
"symmetric": True,
"dynamic": False,
"actorder": actorder,
},
}
},
}
@pytest.mark.parametrize(
"actorder", [ActivationOrdering.GROUP, ActivationOrdering.DYNAMIC, True]
)
def test_compressed_tensors_rejects_runtime_activation_order(actorder):
with pytest.raises(ValueError, match="group/dynamic activation ordering"):
CompressedTensorsConfig.from_config(
_compressed_tensors_actorder_config(actorder)
)
@pytest.mark.parametrize(
"actorder", [ActivationOrdering.STATIC, ActivationOrdering.WEIGHT, False, None]
)
def test_compressed_tensors_accepts_preordered_weights(actorder):
config = CompressedTensorsConfig.from_config(
_compressed_tensors_actorder_config(actorder)
)
assert config.target_scheme_map["Linear"]["weights"].actorder == (
actorder.value if isinstance(actorder, ActivationOrdering) else None
)
@pytest.mark.parametrize(
"model_args",
[
@@ -271,15 +313,6 @@ def test_compressed_tensors_w8a8_dynamic_per_token(
None,
8,
True,
False,
),
(
"nm-testing/TinyLlama-1.1B-Chat-v1.0-W4A16-G128-Asym-Updated-ActOrder",
"group",
128,
8,
False,
True,
),
],
)
@@ -287,7 +320,7 @@ def test_compressed_tensors_w8a8_dynamic_per_token(
not current_platform.is_cuda(), reason="The tests are skipped on non-CUDA platform."
)
def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
model, strategy, group, pack_factor, symmetric, has_g_idx = wNa16_args
model, strategy, group, pack_factor, symmetric = wNa16_args
with vllm_runner(model, enforce_eager=True) as llm:
def check_model(model):
@@ -302,7 +335,6 @@ def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
assert qkv_proj.scheme.pack_factor == pack_factor
assert qkv_proj.scheme.symmetric == symmetric
assert qkv_proj.scheme.has_g_idx == has_g_idx
llm.apply_model(check_model)
@@ -926,35 +958,6 @@ def test_compressed_tensors_mxfp8_moe_setup(vllm_runner):
assert output
@pytest.mark.parametrize(
"actorder,group_size,part,full,expected",
[
# actorder="group" with real grouping: must load full-K w2 scales and,
# when sharded (part != full), report is_k_full=False.
(ActivationOrdering.GROUP, 32, 64, 128, (True, 128, False)),
# actorder="group" but unsharded (part == full): full scales, k_full.
(ActivationOrdering.GROUP, 32, 128, 128, (True, 128, True)),
# actorder="group" with channel-wise (group_size == -1): no full load.
(ActivationOrdering.GROUP, -1, 64, 128, (False, 64, False)),
# "static"/"weight" reorder at quant time -> shard normally + k_full.
# Regression: static actorder under TP must keep is_k_full=True so the
# Marlin kernel never gets the invalid (group_size=16, is_k_full=0).
("static", 32, 64, 128, (False, 64, True)),
("weight", 32, 64, 128, (False, 64, True)),
(None, 32, 64, 128, (False, 64, True)),
],
)
def test_wna16_moe_w2_scale_sharding(actorder, group_size, part, full, expected):
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe.compressed_tensors_moe_wna16 import ( # noqa: E501
CompressedTensorsWNA16MoEMethod,
)
result = CompressedTensorsWNA16MoEMethod._w2_scale_sharding(
actorder, group_size, part, full
)
assert result == expected
@pytest.mark.parametrize("num_bits", range(2, 9))
def test_humming_supports_compressed_tensors_wna16_quant_key(num_bits):
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
tests/quantization/test_cpu_wna16.py
+1 -2
View File
@@ -9,8 +9,7 @@ if not current_platform.is_cpu():
MODELS = [
"TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ",
"TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", # with g_idx
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", # without g_idx
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4",
"RedHatAI/Qwen3-1.7B-quantized.w4a16", # with zp
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear
tests/quantization/test_cutlass_w4a16.py
+4 -10
View File
@@ -64,7 +64,6 @@ def test_machete_kernel_selected(act_type, weight_type, group_size, zero_points)
weight_type=weight_type,
group_size=group_size,
zero_points=zero_points,
has_g_idx=False,
)
kernel = choose_mp_linear_kernel(config)
assert kernel is MacheteLinearKernel, (
@@ -73,23 +72,21 @@ def test_machete_kernel_selected(act_type, weight_type, group_size, zero_points)
@pytest.mark.parametrize(
"full_shape,part_shape,weight_type,group_size,has_g_idx,expected_reason",
"full_shape,part_shape,weight_type,group_size,expected_reason",
[
((4096, 4096), (2048, 4096), scalar_types.uint4b8, 128, True, "Act reordering"),
(
(4096, 4096),
(4096, 4096),
scalar_types.float6_e3m2f,
128,
False,
"Quant type",
),
((4096, 4096), (4096, 4096), scalar_types.uint4b8, 32, False, "Group size"),
((4096, 4096), (4096, 4096), scalar_types.uint4b8, 32, "Group size"),
],
ids=["partitioned-g_idx", "unsupported-quant-type", "unsupported-group-size"],
ids=["unsupported-quant-type", "unsupported-group-size"],
)
def test_machete_rejects_invalid_config(
full_shape, part_shape, weight_type, group_size, has_g_idx, expected_reason
full_shape, part_shape, weight_type, group_size, expected_reason
):
"""Verify Machete rejects unsupported configurations."""
config = MPLinearLayerConfig(
@@ -99,7 +96,6 @@ def test_machete_rejects_invalid_config(
weight_type=weight_type,
group_size=group_size,
zero_points=False,
has_g_idx=has_g_idx,
)
can_impl, reason = MacheteLinearKernel.can_implement(config)
assert not can_impl
@@ -117,7 +113,6 @@ def test_kernel_selection_with_disabled_machete(monkeypatch):
weight_type=scalar_types.uint4b8,
group_size=128,
zero_points=False,
has_g_idx=False,
)
kernel = choose_mp_linear_kernel(config)
assert kernel is not MacheteLinearKernel, "MacheteLinearKernel should be disabled"
@@ -127,7 +122,6 @@ def test_kernel_selection_with_disabled_machete(monkeypatch):
"model_name",
[
"nm-testing/tinyllama-oneshot-w4a16-channel-v2",
"nm-testing/TinyLlama-1.1B-Chat-v1.0-W4A16-G128-Asym-Updated-ActOrder",
],
)
def test_w4a16_machete_e2e(vllm_runner, model_name):
tests/quantization/test_gptq_dynamic.py
+7 -50
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests whether gptq models with dynamic quantized can be loaded.
"""Tests whether unsupported GPTQ group actorder models are rejected.
Run `pytest tests/quantization/test_gptq_dynamic.py --forked`.
@@ -8,59 +8,16 @@ Note: Only symmetric GPTQ models are supported after consolidation to Marlin.
"""
import pytest
import torch
from tests.quantization.utils import load_model_without_vllm_runner
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
get_dynamic_override,
)
PROMPT = "On the surface of Mars, we found"
# The first layer is quantized using bits=4, group_size=128
# The second layer is quantized using bits=8, group_size=32
# All other layers (layer index >= 2) are not quantized
# Note: Only symmetric models are supported with Marlin kernels
MODELS = [
"ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
]
@pytest.mark.parametrize("model_id", MODELS)
def test_gptq_with_dynamic(model_id: str, monkeypatch, dist_init, workspace_init):
# `LLM.apply_model` requires pickling a function.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
linear_method_cls = AutoGPTQLinearMethod
model, _ = load_model_without_vllm_runner(
model_id,
dtype=torch.float16,
model_config_kwargs={
"max_model_len": 2048,
"hf_overrides": {"num_hidden_layers": 3},
},
)
for name, submodule in model.named_modules():
if name == "lm_head":
assert isinstance(submodule.quant_method, linear_method_cls)
elif name == "model.layers.0.self_attn.qkv_proj":
assert isinstance(submodule.quant_method, linear_method_cls)
config = submodule.quant_method.quant_config
assert config.weight_bits == 4
assert config.group_size == 128
assert config.desc_act
elif name == "model.layers.1.self_attn.qkv_proj":
assert isinstance(submodule.quant_method, linear_method_cls)
config = submodule.quant_method.quant_config
assert get_dynamic_override(config, layer_name=name, key="bits") == 8
assert get_dynamic_override(config, layer_name=name, key="group_size") == 32
assert not get_dynamic_override(config, layer_name=name, key="desc_act")
elif name in (
"model.layers.2.self_attn.qkv_proj",
"model.layers.2.mlp.gate_up_proj",
):
assert isinstance(submodule.quant_method, UnquantizedLinearMethod)
def test_gptq_with_dynamic_desc_act_rejected(vllm_runner, model_id: str):
with (
pytest.raises(ValueError, match="desc_act=True"),
vllm_runner(model_id, max_model_len=2048, enforce_eager=True),
):
pass
tests/quantization/test_lm_head.py
+1 -2
View File
@@ -17,8 +17,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
PROMPT = "On the surface of Mars, we found"
MODELS_QUANT = [
("ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head", True),
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", False),
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", False),
]
tests/quantization/test_moe_wna16.py
+29 -25
View File
@@ -6,7 +6,6 @@ from types import SimpleNamespace
import pytest
import torch
from compressed_tensors.quantization import (
ActivationOrdering,
QuantizationArgs,
QuantizationStrategy,
QuantizationType,
@@ -33,6 +32,35 @@ def test_map_wna16_backend_supports_triton():
assert map_wna16_backend("triton") == WNA16MoEBackend.TRITON
@pytest.mark.parametrize(
"config",
[
{"desc_act": True, "group_size": 128},
{
"desc_act": False,
"group_size": 128,
"dynamic": {r"+:model\.layers\.0\..*": {"desc_act": True}},
},
],
)
def test_moe_wna16_rejects_gptq_group_activation_order(config):
config.update({"quant_method": "gptq", "bits": 4, "sym": True})
with pytest.raises(ValueError, match="group activation ordering"):
MoeWNA16Config.from_config(config)
def test_moe_wna16_accepts_channelwise_gptq_activation_order():
config = {
"quant_method": "gptq",
"bits": 4,
"group_size": -1,
"desc_act": True,
"sym": True,
}
assert MoeWNA16Config.is_moe_wna16_compatible(config)
MoeWNA16Config.from_config(config)
@pytest.mark.parametrize(
("backend", "quant_config", "may_have_zp", "may_have_bias", "expected"),
[
@@ -43,28 +71,6 @@ def test_map_wna16_backend_supports_triton():
False,
"AutoAWQ weight layout",
),
(
WNA16MoEBackend.TRITON,
AutoGPTQConfig(4, 128, True, True, False, {}, {}),
False,
False,
"activation ordering",
),
(
WNA16MoEBackend.TRITON,
QuantizationArgs(
num_bits=4,
type=QuantizationType.INT,
strategy=QuantizationStrategy.GROUP,
symmetric=True,
dynamic=False,
group_size=128,
actorder=ActivationOrdering.GROUP,
),
False,
False,
"activation ordering",
),
(
WNA16MoEBackend.TRITON,
AutoGPTQConfig(4, 128, False, True, False, {}, {}),
@@ -244,7 +250,6 @@ def test_compressed_tensors_wna16_moe_create_weights_uses_ceil_packed_shapes(
hidden_size=256,
intermediate_size_per_partition=512,
params_dtype=torch.float16,
intermediate_size_full=512,
)
packed_hidden = (256 * num_bits + 31) // 32
@@ -296,7 +301,6 @@ def test_compressed_tensors_wna16_moe_converts_and_sets_up_humming_kernel():
num_experts=2,
hidden_size=256,
intermediate_size_per_partition=512,
intermediate_size_full=512,
params_dtype=torch.bfloat16,
)
layer.cuda()