[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)

This commit is contained in:
fxmarty-amd
2026-09-02 08:55:15 -04:00
committed by GitHub
parent 2a4e3cc3db
commit 7894394b07
11 changed files with 892 additions and 172 deletions
+195
View File
@@ -0,0 +1,195 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests quantization configuration matching utilities."""
from unittest.mock import Mock
import pytest
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
should_ignore_layer,
)
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
_find_matching_targets,
)
from vllm.model_executor.layers.quantization.quark.utils import (
should_ignore_layer as quark_should_ignore_layer,
)
from vllm.model_executor.layers.quantization.utils.config_utils import (
find_matching_patterns,
get_layer_name_after_index,
is_equal_or_regex_match,
)
def test_is_equal_or_regex_match():
assert is_equal_or_regex_match(
"model.layers.0.mlp.down_proj", "model.layers.0.mlp.down_proj"
)
assert is_equal_or_regex_match("model.layers.0.mlp.down_proj", r"re:.*down_proj")
assert not is_equal_or_regex_match("model.layers.0.mlp.down_proj", "other")
def test_fnmatch_is_opt_in():
layer_name = "model.layers.0.mlp.experts"
pattern = "*mlp.experts*"
assert not is_equal_or_regex_match(layer_name, pattern)
assert is_equal_or_regex_match(layer_name, pattern, use_fnmatch=True)
assert not should_ignore_layer(layer_name, [pattern])
assert not quark_should_ignore_layer(layer_name, [pattern])
assert should_ignore_layer(layer_name, [pattern], use_fnmatch=True)
def test_online_targets_support_fnmatch_patterns():
layer_name = "model.layers.0.mlp.experts"
targets = {"*mlp.experts*": "mxfp4"}
assert _find_matching_targets(layer_name, targets) == ["*mlp.experts*"]
def test_online_ignore_supports_fnmatch_patterns():
layer_name = "model.layers.0.mlp.experts"
ignore = ["*mlp.experts*"]
config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={"*mlp.experts*": "mxfp4"}, ignore=ignore)
)
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
config._dispatch_target(layer_name, Mock())
@pytest.mark.parametrize(
"layer_name,expected",
[
("model.layers.1.self_attn.qkv_proj", "self_attn.qkv_proj"),
("model.layers.2.mlp.down_proj", "mlp.down_proj"),
("lm_head", "lm_head"),
],
)
def test_get_layer_name_after_index(layer_name, expected):
assert get_layer_name_after_index(layer_name) == expected
@pytest.mark.parametrize(
"patterns",
[
[r"re:.*qkv_proj.*"],
[r"re:.*\.[qkv]_proj$"],
],
ids=["direct_fused_regex", "fused_shard_regexes"],
)
def test_find_matching_patterns_for_fused_regexes(patterns):
layer_name = "model.layers.0.self_attn.qkv_proj"
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
matches = find_matching_patterns(layer_name, patterns, fused_mapping)
assert all(len(shard_matches) == 1 for shard_matches in matches)
@pytest.mark.parametrize(
"patterns,expected",
[
(
[r"re:.*\.qkv_proj$"],
[{r"re:.*\.qkv_proj$"}],
),
(
[r"re:.*\.q_proj$", r"re:.*\.k_proj$", r"re:.*\.v_proj$"],
[
{r"re:.*\.q_proj$"},
{r"re:.*\.k_proj$"},
{r"re:.*\.v_proj$"},
],
),
],
ids=["direct_fused_match", "individual_shard_matches"],
)
def test_find_matching_patterns_distinguishes_direct_and_shard_matches(
patterns, expected
):
layer_name = "model.layers.0.self_attn.qkv_proj"
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
assert find_matching_patterns(layer_name, patterns, fused_mapping) == expected
assert should_ignore_layer(layer_name, patterns, fused_mapping)
def test_should_ignore_layer_rejects_partially_matched_fused_layer():
layer_name = "model.layers.0.self_attn.qkv_proj"
patterns = [r"re:.*\.q_proj$"]
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
assert find_matching_patterns(layer_name, patterns, fused_mapping) == [
{r"re:.*\.q_proj$"},
set(),
set(),
]
with pytest.raises(ValueError, match="different quantization schemes"):
should_ignore_layer(layer_name, patterns, fused_mapping)
def test_should_ignore_layer_returns_false_when_no_fused_pattern_matches():
layer_name = "model.layers.0.self_attn.qkv_proj"
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
assert find_matching_patterns(layer_name, [], fused_mapping) == [
set(),
set(),
set(),
]
assert not should_ignore_layer(layer_name, [], fused_mapping)
@pytest.mark.parametrize(
"patterns",
[
[r"re:.*qkv_proj.*"],
[r"re:.*\.[qkv]_proj$"],
],
ids=["direct_fused_regex", "fused_shard_regexes"],
)
def test_ignore_and_targets_match_fused_regexes_identically(patterns):
layer_name = "model.layers.0.self_attn.qkv_proj"
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
targets = {pattern: "fp8_per_block" for pattern in patterns}
assert should_ignore_layer(layer_name, patterns, fused_mapping)
matches = _find_matching_targets(layer_name, targets, fused_mapping)
assert len(matches) == 1
assert targets[matches[0]] == "fp8_per_block"
def test_ignore_allows_individually_matched_fused_shards():
layer_name = "model.layers.0.self_attn.qkv_proj"
patterns = [
"model.layers.0.self_attn.q_proj",
"model.layers.0.self_attn.k_proj",
"model.layers.0.self_attn.v_proj",
]
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
assert should_ignore_layer(layer_name, patterns, fused_mapping)
@pytest.mark.parametrize(
"patterns",
[
["model.layers.0.self_attn.qkv_proj"],
[r"re:.*\.qkv_proj$"],
[
"model.layers.0.self_attn.q_proj",
"model.layers.0.self_attn.k_proj",
"model.layers.0.self_attn.v_proj",
],
],
ids=["direct_fused_name", "direct_fused_regex", "individual_shards"],
)
def test_quark_and_compressed_tensors_ignore_fused_layers_identically(patterns):
layer_name = "model.layers.0.self_attn.qkv_proj"
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
assert should_ignore_layer(layer_name, patterns, fused_mapping)
assert quark_should_ignore_layer(layer_name, patterns, fused_mapping)
+198 -5
View File
@@ -18,10 +18,21 @@ from tests.quantization.utils import (
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm._custom_ops import scaled_fp4_quant
from vllm.config.quantization import resolve_quantization_config
from vllm.config import ModelConfig
from vllm.config.quantization import (
QuantizationConfigArgs,
resolve_quantization_config,
)
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerBlockOnlineLinearMethod,
Fp8PerBlockOnlineMoEMethod,
@@ -49,6 +60,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
amax_for_tp_weight_quant,
weight_amax,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.base_loader import log_online_quantization
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
from vllm.model_executor.models.granitemoe import (
GraniteMoeModel,
@@ -154,12 +169,23 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
Fp8PerBlockOnlineLinearMethod,
Fp8PerTensorOnlineMoEMethod,
),
# quantization='online' with per-layer target patterns
(
"online",
{
"targets": {
r"re:.*self_attn\.o_proj": "fp8_per_block",
r"re:.*block_sparse_moe\.experts": "fp8_per_tensor",
}
},
Fp8PerBlockOnlineLinearMethod,
Fp8PerTensorOnlineMoEMethod,
),
# ignore with direct layer name
(
"fp8_per_tensor",
# qkv_proj is fused from q_proj/k_proj/v_proj, so currently the
# ignore regex must match the unfused shard names
# TODO(future PR): also make 're:.*qkv_proj.*' work
# qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex
# remains supported alongside direct fused-name regexes.
{"ignore": ["model.layers.1.self_attn.o_proj", "re:.*[qkv]_proj"]},
Fp8PerTensorOnlineLinearMethod,
Fp8PerTensorOnlineMoEMethod,
@@ -171,6 +197,15 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(
Mxfp4OnlineMoEMethod,
),
],
ids=[
"fp8_per_tensor",
"fp8_per_block",
"fp8_per_channel",
"per_layer_kind_overrides",
"targets",
"ignore",
"mxfp4",
],
)
@pytest.mark.parametrize(
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
@@ -266,6 +301,12 @@ def test_online_quantization(
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
)
if isinstance(online_quant_args, dict) and "targets" in online_quant_args:
for layer in model.model.layers:
assert isinstance(
layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod
)
input_ids = torch.tensor([1, 2, 3, 4], device=DEVICE)
positions = torch.arange(input_ids.numel(), device=DEVICE)
with set_forward_context(None, vllm_config, num_tokens=input_ids.numel()):
@@ -321,6 +362,158 @@ def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> Non
assert outputs
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
@pytest.mark.parametrize(
"targets,prefix,expected_method_cls,unmatched_prefix,expected_metadata",
[
(
{r"re:.*self_attn\.o_proj": "fp8_per_block"},
"model.layers.0.self_attn.o_proj",
Fp8PerBlockOnlineLinearMethod,
"model.layers.0.self_attn.qkv_proj",
("targets", "fp8_per_block", r"re:.*self_attn\.o_proj"),
),
(
{r"re:.*qkv_proj.*": "fp8_per_tensor"},
"model.layers.0.self_attn.qkv_proj",
Fp8PerTensorOnlineLinearMethod,
"model.layers.0.self_attn.o_proj",
("targets", "fp8_per_tensor", r"re:.*qkv_proj.*"),
),
(
{r"re:.*[qkv]_proj": "fp8_per_tensor"},
"model.layers.0.self_attn.qkv_proj",
Fp8PerTensorOnlineLinearMethod,
"model.layers.0.self_attn.o_proj",
("targets", "fp8_per_tensor", r"re:.*[qkv]_proj"),
),
],
ids=["linear_regex", "direct_fused_regex", "legacy_fused_regex"],
)
def test_online_quantization_targets(
default_vllm_config,
dist_init,
targets: dict[str, str],
prefix: str,
expected_method_cls,
unmatched_prefix: str,
expected_metadata: tuple[str, str, str],
) -> None:
"""Target patterns select the real online linear methods."""
default_vllm_config.model_config = ModelConfig()
config = OnlineQuantizationConfig(QuantizationConfigArgs(targets=targets))
config.packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
layer = ColumnParallelLinear(
input_size=1,
output_size=1,
bias=False,
disable_tp=True,
)
method = config.get_quant_method(layer, prefix)
assert isinstance(method, expected_method_cls)
assert config.quantized_layers == {prefix: expected_metadata}
unmatched_method = config.get_quant_method(layer, unmatched_prefix)
assert isinstance(unmatched_method, UnquantizedLinearMethod)
assert config.quantized_layers == {prefix: expected_metadata}
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
def test_online_quantization_records_global_config(
default_vllm_config, dist_init
) -> None:
default_vllm_config.model_config = ModelConfig()
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_block"))
prefix = "model.layers.0.self_attn.o_proj"
layer = ColumnParallelLinear(
input_size=1,
output_size=1,
bias=False,
disable_tp=True,
)
method = config.get_quant_method(layer, prefix)
assert isinstance(method, Fp8PerBlockOnlineLinearMethod)
assert config.quantized_layers == {
prefix: ("linear", str(config.args.linear), None)
}
def test_online_quantization_targets_ignore_collision() -> None:
"""A targets/ignore collision is reported when the layer is dispatched."""
config = OnlineQuantizationConfig(
QuantizationConfigArgs(
targets={"model.layers.0.self_attn.o_proj": "fp8_per_tensor"},
ignore=["model.layers.0.self_attn.o_proj"],
)
)
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
config._dispatch_target(
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
)
def test_online_quantization_targets_reject_unsupported_layer() -> None:
"""A targets match on a non-linear, non-MoE layer is rejected."""
config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={"lm_head": "fp8_per_tensor"})
)
lm_head = VocabParallelEmbedding(
num_embeddings=1,
embedding_dim=1,
disable_tp=True,
)
with pytest.raises(
ValueError,
match=(
"Layer lm_head was matched by quantization_config.targets "
r"\(lm_head\), but online quantization is not supported for "
"VocabParallelEmbedding."
),
):
config.get_quant_method(lm_head, "lm_head")
def test_log_online_quantization(default_vllm_config, monkeypatch) -> None:
config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor"))
config.quantized_layers = {
"model.layers.0.mlp.down_proj": ("linear", "fp8_per_tensor", None),
"model.layers.1.mlp.down_proj": ("linear", "fp8_per_tensor", None),
"model.layers.0.self_attn.qkv_proj": (
"targets",
"mxfp4",
r"re:.*qkv_proj.*",
),
}
default_vllm_config.quant_config = config
logged_messages: list[str] = []
def record_info(message: str, *args: object) -> None:
logged_messages.append(message % args)
monkeypatch.setattr(
"vllm.model_executor.model_loader.base_loader.logger.info", record_info
)
log_online_quantization(default_vllm_config)
assert logged_messages == [
"Quantized 3 layers of types: mlp.down_proj: 2 (from linear: "
"fp8_per_tensor); self_attn.qkv_proj: 1 (from targets: "
"re:.*qkv_proj.*, mxfp4)"
]
@pytest.mark.skipif(
not (
current_platform.is_cuda()
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for QuantizationConfigArgs parsing."""
from unittest.mock import Mock
import pytest
from vllm.config.quantization import (
@@ -10,6 +12,11 @@ from vllm.config.quantization import (
QuantSpec,
resolve_quantization_config,
)
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
_find_matching_targets,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8DynamicTokenSym,
@@ -34,6 +41,10 @@ def test_quant_spec_accepts_quant_key_directly():
assert spec.activation is None
def test_quant_spec_string_representation_uses_quantization_name():
assert str(QuantSpec(weight="mxfp4")) == "mxfp4"
def test_quant_spec_rejects_unknown_name():
with pytest.raises(ValueError, match="unknown quantization name"):
QuantSpec(weight="not_a_real_format")
@@ -70,6 +81,11 @@ def test_args_accepts_dict_form():
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
def test_targets_reject_non_string_keys():
with pytest.raises(ValueError, match="targets keys must be strings"):
QuantizationConfigArgs._validate_targets({123: "mxfp8"})
# ---- resolve_quantization_config -----------------------------------------
@@ -132,3 +148,64 @@ def test_static_block_weight_paired_with_dynamic_block_activation():
spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
assert spec.weight == kFp8Static128BlockSym
assert spec.activation == kFp8Dynamic128Sym
def test_targets_allow_distinct_patterns_with_the_same_shorthand():
layer_name = "model.layers.0.self_attn.qkv_proj"
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
targets = {
r"re:.*\.q_proj$": "mxfp8",
r"re:.*\.k_proj$": "mxfp8",
r"re:.*\.v_proj$": "mxfp8",
}
matches = _find_matching_targets(layer_name, targets, fused_mapping)
assert len(matches) == 1
assert targets[matches[0]] == "mxfp8"
def test_targets_reject_overlapping_patterns():
targets = {
r"re:.*o_proj": "fp8_per_tensor",
"model.layers.0.self_attn.o_proj": "fp8_per_block",
}
with pytest.raises(ValueError, match="multiple quantization_config.targets"):
_find_matching_targets("model.layers.0.self_attn.o_proj", targets)
def test_targets_reject_partially_matched_fused_layer():
targets = {r"re:.*q_proj": "fp8_per_tensor"}
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
with pytest.raises(ValueError, match="unmatched shards"):
_find_matching_targets(
"model.layers.0.self_attn.qkv_proj", targets, fused_mapping
)
def test_targets_reject_fused_shards_with_different_schemes():
targets = {
r"re:.*\.q_proj$": "fp8_per_tensor",
r"re:.*\.k_proj$": "fp8_per_block",
r"re:.*\.v_proj$": "fp8_per_tensor",
}
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
with pytest.raises(ValueError, match="different quantization_config.targets"):
_find_matching_targets(
"model.layers.0.self_attn.qkv_proj", targets, fused_mapping
)
def test_targets_reject_moe_only_shorthand_for_linear_layer():
config = OnlineQuantizationConfig(
QuantizationConfigArgs(
targets={"model.layers.0.self_attn.o_proj": "nvfp4_per_token"}
)
)
with pytest.raises(ValueError, match="does not define a QuantSpec"):
config._dispatch_target(
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
)
+21 -1
View File
@@ -50,7 +50,10 @@ from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E
QuarkW8A8Int8MoEMethod,
)
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
from vllm.model_executor.layers.quantization.quark.utils import (
QuarkQTensorHint,
should_ignore_layer,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
@@ -1294,6 +1297,23 @@ FUSED_MAPPING = {
}
def test_quark_should_ignore_layer_checks_children():
assert should_ignore_layer(
"model.layers.78.mlp.experts",
["model.layers.78.mlp.experts.0.down_proj"],
check_children=True,
)
def test_quark_should_ignore_layer_rejects_partial_fused_matches():
with pytest.raises(ValueError, match="different quantization schemes"):
should_ignore_layer(
"model.layers.0.self_attn.qkv_proj",
["model.layers.0.self_attn.q_proj"],
FUSED_MAPPING,
)
def test_fused_name_listed_directly_is_skipped():
# Regression for Step-3.5-Flash-FP8: the checkpoint lists the fused
# name (``qkv_proj``) directly in ``modules_to_not_convert``. When a