[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)

This commit is contained in:
fxmarty-amd
2026-09-02 08:55:15 -04:00
committed by GitHub
parent 2a4e3cc3db
commit 7894394b07
11 changed files with 892 additions and 172 deletions
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for QuantizationConfigArgs parsing."""
from unittest.mock import Mock
import pytest
from vllm.config.quantization import (
@@ -10,6 +12,11 @@ from vllm.config.quantization import (
QuantSpec,
resolve_quantization_config,
)
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
_find_matching_targets,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8DynamicTokenSym,
@@ -34,6 +41,10 @@ def test_quant_spec_accepts_quant_key_directly():
assert spec.activation is None
def test_quant_spec_string_representation_uses_quantization_name():
assert str(QuantSpec(weight="mxfp4")) == "mxfp4"
def test_quant_spec_rejects_unknown_name():
with pytest.raises(ValueError, match="unknown quantization name"):
QuantSpec(weight="not_a_real_format")
@@ -70,6 +81,11 @@ def test_args_accepts_dict_form():
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
def test_targets_reject_non_string_keys():
with pytest.raises(ValueError, match="targets keys must be strings"):
QuantizationConfigArgs._validate_targets({123: "mxfp8"})
# ---- resolve_quantization_config -----------------------------------------
@@ -132,3 +148,64 @@ def test_static_block_weight_paired_with_dynamic_block_activation():
spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
assert spec.weight == kFp8Static128BlockSym
assert spec.activation == kFp8Dynamic128Sym
def test_targets_allow_distinct_patterns_with_the_same_shorthand():
layer_name = "model.layers.0.self_attn.qkv_proj"
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
targets = {
r"re:.*\.q_proj$": "mxfp8",
r"re:.*\.k_proj$": "mxfp8",
r"re:.*\.v_proj$": "mxfp8",
}
matches = _find_matching_targets(layer_name, targets, fused_mapping)
assert len(matches) == 1
assert targets[matches[0]] == "mxfp8"
def test_targets_reject_overlapping_patterns():
targets = {
r"re:.*o_proj": "fp8_per_tensor",
"model.layers.0.self_attn.o_proj": "fp8_per_block",
}
with pytest.raises(ValueError, match="multiple quantization_config.targets"):
_find_matching_targets("model.layers.0.self_attn.o_proj", targets)
def test_targets_reject_partially_matched_fused_layer():
targets = {r"re:.*q_proj": "fp8_per_tensor"}
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
with pytest.raises(ValueError, match="unmatched shards"):
_find_matching_targets(
"model.layers.0.self_attn.qkv_proj", targets, fused_mapping
)
def test_targets_reject_fused_shards_with_different_schemes():
targets = {
r"re:.*\.q_proj$": "fp8_per_tensor",
r"re:.*\.k_proj$": "fp8_per_block",
r"re:.*\.v_proj$": "fp8_per_tensor",
}
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
with pytest.raises(ValueError, match="different quantization_config.targets"):
_find_matching_targets(
"model.layers.0.self_attn.qkv_proj", targets, fused_mapping
)
def test_targets_reject_moe_only_shorthand_for_linear_layer():
config = OnlineQuantizationConfig(
QuantizationConfigArgs(
targets={"model.layers.0.self_attn.o_proj": "nvfp4_per_token"}
)
)
with pytest.raises(ValueError, match="does not define a QuantSpec"):
config._dispatch_target(
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
)