[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)
This commit is contained in:
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for QuantizationConfigArgs parsing."""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.quantization import (
|
||||
@@ -10,6 +12,11 @@ from vllm.config.quantization import (
|
||||
QuantSpec,
|
||||
resolve_quantization_config,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
_find_matching_targets,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTokenSym,
|
||||
@@ -34,6 +41,10 @@ def test_quant_spec_accepts_quant_key_directly():
|
||||
assert spec.activation is None
|
||||
|
||||
|
||||
def test_quant_spec_string_representation_uses_quantization_name():
|
||||
assert str(QuantSpec(weight="mxfp4")) == "mxfp4"
|
||||
|
||||
|
||||
def test_quant_spec_rejects_unknown_name():
|
||||
with pytest.raises(ValueError, match="unknown quantization name"):
|
||||
QuantSpec(weight="not_a_real_format")
|
||||
@@ -70,6 +81,11 @@ def test_args_accepts_dict_form():
|
||||
assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic)
|
||||
|
||||
|
||||
def test_targets_reject_non_string_keys():
|
||||
with pytest.raises(ValueError, match="targets keys must be strings"):
|
||||
QuantizationConfigArgs._validate_targets({123: "mxfp8"})
|
||||
|
||||
|
||||
# ---- resolve_quantization_config -----------------------------------------
|
||||
|
||||
|
||||
@@ -132,3 +148,64 @@ def test_static_block_weight_paired_with_dynamic_block_activation():
|
||||
spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic")
|
||||
assert spec.weight == kFp8Static128BlockSym
|
||||
assert spec.activation == kFp8Dynamic128Sym
|
||||
|
||||
|
||||
def test_targets_allow_distinct_patterns_with_the_same_shorthand():
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
targets = {
|
||||
r"re:.*\.q_proj$": "mxfp8",
|
||||
r"re:.*\.k_proj$": "mxfp8",
|
||||
r"re:.*\.v_proj$": "mxfp8",
|
||||
}
|
||||
|
||||
matches = _find_matching_targets(layer_name, targets, fused_mapping)
|
||||
assert len(matches) == 1
|
||||
assert targets[matches[0]] == "mxfp8"
|
||||
|
||||
|
||||
def test_targets_reject_overlapping_patterns():
|
||||
targets = {
|
||||
r"re:.*o_proj": "fp8_per_tensor",
|
||||
"model.layers.0.self_attn.o_proj": "fp8_per_block",
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError, match="multiple quantization_config.targets"):
|
||||
_find_matching_targets("model.layers.0.self_attn.o_proj", targets)
|
||||
|
||||
|
||||
def test_targets_reject_partially_matched_fused_layer():
|
||||
targets = {r"re:.*q_proj": "fp8_per_tensor"}
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
with pytest.raises(ValueError, match="unmatched shards"):
|
||||
_find_matching_targets(
|
||||
"model.layers.0.self_attn.qkv_proj", targets, fused_mapping
|
||||
)
|
||||
|
||||
|
||||
def test_targets_reject_fused_shards_with_different_schemes():
|
||||
targets = {
|
||||
r"re:.*\.q_proj$": "fp8_per_tensor",
|
||||
r"re:.*\.k_proj$": "fp8_per_block",
|
||||
r"re:.*\.v_proj$": "fp8_per_tensor",
|
||||
}
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
with pytest.raises(ValueError, match="different quantization_config.targets"):
|
||||
_find_matching_targets(
|
||||
"model.layers.0.self_attn.qkv_proj", targets, fused_mapping
|
||||
)
|
||||
|
||||
|
||||
def test_targets_reject_moe_only_shorthand_for_linear_layer():
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(
|
||||
targets={"model.layers.0.self_attn.o_proj": "nvfp4_per_token"}
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="does not define a QuantSpec"):
|
||||
config._dispatch_target(
|
||||
"model.layers.0.self_attn.o_proj", Mock(spec=LinearBase)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user