diff --git a/docs/features/quantization/online.md b/docs/features/quantization/online.md index c7257c8d39..342205d000 100644 --- a/docs/features/quantization/online.md +++ b/docs/features/quantization/online.md @@ -67,7 +67,7 @@ quantization_config: moe: weight: activation: - ignore: [, ...] + ignore: [, ...] ``` `linear` and `moe` accept a full `{weight, activation}` dict, or a bare @@ -133,7 +133,7 @@ llm = LLM( ### Excluding Layers from Quantization -Use the `ignore` parameter to skip specific layers. It accepts exact layer names and regex patterns (prefixed with `re:`): +Use the `ignore` parameter to skip specific layers. It accepts exact layer names, regex patterns (prefixed with `re:`), and patterns understood by [`fnmatch.fnmatch`](https://docs.python.org/3/library/fnmatch.html#fnmatch.fnmatch): ```python from vllm import LLM @@ -147,10 +147,54 @@ llm = LLM( "model.layers.1.self_attn.o_proj", # regex: skip all QKV projections "re:.*[qkv]_proj", + # fnmatch: skip all MoE experts + "*mlp.experts*", ], }, ) ``` !!! note - For fused layers (e.g., `qkv_proj` which fuses `q_proj`, `k_proj`, `v_proj`), the ignore pattern must match the **unfused** shard names (`q_proj`, `k_proj`, `v_proj`), not the fused name. + For fused layers (e.g., `qkv_proj` which fuses `q_proj`, `k_proj`, `v_proj`), patterns may match the fused name directly or all of its unfused shard names. + +### Fine-Grained Per-Layer Quantization Schemes + +Use the `targets` parameter to apply different online shorthands to different layers, instead of one scheme applied everywhere via `linear`/`moe`. Keys are exact layer names, regex patterns (prefixed with `re:`), or patterns understood by [`fnmatch.fnmatch`](https://docs.python.org/3/library/fnmatch.html#fnmatch.fnmatch); values are shorthand names (`fp8_per_tensor`, `fp8_per_block`, `fp8_per_channel`, `mxfp8`, `int8_per_channel_weight_only`, `nvfp4_per_token`). + +Example: + +```python +from vllm import LLM + +llm = LLM( + "Qwen/Qwen3.5-35B-A3B", + quantization="online", + quantization_config={ + "targets": { + # exact layer name + "model.layers.0.self_attn.o_proj": "fp8_per_tensor", + # regex: quantize all QKV projections + r"re:.*self_attn\.qkv_proj.*": "mxfp8", + # fnmatch: quantize all MoE experts + "*mlp.experts*": "mxfp4", + }, + }, +) +``` + +Or with the CLI: + +```bash +vllm serve Qwen/Qwen3.5-35B-A3B \ + --quantization online \ + --quantization-config '{"targets":{"model.layers.0.self_attn.o_proj":"fp8_per_tensor","re:.*self_attn\\.qkv_proj.*":"mxfp8","*mlp.experts*":"mxfp4"}}' +``` + +!!! info + + - `targets` is mutually exclusive with online `linear` and `moe`: set one or the other, not both. + - A layer that matches no `targets` pattern is left unchanged from its checkpoint dtype. + - A layer name may not match both `targets` and `ignore`, raising an error. + - A layer may not match more than one `targets` pattern, raising an error. + - fnmatch-style patterns are supported only by online quantization; they are + not applied to Quark or compressed-tensors configurations. diff --git a/tests/quantization/test_config_utils.py b/tests/quantization/test_config_utils.py new file mode 100644 index 0000000000..cdab0fbd45 --- /dev/null +++ b/tests/quantization/test_config_utils.py @@ -0,0 +1,195 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests quantization configuration matching utilities.""" + +from unittest.mock import Mock + +import pytest + +from vllm.config.quantization import QuantizationConfigArgs +from vllm.model_executor.layers.quantization.compressed_tensors.utils import ( + should_ignore_layer, +) +from vllm.model_executor.layers.quantization.online.base import ( + OnlineQuantizationConfig, + _find_matching_targets, +) +from vllm.model_executor.layers.quantization.quark.utils import ( + should_ignore_layer as quark_should_ignore_layer, +) +from vllm.model_executor.layers.quantization.utils.config_utils import ( + find_matching_patterns, + get_layer_name_after_index, + is_equal_or_regex_match, +) + + +def test_is_equal_or_regex_match(): + assert is_equal_or_regex_match( + "model.layers.0.mlp.down_proj", "model.layers.0.mlp.down_proj" + ) + assert is_equal_or_regex_match("model.layers.0.mlp.down_proj", r"re:.*down_proj") + assert not is_equal_or_regex_match("model.layers.0.mlp.down_proj", "other") + + +def test_fnmatch_is_opt_in(): + layer_name = "model.layers.0.mlp.experts" + pattern = "*mlp.experts*" + + assert not is_equal_or_regex_match(layer_name, pattern) + assert is_equal_or_regex_match(layer_name, pattern, use_fnmatch=True) + assert not should_ignore_layer(layer_name, [pattern]) + assert not quark_should_ignore_layer(layer_name, [pattern]) + assert should_ignore_layer(layer_name, [pattern], use_fnmatch=True) + + +def test_online_targets_support_fnmatch_patterns(): + layer_name = "model.layers.0.mlp.experts" + targets = {"*mlp.experts*": "mxfp4"} + + assert _find_matching_targets(layer_name, targets) == ["*mlp.experts*"] + + +def test_online_ignore_supports_fnmatch_patterns(): + layer_name = "model.layers.0.mlp.experts" + ignore = ["*mlp.experts*"] + config = OnlineQuantizationConfig( + QuantizationConfigArgs(targets={"*mlp.experts*": "mxfp4"}, ignore=ignore) + ) + + with pytest.raises(ValueError, match="matches both quantization_config.ignore"): + config._dispatch_target(layer_name, Mock()) + + +@pytest.mark.parametrize( + "layer_name,expected", + [ + ("model.layers.1.self_attn.qkv_proj", "self_attn.qkv_proj"), + ("model.layers.2.mlp.down_proj", "mlp.down_proj"), + ("lm_head", "lm_head"), + ], +) +def test_get_layer_name_after_index(layer_name, expected): + assert get_layer_name_after_index(layer_name) == expected + + +@pytest.mark.parametrize( + "patterns", + [ + [r"re:.*qkv_proj.*"], + [r"re:.*\.[qkv]_proj$"], + ], + ids=["direct_fused_regex", "fused_shard_regexes"], +) +def test_find_matching_patterns_for_fused_regexes(patterns): + layer_name = "model.layers.0.self_attn.qkv_proj" + fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + + matches = find_matching_patterns(layer_name, patterns, fused_mapping) + assert all(len(shard_matches) == 1 for shard_matches in matches) + + +@pytest.mark.parametrize( + "patterns,expected", + [ + ( + [r"re:.*\.qkv_proj$"], + [{r"re:.*\.qkv_proj$"}], + ), + ( + [r"re:.*\.q_proj$", r"re:.*\.k_proj$", r"re:.*\.v_proj$"], + [ + {r"re:.*\.q_proj$"}, + {r"re:.*\.k_proj$"}, + {r"re:.*\.v_proj$"}, + ], + ), + ], + ids=["direct_fused_match", "individual_shard_matches"], +) +def test_find_matching_patterns_distinguishes_direct_and_shard_matches( + patterns, expected +): + layer_name = "model.layers.0.self_attn.qkv_proj" + fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + + assert find_matching_patterns(layer_name, patterns, fused_mapping) == expected + assert should_ignore_layer(layer_name, patterns, fused_mapping) + + +def test_should_ignore_layer_rejects_partially_matched_fused_layer(): + layer_name = "model.layers.0.self_attn.qkv_proj" + patterns = [r"re:.*\.q_proj$"] + fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + + assert find_matching_patterns(layer_name, patterns, fused_mapping) == [ + {r"re:.*\.q_proj$"}, + set(), + set(), + ] + with pytest.raises(ValueError, match="different quantization schemes"): + should_ignore_layer(layer_name, patterns, fused_mapping) + + +def test_should_ignore_layer_returns_false_when_no_fused_pattern_matches(): + layer_name = "model.layers.0.self_attn.qkv_proj" + fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + + assert find_matching_patterns(layer_name, [], fused_mapping) == [ + set(), + set(), + set(), + ] + assert not should_ignore_layer(layer_name, [], fused_mapping) + + +@pytest.mark.parametrize( + "patterns", + [ + [r"re:.*qkv_proj.*"], + [r"re:.*\.[qkv]_proj$"], + ], + ids=["direct_fused_regex", "fused_shard_regexes"], +) +def test_ignore_and_targets_match_fused_regexes_identically(patterns): + layer_name = "model.layers.0.self_attn.qkv_proj" + fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + targets = {pattern: "fp8_per_block" for pattern in patterns} + + assert should_ignore_layer(layer_name, patterns, fused_mapping) + matches = _find_matching_targets(layer_name, targets, fused_mapping) + assert len(matches) == 1 + assert targets[matches[0]] == "fp8_per_block" + + +def test_ignore_allows_individually_matched_fused_shards(): + layer_name = "model.layers.0.self_attn.qkv_proj" + patterns = [ + "model.layers.0.self_attn.q_proj", + "model.layers.0.self_attn.k_proj", + "model.layers.0.self_attn.v_proj", + ] + fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + + assert should_ignore_layer(layer_name, patterns, fused_mapping) + + +@pytest.mark.parametrize( + "patterns", + [ + ["model.layers.0.self_attn.qkv_proj"], + [r"re:.*\.qkv_proj$"], + [ + "model.layers.0.self_attn.q_proj", + "model.layers.0.self_attn.k_proj", + "model.layers.0.self_attn.v_proj", + ], + ], + ids=["direct_fused_name", "direct_fused_regex", "individual_shards"], +) +def test_quark_and_compressed_tensors_ignore_fused_layers_identically(patterns): + layer_name = "model.layers.0.self_attn.qkv_proj" + fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + + assert should_ignore_layer(layer_name, patterns, fused_mapping) + assert quark_should_ignore_layer(layer_name, patterns, fused_mapping) diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index 081a5372d8..4903878bd6 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -18,10 +18,21 @@ from tests.quantization.utils import ( from vllm import _custom_ops as ops from vllm._aiter_ops import rocm_aiter_ops from vllm._custom_ops import scaled_fp4_quant -from vllm.config.quantization import resolve_quantization_config +from vllm.config import ModelConfig +from vllm.config.quantization import ( + QuantizationConfigArgs, + resolve_quantization_config, +) from vllm.forward_context import set_forward_context from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.linear import UnquantizedLinearMethod +from vllm.model_executor.layers.linear import ( + ColumnParallelLinear, + LinearBase, + UnquantizedLinearMethod, +) +from vllm.model_executor.layers.quantization.online.base import ( + OnlineQuantizationConfig, +) from vllm.model_executor.layers.quantization.online.fp8 import ( Fp8PerBlockOnlineLinearMethod, Fp8PerBlockOnlineMoEMethod, @@ -49,6 +60,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( amax_for_tp_weight_quant, weight_amax, ) +from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.base_loader import log_online_quantization from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader from vllm.model_executor.models.granitemoe import ( GraniteMoeModel, @@ -154,12 +169,23 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( Fp8PerBlockOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, ), + # quantization='online' with per-layer target patterns + ( + "online", + { + "targets": { + r"re:.*self_attn\.o_proj": "fp8_per_block", + r"re:.*block_sparse_moe\.experts": "fp8_per_tensor", + } + }, + Fp8PerBlockOnlineLinearMethod, + Fp8PerTensorOnlineMoEMethod, + ), # ignore with direct layer name ( "fp8_per_tensor", - # qkv_proj is fused from q_proj/k_proj/v_proj, so currently the - # ignore regex must match the unfused shard names - # TODO(future PR): also make 're:.*qkv_proj.*' work + # qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex + # remains supported alongside direct fused-name regexes. {"ignore": ["model.layers.1.self_attn.o_proj", "re:.*[qkv]_proj"]}, Fp8PerTensorOnlineLinearMethod, Fp8PerTensorOnlineMoEMethod, @@ -171,6 +197,15 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( Mxfp4OnlineMoEMethod, ), ], + ids=[ + "fp8_per_tensor", + "fp8_per_block", + "fp8_per_channel", + "per_layer_kind_overrides", + "targets", + "ignore", + "mxfp4", + ], ) @pytest.mark.parametrize( "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False] @@ -266,6 +301,12 @@ def test_online_quantization( layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod ) + if isinstance(online_quant_args, dict) and "targets" in online_quant_args: + for layer in model.model.layers: + assert isinstance( + layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod + ) + input_ids = torch.tensor([1, 2, 3, 4], device=DEVICE) positions = torch.arange(input_ids.numel(), device=DEVICE) with set_forward_context(None, vllm_config, num_tokens=input_ids.numel()): @@ -321,6 +362,158 @@ def test_online_quantization_loads_real_weights(vllm_runner, monkeypatch) -> Non assert outputs +@pytest.mark.skipif( + not is_quant_method_supported("fp8"), + reason="FP8 is not supported on this GPU type.", +) +@pytest.mark.parametrize( + "targets,prefix,expected_method_cls,unmatched_prefix,expected_metadata", + [ + ( + {r"re:.*self_attn\.o_proj": "fp8_per_block"}, + "model.layers.0.self_attn.o_proj", + Fp8PerBlockOnlineLinearMethod, + "model.layers.0.self_attn.qkv_proj", + ("targets", "fp8_per_block", r"re:.*self_attn\.o_proj"), + ), + ( + {r"re:.*qkv_proj.*": "fp8_per_tensor"}, + "model.layers.0.self_attn.qkv_proj", + Fp8PerTensorOnlineLinearMethod, + "model.layers.0.self_attn.o_proj", + ("targets", "fp8_per_tensor", r"re:.*qkv_proj.*"), + ), + ( + {r"re:.*[qkv]_proj": "fp8_per_tensor"}, + "model.layers.0.self_attn.qkv_proj", + Fp8PerTensorOnlineLinearMethod, + "model.layers.0.self_attn.o_proj", + ("targets", "fp8_per_tensor", r"re:.*[qkv]_proj"), + ), + ], + ids=["linear_regex", "direct_fused_regex", "legacy_fused_regex"], +) +def test_online_quantization_targets( + default_vllm_config, + dist_init, + targets: dict[str, str], + prefix: str, + expected_method_cls, + unmatched_prefix: str, + expected_metadata: tuple[str, str, str], +) -> None: + """Target patterns select the real online linear methods.""" + default_vllm_config.model_config = ModelConfig() + config = OnlineQuantizationConfig(QuantizationConfigArgs(targets=targets)) + config.packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + + layer = ColumnParallelLinear( + input_size=1, + output_size=1, + bias=False, + disable_tp=True, + ) + + method = config.get_quant_method(layer, prefix) + assert isinstance(method, expected_method_cls) + assert config.quantized_layers == {prefix: expected_metadata} + + unmatched_method = config.get_quant_method(layer, unmatched_prefix) + assert isinstance(unmatched_method, UnquantizedLinearMethod) + assert config.quantized_layers == {prefix: expected_metadata} + + +@pytest.mark.skipif( + not is_quant_method_supported("fp8"), + reason="FP8 is not supported on this GPU type.", +) +def test_online_quantization_records_global_config( + default_vllm_config, dist_init +) -> None: + default_vllm_config.model_config = ModelConfig() + config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_block")) + prefix = "model.layers.0.self_attn.o_proj" + layer = ColumnParallelLinear( + input_size=1, + output_size=1, + bias=False, + disable_tp=True, + ) + + method = config.get_quant_method(layer, prefix) + + assert isinstance(method, Fp8PerBlockOnlineLinearMethod) + assert config.quantized_layers == { + prefix: ("linear", str(config.args.linear), None) + } + + +def test_online_quantization_targets_ignore_collision() -> None: + """A targets/ignore collision is reported when the layer is dispatched.""" + config = OnlineQuantizationConfig( + QuantizationConfigArgs( + targets={"model.layers.0.self_attn.o_proj": "fp8_per_tensor"}, + ignore=["model.layers.0.self_attn.o_proj"], + ) + ) + with pytest.raises(ValueError, match="matches both quantization_config.ignore"): + config._dispatch_target( + "model.layers.0.self_attn.o_proj", Mock(spec=LinearBase) + ) + + +def test_online_quantization_targets_reject_unsupported_layer() -> None: + """A targets match on a non-linear, non-MoE layer is rejected.""" + config = OnlineQuantizationConfig( + QuantizationConfigArgs(targets={"lm_head": "fp8_per_tensor"}) + ) + lm_head = VocabParallelEmbedding( + num_embeddings=1, + embedding_dim=1, + disable_tp=True, + ) + + with pytest.raises( + ValueError, + match=( + "Layer lm_head was matched by quantization_config.targets " + r"\(lm_head\), but online quantization is not supported for " + "VocabParallelEmbedding." + ), + ): + config.get_quant_method(lm_head, "lm_head") + + +def test_log_online_quantization(default_vllm_config, monkeypatch) -> None: + config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor")) + config.quantized_layers = { + "model.layers.0.mlp.down_proj": ("linear", "fp8_per_tensor", None), + "model.layers.1.mlp.down_proj": ("linear", "fp8_per_tensor", None), + "model.layers.0.self_attn.qkv_proj": ( + "targets", + "mxfp4", + r"re:.*qkv_proj.*", + ), + } + default_vllm_config.quant_config = config + + logged_messages: list[str] = [] + + def record_info(message: str, *args: object) -> None: + logged_messages.append(message % args) + + monkeypatch.setattr( + "vllm.model_executor.model_loader.base_loader.logger.info", record_info + ) + log_online_quantization(default_vllm_config) + + assert logged_messages == [ + "Quantized 3 layers of types: mlp.down_proj: 2 (from linear: " + "fp8_per_tensor); self_attn.qkv_proj: 1 (from targets: " + "re:.*qkv_proj.*, mxfp4)" + ] + + @pytest.mark.skipif( not ( current_platform.is_cuda() diff --git a/tests/quantization/test_quantization_config_args.py b/tests/quantization/test_quantization_config_args.py index 7e881d6b0b..1b67c18caa 100644 --- a/tests/quantization/test_quantization_config_args.py +++ b/tests/quantization/test_quantization_config_args.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Unit tests for QuantizationConfigArgs parsing.""" +from unittest.mock import Mock + import pytest from vllm.config.quantization import ( @@ -10,6 +12,11 @@ from vllm.config.quantization import ( QuantSpec, resolve_quantization_config, ) +from vllm.model_executor.layers.linear import LinearBase +from vllm.model_executor.layers.quantization.online.base import ( + OnlineQuantizationConfig, + _find_matching_targets, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Dynamic128Sym, kFp8DynamicTokenSym, @@ -34,6 +41,10 @@ def test_quant_spec_accepts_quant_key_directly(): assert spec.activation is None +def test_quant_spec_string_representation_uses_quantization_name(): + assert str(QuantSpec(weight="mxfp4")) == "mxfp4" + + def test_quant_spec_rejects_unknown_name(): with pytest.raises(ValueError, match="unknown quantization name"): QuantSpec(weight="not_a_real_format") @@ -70,6 +81,11 @@ def test_args_accepts_dict_form(): assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic) +def test_targets_reject_non_string_keys(): + with pytest.raises(ValueError, match="targets keys must be strings"): + QuantizationConfigArgs._validate_targets({123: "mxfp8"}) + + # ---- resolve_quantization_config ----------------------------------------- @@ -132,3 +148,64 @@ def test_static_block_weight_paired_with_dynamic_block_activation(): spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic") assert spec.weight == kFp8Static128BlockSym assert spec.activation == kFp8Dynamic128Sym + + +def test_targets_allow_distinct_patterns_with_the_same_shorthand(): + layer_name = "model.layers.0.self_attn.qkv_proj" + fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + targets = { + r"re:.*\.q_proj$": "mxfp8", + r"re:.*\.k_proj$": "mxfp8", + r"re:.*\.v_proj$": "mxfp8", + } + + matches = _find_matching_targets(layer_name, targets, fused_mapping) + assert len(matches) == 1 + assert targets[matches[0]] == "mxfp8" + + +def test_targets_reject_overlapping_patterns(): + targets = { + r"re:.*o_proj": "fp8_per_tensor", + "model.layers.0.self_attn.o_proj": "fp8_per_block", + } + + with pytest.raises(ValueError, match="multiple quantization_config.targets"): + _find_matching_targets("model.layers.0.self_attn.o_proj", targets) + + +def test_targets_reject_partially_matched_fused_layer(): + targets = {r"re:.*q_proj": "fp8_per_tensor"} + fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + + with pytest.raises(ValueError, match="unmatched shards"): + _find_matching_targets( + "model.layers.0.self_attn.qkv_proj", targets, fused_mapping + ) + + +def test_targets_reject_fused_shards_with_different_schemes(): + targets = { + r"re:.*\.q_proj$": "fp8_per_tensor", + r"re:.*\.k_proj$": "fp8_per_block", + r"re:.*\.v_proj$": "fp8_per_tensor", + } + fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + + with pytest.raises(ValueError, match="different quantization_config.targets"): + _find_matching_targets( + "model.layers.0.self_attn.qkv_proj", targets, fused_mapping + ) + + +def test_targets_reject_moe_only_shorthand_for_linear_layer(): + config = OnlineQuantizationConfig( + QuantizationConfigArgs( + targets={"model.layers.0.self_attn.o_proj": "nvfp4_per_token"} + ) + ) + + with pytest.raises(ValueError, match="does not define a QuantSpec"): + config._dispatch_target( + "model.layers.0.self_attn.o_proj", Mock(spec=LinearBase) + ) diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index 1c1537a122..bbb1702502 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -50,7 +50,10 @@ from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E QuarkW8A8Int8MoEMethod, ) from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme -from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint +from vllm.model_executor.layers.quantization.quark.utils import ( + QuarkQTensorHint, + should_ignore_layer, +) from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( quant_dequant_mxfp4, ) @@ -1294,6 +1297,23 @@ FUSED_MAPPING = { } +def test_quark_should_ignore_layer_checks_children(): + assert should_ignore_layer( + "model.layers.78.mlp.experts", + ["model.layers.78.mlp.experts.0.down_proj"], + check_children=True, + ) + + +def test_quark_should_ignore_layer_rejects_partial_fused_matches(): + with pytest.raises(ValueError, match="different quantization schemes"): + should_ignore_layer( + "model.layers.0.self_attn.qkv_proj", + ["model.layers.0.self_attn.q_proj"], + FUSED_MAPPING, + ) + + def test_fused_name_listed_directly_is_skipped(): # Regression for Step-3.5-Flash-FP8: the checkpoint lists the fused # name (``qkv_proj``) directly in ``modules_to_not_convert``. When a diff --git a/vllm/config/quantization.py b/vllm/config/quantization.py index 93ea6736b7..2d78e37a79 100644 --- a/vllm/config/quantization.py +++ b/vllm/config/quantization.py @@ -3,7 +3,14 @@ from typing import Annotated, Any -from pydantic import Field, GetPydanticSchema, ValidationInfo, field_validator +import regex as re +from pydantic import ( + Field, + GetPydanticSchema, + ValidationInfo, + field_validator, + model_validator, +) from pydantic_core import core_schema from vllm.config.utils import config @@ -76,6 +83,21 @@ class QuantSpec: activation: QuantKeyField = None """Activation quantization key, or a name from QUANT_KEY_NAMES.""" + def __str__(self) -> str: + def quant_key_str(quant_key: QuantKey | None) -> str: + if quant_key is None: + return "None" + return next( + ( + name + for name, known_quant_key in QUANT_KEY_NAMES.items() + if known_quant_key == quant_key + ), + str(quant_key), + ) + + return quant_key_str(self.weight) + @config class QuantizationConfigArgs: @@ -92,7 +114,16 @@ class QuantizationConfigArgs: """Spec applied to ``FusedMoEFactory`` layers.""" ignore: list[str] = Field(default_factory=list) - """Layers to skip quantization for.""" + """Layers to skip quantization for. Online quantization also supports + fnmatch-style patterns.""" + + targets: dict[str, str] | None = None + """Per-layer online quantization overrides, keyed by exact layer name or + regex patterns with a `re:`, or fnmatch-style patterns for online + quantization, mapping to an online shorthand name (see + `_ONLINE_SHORTHANDS`). A layer that matches no pattern is left unquantized. + Mutually exclusive with `linear` and `moe`. + """ @field_validator("linear", "moe", mode="before") @classmethod @@ -110,6 +141,46 @@ class QuantizationConfigArgs: return spec return QuantSpec(weight=_coerce_quant_key(v)) + @field_validator("targets", mode="before") + @classmethod + def _validate_targets(cls, v: Any) -> Any: + if v is None: + return v + if not isinstance(v, dict): + raise TypeError(f"targets must be a dict, got {type(v).__name__}") + for pattern, shorthand in v.items(): + if not isinstance(pattern, str): + raise ValueError( + f"targets keys must be strings, got {type(pattern).__name__}" + ) + if not isinstance(shorthand, str) or shorthand not in _ONLINE_SHORTHANDS: + raise ValueError( + f"targets[{pattern}] = {shorthand} is not a valid " + f"online shorthand name; expected one of " + f"{sorted(_ONLINE_SHORTHANDS)}" + ) + if pattern.startswith("re:"): + try: + re.compile(pattern[3:]) + except re.error as e: + raise ValueError( + f"targets key {pattern} is not a valid regex: {e}" + ) from e + return v + + @model_validator(mode="after") + def _validate_targets_exclusivity(self) -> "QuantizationConfigArgs": + if self.targets is None: + return self + if self.linear is not None or self.moe is not None: + raise ValueError( + "quantization_config.targets is mutually exclusive with " + f"quantization_config.linear/moe, got " + f"targets={self.targets}, linear={self.linear}, " + f"moe={self.moe}." + ) + return self + # CLI shorthands accepted by `--quantization`. Each desugars to a full # QuantizationConfigArgs; activation overrides go through quantization_config. @@ -191,4 +262,5 @@ def resolve_quantization_config( linear=quantization_config.linear or base.linear, moe=quantization_config.moe or base.moe, ignore=quantization_config.ignore or base.ignore, + targets=quantization_config.targets or base.targets, ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/utils.py b/vllm/model_executor/layers/quantization/compressed_tensors/utils.py index 872771af9a..188ee8c4c9 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/utils.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/utils.py @@ -4,11 +4,14 @@ from collections.abc import Iterable, Mapping from types import MappingProxyType -import regex as re from compressed_tensors import CompressionFormat from compressed_tensors.quantization import QuantizationStrategy from torch.nn import Module +from vllm.model_executor.layers.quantization.utils.config_utils import ( + find_matching_patterns, + is_equal_or_regex_match, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, @@ -51,63 +54,20 @@ def should_ignore_layer( layer_name: str | None, ignore: Iterable[str] = tuple(), fused_mapping: Mapping[str, list[str]] = MappingProxyType({}), + use_fnmatch: bool = False, ) -> bool: if layer_name is None: return False - - # layer_name = model.layers.0.self_attn.qkv_proj - # proj_name = qkv_proj - proj_name = layer_name.split(".")[-1] - - # Fused layers like gate_up_proj or qkv_proj will not be fused - # in the safetensors checkpoint. So, we convert the name - # from the fused version to unfused + check to make sure that - # each shard of the fused layer has the same scheme. - if proj_name in fused_mapping and layer_name not in ignore: - shard_proj_names = fused_mapping[proj_name] - - # Convert fused_name --> [shard_names] - shard_names = [ - layer_name.replace(proj_name, shard_proj_name) - for shard_proj_name in shard_proj_names - ] - - # Layer should be ignored if shards are ignored. - should_ignore_layer = None - for shard_name in shard_names: - should_ignore_shard = check_equal_or_regex_match( - layer_name=shard_name, targets=ignore - ) - - # If shard_idx=0, set layer ignore to match shard. - if should_ignore_layer is None: - should_ignore_layer = should_ignore_shard - - # If shard_idx=1+ confirm scheme matches prior shards. - elif should_ignore_shard != should_ignore_layer: - raise ValueError( - f"Found a different quantization schemes for " - f"{shard_proj_names} in {layer_name}. vLLM " - "requires all to use the same scheme." - ) - - # Unfused layers like down_proj and o_proj will match - # the safetensors checkpoint already. - else: - should_ignore_layer = check_equal_or_regex_match( - layer_name=layer_name, targets=ignore + per_shard_matches = find_matching_patterns( + layer_name, ignore, fused_mapping, use_fnmatch=use_fnmatch + ) + shards_ignored = [len(matches) > 0 for matches in per_shard_matches] + if any(shards_ignored) and not all(shards_ignored): + raise ValueError( + f"Found different quantization schemes for the shards of " + f"{layer_name}. vLLM requires all to use the same scheme." ) - - assert should_ignore_layer is not None - return should_ignore_layer - - -def check_equal_or_regex_match(layer_name: str, targets: Iterable[str]) -> bool: - """ - Checks whether a layer_name is exactly equal or a regex match for - if target starts with 're:' to any target in list. - """ - return any(_is_equal_or_regex_match(layer_name, target) for target in targets) + return all(shards_ignored) def find_matched_target( @@ -167,32 +127,11 @@ def _find_first_match( """ for target in targets: - if _is_equal_or_regex_match(value, target, check_contains=check_contains): + if is_equal_or_regex_match(value, target, check_contains=check_contains): return target return None -def _is_equal_or_regex_match( - value: str, target: str, check_contains: bool = False -) -> bool: - """ - Checks whether a value is exactly equal or a regex match for target - if target starts with 're:'. If check_contains is set to True, - additionally checks if the target string is contained within the value. - """ - - if target.startswith("re:"): - pattern = target[3:] - if re.match(pattern, value): - return True - elif check_contains: - if target.lower() in value.lower(): - return True - elif target == value: - return True - return False - - def _match_fused_layer( layer_name: str, target_layers: Iterable[str], @@ -230,7 +169,7 @@ def _match_fused_layer( unfused_matches: list[str | None] = [] for unfused in unfused_paths: for target in target_layers: - if _is_equal_or_regex_match(unfused, target): + if is_equal_or_regex_match(unfused, target): unfused_matches.append(target) break else: diff --git a/vllm/model_executor/layers/quantization/online/base.py b/vllm/model_executor/layers/quantization/online/base.py index c9e7a43e3e..a9f994daef 100644 --- a/vllm/model_executor/layers/quantization/online/base.py +++ b/vllm/model_executor/layers/quantization/online/base.py @@ -1,11 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Mapping +from types import MappingProxyType from typing import Any import torch -from vllm.config.quantization import QuantizationConfigArgs, QuantSpec +from vllm.config.quantization import ( + _ONLINE_SHORTHANDS, + QuantizationConfigArgs, + QuantSpec, +) from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( RoutedExperts, @@ -47,6 +53,10 @@ from vllm.model_executor.layers.quantization.online.mxfp8 import ( from vllm.model_executor.layers.quantization.online.nvfp4 import ( Nvfp4OnlineMoEMethod, ) +from vllm.model_executor.layers.quantization.utils.config_utils import ( + find_matching_patterns, + get_layer_name_after_index, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kFp8Static128BlockSym, @@ -83,6 +93,39 @@ _ONLINE_MOE_METHODS: dict[QuantKey, type] = { } +def _find_matching_targets( + prefix: str, + targets: Mapping[str, str], + fused_mapping: Mapping[str, list[str]] = MappingProxyType({}), +) -> list[str]: + per_shard_matches = find_matching_patterns( + prefix, targets, fused_mapping, use_fnmatch=True + ) + if all(len(matches) == 0 for matches in per_shard_matches): + return [] + if any(len(matches) == 0 for matches in per_shard_matches): + raise ValueError( + f"Found unmatched shards for {prefix}: {per_shard_matches}. vLLM " + "requires all shards of a fused layer to match a target." + ) + if any(len(matches) > 1 for matches in per_shard_matches): + raise ValueError( + f"Found multiple quantization_config.targets matches for the " + f"shards of {prefix}: {per_shard_matches}. Each shard may match " + "at most one target." + ) + + matched_patterns = [next(iter(matches)) for matches in per_shard_matches] + quant_key_strs = {targets[pattern] for pattern in matched_patterns} + if len(quant_key_strs) > 1: + raise ValueError( + f"Found different quantization_config.targets values for the " + f"shards of {prefix}: {matched_patterns}. vLLM requires all " + "shards of a fused layer to use the same target." + ) + return [matched_patterns[0]] + + class OnlineQuantizationConfig(QuantizationConfig): """Model-level config for online quantization (quantize fp16/bf16 weights during model loading, without requiring a pre-quantized checkpoint).""" @@ -92,14 +135,44 @@ class OnlineQuantizationConfig(QuantizationConfig): args: QuantizationConfigArgs, ) -> None: super().__init__() - if args.linear is None and args.moe is None: + if args.linear is None and args.moe is None and args.targets is None: raise ValueError( "OnlineQuantizationConfig requires at least one of " - "quantization_config.linear or quantization_config.moe " - "to be set." + "quantization_config.linear, quantization_config.moe, or " + "quantization_config.targets to be set." ) self.args = args self.ignored_layers: list[str] = args.ignore + self.quantized_layers: dict[str, tuple[str, str, str | None]] = {} + + @property + def quantized_layer_summaries(self) -> list[str]: + counts: dict[tuple[str, str, str | None, str], int] = {} + for layer_name, ( + source, + quant_key_str, + target_pattern, + ) in self.quantized_layers.items(): + key = ( + get_layer_name_after_index(layer_name), + source, + target_pattern, + quant_key_str, + ) + counts[key] = counts.get(key, 0) + 1 + + summaries = [] + # Build summary entries as + # `self_attn.o_proj: 24 (from targets: re:.*self_attn\.o_proj, mxfp4` + for (layer_type, source, target_pattern, quant_key_str), count in sorted( + counts.items() + ): + pattern_prefix = f"{target_pattern}, " if target_pattern else "" + summaries.append( + f"{layer_type}: {count} " + f"(from {source}: {pattern_prefix}{quant_key_str})" + ) + return summaries @classmethod def get_name(cls) -> QuantizationMethods: @@ -154,29 +227,101 @@ class OnlineQuantizationConfig(QuantizationConfig): return cls(layer=layer) return cls() + def _dispatch_target( + self, prefix: str, layer: torch.nn.Module + ) -> "QuantizeMethodBase | None": + assert self.args.targets is not None + ignored = should_ignore_layer( + prefix, + ignore=self.ignored_layers, + fused_mapping=self.packed_modules_mapping, + use_fnmatch=True, + ) + matches = _find_matching_targets( + prefix, self.args.targets, fused_mapping=self.packed_modules_mapping + ) + if ignored and matches: + raise ValueError( + f"Layer {prefix} matches both quantization_config.ignore " + f"and quantization_config.targets ({matches}); a layer may " + f"not be referenced by both." + ) + if ignored or not matches: + return None + if len(matches) > 1: + raise ValueError( + f"Layer {prefix} matches multiple quantization_config." + f"targets patterns: {matches}. Each layer may match at most " + f"one target." + ) + quant_key_str = self.args.targets[matches[0]] + shorthand = _ONLINE_SHORTHANDS[quant_key_str] + if isinstance(layer, LinearBase): + quant_spec = shorthand.linear + table = _ONLINE_LINEAR_METHODS + elif isinstance(layer, RoutedExperts): + quant_spec = shorthand.moe + table = _ONLINE_MOE_METHODS + else: + raise ValueError( + f"Layer {prefix} was matched by quantization_config.targets " + f"({matches[0]}), but online quantization is not supported for " + f"{type(layer).__name__}." + ) + if quant_spec is None: + raise ValueError( + f"targets pattern {matches[0]} = {quant_key_str} does " + f"not define a QuantSpec for {type(layer).__name__} layers " + f"(matched at {prefix})." + ) + method = self._dispatch(quant_spec, table, layer) + if method is not None: + self.quantized_layers[prefix] = ("targets", quant_key_str, matches[0]) + return method + def get_quant_method( self, layer: torch.nn.Module, prefix: str ) -> "QuantizeMethodBase | None": + # `targets` takes precedence over `moe` and `linear` and is exclusive. + if self.args.targets is not None: + method = self._dispatch_target(prefix, layer) + if method is not None: + return method + + if isinstance(layer, LinearBase): + return UnquantizedLinearMethod() + if isinstance(layer, RoutedExperts): + return UnquantizedFusedMoEMethod(layer.moe_config) + return None + if isinstance(layer, LinearBase): if should_ignore_layer( prefix, ignore=self.ignored_layers, fused_mapping=self.packed_modules_mapping, + use_fnmatch=True, ): return UnquantizedLinearMethod() method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer) - return method if method is not None else UnquantizedLinearMethod() + if method is not None: + self.quantized_layers[prefix] = ( + "linear", + str(self.args.linear), + None, + ) + return method + return UnquantizedLinearMethod() elif isinstance(layer, RoutedExperts): if should_ignore_layer( prefix, ignore=self.ignored_layers, fused_mapping=self.packed_modules_mapping, + use_fnmatch=True, ): return UnquantizedFusedMoEMethod(layer.moe_config) method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer) - return ( - method - if method is not None - else UnquantizedFusedMoEMethod(layer.moe_config) - ) + if method is not None: + self.quantized_layers[prefix] = ("moe", str(self.args.moe), None) + return method + return UnquantizedFusedMoEMethod(layer.moe_config) return None diff --git a/vllm/model_executor/layers/quantization/quark/utils.py b/vllm/model_executor/layers/quantization/quark/utils.py index 7105561949..48c978263e 100644 --- a/vllm/model_executor/layers/quantization/quark/utils.py +++ b/vllm/model_executor/layers/quantization/quark/utils.py @@ -5,9 +5,12 @@ from collections.abc import Iterable, Mapping from types import MappingProxyType from typing import Any -import regex as re import torch +from vllm.model_executor.layers.quantization.utils.config_utils import ( + find_matching_patterns, +) + QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None @@ -49,80 +52,18 @@ def should_ignore_layer( ): return True - # layer_name = model.layers.0.self_attn.qkv_proj - # proj_name = qkv_proj - proj_name = layer_name.split(".")[-1] - - # Fused layers like gate_up_proj or qkv_proj will not be fused - # in the safetensors checkpoint. So, we convert the name - # from the fused version to unfused + check to make sure that - # each shard of the fused layer has the same scheme. - if proj_name in fused_mapping: - shard_proj_names = fused_mapping[proj_name] - - # Convert fused_name --> [shard_names] - shard_names = [ - layer_name.replace(proj_name, shard_proj_name) - for shard_proj_name in shard_proj_names - ] - - # Layer should be ignored if shards are ignored. - should_ignore_layer = None - for shard_name in shard_names: - should_ignore_shard = check_equal_or_regex_match( - layer_name=shard_name, targets=ignore - ) - - # If shard_idx=0, set layer ignore to match shard. - if should_ignore_layer is None: - should_ignore_layer = should_ignore_shard - - # If shard_idx=1+ confirm scheme matches prior shards. - elif should_ignore_shard != should_ignore_layer: - raise ValueError( - f"Found a different quantization schemes for " - f"{shard_proj_names} in {layer_name}. vLLM " - "requires all to use the same scheme." - ) - - # Unfused layers like down_proj and o_proj will match - # the safetensors checkpoint already. - else: - should_ignore_layer = check_equal_or_regex_match( - layer_name=layer_name, targets=ignore + # A direct fused-layer pattern takes precedence over expansion. For + # model.layers.0.self_attn.qkv_proj, + # ignore=["re:.*qkv_proj.*"] yields [{"re:.*qkv_proj.*"}]. In contrast, + # ignore=["re:.*[qkv]_proj"] yields one matching set per expanded shard. + per_shard_matches = find_matching_patterns(layer_name, ignore, fused_mapping) + shards_ignored = [len(matches) > 0 for matches in per_shard_matches] + if any(shards_ignored) and not all(shards_ignored): + raise ValueError( + f"Found different quantization schemes for the shards of " + f"{layer_name}. vLLM requires all to use the same scheme." ) - - assert should_ignore_layer is not None - return should_ignore_layer - - -def check_equal_or_regex_match(layer_name: str, targets: Iterable[str]) -> bool: - """ - Checks whether a layer_name is exactly equal or a regex match for - if target starts with 're:' to any target in list. - """ - return any(_is_equal_or_regex_match(layer_name, target) for target in targets) - - -def _is_equal_or_regex_match( - value: str, target: str, check_contains: bool = False -) -> bool: - """ - Checks whether a value is exactly equal or a regex match for target - if target starts with 're:'. If check_contains is set to True, - additionally checks if the target string is contained within the value. - """ - - if target.startswith("re:"): - pattern = target[3:] - if re.match(pattern, value): - return True - elif check_contains: - if target.lower() in value.lower(): - return True - elif target == value: - return True - return False + return all(shards_ignored) # utility for tensor dims > 2 cases diff --git a/vllm/model_executor/layers/quantization/utils/config_utils.py b/vllm/model_executor/layers/quantization/utils/config_utils.py index f31fdde6e9..cfee80da96 100644 --- a/vllm/model_executor/layers/quantization/utils/config_utils.py +++ b/vllm/model_executor/layers/quantization/utils/config_utils.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Iterable, Mapping +from fnmatch import fnmatch +from types import MappingProxyType from typing import TYPE_CHECKING import regex as re @@ -244,3 +247,76 @@ def is_shared_expert_quant_fse_compatible( "shared-expert FSE quantization compatibility is not implemented for " f"{type(quant_config).__name__}", ) + + +def find_matching_patterns( + layer_name: str, + patterns: Iterable[str], + fused_mapping: Mapping[str, list[str]] = MappingProxyType({}), + use_fnmatch: bool = False, +) -> list[set[str]]: + """Return matching patterns for a layer or each shard of a fused layer. + + A pattern matching the fused layer directly takes precedence. Otherwise, + return one set of matching patterns for every shard. + """ + patterns = list(patterns) + matches = [ + pattern + for pattern in patterns + if is_equal_or_regex_match(layer_name, pattern, use_fnmatch=use_fnmatch) + ] + if matches: + return [set(matches)] + + proj_name = layer_name.split(".")[-1] + if proj_name not in fused_mapping: + return [set()] + + shard_names = [ + layer_name.replace(proj_name, shard_proj_name) + for shard_proj_name in fused_mapping[proj_name] + ] + per_shard_matches = [ + { + pattern + for pattern in patterns + if is_equal_or_regex_match(shard_name, pattern, use_fnmatch=use_fnmatch) + } + for shard_name in shard_names + ] + return per_shard_matches + + +def get_layer_name_after_index(layer_name: str) -> str: + """Return the suffix following the final numeric component of a layer name.""" + parts = layer_name.split(".") + for index in range(len(parts) - 1, -1, -1): + if parts[index].isdigit(): + return ".".join(parts[index + 1 :]) + return layer_name + + +def is_equal_or_regex_match( + value: str, + target: str, + check_contains: bool = False, + use_fnmatch: bool = False, +) -> bool: + """ + Checks whether a value is exactly equal or a regex match for target + if target starts with 're:'. If check_contains is set to True, + additionally checks if the target string is contained within the value. + If use_fnmatch is set, supports shell-style patterns in target. + """ + + if target.startswith("re:"): + pattern = target[3:] + if re.match(pattern, value): + return True + elif check_contains: + if target.lower() in value.lower(): + return True + elif target == value or use_fnmatch and fnmatch(value, target): + return True + return False diff --git a/vllm/model_executor/model_loader/base_loader.py b/vllm/model_executor/model_loader/base_loader.py index d1f44666c5..3c03be2071 100644 --- a/vllm/model_executor/model_loader/base_loader.py +++ b/vllm/model_executor/model_loader/base_loader.py @@ -58,6 +58,7 @@ class BaseModelLoader(ABC): prefix=prefix, ) + log_online_quantization(vllm_config) log_model_inspection(model) logger.debug("Loading weights on %s ...", load_device) @@ -92,6 +93,23 @@ def log_model_inspection(model: nn.Module) -> None: logger.info("vLLM model structure:\n%s", format_model_inspection(model)) +def log_online_quantization(vllm_config: VllmConfig) -> None: + """Log the online-quantized layer count and types, when applicable.""" + from vllm.model_executor.layers.quantization.online.base import ( + OnlineQuantizationConfig, + ) + + quant_config = vllm_config.quant_config + if not isinstance(quant_config, OnlineQuantizationConfig): + return + + logger.info( + "Quantized %d layers of types: %s", + len(quant_config.quantized_layers), + "; ".join(quant_config.quantized_layer_summaries), + ) + + def _has_online_quant(model: nn.Module): for module in model.modules(): quant_method = getattr(module, "quant_method", None)